From 1cda7318c1f74c0ac8bf612a480d6d7cfac7fee6 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Thu, 13 Jan 2022 08:24:45 +0000 Subject: [PATCH 001/609] CHANGELOG: release v4.14.0 --- CHANGELOG.md | 3 +++ pom.xml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d077037c..f73e2538 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +## [4.14.0] - 2022-01-13 + ### Added - Mapper_Postgres_CSV_Test: Add test RMLTC1027 to check whether NULLs are ignored in PostgreSQL (see [issue 159](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/159)) - IDLabFunctions: add isNull function (see [issue 219](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/219)). @@ -433,6 +435,7 @@ and [169](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/169)) - support for accessing remote files (via HTTP GET) - basic support for functions +[4.14.0]: https://github.com/RMLio/rmlmapper-java/compare/v4.13.0...v4.14.0 [4.13.0]: https://github.com/RMLio/rmlmapper-java/compare/v4.12.0...v4.13.0 [4.12.0]: https://github.com/RMLio/rmlmapper-java/compare/v4.11.0...v4.12.0 [4.11.0]: https://github.com/RMLio/rmlmapper-java/compare/v4.10.1...v4.11.0 diff --git a/pom.xml b/pom.xml index 26eb4646..b4e567b3 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ be.ugent.rml rmlmapper RMLMapper - 4.13.0 + 4.14.0 The RMLMapper executes RML rules to generate high quality Linked Data from multiple originally (semi-)structured data sources. From e93e24cc75d22afbaf71099df07e68ece84130f9 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Thu, 13 Jan 2022 09:29:38 +0100 Subject: [PATCH 002/609] CI: add 'v' for tag creation --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8c047104..cf27002f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -142,7 +142,7 @@ Create Release: - cd "${CI_COMMIT_SHA}" # Create git tag - - git tag "$RELEASE_TAG_NAME" + - git tag "v$RELEASE_TAG_NAME" # Add all generated files to Git - git add . From fc6186610226232bfd43c8a685a39abc46c9edf2 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Thu, 13 Jan 2022 09:37:40 +0100 Subject: [PATCH 003/609] CI: skip Oracle DB tests on tags Gitlab CI bot cannot access the private Docker images --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cf27002f..e6698b76 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -56,6 +56,7 @@ Oracle DB: except: - master - development + - tags # Gitlab CI bot cannot access Docker images Docker Build: stage: unittests From b61d4d75f77db64e031c504c251a6be570a31b4c Mon Sep 17 00:00:00 2001 From: Ben De Meester Date: Thu, 3 Feb 2022 13:02:26 +0000 Subject: [PATCH 004/609] README: add quick start guide Make it clear for people how to easily start using the RMLMapper. --- CHANGELOG.md | 4 ++++ README.md | 18 +++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17527c75..d19f5e8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +### Fixed + +- Clarified Readme for quick start + ## [4.15.0] - 2022-02-01 ### Fixed diff --git a/README.md b/README.md index 52484def..272d49de 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,9 @@ The RMLMapper execute RML rules to generate Linked Data. It is a Java library, which is available via the command line ([API docs online](https://javadoc.io/doc/be.ugent.rml/rmlmapper)). The RMLMapper loads all data in memory, so be aware when working with big datasets. +Want to get started quickly? Check out [Releases](#releases) on where to find the latest CLI build as a jar, +and see [Usage](#cli) on how to use the commandline interface! + ## Table of contents - [Features](#features) @@ -21,13 +24,14 @@ The RMLMapper loads all data in memory, so be aware when working with big datase - [Generating metadata](#generating-metadata) - [Testing](#testing) - [RDBs](#rdbs) -- [Deploy on Central Repository](#deploy-on-central-repository) - [Dependencies](#dependencies) - [Commercial Support](#commercial-support) - [Remarks](#remarks) + - [Typed spreadsheet files](#typed-spreadsheet-files) - [XML file parsing performance](#xml-file-parsing-performance) - [Language tag support](#language-tag-support) - [Duplicate removal and serialization format](#duplicate-removal-and-serialization-format) + - [I have a question! Where can I get help?](#i-have-a-question-where-can-i-get-help) - [Documentation](#documentation) - [UML Diagrams](#uml-diagrams) @@ -67,11 +71,19 @@ The RMLMapper loads all data in memory, so be aware when working with big datase - TPF servers ## Releases -The standalone jar file for every release can be found on the release's page on GitHub. + +The standalone jar file (that has a [commandline interface](#cli)) for every release can be found on the release's page on GitHub. You can find the latest release [here](https://github.com/RMLio/rmlmapper-java/releases/latest). +This is the recommended way to get started with RMLMapper. +Do you want to build from source yourself? Check [Build](#build). ## Build -The RMLMapper is build using Maven: `mvn install`. +The RMLMapper is build using Maven. +As it is also tested against Oracle (check [here](#accessing-oracle-database) for details), +it needs a specific set-up to run all tests. +That's why we recommend to build without testing: `mvn install -DskipTests=true`. +If you want, you can install with tests, and just skip the Oracle tests: `mvn test -Dtest=!Mapper_OracleDB_Test`. + A standalone jar can be found in `/target`. Two jars are found in `/target`: a slim jar without bundled dependencies, and a standalone jar (suffixed with `-all.jar`) with all dependencies bundled. From fd8ce35307457400193757fff49ea1fba3b2794b Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Tue, 8 Feb 2022 16:52:14 +0100 Subject: [PATCH 005/609] pom: upgrade postgresql JDBC driver to 42.3.2 Fixes https://github.com/RMLio/rmlmapper-java/issues/146 --- CHANGELOG.md | 4 ++++ pom.xml | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d19f5e8a..ada1946f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Clarified Readme for quick start +### Changed + +- Upgrade postgresql JDBC driver to 42.3.2 (see [issue 146](https://github.com/RMLio/rmlmapper-java/issues/146)) + ## [4.15.0] - 2022-02-01 ### Fixed diff --git a/pom.xml b/pom.xml index bbb7d810..712dcad5 100644 --- a/pom.xml +++ b/pom.xml @@ -176,9 +176,9 @@ test - postgresql + org.postgresql postgresql - 9.1-901-1.jdbc4 + 42.3.2 From 17c0076cddb4d7a15d58609a137153f29f4e9b65 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Sun, 20 Feb 2022 13:54:37 +0100 Subject: [PATCH 006/609] Executor: expose rmlStore Necessary for unittests --- src/main/java/be/ugent/rml/Executor.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/be/ugent/rml/Executor.java b/src/main/java/be/ugent/rml/Executor.java index f1586d2a..2ebbabab 100644 --- a/src/main/java/be/ugent/rml/Executor.java +++ b/src/main/java/be/ugent/rml/Executor.java @@ -464,4 +464,8 @@ public static String getNewBlankNodeID() { public List getTriplesMaps() { return initializer.getTriplesMaps(); } + + public QuadStore getRMLStore() { + return this.rmlStore; + } } \ No newline at end of file From 2a94a971fbec7ef776f89c3d2f3f4b8588e91820 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Sun, 20 Feb 2022 13:55:02 +0100 Subject: [PATCH 007/609] NAMESPACES: add LDES and TREE --- src/main/java/be/ugent/rml/NAMESPACES.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/be/ugent/rml/NAMESPACES.java b/src/main/java/be/ugent/rml/NAMESPACES.java index dcf4e63f..13c9a3f2 100644 --- a/src/main/java/be/ugent/rml/NAMESPACES.java +++ b/src/main/java/be/ugent/rml/NAMESPACES.java @@ -30,4 +30,6 @@ public class NAMESPACES { public static final String RMLT = "http://semweb.mmlab.be/ns/rml-target#"; public static final String FORMATS = "http://www.w3.org/ns/formats/"; public static final String COMP = "http://semweb.mmlab.be/ns/rml-compression#"; + public static final String LDES = "https://w3id.org/ldes#"; + public static final String TREE = "https://w3id.org/tree#"; } From 468c1c5fe00a20fcac545f5028dcd56d66f4e6c8 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Sun, 20 Feb 2022 13:55:50 +0100 Subject: [PATCH 008/609] target: add LDES Logical Target An LDES Logical Target is a regular RML Logical Target with only some metadata added to encapsulate the exported RDF as an LDES. The LDES ecosystem such as the LDES Server and bucketizers can directly use this even stream thanks to this metadata. --- .../be/ugent/rml/target/LocalFileTarget.java | 13 +- .../rml/target/SPARQLEndpointTarget.java | 12 +- src/main/java/be/ugent/rml/target/Target.java | 8 ++ .../be/ugent/rml/target/TargetFactory.java | 112 +++++++++++++++++- 4 files changed, 138 insertions(+), 7 deletions(-) diff --git a/src/main/java/be/ugent/rml/target/LocalFileTarget.java b/src/main/java/be/ugent/rml/target/LocalFileTarget.java index 7fbacea3..e864e84b 100644 --- a/src/main/java/be/ugent/rml/target/LocalFileTarget.java +++ b/src/main/java/be/ugent/rml/target/LocalFileTarget.java @@ -1,6 +1,8 @@ package be.ugent.rml.target; import be.ugent.rml.access.COMPRESSION; +import be.ugent.rml.store.Quad; +import be.ugent.rml.term.Term; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -8,6 +10,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.util.List; import java.util.zip.GZIPOutputStream; import static org.apache.commons.io.FileUtils.getFile; @@ -21,6 +24,7 @@ public class LocalFileTarget implements Target { private String basePath; private String serializationFormat; private String compression; + private List metadata; private OutputStream outputStream; private static final Logger logger = LoggerFactory.getLogger(LocalFileTarget.class); @@ -30,12 +34,14 @@ public class LocalFileTarget implements Target { * @param basePath the used base path. * @param serializationFormat serialization format to use. * @param compression compression to apply. + * @param metadata additional metadata to add when writing. */ - public LocalFileTarget(String path, String basePath, String serializationFormat, String compression) { + public LocalFileTarget(String path, String basePath, String serializationFormat, String compression, List metadata) { this.path = path; this.basePath = basePath; this.serializationFormat = serializationFormat; this.compression = compression; + this.metadata = metadata; } /** @@ -119,4 +125,9 @@ public void close() { logger.error("Failed to close target: " + e); } } + + @Override + public List getMetadata() { + return this.metadata; + } } \ No newline at end of file diff --git a/src/main/java/be/ugent/rml/target/SPARQLEndpointTarget.java b/src/main/java/be/ugent/rml/target/SPARQLEndpointTarget.java index 50d52e3f..cf1fa789 100644 --- a/src/main/java/be/ugent/rml/target/SPARQLEndpointTarget.java +++ b/src/main/java/be/ugent/rml/target/SPARQLEndpointTarget.java @@ -1,5 +1,7 @@ package be.ugent.rml.target; +import be.ugent.rml.store.Quad; +import be.ugent.rml.term.Term; import org.apache.http.client.HttpResponseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -8,10 +10,12 @@ import java.net.HttpURLConnection; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.util.List; public class SPARQLEndpointTarget implements Target { private final String url; + private final List metadata; private final File tempFile; private static final Logger logger = LoggerFactory.getLogger(SPARQLEndpointTarget.class); private static final int BUFFER_SIZE = 8192; @@ -21,8 +25,9 @@ public class SPARQLEndpointTarget implements Target { * This constructor takes an URL of the SPARQL endpoint as argument. * @param url URL of the SPARQL endpoint */ - public SPARQLEndpointTarget(String url) throws IOException { + public SPARQLEndpointTarget(String url, List metadata) throws IOException { this.url = url; + this.metadata = metadata; this.tempFile = File.createTempFile("rmlmapper-", ".nt"); this.tempFile.deleteOnExit(); } @@ -120,4 +125,9 @@ public void close() { logger.error("Failed to close target: " + e); } } + + @Override + public List getMetadata() { + return this.metadata; + } } \ No newline at end of file diff --git a/src/main/java/be/ugent/rml/target/Target.java b/src/main/java/be/ugent/rml/target/Target.java index 35cb0dc8..7945af6f 100644 --- a/src/main/java/be/ugent/rml/target/Target.java +++ b/src/main/java/be/ugent/rml/target/Target.java @@ -1,7 +1,10 @@ package be.ugent.rml.target; +import be.ugent.rml.store.Quad; + import java.io.IOException; import java.io.OutputStream; +import java.util.List; /** * This interface represents the target of a knowledge graph. @@ -26,4 +29,9 @@ public interface Target { * This method closes the target. */ void close(); + + /** + * This method returns the metadata associated with the target. + */ + List getMetadata(); } diff --git a/src/main/java/be/ugent/rml/target/TargetFactory.java b/src/main/java/be/ugent/rml/target/TargetFactory.java index bd32cfe8..eb227cd1 100644 --- a/src/main/java/be/ugent/rml/target/TargetFactory.java +++ b/src/main/java/be/ugent/rml/target/TargetFactory.java @@ -2,15 +2,19 @@ import be.ugent.rml.NAMESPACES; import be.ugent.rml.Utils; +import be.ugent.rml.store.Quad; import be.ugent.rml.store.QuadStore; import be.ugent.rml.term.Literal; import be.ugent.rml.term.NamedNode; import be.ugent.rml.term.Term; import org.apache.commons.lang3.NotImplementedException; +import org.apache.poi.ss.formula.functions.Na; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; public class TargetFactory { @@ -30,11 +34,14 @@ public TargetFactory(String basePath) { * This method returns a Target instance based on the RML rules in rmlStore. * @param logicalTarget the Logical Target for which the Target needs to be created. * @param rmlStore a QuadStore with RML rules. + * @param outputStore a QuadStore with the RDF triples to write to the target. */ - public Target getTarget(Term logicalTarget, QuadStore rmlStore) throws NotImplementedException, IOException { + public Target getTarget(Term logicalTarget, QuadStore rmlStore, QuadStore outputStore) throws NotImplementedException, IOException { Target target = null; String serializationFormat = "nquads"; String compression = null; + boolean isLDES = false; + List metadata = new ArrayList<>(); // Old Logical Source reference is supported for Logical Targets as well for backwards compatibility if (logicalTarget instanceof Literal) { @@ -55,7 +62,7 @@ else if (location.endsWith(".jsonld")) { else { throw new NotImplementedException("Serialization format for " + location + " not implemented!"); } - target = new LocalFileTarget(location, this.basePath, serializationFormat, null); + target = new LocalFileTarget(location, this.basePath, serializationFormat, null, metadata); return target; } @@ -106,6 +113,101 @@ else if (location.endsWith(".jsonld")) { logger.debug("Compression disabled"); } + // Detect LDES EventStreamTarget + List types = Utils.getObjectsFromQuads(rmlStore.getQuads(logicalTarget, + new NamedNode(NAMESPACES.RDF + "type"), null)); + for (Term type: types) { + // Target has LDES features, read them + if (type.getValue().equals(NAMESPACES.LDES + "EventStreamTarget")) { + logger.debug("Found LDES EventStreamTarget"); + isLDES = true; + Term iri; + Term retention_iri; + Term eventstream_iri; + Term versionOfPathObj = null; + Term timestampPathObj = null; + Term retentionPolicyType; + Term retentionPolicyAmount; + + // Required LDES IRI + try { + iri = Utils.getObjectsFromQuads(rmlStore.getQuads(logicalTarget, + new NamedNode(NAMESPACES.LDES + "baseIRI"), null)).get(0); + retention_iri = new NamedNode(iri.getValue() + "#retention"); + eventstream_iri = new NamedNode(iri.getValue() + "#eventstream"); + logger.debug("LDES base IRI: " + iri.getValue()); + } + catch (IndexOutOfBoundsException e) { + throw new IllegalArgumentException("No base IRI specified for LDES!"); + } + + // Required SHACL shape + try { + Term shape = Utils.getObjectsFromQuads(rmlStore.getQuads(logicalTarget, + new NamedNode(NAMESPACES.TREE + "shape"), null)).get(0); + logger.debug("SHACL shape: " + shape.getValue()); + + // TODO: Handle embedded SHACL shapes in RML mapping rules as well. + metadata.add(new Quad(eventstream_iri, new NamedNode(NAMESPACES.TREE + "shape"), shape)); + } + catch (IndexOutOfBoundsException e) { + logger.error("No SHACL shape specified for LDES!"); + } + + // LDES tree:view + metadata.add(new Quad(eventstream_iri, new NamedNode(NAMESPACES.RDF + "type"), + new NamedNode(NAMESPACES.LDES + "EventStream"))); + List subjects = new ArrayList(new HashSet(Utils.getSubjectsFromQuads(outputStore.getQuads(null, null, null)))); + for (Term s: subjects) { + metadata.add(new Quad(eventstream_iri, new NamedNode(NAMESPACES.TREE + "member"), s)); + } + metadata.add(new Quad(eventstream_iri, new NamedNode(NAMESPACES.TREE + "view"), iri)); + + // Optional versionOf path + try { + versionOfPathObj = Utils.getObjectsFromQuads(rmlStore.getQuads(logicalTarget, + new NamedNode(NAMESPACES.LDES + "versionOfPath"), null)).get(0); + } + catch (IndexOutOfBoundsException e) { + logger.debug("No versionOfPath found"); + } + + // Optional timestamp path + try { + timestampPathObj = Utils.getObjectsFromQuads(rmlStore.getQuads(logicalTarget, + new NamedNode(NAMESPACES.LDES + "timestampPath"), null)).get(0); + } + catch (IndexOutOfBoundsException e) { + logger.debug("No timestampPath found"); + } + + // Optional retention policy + try { + Term retentionPolicy = Utils.getObjectsFromQuads(rmlStore.getQuads(logicalTarget, + new NamedNode(NAMESPACES.LDES + "retentionPolicy"), null)).get(0); + // TODO parse retention policies + throw new NotImplementedException("Parsing LDES retention policies not implemented yet!"); + } + catch (IndexOutOfBoundsException e) { + logger.debug("No retention policy specified, default to latest only"); + retentionPolicyType = new NamedNode(NAMESPACES.LDES + "LatestVersionSubject"); + retentionPolicyAmount = new Literal("1"); + } + + metadata.add(new Quad(iri, new NamedNode(NAMESPACES.LDES + "retentionPolicy"), retention_iri)); + metadata.add(new Quad(retention_iri, new NamedNode(NAMESPACES.RDF + "type"), retentionPolicyType)); + metadata.add(new Quad(retention_iri, new NamedNode(NAMESPACES.LDES + "amount"), retentionPolicyAmount)); + if (versionOfPathObj != null) { + metadata.add(new Quad(retention_iri, new NamedNode(NAMESPACES.LDES + "versionOfPath"), versionOfPathObj)); + } + if (timestampPathObj != null) { + metadata.add(new Quad(retention_iri, new NamedNode(NAMESPACES.LDES + "timestampPath"), timestampPathObj)); + } + + break; + } + } + // Build target if (!targets.isEmpty()) { Term t = targets.get(0); @@ -123,7 +225,7 @@ else if (location.endsWith(".jsonld")) { new NamedNode(NAMESPACES.VOID + "dataDump"), null)).get(0).getValue(); location = location.replace("file://", ""); // Local file starts with file:// logger.debug("VoID datadump location: " + location); - target = new LocalFileTarget(location, this.basePath, serializationFormat, compression); + target = new LocalFileTarget(location, this.basePath, serializationFormat, compression, metadata); break; } case NAMESPACES.DCAT + "Dataset": { // DCAT Dataset @@ -132,7 +234,7 @@ else if (location.endsWith(".jsonld")) { new NamedNode(NAMESPACES.DCAT + "dataDump"), null)).get(0).getValue(); location = location.replace("file://", ""); // Local file starts with file:// logger.debug("DCAT datadump location: " + location); - target = new LocalFileTarget(location, this.basePath, serializationFormat, compression); + target = new LocalFileTarget(location, this.basePath, serializationFormat, compression, metadata); break; } case NAMESPACES.SD + "Service": { // SPARQL Service @@ -150,7 +252,7 @@ else if (location.endsWith(".jsonld")) { } // Try to instantiate a SPARQL endpoint - target = new SPARQLEndpointTarget(endpoint); + target = new SPARQLEndpointTarget(endpoint, metadata); break; } default: { From 9febf7c34a2efb1710d6a71f489257623b0e087b Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Sun, 20 Feb 2022 13:58:48 +0100 Subject: [PATCH 009/609] test: add LDES Logical Target tests --- src/test/java/be/ugent/rml/Mapper_Target.java | 32 + .../java/be/ugent/rml/Mapper_WoT_Test.java | 21 + src/test/java/be/ugent/rml/TestCore.java | 10 + .../web-of-things/ldes/defaults/bluebike.ttl | 1142 +++++++++ .../web-of-things/ldes/defaults/mapping.ttl | 159 ++ .../ldes/defaults/out-default.nq | 0 .../ldes/defaults/out-local-file.nq | 1073 ++++++++ .../web-of-things/ldes/paths/mapping.ttl | 161 ++ .../web-of-things/ldes/paths/out-default.nq | 0 .../ldes/paths/out-local-file.nq | 1075 ++++++++ .../ldes/private-security-data.ttl | 2 + .../web-of-things/ldes/stations.jsonld | 2178 +++++++++++++++++ 12 files changed, 5853 insertions(+) create mode 100644 src/test/resources/web-of-things/ldes/defaults/bluebike.ttl create mode 100644 src/test/resources/web-of-things/ldes/defaults/mapping.ttl create mode 100644 src/test/resources/web-of-things/ldes/defaults/out-default.nq create mode 100644 src/test/resources/web-of-things/ldes/defaults/out-local-file.nq create mode 100644 src/test/resources/web-of-things/ldes/paths/mapping.ttl create mode 100644 src/test/resources/web-of-things/ldes/paths/out-default.nq create mode 100644 src/test/resources/web-of-things/ldes/paths/out-local-file.nq create mode 100644 src/test/resources/web-of-things/ldes/private-security-data.ttl create mode 100644 src/test/resources/web-of-things/ldes/stations.jsonld diff --git a/src/test/java/be/ugent/rml/Mapper_Target.java b/src/test/java/be/ugent/rml/Mapper_Target.java index 51b19161..b0f572ad 100644 --- a/src/test/java/be/ugent/rml/Mapper_Target.java +++ b/src/test/java/be/ugent/rml/Mapper_Target.java @@ -86,6 +86,38 @@ public void evaluate_local_file_target_dcat() throws Exception { webApi.stop(0); } + @Test + public void evaluate_ldes_default_target_dcat() throws Exception { + // Create Web API + HttpServer webApi = HttpServer.create(new InetSocketAddress(8000), 0); + webApi.createContext("/bluebike", new Mapper_WoT_Test.BlueBikeStationHandler()); + webApi.setExecutor(null); // creates a default executor + webApi.start(); + + HashMap outPaths = new HashMap(); + outPaths.put(new NamedNode("http://example.com/rules/#TargetDump"), "web-of-things/ldes/defaults/out-local-file.nq"); + outPaths.put(new NamedNode("rmlmapper://default.store"), "web-of-things/ldes/defaults/out-default.nq"); + doMapping("web-of-things/ldes/defaults/mapping.ttl", outPaths, "./web-of-things/ldes/private-security-data.ttl"); + + webApi.stop(0); + } + + @Test + public void evaluate_ldes_paths_target_dcat() throws Exception { + // Create Web API + HttpServer webApi = HttpServer.create(new InetSocketAddress(8000), 0); + webApi.createContext("/bluebike", new Mapper_WoT_Test.BlueBikeStationHandler()); + webApi.setExecutor(null); // creates a default executor + webApi.start(); + + HashMap outPaths = new HashMap(); + outPaths.put(new NamedNode("http://example.com/rules/#TargetDump"), "web-of-things/ldes/paths/out-local-file.nq"); + outPaths.put(new NamedNode("rmlmapper://default.store"), "web-of-things/ldes/paths/out-default.nq"); + doMapping("web-of-things/ldes/paths/mapping.ttl", outPaths, "./web-of-things/ldes/private-security-data.ttl"); + + webApi.stop(0); + } + @Test public void evaluate_nquads_serialization() throws Exception { // Create Web API diff --git a/src/test/java/be/ugent/rml/Mapper_WoT_Test.java b/src/test/java/be/ugent/rml/Mapper_WoT_Test.java index d80d9e5f..78c7ba0c 100644 --- a/src/test/java/be/ugent/rml/Mapper_WoT_Test.java +++ b/src/test/java/be/ugent/rml/Mapper_WoT_Test.java @@ -81,6 +81,27 @@ public void handle(HttpExchange t) throws IOException { } } + static class BlueBikeStationHandler implements HttpHandler { + @Override + public void handle(HttpExchange t) throws IOException { + String response = "couldn't load iRail stations JSON file"; + try { + response = Utils.fileToString(Utils.getFile("./web-of-things/ldes/stations.jsonld")); + } catch (IOException ex) { + ex.printStackTrace(); + } + List contentType = new ArrayList<>(); + contentType.add("application/json"); + + // Return stations if not redirected + t.getResponseHeaders().put("Content-Type", contentType); + t.sendResponseHeaders(200, response.length()); + OutputStream os = t.getResponseBody(); + os.write(response.getBytes()); + os.close(); + } + } + static class IRailStationHandler1 implements HttpHandler { @Override public void handle(HttpExchange t) throws IOException { diff --git a/src/test/java/be/ugent/rml/TestCore.java b/src/test/java/be/ugent/rml/TestCore.java index b8c48a53..f131ad0d 100644 --- a/src/test/java/be/ugent/rml/TestCore.java +++ b/src/test/java/be/ugent/rml/TestCore.java @@ -7,6 +7,8 @@ import be.ugent.rml.store.Quad; import be.ugent.rml.store.QuadStore; import be.ugent.rml.store.QuadStoreFactory; +import be.ugent.rml.target.Target; +import be.ugent.rml.target.TargetFactory; import be.ugent.rml.term.NamedNode; import be.ugent.rml.term.Term; import org.eclipse.rdf4j.rio.RDFFormat; @@ -251,7 +253,9 @@ void doMapping(Executor executor, String outPath) throws Exception { */ void doMapping(Executor executor, HashMap outPaths) throws Exception { logger.debug("Comparing target outputs"); + TargetFactory targetFactory = new TargetFactory("http://example.org/rules/"); HashMap results = executor.executeV5(null); + for (Map.Entry entry: outPaths.entrySet()) { Term target = entry.getKey(); String outPath = entry.getValue(); @@ -260,6 +264,12 @@ void doMapping(Executor executor, HashMap outPaths) throws Excepti logger.debug("\tSize:" + results.get(target).size()); results.get(target).removeDuplicates(); + // Targets may have additional metadata that needs to be included such as LDES encapsulation + if (!target.getValue().equals("rmlmapper://default.store")) { + Target t = targetFactory.getTarget(target, executor.getRMLStore(), results.get(target)); + results.get(target).addQuads(t.getMetadata()); + } + compareStores(filePathToStore(outPath), results.get(target)); } } diff --git a/src/test/resources/web-of-things/ldes/defaults/bluebike.ttl b/src/test/resources/web-of-things/ldes/defaults/bluebike.ttl new file mode 100644 index 00000000..f451af12 --- /dev/null +++ b/src/test/resources/web-of-things/ldes/defaults/bluebike.ttl @@ -0,0 +1,1142 @@ + + a ; + 11; + 11; + 0; + ; + "2021-10-12T06:20:08.208Z"^^; + ; + "50.942901611328"^^; + "4.038559913635"^^; + "Aalst" . + + a ; + 6; + 6; + 0; + ; + "2021-10-12T06:20:44.789Z"^^; + ; + "51.092498779297"^^; + "3.449820041656"^^; + "Aalter" . + + a ; + 9; + 14; + 5; + ; + "2021-10-12T06:20:12.927Z"^^; + ; + "50.984699249268"^^; + "4.824250221252"^^; + "Aarschot" . + + a ; + 51; + 60; + 9; + ; + "2021-10-12T06:20:11.221Z"^^; + ; + "51.199600219727"^^; + "4.431789875031"^^; + "Antwerpen-Berchem" . + + a + ; + 31; + 43; + 12; + ; + "2021-10-12T06:20:06.338Z"^^; + ; + "51.217498779297"^^; + "4.421050071716"^^; + "Antwerpen-Centraal" . + + a + ; + 5; + 5; + 0; + ; + "2021-10-12T06:21:00.798Z"^^; + ; + "51.245498657227"^^; + "4.424610137939"^^; + "Antwerpen-Luchtbal" . + + + a ; + 7; + 7; + 0; + ; + "2021-10-12T06:21:01.494Z"^^; + ; + "51.2617"^^; + "4.42755"^^; + "Antwerpen-Noorderdokken" . + + a ; + 8; + 8; + 0; + ; + "2021-10-12T06:20:51.829Z"^^; + ; + "51.198001861572"^^; + "4.390810012817"^^; + "Antwerpen-Zuid" . + + a ; + 7; + 10; + 3; + ; + "2021-10-12T06:20:09.208Z"^^; + ; + "50.906700134277"^^; + "4.207960128784"^^; + "Asse" . + + a ; + 5; + 5; + 0; + ; + "2021-10-12T06:20:09.940Z"^^; + ; + "50.455600738525"^^; + "3.944180011749"^^; + "Bergen" . + + a ; + 10; + 11; + 1; + ; + "2021-10-12T06:20:55.643Z"^^; + ; + "51.208499908447"^^; + "4.260260105133"^^; + "Beveren" . + + a ; + 6; + 7; + 1; + ; + "2021-10-12T06:21:14.201Z"^^; + ; + "51.3119"^^; + "3.1338"^^; + "Blankenberge" . + + + a ; + 5; + 5; + 0; + ; + "2021-10-12T06:20:44.070Z"^^; + ; + "51.16849899292"^^; + "4.494080066681"^^; + "Boechout P+R Capenberg" . + + a ; + 6; + 6; + 0; + ; + "2021-10-12T06:21:05.822Z"^^; + ; + "51.165508"^^; + "4.746427"^^; + "Bouwel" . + + a ; + 34; + 63; + 29; + ; + "2021-10-12T06:20:10.484Z"^^; + ; + "51.196998596191"^^; + "3.218539953232"^^; + "Brugge" . + + a ; + 17; + 27; + 10; + ; + "2021-10-12T06:20:54.646Z"^^; + ; + "51.196800231934"^^; + "3.216079950333"^^; + "Brugge (Kamgebouw)" . + + a ; + 13; + 14; + 1; + ; + "2021-10-12T06:20:07.205Z"^^; + ; + "50.844898223877"^^; + "4.35586977005"^^; + "Brussel-Centraal" . + + a ; + 4; + 6; + 2; + ; + "2021-10-12T06:20:13.845Z"^^; + ; + "50.838199615479"^^; + "4.372889995575"^^; + "Brussel-Luxemburg" . + + a ; + 4; + 6; + 2; + ; + "2021-10-12T06:21:19.556Z"^^; + ; + "50.835529468767"^^; + "4.333756181866"^^; + "Brussel-Zuid" . + + a ; + 27; + 28; + 1; + ; + "2021-10-12T06:20:14.572Z"^^; + ; + "50.86009979248"^^; + "4.360489845276"^^; + "Brussel Noord" . + + a ; + 8; + 8; + 0; + ; + "2021-10-12T06:21:13.294Z"^^; + ; + "51.077376"^^; + "2.601611"^^; + "De Panne" . + + a ; + 4; + 62; + 58; + ; + "2021-10-12T06:20:15.386Z"^^; + ; + "50.978199005127"^^; + "3.534009933472"^^; + "Deinze" . + + a ; + 9; + 11; + 2; + ; + "2021-10-12T06:20:45.522Z"^^; + ; + "50.982398986816"^^; + "3.525919914246"^^; + "Deinze Leiespiegel" . + + a ; + 22; + 27; + 5; + ; + "2021-10-12T06:20:16.315Z"^^; + ; + "51.023498535156"^^; + "4.101920127869"^^; + "Dendermonde" . + + a ; + 4; + 4; + 0; + ; + "2021-10-12T06:21:02.217Z"^^; + ; + "50.889099121094"^^; + "4.447710037231"^^; + "Diegem" . + + a ; + 10; + 12; + 2; + ; + "2021-10-12T06:20:17.089Z"^^; + ; + "50.993000030518"^^; + "5.051129817963"^^; + "Diest" . + + a ; + 1; + 11; + 10; + ; + "2021-10-12T06:20:34.600Z"^^; + ; + "51.180801391602"^^; + "3.575040102005"^^; + "Eeklo" . + + a + ; + 3; + 4; + 1; + ; + "2021-10-12T06:20:59.366Z"^^; + ; + "51.284198760986"^^; + "4.43501996994"^^; + "Ekeren (perron 1)" . + + a + ; + 7; + 7; + 0; + ; + "2021-10-12T06:21:00.100Z"^^; + ; + "51.281398773193"^^; + "4.434390068054"^^; + "Ekeren (perron 2)" . + + a ; + 7; + 8; + 1; + ; + "2021-10-12T06:21:11.779Z"^^; + ; + "51.462176"^^; + "4.451532"^^; + "Essen" . + + a ; + 7; + 7; + 0; + ; + "2021-10-12T06:20:57.946Z"^^; + ; + "50.929401397705"^^; + "3.64013004303"^^; + "Gavere-Asper" . + + a ; + 4; + 4; + 0; + ; + "2021-10-12T06:20:58.653Z"^^; + ; + "50.930500030518"^^; + "3.654690027237"^^; + "Gavere mobipunt" . + + a ; + 8; + 15; + 7; + ; + "2021-10-12T06:20:35.295Z"^^; + ; + "51.168800354004"^^; + "4.989160060883"^^; + "Geel" . + + a ; + 5; + 6; + 1; + ; + "2021-10-12T06:20:17.608Z"^^; + ; + "50.967098236084"^^; + "5.496369838715"^^; + "Genk" . + + a ; + 31; + 41; + 10; + ; + "2021-10-12T06:20:18.137Z"^^; + ; + "51.055999755859"^^; + "3.740159988403"^^; + "Gent-Dampoort" . + + + a ; + 16; + 26; + 10; + ; + "2021-10-12T06:20:53.258Z"^^; + ; + "51.035999298096"^^; + "3.711899995804"^^; + "Gent-Sint-Pieters (M. Hendrikaplein)" . + + + a ; + 52; + 70; + 18; + ; + "2021-10-12T06:20:18.962Z"^^; + ; + "51.034900665283"^^; + "3.709640026093"^^; + "Gent-Sint-Pieters (St.-Denijslaan)" . + + a ; + 2; + 2; + 0; + ; + "2021-10-12T06:20:39.838Z"^^; + ; + "50.770500183105"^^; + "3.872560024261"^^; + "Geraardsbergen " . + + a ; + 7; + 8; + 1; + ; + "2021-10-12T06:21:06.544Z"^^; + ; + "50.966578"^^; + "4.613315"^^; + "Haacht" . + + a ; + 7; + 12; + 5; + ; + "2021-10-12T06:20:19.671Z"^^; + ; + "50.7333984375"^^; + "4.239940166473"^^; + "Halle" . + + a ; + 8; + 9; + 1; + ; + "2021-10-12T06:20:57.256Z"^^; + ; + "50.855701446533"^^; + "3.31374001503"^^; + "Harelbeke" . + + a ; + 35; + 59; + 24; + ; + "2021-10-12T06:20:20.529Z"^^; + ; + "50.930999755859"^^; + "5.32742023468"^^; + "Hasselt" . + + a ; + 11; + 12; + 1; + ; + "2021-10-12T06:20:48.339Z"^^; + ; + "51.365100860596"^^; + "4.460559844971"^^; + "Heide" . + + a ; + 5; + 5; + 0; + ; + "2021-10-12T06:21:15.934Z"^^; + ; + "51.334168"^^; + "3.239668"^^; + "Heist" . + + a ; + 12; + 12; + 0; + ; + "2021-10-12T06:20:36.712Z"^^; + ; + "51.074199676514"^^; + "4.708779811859"^^; + "Heist-op-den-Berg" . + + a ; + 11; + 12; + 1; + ; + "2021-10-12T06:20:21.475Z"^^; + ; + "51.180999755859"^^; + "4.828859806061"^^; + "Herentals" . + + a ; + 8; + 8; + 0; + ; + "2021-10-12T06:21:03.654Z"^^; + ; + "51.03833"^^; + "5.281699"^^; + "Heusden" . + + a ; + 7; + 9; + 2; + ; + "2021-10-12T06:20:56.370Z"^^; + ; + "50.862598419189"^^; + "4.695439815521"^^; + "Heverlee " . + + a ; + 7; + 7; + 0; + ; + "2021-10-12T06:21:10.807Z"^^; + ; + "51.1541195"^^; + "4.464733"^^; + "Hove" . + + a ; + 5; + 10; + 5; + ; + "2021-10-12T06:20:41.845Z"^^; + ; + "50.848400115967"^^; + "2.876270055771"^^; + "Ieper" . + + a ; + 6; + 8; + 2; + ; + "2021-10-12T06:20:52.530Z"^^; + ; + "50.9208984375"^^; + "3.214819908142"^^; + "Izegem" . + + a ; + 7; + 8; + 1; + ; + "2021-10-12T06:21:16.629Z"^^; + ; + "51.34018"^^; + "3.28454"^^; + "Knokke" . + + a ; + 5; + 7; + 2; + ; + "2021-10-12T06:20:49.729Z"^^; + ; + "51.134300231934"^^; + "4.476119995117"^^; + "Kontich-Lint" . + + a ; + 28; + 47; + 19; + ; + "2021-10-12T06:20:21.952Z"^^; + ; + "50.823799133301"^^; + "3.263269901276"^^; + "Kortrijk" . + + a ; + 8; + 8; + 0; + ; + "2021-10-12T06:21:18.066Z"^^; + ; + "51.064208"^^; + "3.575647"^^; + "Landegem" . + + a ; + 7; + 8; + 1; + ; + "2021-10-12T06:21:12.535Z"^^; + ; + "50.74829"^^; + "5.079402"^^; + "Landen" . + + + a ; + 16; + 24; + 8; + ; + "2021-10-12T06:20:53.955Z"^^; + ; + "50.882499694824"^^; + "4.716639995575"^^; + "Leuven (Kop van Kessel-Lo)" . + + a + ; + 57; + 86; + 29; + ; + "2021-10-12T06:20:22.664Z"^^; + ; + "50.87979888916"^^; + "4.715939998627"^^; + "Leuven (Tiensevest)" . + + a ; + 21; + 26; + 5; + ; + "2021-10-12T06:20:23.383Z"^^; + ; + "51.135799407959"^^; + "4.56152009964"^^; + "Lier" . + + + a ; + 8; + 8; + 0; + ; + "2021-10-12T06:20:43.366Z"^^; + ; + "51.125301361084"^^; + "4.573339939117"^^; + "Lier P+R De Mol" . + + a ; + 7; + 7; + 0; + ; + "2021-10-12T06:20:42.586Z"^^; + ; + "51.128299713135"^^; + "4.583220005035"^^; + "Lier Veemarkt" . + + a ; + 13; + 17; + 4; + ; + "2021-10-12T06:20:24.095Z"^^; + ; + "51.107398986816"^^; + "3.986469984055"^^; + "Lokeren" . + + a ; + 6; + 6; + 0; + ; + "2021-10-12T06:21:10.091Z"^^; + ; + "51.21177"^^; + "5.312579"^^; + "Lommel" . + + a ; + 0; + 4; + 4; + ; + "2021-10-12T06:20:24.831Z"^^; + ; + "50.625801086426"^^; + "5.565639972687"^^; + "Luik" . + + a ; + 47; + 76; + 29; + ; + "2021-10-12T06:20:25.537Z"^^; + ; + "51.0178"^^; + "4.48299"^^; + "Mechelen" . + + + a ; + 17; + 20; + 3; + ; + "2021-10-12T06:20:38.128Z"^^; + ; + "51.030200958252"^^; + "4.489570140839"^^; + "Mechelen-Nekkerspoel" . + + a ; + 6; + 7; + 1; + ; + "2021-10-12T06:20:49.040Z"^^; + ; + "51.029300689697"^^; + "4.484320163727"^^; + "Mechelen Veemarkt" . + + a ; + 8; + 8; + 0; + ; + "2021-10-12T06:21:05.094Z"^^; + ; + "50.932948204381"^^; + "4.329820851913"^^; + "Meise" . + + a ; + 10; + 15; + 5; + ; + "2021-10-12T06:20:26.232Z"^^; + ; + "51.190498352051"^^; + "5.115129947662"^^; + "Mol" . + + a ; + 7; + 7; + 0; + ; + "2021-10-12T06:21:20.277Z"^^; + ; + "51.18119"^^; + "5.112421"^^; + "Mol - De Nete" . + + a ; + 4; + 6; + 2; + ; + "2021-10-12T06:20:26.934Z"^^; + ; + "51.171199798584"^^; + "4.455999851227"^^; + "Mortsel-Oude-God" . + + a ; + 6; + 6; + 0; + ; + "2021-10-12T06:20:27.637Z"^^; + ; + "50.468601226807"^^; + "4.862239837646"^^; + "Namen" . + + a ; + 7; + 10; + 3; + ; + "2021-10-12T06:20:36.006Z"^^; + ; + "50.839500427246"^^; + "4.026179790497"^^; + "Ninove" . + + a ; + 10; + 10; + 0; + ; + "2021-10-12T06:20:47.633Z"^^; + ; + "51.35710144043"^^; + "4.632410049438"^^; + "Noorderkempen" . + + a ; + 53; + 69; + 16; + ; + "2021-10-12T06:20:28.362Z"^^; + ; + "51.228298187256"^^; + "2.925649881363"^^; + "Oostende" . + + a ; + 0; + 2; + 2; + ; + "2021-10-12T06:20:39.151Z"^^; + ; + "50.851299285889"^^; + "3.601880073547"^^; + "Oudenaarde " . + + a ; + 7; + 7; + 0; + ; + "2021-10-12T06:20:46.915Z"^^; + ; + "50.854499816895"^^; + "2.735719919205"^^; + "Poperinge" . + + a ; + 14; + 27; + 13; + ; + "2021-10-12T06:20:29.065Z"^^; + ; + "50.949001312256"^^; + "3.130480051041"^^; + "Roeselare" . + + a + ; + 9; + 9; + 0; + ; + "2021-10-12T06:21:17.346Z"^^; + ; + "50.747786"^^; + "4.361626"^^; + "Sint-Genesius-Rode" . + + a ; + 15; + 30; + 15; + ; + "2021-10-12T06:20:29.783Z"^^; + ; + "51.171398162842"^^; + "4.143700122833"^^; + "Sint-Niklaas" . + + a ; + 9; + 9; + 0; + ; + "2021-10-12T06:20:30.506Z"^^; + ; + "50.817401885986"^^; + "5.176330089569"^^; + "Sint-Truiden" . + + a ; + 5; + 5; + 0; + ; + "2021-10-12T06:21:07.957Z"^^; + ; + "51.125867"^^; + "4.221333"^^; + "Temse" . + + a ; + 7; + 7; + 0; + ; + "2021-10-12T06:20:50.446Z"^^; + ; + "50.990600585938"^^; + "3.329819917679"^^; + "Tielt" . + + a ; + 5; + 13; + 8; + ; + "2021-10-12T06:20:46.217Z"^^; + ; + "50.808200836182"^^; + "4.92561006546"^^; + "Tienen " . + + a ; + 8; + 8; + 0; + ; + "2021-10-12T06:20:37.434Z"^^; + ; + "50.78450012207"^^; + "5.473400115967"^^; + "Tongeren" . + + a ; + 3; + 4; + 1; + ; + "2021-10-12T06:20:31.284Z"^^; + ; + "51.064800262451"^^; + "3.105509996414"^^; + "Torhout" . + + a ; + 9; + 13; + 4; + ; + "2021-10-12T06:20:32.245Z"^^; + ; + "51.322101593018"^^; + "4.937900066376"^^; + "Turnhout" . + + a ; + 20; + 40; + 20; + ; + "2021-10-12T06:20:33.067Z"^^; + ; + "50.924701690674"^^; + "4.433000087738"^^; + "Vilvoorde" . + + + a ; + 6; + 7; + 1; + ; + "2021-10-12T06:21:18.772Z"^^; + ; + "50.923187"^^; + "4.413591"^^; + "Vilvoorde Hoppin De Kassei" . + + a ; + 12; + 16; + 4; + ; + "2021-10-12T06:20:33.881Z"^^; + ; + "50.891799926758"^^; + "3.424420118332"^^; + "Waregem" . + + a + ; + 4; + 4; + 0; + ; + "2021-10-12T06:21:02.941Z"^^; + ; + "50.881999969482"^^; + "3.426719903946"^^; + "Waregem Expo P+R" . + + a ; + 8; + 8; + 0; + ; + "2021-10-12T06:21:09.366Z"^^; + ; + "50.810926"^^; + "3.182521"^^; + "Wevelgem" . + + a ; + 8; + 8; + 0; + ; + "2021-10-12T06:20:51.139Z"^^; + ; + "50.885200500488"^^; + "4.470870018005"^^; + "Zaventem" . + + a ; + 2; + 2; + 0; + ; + "2021-10-12T06:21:14.998Z"^^; + ; + "51.32651"^^; + "3.195476"^^; + "Zeebrugge-dorp" . + + a ; + 6; + 7; + 1; + ; + "2021-10-12T06:21:08.664Z"^^; + ; + "51.073108"^^; + "4.041885"^^; + "Zele" . + + a ; + 8; + 8; + 0; + ; + "2021-10-12T06:21:04.377Z"^^; + ; + "51.033617"^^; + "5.329922"^^; + "Zolder" . + + a ; + 7; + 8; + 1; + ; + "2021-10-12T06:21:07.245Z"^^; + ; + "50.86948"^^; + "3.814341"^^; + "Zottegem " . + + + . + + a ; + , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , + , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , + , , + , , + , , + , ; + . + + a ; + "1" . diff --git a/src/test/resources/web-of-things/ldes/defaults/mapping.ttl b/src/test/resources/web-of-things/ldes/defaults/mapping.ttl new file mode 100644 index 00000000..8b1c6f8e --- /dev/null +++ b/src/test/resources/web-of-things/ldes/defaults/mapping.ttl @@ -0,0 +1,159 @@ +# +# RML mapping rules for https://datapiloten.be/bluebike/availabilities.geojson +# (c) Dylan Van Assche (2021) +# IDLab - Ghent University - imec +# +@prefix rr: . +@prefix ex: . +@prefix rdfs: . +@prefix xsd: . +@prefix rml: . +@prefix rmlt: . +@prefix comp: . +@prefix formats: . +@prefix ql: . +@prefix rdf: . +@prefix geo: . +@prefix fnml: . +@prefix fno: . +@prefix grel: . +@prefix dcterms: . +@prefix tree: . +@prefix ldes: . +@prefix purl: . +@prefix td: . +@prefix htv: . +@prefix hctl: . +@base . + +<#TargetDump> a rmlt:LogicalTarget, ldes:EventStreamTarget; + rmlt:target [ a void:Dataset; + void:dataDump ; + ]; + rmlt:serialization formats:N-Triples; + ldes:baseIRI ; + tree:shape ; +. + +<#WoTWebResource> a td:PropertyAffordance; + td:hasForm [ + # URL and content type + hctl:hasTarget "http://localhost:8000/bluebike"; + hctl:forContentType "application/json"; + # Read only + hctl:hasOperationType td:readproperty ; + # Set HTTP method and headers + htv:methodName "GET"; + htv:headers ([ + htv:fieldName "User-Agent"; + htv:fieldValue "RML Processor"; + ]); + ]; +. + +<#WoTWebAPI> a td:Thing; + td:hasPropertyAffordance <#WoTWebResource>; +. + +<#StationsTriplesMap> a rr:TriplesMap; + rml:logicalSource [ + rml:source <#WoTWebResource>; + rml:referenceFormulation ql:JSONPath; + rml:iterator "$.features[*]" + ]; + + # Unique IRI generation: $stationId#$generatedAtTime + rr:subjectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant grel:array_join ] + ]; + rr:predicateObjectMap [ + rr:predicate grel:p_array_a ; + rr:objectMap [ rml:reference "properties.@id" ] + ]; + rr:predicateObjectMap [ + rr:predicate grel:p_array_a ; + rr:objectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant grel:array_join ] + ]; + + rr:predicateObjectMap [ + rr:predicate grel:p_array_a ; + rr:objectMap [ rr:constant "#" ] + ]; + + rr:predicateObjectMap [ + rr:predicate grel:p_array_a ; + rr:objectMap [ rml:reference "generatedAtTime" ] + ]; + ]; + ]; + ]; + ]; + rr:termType rr:IRI; + rml:logicalTarget <#TargetDump>; + ]; + + # rdf:type + rr:predicateObjectMap [ + rr:predicate rdf:type ; + rr:objectMap [ rml:reference "properties.@type"; rr:termType rr:IRI ] + ]; + + # BlueBike station: name + rr:predicateObjectMap [ + rr:predicate rdfs:label ; + rr:objectMap [ rml:reference "properties.name"; rr:datatype xsd:string; ] + ]; + + # BlueBike station: available bicycles + rr:predicateObjectMap [ + rr:predicate ex:bikesAvailable ; + rr:objectMap [ rml:reference "properties.bikes_available"; rr:datatype xsd:integer; ] + ]; + + # BlueBike station: maximum capacity + rr:predicateObjectMap [ + rr:predicate ex:capacity ; + rr:objectMap [ rml:reference "properties.capacity"; rr:datatype xsd:integer; ] + ]; + + # BlueBike station: available docks + rr:predicateObjectMap [ + rr:predicate ex:docksAvailable ; + rr:objectMap [ rml:reference "properties.docks_available"; rr:datatype xsd:integer; ] + ]; + + # Nearby NMBS station + rr:predicateObjectMap [ + rr:predicate ex:nearby ; + rr:objectMap [ rml:reference "properties.nearby"; rr:termType rr:IRI; ] + ]; + + # Geo location + rr:predicateObjectMap [ + rr:predicate geo:latitude ; + rr:objectMap [ rml:reference "properties.latitude"; rr:datatype xsd:float; ] + ]; + + rr:predicateObjectMap [ + rr:predicate geo:longitude ; + rr:objectMap [ rml:reference "properties.longitude"; rr:datatype xsd:float; ] + ]; + + # Versioning + rr:predicateObjectMap [ + rr:predicate dcterms:created ; + rr:objectMap [ rml:reference "generatedAtTime"; rr:datatype xsd:dateTime; ] + ]; + + rr:predicateObjectMap [ + rr:predicate dcterms:isVersionOf ; + rr:objectMap [ rml:reference "properties.@id"; rr:termType rr:IRI; ] + ]; +. diff --git a/src/test/resources/web-of-things/ldes/defaults/out-default.nq b/src/test/resources/web-of-things/ldes/defaults/out-default.nq new file mode 100644 index 00000000..e69de29b diff --git a/src/test/resources/web-of-things/ldes/defaults/out-local-file.nq b/src/test/resources/web-of-things/ldes/defaults/out-local-file.nq new file mode 100644 index 00000000..7f1c789e --- /dev/null +++ b/src/test/resources/web-of-things/ldes/defaults/out-local-file.nq @@ -0,0 +1,1073 @@ + "11"^^ . + "11"^^ . + "0"^^ . + . + "2021-10-12T06:20:08.208Z"^^ . + . + . + "50.942901611328"^^ . + "4.038559913635"^^ . + "Aalst" . + "6"^^ . + "6"^^ . + "0"^^ . + . + "2021-10-12T06:20:44.789Z"^^ . + . + . + "51.092498779297"^^ . + "3.449820041656"^^ . + "Aalter" . + "9"^^ . + "14"^^ . + "5"^^ . + . + "2021-10-12T06:20:12.927Z"^^ . + . + . + "50.984699249268"^^ . + "4.824250221252"^^ . + "Aarschot" . + "51"^^ . + "60"^^ . + "9"^^ . + . + "2021-10-12T06:20:11.221Z"^^ . + . + . + "51.199600219727"^^ . + "4.431789875031"^^ . + "Antwerpen-Berchem" . + "31"^^ . + "43"^^ . + "12"^^ . + . + "2021-10-12T06:20:06.338Z"^^ . + . + . + "51.217498779297"^^ . + "4.421050071716"^^ . + "Antwerpen-Centraal" . + "5"^^ . + "5"^^ . + "0"^^ . + . + "2021-10-12T06:21:00.798Z"^^ . + . + . + "51.245498657227"^^ . + "4.424610137939"^^ . + "Antwerpen-Luchtbal" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:21:01.494Z"^^ . + . + . + "51.2617"^^ . + "4.42755"^^ . + "Antwerpen-Noorderdokken" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:20:51.829Z"^^ . + . + . + "51.198001861572"^^ . + "4.390810012817"^^ . + "Antwerpen-Zuid" . + "7"^^ . + "10"^^ . + "3"^^ . + . + "2021-10-12T06:20:09.208Z"^^ . + . + . + "50.906700134277"^^ . + "4.207960128784"^^ . + "Asse" . + "5"^^ . + "5"^^ . + "0"^^ . + . + "2021-10-12T06:20:09.940Z"^^ . + . + . + "50.455600738525"^^ . + "3.944180011749"^^ . + "Bergen" . + "10"^^ . + "11"^^ . + "1"^^ . + . + "2021-10-12T06:20:55.643Z"^^ . + . + . + "51.208499908447"^^ . + "4.260260105133"^^ . + "Beveren" . + "6"^^ . + "7"^^ . + "1"^^ . + . + "2021-10-12T06:21:14.201Z"^^ . + . + . + "51.3119"^^ . + "3.1338"^^ . + "Blankenberge" . + "5"^^ . + "5"^^ . + "0"^^ . + . + "2021-10-12T06:20:44.070Z"^^ . + . + . + "51.16849899292"^^ . + "4.494080066681"^^ . + "Boechout P+R Capenberg" . + "6"^^ . + "6"^^ . + "0"^^ . + . + "2021-10-12T06:21:05.822Z"^^ . + . + . + "51.165508"^^ . + "4.746427"^^ . + "Bouwel" . + "34"^^ . + "63"^^ . + "29"^^ . + . + "2021-10-12T06:20:10.484Z"^^ . + . + . + "51.196998596191"^^ . + "3.218539953232"^^ . + "Brugge" . + "17"^^ . + "27"^^ . + "10"^^ . + . + "2021-10-12T06:20:54.646Z"^^ . + . + . + "51.196800231934"^^ . + "3.216079950333"^^ . + "Brugge (Kamgebouw)" . + "13"^^ . + "14"^^ . + "1"^^ . + . + "2021-10-12T06:20:07.205Z"^^ . + . + . + "50.844898223877"^^ . + "4.35586977005"^^ . + "Brussel-Centraal" . + "4"^^ . + "6"^^ . + "2"^^ . + . + "2021-10-12T06:20:13.845Z"^^ . + . + . + "50.838199615479"^^ . + "4.372889995575"^^ . + "Brussel-Luxemburg" . + "4"^^ . + "6"^^ . + "2"^^ . + . + "2021-10-12T06:21:19.556Z"^^ . + . + . + "50.835529468767"^^ . + "4.333756181866"^^ . + "Brussel-Zuid" . + "27"^^ . + "28"^^ . + "1"^^ . + . + "2021-10-12T06:20:14.572Z"^^ . + . + . + "50.86009979248"^^ . + "4.360489845276"^^ . + "Brussel Noord" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:13.294Z"^^ . + . + . + "51.077376"^^ . + "2.601611"^^ . + "De Panne" . + "4"^^ . + "62"^^ . + "58"^^ . + . + "2021-10-12T06:20:15.386Z"^^ . + . + . + "50.978199005127"^^ . + "3.534009933472"^^ . + "Deinze" . + "9"^^ . + "11"^^ . + "2"^^ . + . + "2021-10-12T06:20:45.522Z"^^ . + . + . + "50.982398986816"^^ . + "3.525919914246"^^ . + "Deinze Leiespiegel" . + "22"^^ . + "27"^^ . + "5"^^ . + . + "2021-10-12T06:20:16.315Z"^^ . + . + . + "51.023498535156"^^ . + "4.101920127869"^^ . + "Dendermonde" . + "4"^^ . + "4"^^ . + "0"^^ . + . + "2021-10-12T06:21:02.217Z"^^ . + . + . + "50.889099121094"^^ . + "4.447710037231"^^ . + "Diegem" . + "10"^^ . + "12"^^ . + "2"^^ . + . + "2021-10-12T06:20:17.089Z"^^ . + . + . + "50.993000030518"^^ . + "5.051129817963"^^ . + "Diest" . + "1"^^ . + "11"^^ . + "10"^^ . + . + "2021-10-12T06:20:34.600Z"^^ . + . + . + "51.180801391602"^^ . + "3.575040102005"^^ . + "Eeklo" . + "3"^^ . + "4"^^ . + "1"^^ . + . + "2021-10-12T06:20:59.366Z"^^ . + . + . + "51.284198760986"^^ . + "4.43501996994"^^ . + "Ekeren (perron 1)" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:21:00.100Z"^^ . + . + . + "51.281398773193"^^ . + "4.434390068054"^^ . + "Ekeren (perron 2)" . + "7"^^ . + "8"^^ . + "1"^^ . + . + "2021-10-12T06:21:11.779Z"^^ . + . + . + "51.462176"^^ . + "4.451532"^^ . + "Essen" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:20:57.946Z"^^ . + . + . + "50.929401397705"^^ . + "3.64013004303"^^ . + "Gavere-Asper" . + "4"^^ . + "4"^^ . + "0"^^ . + . + "2021-10-12T06:20:58.653Z"^^ . + . + . + "50.930500030518"^^ . + "3.654690027237"^^ . + "Gavere mobipunt" . + "8"^^ . + "15"^^ . + "7"^^ . + . + "2021-10-12T06:20:35.295Z"^^ . + . + . + "51.168800354004"^^ . + "4.989160060883"^^ . + "Geel" . + "5"^^ . + "6"^^ . + "1"^^ . + . + "2021-10-12T06:20:17.608Z"^^ . + . + . + "50.967098236084"^^ . + "5.496369838715"^^ . + "Genk" . + "31"^^ . + "41"^^ . + "10"^^ . + . + "2021-10-12T06:20:18.137Z"^^ . + . + . + "51.055999755859"^^ . + "3.740159988403"^^ . + "Gent-Dampoort" . + "16"^^ . + "26"^^ . + "10"^^ . + . + "2021-10-12T06:20:53.258Z"^^ . + . + . + "51.035999298096"^^ . + "3.711899995804"^^ . + "Gent-Sint-Pieters (M. Hendrikaplein)" . + "52"^^ . + "70"^^ . + "18"^^ . + . + "2021-10-12T06:20:18.962Z"^^ . + . + . + "51.034900665283"^^ . + "3.709640026093"^^ . + "Gent-Sint-Pieters (St.-Denijslaan)" . + "2"^^ . + "2"^^ . + "0"^^ . + . + "2021-10-12T06:20:39.838Z"^^ . + . + . + "50.770500183105"^^ . + "3.872560024261"^^ . + "Geraardsbergen " . + "7"^^ . + "8"^^ . + "1"^^ . + . + "2021-10-12T06:21:06.544Z"^^ . + . + . + "50.966578"^^ . + "4.613315"^^ . + "Haacht" . + "7"^^ . + "12"^^ . + "5"^^ . + . + "2021-10-12T06:20:19.671Z"^^ . + . + . + "50.7333984375"^^ . + "4.239940166473"^^ . + "Halle" . + "8"^^ . + "9"^^ . + "1"^^ . + . + "2021-10-12T06:20:57.256Z"^^ . + . + . + "50.855701446533"^^ . + "3.31374001503"^^ . + "Harelbeke" . + "35"^^ . + "59"^^ . + "24"^^ . + . + "2021-10-12T06:20:20.529Z"^^ . + . + . + "50.930999755859"^^ . + "5.32742023468"^^ . + "Hasselt" . + "11"^^ . + "12"^^ . + "1"^^ . + . + "2021-10-12T06:20:48.339Z"^^ . + . + . + "51.365100860596"^^ . + "4.460559844971"^^ . + "Heide" . + "5"^^ . + "5"^^ . + "0"^^ . + . + "2021-10-12T06:21:15.934Z"^^ . + . + . + "51.334168"^^ . + "3.239668"^^ . + "Heist" . + "12"^^ . + "12"^^ . + "0"^^ . + . + "2021-10-12T06:20:36.712Z"^^ . + . + . + "51.074199676514"^^ . + "4.708779811859"^^ . + "Heist-op-den-Berg" . + "11"^^ . + "12"^^ . + "1"^^ . + . + "2021-10-12T06:20:21.475Z"^^ . + . + . + "51.180999755859"^^ . + "4.828859806061"^^ . + "Herentals" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:03.654Z"^^ . + . + . + "51.03833"^^ . + "5.281699"^^ . + "Heusden" . + "7"^^ . + "9"^^ . + "2"^^ . + . + "2021-10-12T06:20:56.370Z"^^ . + . + . + "50.862598419189"^^ . + "4.695439815521"^^ . + "Heverlee " . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:21:10.807Z"^^ . + . + . + "51.1541195"^^ . + "4.464733"^^ . + "Hove" . + "5"^^ . + "10"^^ . + "5"^^ . + . + "2021-10-12T06:20:41.845Z"^^ . + . + . + "50.848400115967"^^ . + "2.876270055771"^^ . + "Ieper" . + "6"^^ . + "8"^^ . + "2"^^ . + . + "2021-10-12T06:20:52.530Z"^^ . + . + . + "50.9208984375"^^ . + "3.214819908142"^^ . + "Izegem" . + "7"^^ . + "8"^^ . + "1"^^ . + . + "2021-10-12T06:21:16.629Z"^^ . + . + . + "51.34018"^^ . + "3.28454"^^ . + "Knokke" . + "5"^^ . + "7"^^ . + "2"^^ . + . + "2021-10-12T06:20:49.729Z"^^ . + . + . + "51.134300231934"^^ . + "4.476119995117"^^ . + "Kontich-Lint" . + "28"^^ . + "47"^^ . + "19"^^ . + . + "2021-10-12T06:20:21.952Z"^^ . + . + . + "50.823799133301"^^ . + "3.263269901276"^^ . + "Kortrijk" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:18.066Z"^^ . + . + . + "51.064208"^^ . + "3.575647"^^ . + "Landegem" . + "7"^^ . + "8"^^ . + "1"^^ . + . + "2021-10-12T06:21:12.535Z"^^ . + . + . + "50.74829"^^ . + "5.079402"^^ . + "Landen" . + "16"^^ . + "24"^^ . + "8"^^ . + . + "2021-10-12T06:20:53.955Z"^^ . + . + . + "50.882499694824"^^ . + "4.716639995575"^^ . + "Leuven (Kop van Kessel-Lo)" . + "57"^^ . + "86"^^ . + "29"^^ . + . + "2021-10-12T06:20:22.664Z"^^ . + . + . + "50.87979888916"^^ . + "4.715939998627"^^ . + "Leuven (Tiensevest)" . + "21"^^ . + "26"^^ . + "5"^^ . + . + "2021-10-12T06:20:23.383Z"^^ . + . + . + "51.135799407959"^^ . + "4.56152009964"^^ . + "Lier" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:20:43.366Z"^^ . + . + . + "51.125301361084"^^ . + "4.573339939117"^^ . + "Lier P+R De Mol" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:20:42.586Z"^^ . + . + . + "51.128299713135"^^ . + "4.583220005035"^^ . + "Lier Veemarkt" . + "13"^^ . + "17"^^ . + "4"^^ . + . + "2021-10-12T06:20:24.095Z"^^ . + . + . + "51.107398986816"^^ . + "3.986469984055"^^ . + "Lokeren" . + "6"^^ . + "6"^^ . + "0"^^ . + . + "2021-10-12T06:21:10.091Z"^^ . + . + . + "51.21177"^^ . + "5.312579"^^ . + "Lommel" . + "0"^^ . + "4"^^ . + "4"^^ . + . + "2021-10-12T06:20:24.831Z"^^ . + . + . + "50.625801086426"^^ . + "5.565639972687"^^ . + "Luik" . + "47"^^ . + "76"^^ . + "29"^^ . + . + "2021-10-12T06:20:25.537Z"^^ . + . + . + "51.0178"^^ . + "4.48299"^^ . + "Mechelen" . + "17"^^ . + "20"^^ . + "3"^^ . + . + "2021-10-12T06:20:38.128Z"^^ . + . + . + "51.030200958252"^^ . + "4.489570140839"^^ . + "Mechelen-Nekkerspoel" . + "6"^^ . + "7"^^ . + "1"^^ . + . + "2021-10-12T06:20:49.040Z"^^ . + . + . + "51.029300689697"^^ . + "4.484320163727"^^ . + "Mechelen Veemarkt" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:05.094Z"^^ . + . + . + "50.932948204381"^^ . + "4.329820851913"^^ . + "Meise" . + "10"^^ . + "15"^^ . + "5"^^ . + . + "2021-10-12T06:20:26.232Z"^^ . + . + . + "51.190498352051"^^ . + "5.115129947662"^^ . + "Mol" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:21:20.277Z"^^ . + . + . + "51.18119"^^ . + "5.112421"^^ . + "Mol - De Nete" . + "4"^^ . + "6"^^ . + "2"^^ . + . + "2021-10-12T06:20:26.934Z"^^ . + . + . + "51.171199798584"^^ . + "4.455999851227"^^ . + "Mortsel-Oude-God" . + "6"^^ . + "6"^^ . + "0"^^ . + . + "2021-10-12T06:20:27.637Z"^^ . + . + . + "50.468601226807"^^ . + "4.862239837646"^^ . + "Namen" . + "7"^^ . + "10"^^ . + "3"^^ . + . + "2021-10-12T06:20:36.006Z"^^ . + . + . + "50.839500427246"^^ . + "4.026179790497"^^ . + "Ninove" . + "10"^^ . + "10"^^ . + "0"^^ . + . + "2021-10-12T06:20:47.633Z"^^ . + . + . + "51.35710144043"^^ . + "4.632410049438"^^ . + "Noorderkempen" . + "53"^^ . + "69"^^ . + "16"^^ . + . + "2021-10-12T06:20:28.362Z"^^ . + . + . + "51.228298187256"^^ . + "2.925649881363"^^ . + "Oostende" . + "0"^^ . + "2"^^ . + "2"^^ . + . + "2021-10-12T06:20:39.151Z"^^ . + . + . + "50.851299285889"^^ . + "3.601880073547"^^ . + "Oudenaarde " . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:20:46.915Z"^^ . + . + . + "50.854499816895"^^ . + "2.735719919205"^^ . + "Poperinge" . + "14"^^ . + "27"^^ . + "13"^^ . + . + "2021-10-12T06:20:29.065Z"^^ . + . + . + "50.949001312256"^^ . + "3.130480051041"^^ . + "Roeselare" . + "9"^^ . + "9"^^ . + "0"^^ . + . + "2021-10-12T06:21:17.346Z"^^ . + . + . + "50.747786"^^ . + "4.361626"^^ . + "Sint-Genesius-Rode" . + "15"^^ . + "30"^^ . + "15"^^ . + . + "2021-10-12T06:20:29.783Z"^^ . + . + . + "51.171398162842"^^ . + "4.143700122833"^^ . + "Sint-Niklaas" . + "9"^^ . + "9"^^ . + "0"^^ . + . + "2021-10-12T06:20:30.506Z"^^ . + . + . + "50.817401885986"^^ . + "5.176330089569"^^ . + "Sint-Truiden" . + "5"^^ . + "5"^^ . + "0"^^ . + . + "2021-10-12T06:21:07.957Z"^^ . + . + . + "51.125867"^^ . + "4.221333"^^ . + "Temse" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:20:50.446Z"^^ . + . + . + "50.990600585938"^^ . + "3.329819917679"^^ . + "Tielt" . + "5"^^ . + "13"^^ . + "8"^^ . + . + "2021-10-12T06:20:46.217Z"^^ . + . + . + "50.808200836182"^^ . + "4.92561006546"^^ . + "Tienen " . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:20:37.434Z"^^ . + . + . + "50.78450012207"^^ . + "5.473400115967"^^ . + "Tongeren" . + "3"^^ . + "4"^^ . + "1"^^ . + . + "2021-10-12T06:20:31.284Z"^^ . + . + . + "51.064800262451"^^ . + "3.105509996414"^^ . + "Torhout" . + "9"^^ . + "13"^^ . + "4"^^ . + . + "2021-10-12T06:20:32.245Z"^^ . + . + . + "51.322101593018"^^ . + "4.937900066376"^^ . + "Turnhout" . + "20"^^ . + "40"^^ . + "20"^^ . + . + "2021-10-12T06:20:33.067Z"^^ . + . + . + "50.924701690674"^^ . + "4.433000087738"^^ . + "Vilvoorde" . + "6"^^ . + "7"^^ . + "1"^^ . + . + "2021-10-12T06:21:18.772Z"^^ . + . + . + "50.923187"^^ . + "4.413591"^^ . + "Vilvoorde Hoppin De Kassei" . + "12"^^ . + "16"^^ . + "4"^^ . + . + "2021-10-12T06:20:33.881Z"^^ . + . + . + "50.891799926758"^^ . + "3.424420118332"^^ . + "Waregem" . + "4"^^ . + "4"^^ . + "0"^^ . + . + "2021-10-12T06:21:02.941Z"^^ . + . + . + "50.881999969482"^^ . + "3.426719903946"^^ . + "Waregem Expo P+R" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:09.366Z"^^ . + . + . + "50.810926"^^ . + "3.182521"^^ . + "Wevelgem" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:20:51.139Z"^^ . + . + . + "50.885200500488"^^ . + "4.470870018005"^^ . + "Zaventem" . + "2"^^ . + "2"^^ . + "0"^^ . + . + "2021-10-12T06:21:14.998Z"^^ . + . + . + "51.32651"^^ . + "3.195476"^^ . + "Zeebrugge-dorp" . + "6"^^ . + "7"^^ . + "1"^^ . + . + "2021-10-12T06:21:08.664Z"^^ . + . + . + "51.073108"^^ . + "4.041885"^^ . + "Zele" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:04.377Z"^^ . + . + . + "51.033617"^^ . + "5.329922"^^ . + "Zolder" . + "7"^^ . + "8"^^ . + "1"^^ . + . + "2021-10-12T06:21:07.245Z"^^ . + . + . + "50.86948"^^ . + "3.814341"^^ . + "Zottegem " . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + "1" . diff --git a/src/test/resources/web-of-things/ldes/paths/mapping.ttl b/src/test/resources/web-of-things/ldes/paths/mapping.ttl new file mode 100644 index 00000000..603e4e45 --- /dev/null +++ b/src/test/resources/web-of-things/ldes/paths/mapping.ttl @@ -0,0 +1,161 @@ +# +# RML mapping rules for https://datapiloten.be/bluebike/availabilities.geojson +# (c) Dylan Van Assche (2021) +# IDLab - Ghent University - imec +# +@prefix rr: . +@prefix ex: . +@prefix rdfs: . +@prefix xsd: . +@prefix rml: . +@prefix rmlt: . +@prefix comp: . +@prefix formats: . +@prefix ql: . +@prefix rdf: . +@prefix geo: . +@prefix fnml: . +@prefix fno: . +@prefix grel: . +@prefix dcterms: . +@prefix tree: . +@prefix ldes: . +@prefix purl: . +@prefix td: . +@prefix htv: . +@prefix hctl: . +@base . + +<#TargetDump> a rmlt:LogicalTarget, ldes:EventStreamTarget; + rmlt:target [ a void:Dataset; + void:dataDump ; + ]; + rmlt:serialization formats:N-Triples; + ldes:baseIRI ; + ldes:timestampPath purl:created; + ldes:versionOfPath purl:isVersionOf; + tree:shape ; +. + +<#WoTWebResource> a td:PropertyAffordance; + td:hasForm [ + # URL and content type + hctl:hasTarget "http://localhost:8000/bluebike"; + hctl:forContentType "application/json"; + # Read only + hctl:hasOperationType td:readproperty ; + # Set HTTP method and headers + htv:methodName "GET"; + htv:headers ([ + htv:fieldName "User-Agent"; + htv:fieldValue "RML Processor"; + ]); + ]; +. + +<#WoTWebAPI> a td:Thing; + td:hasPropertyAffordance <#WoTWebResource>; +. + +<#StationsTriplesMap> a rr:TriplesMap; + rml:logicalSource [ + rml:source <#WoTWebResource>; + rml:referenceFormulation ql:JSONPath; + rml:iterator "$.features[*]" + ]; + + # Unique IRI generation: $stationId#$generatedAtTime + rr:subjectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant grel:array_join ] + ]; + rr:predicateObjectMap [ + rr:predicate grel:p_array_a ; + rr:objectMap [ rml:reference "properties.@id" ] + ]; + rr:predicateObjectMap [ + rr:predicate grel:p_array_a ; + rr:objectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant grel:array_join ] + ]; + + rr:predicateObjectMap [ + rr:predicate grel:p_array_a ; + rr:objectMap [ rr:constant "#" ] + ]; + + rr:predicateObjectMap [ + rr:predicate grel:p_array_a ; + rr:objectMap [ rml:reference "generatedAtTime" ] + ]; + ]; + ]; + ]; + ]; + rr:termType rr:IRI; + rml:logicalTarget <#TargetDump>; + ]; + + # rdf:type + rr:predicateObjectMap [ + rr:predicate rdf:type ; + rr:objectMap [ rml:reference "properties.@type"; rr:termType rr:IRI ] + ]; + + # BlueBike station: name + rr:predicateObjectMap [ + rr:predicate rdfs:label ; + rr:objectMap [ rml:reference "properties.name"; rr:datatype xsd:string; ] + ]; + + # BlueBike station: available bicycles + rr:predicateObjectMap [ + rr:predicate ex:bikesAvailable ; + rr:objectMap [ rml:reference "properties.bikes_available"; rr:datatype xsd:integer; ] + ]; + + # BlueBike station: maximum capacity + rr:predicateObjectMap [ + rr:predicate ex:capacity ; + rr:objectMap [ rml:reference "properties.capacity"; rr:datatype xsd:integer; ] + ]; + + # BlueBike station: available docks + rr:predicateObjectMap [ + rr:predicate ex:docksAvailable ; + rr:objectMap [ rml:reference "properties.docks_available"; rr:datatype xsd:integer; ] + ]; + + # Nearby NMBS station + rr:predicateObjectMap [ + rr:predicate ex:nearby ; + rr:objectMap [ rml:reference "properties.nearby"; rr:termType rr:IRI; ] + ]; + + # Geo location + rr:predicateObjectMap [ + rr:predicate geo:latitude ; + rr:objectMap [ rml:reference "properties.latitude"; rr:datatype xsd:float; ] + ]; + + rr:predicateObjectMap [ + rr:predicate geo:longitude ; + rr:objectMap [ rml:reference "properties.longitude"; rr:datatype xsd:float; ] + ]; + + # Versioning + rr:predicateObjectMap [ + rr:predicate dcterms:created ; + rr:objectMap [ rml:reference "generatedAtTime"; rr:datatype xsd:dateTime; ] + ]; + + rr:predicateObjectMap [ + rr:predicate dcterms:isVersionOf ; + rr:objectMap [ rml:reference "properties.@id"; rr:termType rr:IRI; ] + ]; +. diff --git a/src/test/resources/web-of-things/ldes/paths/out-default.nq b/src/test/resources/web-of-things/ldes/paths/out-default.nq new file mode 100644 index 00000000..e69de29b diff --git a/src/test/resources/web-of-things/ldes/paths/out-local-file.nq b/src/test/resources/web-of-things/ldes/paths/out-local-file.nq new file mode 100644 index 00000000..d00db8e4 --- /dev/null +++ b/src/test/resources/web-of-things/ldes/paths/out-local-file.nq @@ -0,0 +1,1075 @@ + "11"^^ . + "11"^^ . + "0"^^ . + . + "2021-10-12T06:20:08.208Z"^^ . + . + . + "50.942901611328"^^ . + "4.038559913635"^^ . + "Aalst" . + "6"^^ . + "6"^^ . + "0"^^ . + . + "2021-10-12T06:20:44.789Z"^^ . + . + . + "51.092498779297"^^ . + "3.449820041656"^^ . + "Aalter" . + "9"^^ . + "14"^^ . + "5"^^ . + . + "2021-10-12T06:20:12.927Z"^^ . + . + . + "50.984699249268"^^ . + "4.824250221252"^^ . + "Aarschot" . + "51"^^ . + "60"^^ . + "9"^^ . + . + "2021-10-12T06:20:11.221Z"^^ . + . + . + "51.199600219727"^^ . + "4.431789875031"^^ . + "Antwerpen-Berchem" . + "31"^^ . + "43"^^ . + "12"^^ . + . + "2021-10-12T06:20:06.338Z"^^ . + . + . + "51.217498779297"^^ . + "4.421050071716"^^ . + "Antwerpen-Centraal" . + "5"^^ . + "5"^^ . + "0"^^ . + . + "2021-10-12T06:21:00.798Z"^^ . + . + . + "51.245498657227"^^ . + "4.424610137939"^^ . + "Antwerpen-Luchtbal" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:21:01.494Z"^^ . + . + . + "51.2617"^^ . + "4.42755"^^ . + "Antwerpen-Noorderdokken" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:20:51.829Z"^^ . + . + . + "51.198001861572"^^ . + "4.390810012817"^^ . + "Antwerpen-Zuid" . + "7"^^ . + "10"^^ . + "3"^^ . + . + "2021-10-12T06:20:09.208Z"^^ . + . + . + "50.906700134277"^^ . + "4.207960128784"^^ . + "Asse" . + "5"^^ . + "5"^^ . + "0"^^ . + . + "2021-10-12T06:20:09.940Z"^^ . + . + . + "50.455600738525"^^ . + "3.944180011749"^^ . + "Bergen" . + "10"^^ . + "11"^^ . + "1"^^ . + . + "2021-10-12T06:20:55.643Z"^^ . + . + . + "51.208499908447"^^ . + "4.260260105133"^^ . + "Beveren" . + "6"^^ . + "7"^^ . + "1"^^ . + . + "2021-10-12T06:21:14.201Z"^^ . + . + . + "51.3119"^^ . + "3.1338"^^ . + "Blankenberge" . + "5"^^ . + "5"^^ . + "0"^^ . + . + "2021-10-12T06:20:44.070Z"^^ . + . + . + "51.16849899292"^^ . + "4.494080066681"^^ . + "Boechout P+R Capenberg" . + "6"^^ . + "6"^^ . + "0"^^ . + . + "2021-10-12T06:21:05.822Z"^^ . + . + . + "51.165508"^^ . + "4.746427"^^ . + "Bouwel" . + "34"^^ . + "63"^^ . + "29"^^ . + . + "2021-10-12T06:20:10.484Z"^^ . + . + . + "51.196998596191"^^ . + "3.218539953232"^^ . + "Brugge" . + "17"^^ . + "27"^^ . + "10"^^ . + . + "2021-10-12T06:20:54.646Z"^^ . + . + . + "51.196800231934"^^ . + "3.216079950333"^^ . + "Brugge (Kamgebouw)" . + "13"^^ . + "14"^^ . + "1"^^ . + . + "2021-10-12T06:20:07.205Z"^^ . + . + . + "50.844898223877"^^ . + "4.35586977005"^^ . + "Brussel-Centraal" . + "4"^^ . + "6"^^ . + "2"^^ . + . + "2021-10-12T06:20:13.845Z"^^ . + . + . + "50.838199615479"^^ . + "4.372889995575"^^ . + "Brussel-Luxemburg" . + "4"^^ . + "6"^^ . + "2"^^ . + . + "2021-10-12T06:21:19.556Z"^^ . + . + . + "50.835529468767"^^ . + "4.333756181866"^^ . + "Brussel-Zuid" . + "27"^^ . + "28"^^ . + "1"^^ . + . + "2021-10-12T06:20:14.572Z"^^ . + . + . + "50.86009979248"^^ . + "4.360489845276"^^ . + "Brussel Noord" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:13.294Z"^^ . + . + . + "51.077376"^^ . + "2.601611"^^ . + "De Panne" . + "4"^^ . + "62"^^ . + "58"^^ . + . + "2021-10-12T06:20:15.386Z"^^ . + . + . + "50.978199005127"^^ . + "3.534009933472"^^ . + "Deinze" . + "9"^^ . + "11"^^ . + "2"^^ . + . + "2021-10-12T06:20:45.522Z"^^ . + . + . + "50.982398986816"^^ . + "3.525919914246"^^ . + "Deinze Leiespiegel" . + "22"^^ . + "27"^^ . + "5"^^ . + . + "2021-10-12T06:20:16.315Z"^^ . + . + . + "51.023498535156"^^ . + "4.101920127869"^^ . + "Dendermonde" . + "4"^^ . + "4"^^ . + "0"^^ . + . + "2021-10-12T06:21:02.217Z"^^ . + . + . + "50.889099121094"^^ . + "4.447710037231"^^ . + "Diegem" . + "10"^^ . + "12"^^ . + "2"^^ . + . + "2021-10-12T06:20:17.089Z"^^ . + . + . + "50.993000030518"^^ . + "5.051129817963"^^ . + "Diest" . + "1"^^ . + "11"^^ . + "10"^^ . + . + "2021-10-12T06:20:34.600Z"^^ . + . + . + "51.180801391602"^^ . + "3.575040102005"^^ . + "Eeklo" . + "3"^^ . + "4"^^ . + "1"^^ . + . + "2021-10-12T06:20:59.366Z"^^ . + . + . + "51.284198760986"^^ . + "4.43501996994"^^ . + "Ekeren (perron 1)" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:21:00.100Z"^^ . + . + . + "51.281398773193"^^ . + "4.434390068054"^^ . + "Ekeren (perron 2)" . + "7"^^ . + "8"^^ . + "1"^^ . + . + "2021-10-12T06:21:11.779Z"^^ . + . + . + "51.462176"^^ . + "4.451532"^^ . + "Essen" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:20:57.946Z"^^ . + . + . + "50.929401397705"^^ . + "3.64013004303"^^ . + "Gavere-Asper" . + "4"^^ . + "4"^^ . + "0"^^ . + . + "2021-10-12T06:20:58.653Z"^^ . + . + . + "50.930500030518"^^ . + "3.654690027237"^^ . + "Gavere mobipunt" . + "8"^^ . + "15"^^ . + "7"^^ . + . + "2021-10-12T06:20:35.295Z"^^ . + . + . + "51.168800354004"^^ . + "4.989160060883"^^ . + "Geel" . + "5"^^ . + "6"^^ . + "1"^^ . + . + "2021-10-12T06:20:17.608Z"^^ . + . + . + "50.967098236084"^^ . + "5.496369838715"^^ . + "Genk" . + "31"^^ . + "41"^^ . + "10"^^ . + . + "2021-10-12T06:20:18.137Z"^^ . + . + . + "51.055999755859"^^ . + "3.740159988403"^^ . + "Gent-Dampoort" . + "16"^^ . + "26"^^ . + "10"^^ . + . + "2021-10-12T06:20:53.258Z"^^ . + . + . + "51.035999298096"^^ . + "3.711899995804"^^ . + "Gent-Sint-Pieters (M. Hendrikaplein)" . + "52"^^ . + "70"^^ . + "18"^^ . + . + "2021-10-12T06:20:18.962Z"^^ . + . + . + "51.034900665283"^^ . + "3.709640026093"^^ . + "Gent-Sint-Pieters (St.-Denijslaan)" . + "2"^^ . + "2"^^ . + "0"^^ . + . + "2021-10-12T06:20:39.838Z"^^ . + . + . + "50.770500183105"^^ . + "3.872560024261"^^ . + "Geraardsbergen " . + "7"^^ . + "8"^^ . + "1"^^ . + . + "2021-10-12T06:21:06.544Z"^^ . + . + . + "50.966578"^^ . + "4.613315"^^ . + "Haacht" . + "7"^^ . + "12"^^ . + "5"^^ . + . + "2021-10-12T06:20:19.671Z"^^ . + . + . + "50.7333984375"^^ . + "4.239940166473"^^ . + "Halle" . + "8"^^ . + "9"^^ . + "1"^^ . + . + "2021-10-12T06:20:57.256Z"^^ . + . + . + "50.855701446533"^^ . + "3.31374001503"^^ . + "Harelbeke" . + "35"^^ . + "59"^^ . + "24"^^ . + . + "2021-10-12T06:20:20.529Z"^^ . + . + . + "50.930999755859"^^ . + "5.32742023468"^^ . + "Hasselt" . + "11"^^ . + "12"^^ . + "1"^^ . + . + "2021-10-12T06:20:48.339Z"^^ . + . + . + "51.365100860596"^^ . + "4.460559844971"^^ . + "Heide" . + "5"^^ . + "5"^^ . + "0"^^ . + . + "2021-10-12T06:21:15.934Z"^^ . + . + . + "51.334168"^^ . + "3.239668"^^ . + "Heist" . + "12"^^ . + "12"^^ . + "0"^^ . + . + "2021-10-12T06:20:36.712Z"^^ . + . + . + "51.074199676514"^^ . + "4.708779811859"^^ . + "Heist-op-den-Berg" . + "11"^^ . + "12"^^ . + "1"^^ . + . + "2021-10-12T06:20:21.475Z"^^ . + . + . + "51.180999755859"^^ . + "4.828859806061"^^ . + "Herentals" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:03.654Z"^^ . + . + . + "51.03833"^^ . + "5.281699"^^ . + "Heusden" . + "7"^^ . + "9"^^ . + "2"^^ . + . + "2021-10-12T06:20:56.370Z"^^ . + . + . + "50.862598419189"^^ . + "4.695439815521"^^ . + "Heverlee " . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:21:10.807Z"^^ . + . + . + "51.1541195"^^ . + "4.464733"^^ . + "Hove" . + "5"^^ . + "10"^^ . + "5"^^ . + . + "2021-10-12T06:20:41.845Z"^^ . + . + . + "50.848400115967"^^ . + "2.876270055771"^^ . + "Ieper" . + "6"^^ . + "8"^^ . + "2"^^ . + . + "2021-10-12T06:20:52.530Z"^^ . + . + . + "50.9208984375"^^ . + "3.214819908142"^^ . + "Izegem" . + "7"^^ . + "8"^^ . + "1"^^ . + . + "2021-10-12T06:21:16.629Z"^^ . + . + . + "51.34018"^^ . + "3.28454"^^ . + "Knokke" . + "5"^^ . + "7"^^ . + "2"^^ . + . + "2021-10-12T06:20:49.729Z"^^ . + . + . + "51.134300231934"^^ . + "4.476119995117"^^ . + "Kontich-Lint" . + "28"^^ . + "47"^^ . + "19"^^ . + . + "2021-10-12T06:20:21.952Z"^^ . + . + . + "50.823799133301"^^ . + "3.263269901276"^^ . + "Kortrijk" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:18.066Z"^^ . + . + . + "51.064208"^^ . + "3.575647"^^ . + "Landegem" . + "7"^^ . + "8"^^ . + "1"^^ . + . + "2021-10-12T06:21:12.535Z"^^ . + . + . + "50.74829"^^ . + "5.079402"^^ . + "Landen" . + "16"^^ . + "24"^^ . + "8"^^ . + . + "2021-10-12T06:20:53.955Z"^^ . + . + . + "50.882499694824"^^ . + "4.716639995575"^^ . + "Leuven (Kop van Kessel-Lo)" . + "57"^^ . + "86"^^ . + "29"^^ . + . + "2021-10-12T06:20:22.664Z"^^ . + . + . + "50.87979888916"^^ . + "4.715939998627"^^ . + "Leuven (Tiensevest)" . + "21"^^ . + "26"^^ . + "5"^^ . + . + "2021-10-12T06:20:23.383Z"^^ . + . + . + "51.135799407959"^^ . + "4.56152009964"^^ . + "Lier" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:20:43.366Z"^^ . + . + . + "51.125301361084"^^ . + "4.573339939117"^^ . + "Lier P+R De Mol" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:20:42.586Z"^^ . + . + . + "51.128299713135"^^ . + "4.583220005035"^^ . + "Lier Veemarkt" . + "13"^^ . + "17"^^ . + "4"^^ . + . + "2021-10-12T06:20:24.095Z"^^ . + . + . + "51.107398986816"^^ . + "3.986469984055"^^ . + "Lokeren" . + "6"^^ . + "6"^^ . + "0"^^ . + . + "2021-10-12T06:21:10.091Z"^^ . + . + . + "51.21177"^^ . + "5.312579"^^ . + "Lommel" . + "0"^^ . + "4"^^ . + "4"^^ . + . + "2021-10-12T06:20:24.831Z"^^ . + . + . + "50.625801086426"^^ . + "5.565639972687"^^ . + "Luik" . + "47"^^ . + "76"^^ . + "29"^^ . + . + "2021-10-12T06:20:25.537Z"^^ . + . + . + "51.0178"^^ . + "4.48299"^^ . + "Mechelen" . + "17"^^ . + "20"^^ . + "3"^^ . + . + "2021-10-12T06:20:38.128Z"^^ . + . + . + "51.030200958252"^^ . + "4.489570140839"^^ . + "Mechelen-Nekkerspoel" . + "6"^^ . + "7"^^ . + "1"^^ . + . + "2021-10-12T06:20:49.040Z"^^ . + . + . + "51.029300689697"^^ . + "4.484320163727"^^ . + "Mechelen Veemarkt" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:05.094Z"^^ . + . + . + "50.932948204381"^^ . + "4.329820851913"^^ . + "Meise" . + "10"^^ . + "15"^^ . + "5"^^ . + . + "2021-10-12T06:20:26.232Z"^^ . + . + . + "51.190498352051"^^ . + "5.115129947662"^^ . + "Mol" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:21:20.277Z"^^ . + . + . + "51.18119"^^ . + "5.112421"^^ . + "Mol - De Nete" . + "4"^^ . + "6"^^ . + "2"^^ . + . + "2021-10-12T06:20:26.934Z"^^ . + . + . + "51.171199798584"^^ . + "4.455999851227"^^ . + "Mortsel-Oude-God" . + "6"^^ . + "6"^^ . + "0"^^ . + . + "2021-10-12T06:20:27.637Z"^^ . + . + . + "50.468601226807"^^ . + "4.862239837646"^^ . + "Namen" . + "7"^^ . + "10"^^ . + "3"^^ . + . + "2021-10-12T06:20:36.006Z"^^ . + . + . + "50.839500427246"^^ . + "4.026179790497"^^ . + "Ninove" . + "10"^^ . + "10"^^ . + "0"^^ . + . + "2021-10-12T06:20:47.633Z"^^ . + . + . + "51.35710144043"^^ . + "4.632410049438"^^ . + "Noorderkempen" . + "53"^^ . + "69"^^ . + "16"^^ . + . + "2021-10-12T06:20:28.362Z"^^ . + . + . + "51.228298187256"^^ . + "2.925649881363"^^ . + "Oostende" . + "0"^^ . + "2"^^ . + "2"^^ . + . + "2021-10-12T06:20:39.151Z"^^ . + . + . + "50.851299285889"^^ . + "3.601880073547"^^ . + "Oudenaarde " . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:20:46.915Z"^^ . + . + . + "50.854499816895"^^ . + "2.735719919205"^^ . + "Poperinge" . + "14"^^ . + "27"^^ . + "13"^^ . + . + "2021-10-12T06:20:29.065Z"^^ . + . + . + "50.949001312256"^^ . + "3.130480051041"^^ . + "Roeselare" . + "9"^^ . + "9"^^ . + "0"^^ . + . + "2021-10-12T06:21:17.346Z"^^ . + . + . + "50.747786"^^ . + "4.361626"^^ . + "Sint-Genesius-Rode" . + "15"^^ . + "30"^^ . + "15"^^ . + . + "2021-10-12T06:20:29.783Z"^^ . + . + . + "51.171398162842"^^ . + "4.143700122833"^^ . + "Sint-Niklaas" . + "9"^^ . + "9"^^ . + "0"^^ . + . + "2021-10-12T06:20:30.506Z"^^ . + . + . + "50.817401885986"^^ . + "5.176330089569"^^ . + "Sint-Truiden" . + "5"^^ . + "5"^^ . + "0"^^ . + . + "2021-10-12T06:21:07.957Z"^^ . + . + . + "51.125867"^^ . + "4.221333"^^ . + "Temse" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:20:50.446Z"^^ . + . + . + "50.990600585938"^^ . + "3.329819917679"^^ . + "Tielt" . + "5"^^ . + "13"^^ . + "8"^^ . + . + "2021-10-12T06:20:46.217Z"^^ . + . + . + "50.808200836182"^^ . + "4.92561006546"^^ . + "Tienen " . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:20:37.434Z"^^ . + . + . + "50.78450012207"^^ . + "5.473400115967"^^ . + "Tongeren" . + "3"^^ . + "4"^^ . + "1"^^ . + . + "2021-10-12T06:20:31.284Z"^^ . + . + . + "51.064800262451"^^ . + "3.105509996414"^^ . + "Torhout" . + "9"^^ . + "13"^^ . + "4"^^ . + . + "2021-10-12T06:20:32.245Z"^^ . + . + . + "51.322101593018"^^ . + "4.937900066376"^^ . + "Turnhout" . + "20"^^ . + "40"^^ . + "20"^^ . + . + "2021-10-12T06:20:33.067Z"^^ . + . + . + "50.924701690674"^^ . + "4.433000087738"^^ . + "Vilvoorde" . + "6"^^ . + "7"^^ . + "1"^^ . + . + "2021-10-12T06:21:18.772Z"^^ . + . + . + "50.923187"^^ . + "4.413591"^^ . + "Vilvoorde Hoppin De Kassei" . + "12"^^ . + "16"^^ . + "4"^^ . + . + "2021-10-12T06:20:33.881Z"^^ . + . + . + "50.891799926758"^^ . + "3.424420118332"^^ . + "Waregem" . + "4"^^ . + "4"^^ . + "0"^^ . + . + "2021-10-12T06:21:02.941Z"^^ . + . + . + "50.881999969482"^^ . + "3.426719903946"^^ . + "Waregem Expo P+R" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:09.366Z"^^ . + . + . + "50.810926"^^ . + "3.182521"^^ . + "Wevelgem" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:20:51.139Z"^^ . + . + . + "50.885200500488"^^ . + "4.470870018005"^^ . + "Zaventem" . + "2"^^ . + "2"^^ . + "0"^^ . + . + "2021-10-12T06:21:14.998Z"^^ . + . + . + "51.32651"^^ . + "3.195476"^^ . + "Zeebrugge-dorp" . + "6"^^ . + "7"^^ . + "1"^^ . + . + "2021-10-12T06:21:08.664Z"^^ . + . + . + "51.073108"^^ . + "4.041885"^^ . + "Zele" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:04.377Z"^^ . + . + . + "51.033617"^^ . + "5.329922"^^ . + "Zolder" . + "7"^^ . + "8"^^ . + "1"^^ . + . + "2021-10-12T06:21:07.245Z"^^ . + . + . + "50.86948"^^ . + "3.814341"^^ . + "Zottegem " . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + "1" . + . + . diff --git a/src/test/resources/web-of-things/ldes/private-security-data.ttl b/src/test/resources/web-of-things/ldes/private-security-data.ttl new file mode 100644 index 00000000..9da7b50c --- /dev/null +++ b/src/test/resources/web-of-things/ldes/private-security-data.ttl @@ -0,0 +1,2 @@ +@prefix idsa: . +@base . diff --git a/src/test/resources/web-of-things/ldes/stations.jsonld b/src/test/resources/web-of-things/ldes/stations.jsonld new file mode 100644 index 00000000..4931763e --- /dev/null +++ b/src/test/resources/web-of-things/ldes/stations.jsonld @@ -0,0 +1,2178 @@ +{ + "@context": { + "mv": "http://schema.mobivoc.org/", + "name": { + "@id": "http://xmlns.com/foaf/0.1/name", + "@type": "http://www.w3.org/2001/XMLSchema#string" + }, + "longitude": { + "@id": "http://www.w3.org/2003/01/geo/wgs84_pos#long", + "@type": "xsd:float" + }, + "latitude": { + "@id": "http://www.w3.org/2003/01/geo/wgs84_pos#lat", + "@type": "xsd:float" + }, + "alternative": { + "@id": "http://purl.org/dc/terms/alternative", + "@type": "http://www.w3.org/1999/02/22-rdf-syntax-ns#langString", + "@container": "@set" + }, + "xsd": "http://www.w3.org/2001/XMLSchema#", + "features": "@graph", + "properties": "@graph", + "generatedAtTime": { + "@id": "http://www.w3.org/ns/prov#generatedAtTime", + "@type": "xsd:date" + }, + "nearby": { + "@id": "http://www.geonames.org/ontology#nearby", + "@type": "@id" + }, + "bikes_available": { + "@id": "mv:capacity", + "@type": "http://www.w3.org/2001/XMLSchema#integer" + }, + "capacity": { + "@id": "mv:totalCapacity", + "@type": "http://www.w3.org/2001/XMLSchema#integer" + } + }, + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.421050071716, + 51.217498779297 + ] + }, + "generatedAtTime": "2021-10-12T06:20:06.338Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Antwerpen-Centraal", + "@type": "http://schema.org/ParkingFacility", + "name": "Antwerpen-Centraal", + "longitude": 4.421050071716, + "latitude": 51.217498779297, + "bikes_available": 31, + "capacity": 43, + "docks_available": 12, + "nearby": "http://irail.be/stations/NMBS/008821006" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.35586977005, + 50.844898223877 + ] + }, + "generatedAtTime": "2021-10-12T06:20:07.205Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Brussel-Centraal", + "@type": "http://schema.org/ParkingFacility", + "name": "Brussel-Centraal", + "longitude": 4.35586977005, + "latitude": 50.844898223877, + "bikes_available": 13, + "capacity": 14, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008813003" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.038559913635, + 50.942901611328 + ] + }, + "generatedAtTime": "2021-10-12T06:20:08.208Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Aalst", + "@type": "http://schema.org/ParkingFacility", + "name": "Aalst", + "longitude": 4.038559913635, + "latitude": 50.942901611328, + "bikes_available": 11, + "capacity": 11, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008895000" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.207960128784, + 50.906700134277 + ] + }, + "generatedAtTime": "2021-10-12T06:20:09.208Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Asse", + "@type": "http://schema.org/ParkingFacility", + "name": "Asse", + "longitude": 4.207960128784, + "latitude": 50.906700134277, + "bikes_available": 7, + "capacity": 10, + "docks_available": 3, + "nearby": "http://irail.be/stations/NMBS/008812070" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.944180011749, + 50.455600738525 + ] + }, + "generatedAtTime": "2021-10-12T06:20:09.940Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Bergen", + "@type": "http://schema.org/ParkingFacility", + "name": "Bergen", + "longitude": 3.944180011749, + "latitude": 50.455600738525, + "bikes_available": 5, + "capacity": 5, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008881000" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.218539953232, + 51.196998596191 + ] + }, + "generatedAtTime": "2021-10-12T06:20:10.484Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Brugge", + "@type": "http://schema.org/ParkingFacility", + "name": "Brugge", + "longitude": 3.218539953232, + "latitude": 51.196998596191, + "bikes_available": 34, + "capacity": 63, + "docks_available": 29, + "nearby": "http://irail.be/stations/NMBS/008891009" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.431789875031, + 51.199600219727 + ] + }, + "generatedAtTime": "2021-10-12T06:20:11.221Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Antwerpen-Berchem", + "@type": "http://schema.org/ParkingFacility", + "name": "Antwerpen-Berchem", + "longitude": 4.431789875031, + "latitude": 51.199600219727, + "bikes_available": 51, + "capacity": 60, + "docks_available": 9, + "nearby": "http://irail.be/stations/NMBS/008821121" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.824250221252, + 50.984699249268 + ] + }, + "generatedAtTime": "2021-10-12T06:20:12.927Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Aarschot", + "@type": "http://schema.org/ParkingFacility", + "name": "Aarschot", + "longitude": 4.824250221252, + "latitude": 50.984699249268, + "bikes_available": 9, + "capacity": 14, + "docks_available": 5, + "nearby": "http://irail.be/stations/NMBS/008833209" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.372889995575, + 50.838199615479 + ] + }, + "generatedAtTime": "2021-10-12T06:20:13.845Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Brussel-Luxemburg", + "@type": "http://schema.org/ParkingFacility", + "name": "Brussel-Luxemburg", + "longitude": 4.372889995575, + "latitude": 50.838199615479, + "bikes_available": 4, + "capacity": 6, + "docks_available": 2, + "nearby": "http://irail.be/stations/NMBS/008811304" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.360489845276, + 50.86009979248 + ] + }, + "generatedAtTime": "2021-10-12T06:20:14.572Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/BrusselNoord", + "@type": "http://schema.org/ParkingFacility", + "name": "Brussel Noord", + "longitude": 4.360489845276, + "latitude": 50.86009979248, + "bikes_available": 27, + "capacity": 28, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008812005" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.534009933472, + 50.978199005127 + ] + }, + "generatedAtTime": "2021-10-12T06:20:15.386Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Deinze", + "@type": "http://schema.org/ParkingFacility", + "name": "Deinze", + "longitude": 3.534009933472, + "latitude": 50.978199005127, + "bikes_available": 4, + "capacity": 62, + "docks_available": 58, + "nearby": "http://irail.be/stations/NMBS/008892106" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.101920127869, + 51.023498535156 + ] + }, + "generatedAtTime": "2021-10-12T06:20:16.315Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Dendermonde", + "@type": "http://schema.org/ParkingFacility", + "name": "Dendermonde", + "longitude": 4.101920127869, + "latitude": 51.023498535156, + "bikes_available": 22, + "capacity": 27, + "docks_available": 5, + "nearby": "http://irail.be/stations/NMBS/008893401" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.051129817963, + 50.993000030518 + ] + }, + "generatedAtTime": "2021-10-12T06:20:17.089Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Diest", + "@type": "http://schema.org/ParkingFacility", + "name": "Diest", + "longitude": 5.051129817963, + "latitude": 50.993000030518, + "bikes_available": 10, + "capacity": 12, + "docks_available": 2, + "nearby": "http://irail.be/stations/NMBS/008831401" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.496369838715, + 50.967098236084 + ] + }, + "generatedAtTime": "2021-10-12T06:20:17.608Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Genk", + "@type": "http://schema.org/ParkingFacility", + "name": "Genk", + "longitude": 5.496369838715, + "latitude": 50.967098236084, + "bikes_available": 5, + "capacity": 6, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008831765" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.740159988403, + 51.055999755859 + ] + }, + "generatedAtTime": "2021-10-12T06:20:18.137Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Gent-Dampoort", + "@type": "http://schema.org/ParkingFacility", + "name": "Gent-Dampoort", + "longitude": 3.740159988403, + "latitude": 51.055999755859, + "bikes_available": 31, + "capacity": 41, + "docks_available": 10, + "nearby": "http://irail.be/stations/NMBS/008893120" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.709640026093, + 51.034900665283 + ] + }, + "generatedAtTime": "2021-10-12T06:20:18.962Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Gent-Sint-Pieters(St.-Denijslaan)", + "@type": "http://schema.org/ParkingFacility", + "name": "Gent-Sint-Pieters (St.-Denijslaan)", + "longitude": 3.709640026093, + "latitude": 51.034900665283, + "bikes_available": 52, + "capacity": 70, + "docks_available": 18, + "nearby": "http://irail.be/stations/NMBS/008892007" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.239940166473, + 50.7333984375 + ] + }, + "generatedAtTime": "2021-10-12T06:20:19.671Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Halle", + "@type": "http://schema.org/ParkingFacility", + "name": "Halle", + "longitude": 4.239940166473, + "latitude": 50.7333984375, + "bikes_available": 7, + "capacity": 12, + "docks_available": 5, + "nearby": "http://irail.be/stations/NMBS/008814308" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.32742023468, + 50.930999755859 + ] + }, + "generatedAtTime": "2021-10-12T06:20:20.529Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Hasselt", + "@type": "http://schema.org/ParkingFacility", + "name": "Hasselt", + "longitude": 5.32742023468, + "latitude": 50.930999755859, + "bikes_available": 35, + "capacity": 59, + "docks_available": 24, + "nearby": "http://irail.be/stations/NMBS/008831005" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.828859806061, + 51.180999755859 + ] + }, + "generatedAtTime": "2021-10-12T06:20:21.475Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Herentals", + "@type": "http://schema.org/ParkingFacility", + "name": "Herentals", + "longitude": 4.828859806061, + "latitude": 51.180999755859, + "bikes_available": 11, + "capacity": 12, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008821717" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.263269901276, + 50.823799133301 + ] + }, + "generatedAtTime": "2021-10-12T06:20:21.952Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Kortrijk", + "@type": "http://schema.org/ParkingFacility", + "name": "Kortrijk", + "longitude": 3.263269901276, + "latitude": 50.823799133301, + "bikes_available": 28, + "capacity": 47, + "docks_available": 19, + "nearby": "http://irail.be/stations/NMBS/008896008" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.715939998627, + 50.87979888916 + ] + }, + "generatedAtTime": "2021-10-12T06:20:22.664Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Leuven(Tiensevest)", + "@type": "http://schema.org/ParkingFacility", + "name": "Leuven (Tiensevest)", + "longitude": 4.715939998627, + "latitude": 50.87979888916, + "bikes_available": 57, + "capacity": 86, + "docks_available": 29, + "nearby": "http://irail.be/stations/NMBS/008833001" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.56152009964, + 51.135799407959 + ] + }, + "generatedAtTime": "2021-10-12T06:20:23.383Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Lier", + "@type": "http://schema.org/ParkingFacility", + "name": "Lier", + "longitude": 4.56152009964, + "latitude": 51.135799407959, + "bikes_available": 21, + "capacity": 26, + "docks_available": 5, + "nearby": "http://irail.be/stations/NMBS/008821600" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.986469984055, + 51.107398986816 + ] + }, + "generatedAtTime": "2021-10-12T06:20:24.095Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Lokeren", + "@type": "http://schema.org/ParkingFacility", + "name": "Lokeren", + "longitude": 3.986469984055, + "latitude": 51.107398986816, + "bikes_available": 13, + "capacity": 17, + "docks_available": 4, + "nearby": "http://irail.be/stations/NMBS/008894201" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.565639972687, + 50.625801086426 + ] + }, + "generatedAtTime": "2021-10-12T06:20:24.831Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Luik", + "@type": "http://schema.org/ParkingFacility", + "name": "Luik", + "longitude": 5.565639972687, + "latitude": 50.625801086426, + "bikes_available": 0, + "capacity": 4, + "docks_available": 4, + "nearby": "http://irail.be/stations/NMBS/008841004" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.48299, + 51.0178 + ] + }, + "generatedAtTime": "2021-10-12T06:20:25.537Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Mechelen", + "@type": "http://schema.org/ParkingFacility", + "name": "Mechelen", + "longitude": 4.48299, + "latitude": 51.0178, + "bikes_available": 47, + "capacity": 76, + "docks_available": 29, + "nearby": "http://irail.be/stations/NMBS/008822004" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.115129947662, + 51.190498352051 + ] + }, + "generatedAtTime": "2021-10-12T06:20:26.232Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Mol", + "@type": "http://schema.org/ParkingFacility", + "name": "Mol", + "longitude": 5.115129947662, + "latitude": 51.190498352051, + "bikes_available": 10, + "capacity": 15, + "docks_available": 5, + "nearby": "http://irail.be/stations/NMBS/008832409" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.455999851227, + 51.171199798584 + ] + }, + "generatedAtTime": "2021-10-12T06:20:26.934Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Mortsel-Oude-God", + "@type": "http://schema.org/ParkingFacility", + "name": "Mortsel-Oude-God", + "longitude": 4.455999851227, + "latitude": 51.171199798584, + "bikes_available": 4, + "capacity": 6, + "docks_available": 2, + "nearby": "http://irail.be/stations/NMBS/008821238" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.862239837646, + 50.468601226807 + ] + }, + "generatedAtTime": "2021-10-12T06:20:27.637Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Namen", + "@type": "http://schema.org/ParkingFacility", + "name": "Namen", + "longitude": 4.862239837646, + "latitude": 50.468601226807, + "bikes_available": 6, + "capacity": 6, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008863008" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 2.925649881363, + 51.228298187256 + ] + }, + "generatedAtTime": "2021-10-12T06:20:28.362Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Oostende", + "@type": "http://schema.org/ParkingFacility", + "name": "Oostende", + "longitude": 2.925649881363, + "latitude": 51.228298187256, + "bikes_available": 53, + "capacity": 69, + "docks_available": 16, + "nearby": "http://irail.be/stations/NMBS/008891702" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.130480051041, + 50.949001312256 + ] + }, + "generatedAtTime": "2021-10-12T06:20:29.065Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Roeselare", + "@type": "http://schema.org/ParkingFacility", + "name": "Roeselare", + "longitude": 3.130480051041, + "latitude": 50.949001312256, + "bikes_available": 14, + "capacity": 27, + "docks_available": 13, + "nearby": "http://irail.be/stations/NMBS/008896800" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.143700122833, + 51.171398162842 + ] + }, + "generatedAtTime": "2021-10-12T06:20:29.783Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Sint-Niklaas", + "@type": "http://schema.org/ParkingFacility", + "name": "Sint-Niklaas", + "longitude": 4.143700122833, + "latitude": 51.171398162842, + "bikes_available": 15, + "capacity": 30, + "docks_available": 15, + "nearby": "http://irail.be/stations/NMBS/008894508" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.176330089569, + 50.817401885986 + ] + }, + "generatedAtTime": "2021-10-12T06:20:30.506Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Sint-Truiden", + "@type": "http://schema.org/ParkingFacility", + "name": "Sint-Truiden", + "longitude": 5.176330089569, + "latitude": 50.817401885986, + "bikes_available": 9, + "capacity": 9, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008831807" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.105509996414, + 51.064800262451 + ] + }, + "generatedAtTime": "2021-10-12T06:20:31.284Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Torhout", + "@type": "http://schema.org/ParkingFacility", + "name": "Torhout", + "longitude": 3.105509996414, + "latitude": 51.064800262451, + "bikes_available": 3, + "capacity": 4, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008891314" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.937900066376, + 51.322101593018 + ] + }, + "generatedAtTime": "2021-10-12T06:20:32.245Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Turnhout", + "@type": "http://schema.org/ParkingFacility", + "name": "Turnhout", + "longitude": 4.937900066376, + "latitude": 51.322101593018, + "bikes_available": 9, + "capacity": 13, + "docks_available": 4, + "nearby": "http://irail.be/stations/NMBS/008821907" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.433000087738, + 50.924701690674 + ] + }, + "generatedAtTime": "2021-10-12T06:20:33.067Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Vilvoorde", + "@type": "http://schema.org/ParkingFacility", + "name": "Vilvoorde", + "longitude": 4.433000087738, + "latitude": 50.924701690674, + "bikes_available": 20, + "capacity": 40, + "docks_available": 20, + "nearby": "http://irail.be/stations/NMBS/008811189" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.424420118332, + 50.891799926758 + ] + }, + "generatedAtTime": "2021-10-12T06:20:33.881Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Waregem", + "@type": "http://schema.org/ParkingFacility", + "name": "Waregem", + "longitude": 3.424420118332, + "latitude": 50.891799926758, + "bikes_available": 12, + "capacity": 16, + "docks_available": 4, + "nearby": "http://irail.be/stations/NMBS/008896149" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.575040102005, + 51.180801391602 + ] + }, + "generatedAtTime": "2021-10-12T06:20:34.600Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Eeklo", + "@type": "http://schema.org/ParkingFacility", + "name": "Eeklo", + "longitude": 3.575040102005, + "latitude": 51.180801391602, + "bikes_available": 1, + "capacity": 11, + "docks_available": 10, + "nearby": "http://irail.be/stations/NMBS/008893708" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.989160060883, + 51.168800354004 + ] + }, + "generatedAtTime": "2021-10-12T06:20:35.295Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Geel", + "@type": "http://schema.org/ParkingFacility", + "name": "Geel", + "longitude": 4.989160060883, + "latitude": 51.168800354004, + "bikes_available": 8, + "capacity": 15, + "docks_available": 7, + "nearby": "http://irail.be/stations/NMBS/008832433" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.026179790497, + 50.839500427246 + ] + }, + "generatedAtTime": "2021-10-12T06:20:36.006Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Ninove", + "@type": "http://schema.org/ParkingFacility", + "name": "Ninove", + "longitude": 4.026179790497, + "latitude": 50.839500427246, + "bikes_available": 7, + "capacity": 10, + "docks_available": 3, + "nearby": "http://irail.be/stations/NMBS/008895760" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.708779811859, + 51.074199676514 + ] + }, + "generatedAtTime": "2021-10-12T06:20:36.712Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Heist-op-den-Berg", + "@type": "http://schema.org/ParkingFacility", + "name": "Heist-op-den-Berg", + "longitude": 4.708779811859, + "latitude": 51.074199676514, + "bikes_available": 12, + "capacity": 12, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821832" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.473400115967, + 50.78450012207 + ] + }, + "generatedAtTime": "2021-10-12T06:20:37.434Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Tongeren", + "@type": "http://schema.org/ParkingFacility", + "name": "Tongeren", + "longitude": 5.473400115967, + "latitude": 50.78450012207, + "bikes_available": 8, + "capacity": 8, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008831310" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.489570140839, + 51.030200958252 + ] + }, + "generatedAtTime": "2021-10-12T06:20:38.128Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Mechelen-Nekkerspoel", + "@type": "http://schema.org/ParkingFacility", + "name": "Mechelen-Nekkerspoel", + "longitude": 4.489570140839, + "latitude": 51.030200958252, + "bikes_available": 17, + "capacity": 20, + "docks_available": 3, + "nearby": "http://irail.be/stations/NMBS/008822343" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.601880073547, + 50.851299285889 + ] + }, + "generatedAtTime": "2021-10-12T06:20:39.151Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Oudenaarde", + "@type": "http://schema.org/ParkingFacility", + "name": "Oudenaarde ", + "longitude": 3.601880073547, + "latitude": 50.851299285889, + "bikes_available": 0, + "capacity": 2, + "docks_available": 2, + "nearby": "http://irail.be/stations/NMBS/008892601" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.872560024261, + 50.770500183105 + ] + }, + "generatedAtTime": "2021-10-12T06:20:39.838Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Geraardsbergen", + "@type": "http://schema.org/ParkingFacility", + "name": "Geraardsbergen ", + "longitude": 3.872560024261, + "latitude": 50.770500183105, + "bikes_available": 2, + "capacity": 2, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008895505" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 2.876270055771, + 50.848400115967 + ] + }, + "generatedAtTime": "2021-10-12T06:20:41.845Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Ieper", + "@type": "http://schema.org/ParkingFacility", + "name": "Ieper", + "longitude": 2.876270055771, + "latitude": 50.848400115967, + "bikes_available": 5, + "capacity": 10, + "docks_available": 5, + "nearby": "http://irail.be/stations/NMBS/008896503" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.583220005035, + 51.128299713135 + ] + }, + "generatedAtTime": "2021-10-12T06:20:42.586Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/LierVeemarkt", + "@type": "http://schema.org/ParkingFacility", + "name": "Lier Veemarkt", + "longitude": 4.583220005035, + "latitude": 51.128299713135, + "bikes_available": 7, + "capacity": 7, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821600" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.573339939117, + 51.125301361084 + ] + }, + "generatedAtTime": "2021-10-12T06:20:43.366Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/LierP%2BR%20De%20Mol", + "@type": "http://schema.org/ParkingFacility", + "name": "Lier P+R De Mol", + "longitude": 4.573339939117, + "latitude": 51.125301361084, + "bikes_available": 8, + "capacity": 8, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821600" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.494080066681, + 51.16849899292 + ] + }, + "generatedAtTime": "2021-10-12T06:20:44.070Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/BoechoutP%2BR%20Capenberg", + "@type": "http://schema.org/ParkingFacility", + "name": "Boechout P+R Capenberg", + "longitude": 4.494080066681, + "latitude": 51.16849899292, + "bikes_available": 5, + "capacity": 5, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821634" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.449820041656, + 51.092498779297 + ] + }, + "generatedAtTime": "2021-10-12T06:20:44.789Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Aalter", + "@type": "http://schema.org/ParkingFacility", + "name": "Aalter", + "longitude": 3.449820041656, + "latitude": 51.092498779297, + "bikes_available": 6, + "capacity": 6, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008891140" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.525919914246, + 50.982398986816 + ] + }, + "generatedAtTime": "2021-10-12T06:20:45.522Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/DeinzeLeiespiegel", + "@type": "http://schema.org/ParkingFacility", + "name": "Deinze Leiespiegel", + "longitude": 3.525919914246, + "latitude": 50.982398986816, + "bikes_available": 9, + "capacity": 11, + "docks_available": 2, + "nearby": "http://irail.be/stations/NMBS/008892106" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.92561006546, + 50.808200836182 + ] + }, + "generatedAtTime": "2021-10-12T06:20:46.217Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Tienen", + "@type": "http://schema.org/ParkingFacility", + "name": "Tienen ", + "longitude": 4.92561006546, + "latitude": 50.808200836182, + "bikes_available": 5, + "capacity": 13, + "docks_available": 8, + "nearby": "http://irail.be/stations/NMBS/008833308" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 2.735719919205, + 50.854499816895 + ] + }, + "generatedAtTime": "2021-10-12T06:20:46.915Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Poperinge", + "@type": "http://schema.org/ParkingFacility", + "name": "Poperinge", + "longitude": 2.735719919205, + "latitude": 50.854499816895, + "bikes_available": 7, + "capacity": 7, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008896735" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.632410049438, + 51.35710144043 + ] + }, + "generatedAtTime": "2021-10-12T06:20:47.633Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Noorderkempen", + "@type": "http://schema.org/ParkingFacility", + "name": "Noorderkempen", + "longitude": 4.632410049438, + "latitude": 51.35710144043, + "bikes_available": 10, + "capacity": 10, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821105" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.460559844971, + 51.365100860596 + ] + }, + "generatedAtTime": "2021-10-12T06:20:48.339Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Heide", + "@type": "http://schema.org/ParkingFacility", + "name": "Heide", + "longitude": 4.460559844971, + "latitude": 51.365100860596, + "bikes_available": 11, + "capacity": 12, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008821519" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.484320163727, + 51.029300689697 + ] + }, + "generatedAtTime": "2021-10-12T06:20:49.040Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/MechelenVeemarkt", + "@type": "http://schema.org/ParkingFacility", + "name": "Mechelen Veemarkt", + "longitude": 4.484320163727, + "latitude": 51.029300689697, + "bikes_available": 6, + "capacity": 7, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008822343" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.476119995117, + 51.134300231934 + ] + }, + "generatedAtTime": "2021-10-12T06:20:49.729Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Kontich-Lint", + "@type": "http://schema.org/ParkingFacility", + "name": "Kontich-Lint", + "longitude": 4.476119995117, + "latitude": 51.134300231934, + "bikes_available": 5, + "capacity": 7, + "docks_available": 2, + "nearby": "http://irail.be/stations/NMBS/008821311" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.329819917679, + 50.990600585938 + ] + }, + "generatedAtTime": "2021-10-12T06:20:50.446Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Tielt", + "@type": "http://schema.org/ParkingFacility", + "name": "Tielt", + "longitude": 3.329819917679, + "latitude": 50.990600585938, + "bikes_available": 7, + "capacity": 7, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008892254" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.470870018005, + 50.885200500488 + ] + }, + "generatedAtTime": "2021-10-12T06:20:51.139Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Zaventem", + "@type": "http://schema.org/ParkingFacility", + "name": "Zaventem", + "longitude": 4.470870018005, + "latitude": 50.885200500488, + "bikes_available": 8, + "capacity": 8, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008811221" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.390810012817, + 51.198001861572 + ] + }, + "generatedAtTime": "2021-10-12T06:20:51.829Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Antwerpen-Zuid", + "@type": "http://schema.org/ParkingFacility", + "name": "Antwerpen-Zuid", + "longitude": 4.390810012817, + "latitude": 51.198001861572, + "bikes_available": 8, + "capacity": 8, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821196" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.214819908142, + 50.9208984375 + ] + }, + "generatedAtTime": "2021-10-12T06:20:52.530Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Izegem", + "@type": "http://schema.org/ParkingFacility", + "name": "Izegem", + "longitude": 3.214819908142, + "latitude": 50.9208984375, + "bikes_available": 6, + "capacity": 8, + "docks_available": 2, + "nearby": "http://irail.be/stations/NMBS/008896909" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.711899995804, + 51.035999298096 + ] + }, + "generatedAtTime": "2021-10-12T06:20:53.258Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Gent-Sint-Pieters(M.%20Hendrikaplein)", + "@type": "http://schema.org/ParkingFacility", + "name": "Gent-Sint-Pieters (M. Hendrikaplein)", + "longitude": 3.711899995804, + "latitude": 51.035999298096, + "bikes_available": 16, + "capacity": 26, + "docks_available": 10, + "nearby": "http://irail.be/stations/NMBS/008892007" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.716639995575, + 50.882499694824 + ] + }, + "generatedAtTime": "2021-10-12T06:20:53.955Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Leuven(Kop%20van%20Kessel-Lo)", + "@type": "http://schema.org/ParkingFacility", + "name": "Leuven (Kop van Kessel-Lo)", + "longitude": 4.716639995575, + "latitude": 50.882499694824, + "bikes_available": 16, + "capacity": 24, + "docks_available": 8, + "nearby": "http://irail.be/stations/NMBS/008833001" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.216079950333, + 51.196800231934 + ] + }, + "generatedAtTime": "2021-10-12T06:20:54.646Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Brugge(Kamgebouw)", + "@type": "http://schema.org/ParkingFacility", + "name": "Brugge (Kamgebouw)", + "longitude": 3.216079950333, + "latitude": 51.196800231934, + "bikes_available": 17, + "capacity": 27, + "docks_available": 10, + "nearby": "http://irail.be/stations/NMBS/008891009" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.260260105133, + 51.208499908447 + ] + }, + "generatedAtTime": "2021-10-12T06:20:55.643Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Beveren", + "@type": "http://schema.org/ParkingFacility", + "name": "Beveren", + "longitude": 4.260260105133, + "latitude": 51.208499908447, + "bikes_available": 10, + "capacity": 11, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008894748" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.695439815521, + 50.862598419189 + ] + }, + "generatedAtTime": "2021-10-12T06:20:56.370Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Heverlee", + "@type": "http://schema.org/ParkingFacility", + "name": "Heverlee ", + "longitude": 4.695439815521, + "latitude": 50.862598419189, + "bikes_available": 7, + "capacity": 9, + "docks_available": 2, + "nearby": "http://irail.be/stations/NMBS/008833126" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.31374001503, + 50.855701446533 + ] + }, + "generatedAtTime": "2021-10-12T06:20:57.256Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Harelbeke", + "@type": "http://schema.org/ParkingFacility", + "name": "Harelbeke", + "longitude": 3.31374001503, + "latitude": 50.855701446533, + "bikes_available": 8, + "capacity": 9, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008896115" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.64013004303, + 50.929401397705 + ] + }, + "generatedAtTime": "2021-10-12T06:20:57.946Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Gavere-Asper", + "@type": "http://schema.org/ParkingFacility", + "name": "Gavere-Asper", + "longitude": 3.64013004303, + "latitude": 50.929401397705, + "bikes_available": 7, + "capacity": 7, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008892643" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.654690027237, + 50.930500030518 + ] + }, + "generatedAtTime": "2021-10-12T06:20:58.653Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Gaveremobipunt", + "@type": "http://schema.org/ParkingFacility", + "name": "Gavere mobipunt", + "longitude": 3.654690027237, + "latitude": 50.930500030518, + "bikes_available": 4, + "capacity": 4, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008892643" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.43501996994, + 51.284198760986 + ] + }, + "generatedAtTime": "2021-10-12T06:20:59.366Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Ekeren(perron%201)", + "@type": "http://schema.org/ParkingFacility", + "name": "Ekeren (perron 1)", + "longitude": 4.43501996994, + "latitude": 51.284198760986, + "bikes_available": 3, + "capacity": 4, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008821071" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.434390068054, + 51.281398773193 + ] + }, + "generatedAtTime": "2021-10-12T06:21:00.100Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Ekeren(perron%202)", + "@type": "http://schema.org/ParkingFacility", + "name": "Ekeren (perron 2)", + "longitude": 4.434390068054, + "latitude": 51.281398773193, + "bikes_available": 7, + "capacity": 7, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821071" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.424610137939, + 51.245498657227 + ] + }, + "generatedAtTime": "2021-10-12T06:21:00.798Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Antwerpen-Luchtbal", + "@type": "http://schema.org/ParkingFacility", + "name": "Antwerpen-Luchtbal", + "longitude": 4.424610137939, + "latitude": 51.245498657227, + "bikes_available": 5, + "capacity": 5, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821063" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.42755, + 51.2617 + ] + }, + "generatedAtTime": "2021-10-12T06:21:01.494Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Antwerpen-Noorderdokken", + "@type": "http://schema.org/ParkingFacility", + "name": "Antwerpen-Noorderdokken", + "longitude": 4.42755, + "latitude": 51.2617, + "bikes_available": 7, + "capacity": 7, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821089" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.447710037231, + 50.889099121094 + ] + }, + "generatedAtTime": "2021-10-12T06:21:02.217Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Diegem", + "@type": "http://schema.org/ParkingFacility", + "name": "Diegem", + "longitude": 4.447710037231, + "latitude": 50.889099121094, + "bikes_available": 4, + "capacity": 4, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008811213" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.426719903946, + 50.881999969482 + ] + }, + "generatedAtTime": "2021-10-12T06:21:02.941Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/WaregemExpo%20P%2BR", + "@type": "http://schema.org/ParkingFacility", + "name": "Waregem Expo P+R", + "longitude": 3.426719903946, + "latitude": 50.881999969482, + "bikes_available": 4, + "capacity": 4, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008896149" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.281699, + 51.03833 + ] + }, + "generatedAtTime": "2021-10-12T06:21:03.654Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Heusden", + "@type": "http://schema.org/ParkingFacility", + "name": "Heusden", + "longitude": 5.281699, + "latitude": 51.03833, + "bikes_available": 8, + "capacity": 8, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008832243" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.329922, + 51.033617 + ] + }, + "generatedAtTime": "2021-10-12T06:21:04.377Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Zolder", + "@type": "http://schema.org/ParkingFacility", + "name": "Zolder", + "longitude": 5.329922, + "latitude": 51.033617, + "bikes_available": 8, + "capacity": 8, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008832250" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.329820851913, + 50.932948204381 + ] + }, + "generatedAtTime": "2021-10-12T06:21:05.094Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Meise", + "@type": "http://schema.org/ParkingFacility", + "name": "Meise", + "longitude": 4.329820851913, + "latitude": 50.932948204381, + "bikes_available": 8, + "capacity": 8, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008812047" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.746427, + 51.165508 + ] + }, + "generatedAtTime": "2021-10-12T06:21:05.822Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Bouwel", + "@type": "http://schema.org/ParkingFacility", + "name": "Bouwel", + "longitude": 4.746427, + "latitude": 51.165508, + "bikes_available": 6, + "capacity": 6, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821725" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.613315, + 50.966578 + ] + }, + "generatedAtTime": "2021-10-12T06:21:06.544Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Haacht", + "@type": "http://schema.org/ParkingFacility", + "name": "Haacht", + "longitude": 4.613315, + "latitude": 50.966578, + "bikes_available": 7, + "capacity": 8, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008822517" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.814341, + 50.86948 + ] + }, + "generatedAtTime": "2021-10-12T06:21:07.245Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Zottegem", + "@type": "http://schema.org/ParkingFacility", + "name": "Zottegem ", + "longitude": 3.814341, + "latitude": 50.86948, + "bikes_available": 7, + "capacity": 8, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008895208" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.221333, + 51.125867 + ] + }, + "generatedAtTime": "2021-10-12T06:21:07.957Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Temse", + "@type": "http://schema.org/ParkingFacility", + "name": "Temse", + "longitude": 4.221333, + "latitude": 51.125867, + "bikes_available": 5, + "capacity": 5, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008894672" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.041885, + 51.073108 + ] + }, + "generatedAtTime": "2021-10-12T06:21:08.664Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Zele", + "@type": "http://schema.org/ParkingFacility", + "name": "Zele", + "longitude": 4.041885, + "latitude": 51.073108, + "bikes_available": 6, + "capacity": 7, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008894235" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.182521, + 50.810926 + ] + }, + "generatedAtTime": "2021-10-12T06:21:09.366Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Wevelgem", + "@type": "http://schema.org/ParkingFacility", + "name": "Wevelgem", + "longitude": 3.182521, + "latitude": 50.810926, + "bikes_available": 8, + "capacity": 8, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008896370" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.312579, + 51.21177 + ] + }, + "generatedAtTime": "2021-10-12T06:21:10.091Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Lommel", + "@type": "http://schema.org/ParkingFacility", + "name": "Lommel", + "longitude": 5.312579, + "latitude": 51.21177, + "bikes_available": 6, + "capacity": 6, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008832565" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.464733, + 51.1541195 + ] + }, + "generatedAtTime": "2021-10-12T06:21:10.807Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Hove", + "@type": "http://schema.org/ParkingFacility", + "name": "Hove", + "longitude": 4.464733, + "latitude": 51.1541195, + "bikes_available": 7, + "capacity": 7, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821337" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.451532, + 51.462176 + ] + }, + "generatedAtTime": "2021-10-12T06:21:11.779Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Essen", + "@type": "http://schema.org/ParkingFacility", + "name": "Essen", + "longitude": 4.451532, + "latitude": 51.462176, + "bikes_available": 7, + "capacity": 8, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008821402" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.079402, + 50.74829 + ] + }, + "generatedAtTime": "2021-10-12T06:21:12.535Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Landen", + "@type": "http://schema.org/ParkingFacility", + "name": "Landen", + "longitude": 5.079402, + "latitude": 50.74829, + "bikes_available": 7, + "capacity": 8, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008833605" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 2.601611, + 51.077376 + ] + }, + "generatedAtTime": "2021-10-12T06:21:13.294Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/DePanne", + "@type": "http://schema.org/ParkingFacility", + "name": "De Panne", + "longitude": 2.601611, + "latitude": 51.077376, + "bikes_available": 8, + "capacity": 8, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008892338" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.1338, + 51.3119 + ] + }, + "generatedAtTime": "2021-10-12T06:21:14.201Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Blankenberge", + "@type": "http://schema.org/ParkingFacility", + "name": "Blankenberge", + "longitude": 3.1338, + "latitude": 51.3119, + "bikes_available": 6, + "capacity": 7, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008891405" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.195476, + 51.32651 + ] + }, + "generatedAtTime": "2021-10-12T06:21:14.998Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Zeebrugge-dorp", + "@type": "http://schema.org/ParkingFacility", + "name": "Zeebrugge-dorp", + "longitude": 3.195476, + "latitude": 51.32651, + "bikes_available": 2, + "capacity": 2, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008891553" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.239668, + 51.334168 + ] + }, + "generatedAtTime": "2021-10-12T06:21:15.934Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Heist", + "@type": "http://schema.org/ParkingFacility", + "name": "Heist", + "longitude": 3.239668, + "latitude": 51.334168, + "bikes_available": 5, + "capacity": 5, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008891645" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.28454, + 51.34018 + ] + }, + "generatedAtTime": "2021-10-12T06:21:16.629Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Knokke", + "@type": "http://schema.org/ParkingFacility", + "name": "Knokke", + "longitude": 3.28454, + "latitude": 51.34018, + "bikes_available": 7, + "capacity": 8, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008891660" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.361626, + 50.747786 + ] + }, + "generatedAtTime": "2021-10-12T06:21:17.346Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Sint-Genesius-Rode", + "@type": "http://schema.org/ParkingFacility", + "name": "Sint-Genesius-Rode", + "longitude": 4.361626, + "latitude": 50.747786, + "bikes_available": 9, + "capacity": 9, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008814167" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.575647, + 51.064208 + ] + }, + "generatedAtTime": "2021-10-12T06:21:18.066Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Landegem", + "@type": "http://schema.org/ParkingFacility", + "name": "Landegem", + "longitude": 3.575647, + "latitude": 51.064208, + "bikes_available": 8, + "capacity": 8, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008892056" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.413591, + 50.923187 + ] + }, + "generatedAtTime": "2021-10-12T06:21:18.772Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/VilvoordeHoppin%20De%20Kassei", + "@type": "http://schema.org/ParkingFacility", + "name": "Vilvoorde Hoppin De Kassei", + "longitude": 4.413591, + "latitude": 50.923187, + "bikes_available": 6, + "capacity": 7, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008811189" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.333756181866, + 50.835529468767 + ] + }, + "generatedAtTime": "2021-10-12T06:21:19.556Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Brussel-Zuid", + "@type": "http://schema.org/ParkingFacility", + "name": "Brussel-Zuid", + "longitude": 4.333756181866, + "latitude": 50.835529468767, + "bikes_available": 4, + "capacity": 6, + "docks_available": 2, + "nearby": "http://irail.be/stations/NMBS/008814001" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.112421, + 51.18119 + ] + }, + "generatedAtTime": "2021-10-12T06:21:20.277Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Mol-%20De%20Nete", + "@type": "http://schema.org/ParkingFacility", + "name": "Mol - De Nete", + "longitude": 5.112421, + "latitude": 51.18119, + "bikes_available": 7, + "capacity": 7, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008832409" + } + } + ] +} From 3bf82131cf583f30500691e19a4e6f17d4a89efb Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Sun, 20 Feb 2022 13:59:17 +0100 Subject: [PATCH 010/609] cli: Main: add Target metadata to output --- src/main/java/be/ugent/rml/cli/Main.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/be/ugent/rml/cli/Main.java b/src/main/java/be/ugent/rml/cli/Main.java index c650ff6a..7b542fc4 100644 --- a/src/main/java/be/ugent/rml/cli/Main.java +++ b/src/main/java/be/ugent/rml/cli/Main.java @@ -8,6 +8,7 @@ import be.ugent.rml.functions.lib.IDLabFunctions; import be.ugent.rml.metadata.MetadataGenerator; import be.ugent.rml.records.RecordsFactory; +import be.ugent.rml.store.Quad; import be.ugent.rml.store.QuadStore; import be.ugent.rml.store.RDF4JStore; import be.ugent.rml.store.SimpleQuadStore; @@ -428,9 +429,10 @@ private static void writeOutputTargets(HashMap targets, QuadSto logger.info(store.size() + " quad was generated " + term + " Target"); } - Target target = targetFactory.getTarget(term, rmlStore); + Target target = targetFactory.getTarget(term, rmlStore, store); String serializationFormat = target.getSerializationFormat(); OutputStream output = target.getOutputStream(); + store.addQuads(target.getMetadata()); // Set character encoding Writer out = new BufferedWriter(new OutputStreamWriter(output, Charset.defaultCharset())); From 6c16849c688a3b139f09ec92e42b2680cc7945f9 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Sun, 20 Feb 2022 13:59:57 +0100 Subject: [PATCH 011/609] CHANGELOG: LDES Logical Target is now supported --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ada1946f..4c14cead 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +### Added + +- Add support for LDES Logical Target + ### Fixed - Clarified Readme for quick start From 6d718e6aace3103ec795efedd9cda7b6ca6b67eb Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Sun, 20 Feb 2022 14:47:36 +0100 Subject: [PATCH 012/609] test: logical-target: sparql: fix test SPARQL Logical Target contained an error in the mapping file. The LDES Logical Target changes increased the test coverage and catched this bug! --- .../resources/web-of-things/logical-target/sparql/mapping.ttl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/resources/web-of-things/logical-target/sparql/mapping.ttl b/src/test/resources/web-of-things/logical-target/sparql/mapping.ttl index fa26fa4a..9d50b514 100644 --- a/src/test/resources/web-of-things/logical-target/sparql/mapping.ttl +++ b/src/test/resources/web-of-things/logical-target/sparql/mapping.ttl @@ -28,7 +28,7 @@ . <#TargetSPARQL> a rmlt:LogicalTarget; - rmlt:target ; + rmlt:target <#SPARQLEndpoint>; . <#WoTWebResource> a td:PropertyAffordance; From 0584ff8442ad066f9deb98711c32c697314b0c4c Mon Sep 17 00:00:00 2001 From: Ben De Meester Date: Thu, 3 Mar 2022 09:22:10 +0000 Subject: [PATCH 013/609] MappingExtractor: No cartesian product for same Logical Sources No more cartesian product when logical source is the same (i.e., has the same URI). If you still want the cartesian product, update your mappings to refer to different logical sources with the same contents. Fixes https://github.com/RMLio/rmlmapper-java/issues/28 --- CHANGELOG.md | 2 + .../java/be/ugent/rml/MappingFactory.java | 41 +++++++++++++ .../be/ugent/rml/extractor/HashExtractor.java | 36 ++++++++++++ .../java/be/ugent/rml/Mapper_JSON_Test.java | 10 ++++ .../test-cases/RMLTC1028-JSON/mapping.ttl | 54 +++++++++++++++++ .../test-cases/RMLTC1028-JSON/output.nq | 12 ++++ .../test-cases/RMLTC1028-JSON/venue.json | 22 +++++++ .../test-cases/RMLTC1028b-JSON/mapping.ttl | 58 +++++++++++++++++++ .../test-cases/RMLTC1028b-JSON/output.nq | 14 +++++ .../test-cases/RMLTC1028b-JSON/venue.json | 22 +++++++ 10 files changed, 271 insertions(+) create mode 100644 src/main/java/be/ugent/rml/extractor/HashExtractor.java create mode 100644 src/test/resources/test-cases/RMLTC1028-JSON/mapping.ttl create mode 100644 src/test/resources/test-cases/RMLTC1028-JSON/output.nq create mode 100644 src/test/resources/test-cases/RMLTC1028-JSON/venue.json create mode 100644 src/test/resources/test-cases/RMLTC1028b-JSON/mapping.ttl create mode 100644 src/test/resources/test-cases/RMLTC1028b-JSON/output.nq create mode 100644 src/test/resources/test-cases/RMLTC1028b-JSON/venue.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c14cead..e3265ae8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed - Clarified Readme for quick start +- No cartesian product when referring to the same logical source (see [issue 28](https://github.com/RMLio/rmlmapper-java/issues/28)]) + - If you still want the cartesian product, update your mappings to refer to different logical sources with the same contents ### Changed diff --git a/src/main/java/be/ugent/rml/MappingFactory.java b/src/main/java/be/ugent/rml/MappingFactory.java index c805e4ca..924a9534 100644 --- a/src/main/java/be/ugent/rml/MappingFactory.java +++ b/src/main/java/be/ugent/rml/MappingFactory.java @@ -1,6 +1,7 @@ package be.ugent.rml; import be.ugent.rml.extractor.ConstantExtractor; +import be.ugent.rml.extractor.HashExtractor; import be.ugent.rml.extractor.ReferenceExtractor; import be.ugent.rml.functions.*; import be.ugent.rml.store.QuadStore; @@ -282,6 +283,26 @@ private void parseObjectMapWithCallback(Term objectmap, BiConsumer logicalSources = Utils.getObjectsFromQuads(store.getQuads(this.triplesMap, new NamedNode(NAMESPACES.RML + "logicalSource"), null)); + Term logicalSource = null; + if (!logicalSources.isEmpty()) { + logicalSource = logicalSources.get(0); + } + + List parentLogicalSources = Utils.getObjectsFromQuads(store.getQuads(parentTriplesMap, new NamedNode(NAMESPACES.RML + "logicalSource"), null)); + Term parentLogicalSource = null; + if (!parentLogicalSources.isEmpty()) { + parentLogicalSource = parentLogicalSources.get(0); + } + // Check if there is at least one Logical Source. + // If logical sources are the same (i.e., have the same IRI): the condition is 'join on same record' + if (logicalSource.equals(parentLogicalSource)) { + // TODO this is a _WILDLY_ inefficient way of handling this: a join is still executed, but then on the hashcode of the record. + // I'm not sure what a more elegant solution would entail in the current architecture: joins are currently hard to optimize + joinConditionFunctionExecutors.add(generateSameLogicalSourceJoinConditionFunctionTermMap()); + } + if (refObjectMapCallback != null) { refObjectMapCallback.accept(parentTriplesMap, joinConditionFunctionExecutors); } @@ -456,6 +477,26 @@ private MultipleRecordsFunctionExecutor parseJoinConditionFunctionTermMap(Term f return new DynamicMultipleRecordsFunctionExecutor(params, functionLoader); } + /** + * Generate a join condition that only returns true if the same record hash is encountered + * @return + * @throws IOException + */ + private MultipleRecordsFunctionExecutor generateSameLogicalSourceJoinConditionFunctionTermMap() throws IOException { + FunctionModel equal = functionLoader.getFunction(new NamedNode("http://example.com/idlab/function/equal")); + Map parameters = new HashMap<>(); + + SingleRecordFunctionExecutor parent = new HashExtractor(); + Object[] detailsParent = {"parent", parent}; + parameters.put("http://users.ugent.be/~bjdmeest/function/grel.ttl#valueParameter", detailsParent); + + SingleRecordFunctionExecutor child = new HashExtractor(); + Object[] detailsChild = {"child", child}; + parameters.put("http://users.ugent.be/~bjdmeest/function/grel.ttl#valueParameter2", detailsChild); + + return new StaticMultipleRecordsFunctionExecutor(equal, parameters); + } + private List parseObjectMapsAndShortcuts(Term pom) throws IOException { List mappingInfos = new ArrayList<>(); diff --git a/src/main/java/be/ugent/rml/extractor/HashExtractor.java b/src/main/java/be/ugent/rml/extractor/HashExtractor.java new file mode 100644 index 00000000..a2d66022 --- /dev/null +++ b/src/main/java/be/ugent/rml/extractor/HashExtractor.java @@ -0,0 +1,36 @@ +package be.ugent.rml.extractor; + +import be.ugent.rml.functions.SingleRecordFunctionExecutor; +import be.ugent.rml.records.Record; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class HashExtractor implements Extractor, SingleRecordFunctionExecutor { + + public HashExtractor() { + } + + @Override + public List extract(Record record) { + ArrayList result = new ArrayList<>(); + result.add("" + record.hashCode()); + return result; + } + + @Override + public Object execute(Record record) throws IOException { + return extract(record); + } + + /** + * to String method + * + * @return string + */ + @Override + public String toString() { + return "HashExtractor"; + } +} diff --git a/src/test/java/be/ugent/rml/Mapper_JSON_Test.java b/src/test/java/be/ugent/rml/Mapper_JSON_Test.java index 73a34814..e4d28f69 100644 --- a/src/test/java/be/ugent/rml/Mapper_JSON_Test.java +++ b/src/test/java/be/ugent/rml/Mapper_JSON_Test.java @@ -274,4 +274,14 @@ public void evaluate_1027_JSON() { public void evaluate_1027_JSONL() { doMapping("./test-cases/RMLTC1027-JSONL/mapping.ttl", "./test-cases/RMLTC1027-JSONL/output.nq"); } + + @Test + public void evaluate_1028_JSON() { + doMapping("./test-cases/RMLTC1028-JSON/mapping.ttl", "./test-cases/RMLTC1028-JSON/output.nq"); + } + + @Test + public void evaluate_1028b_JSON() { + doMapping("./test-cases/RMLTC1028b-JSON/mapping.ttl", "./test-cases/RMLTC1028b-JSON/output.nq"); + } } diff --git a/src/test/resources/test-cases/RMLTC1028-JSON/mapping.ttl b/src/test/resources/test-cases/RMLTC1028-JSON/mapping.ttl new file mode 100644 index 00000000..1816e936 --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1028-JSON/mapping.ttl @@ -0,0 +1,54 @@ +@prefix rml: . +@prefix rr: . +@prefix ql: . +@prefix ex: . +@base . + +<#LogicalSource> rml:source "venue.json"; + rml:referenceFormulation ql:JSONPath ; + rml:iterator "$.venue[*]" . + +<#VenueMapping> a rr:TriplesMap; + rml:logicalSource <#LogicalSource> ; + rr:subjectMap + [ + rr:template "http://loc.example.com/city/{location.city}"; + rr:class ex:City; + ]; + rr:predicateObjectMap [ + rr:predicate ex:latlong; + rr:objectMap [ + rr:parentTriplesMap <#LocationMapping_JSON> + ] + ]; + rr:predicateObjectMap [ + rr:predicate ex:countryCode; + rr:objectMap [ + rml:reference "location.country" + ] + ]; + rr:predicateObjectMap [ + rr:predicate ex:onContinent; + rr:objectMap [ + rml:reference "location.continent" + ] + ]. + +<#LocationMapping_JSON> a rr:TriplesMap; + rml:logicalSource <#LogicalSource> ; + rr:subjectMap [ + rr:template "http://loc.example.com/latlong/{latitude},{longitude}" + ]; + rr:predicateObjectMap [ + rr:predicate ex:lat; + rr:objectMap [ + rml:reference "latitude" + ] + ]; + + rr:predicateObjectMap [ + rr:predicate ex:long; + rr:objectMap [ + rml:reference "longitude" + ] + ]. \ No newline at end of file diff --git a/src/test/resources/test-cases/RMLTC1028-JSON/output.nq b/src/test/resources/test-cases/RMLTC1028-JSON/output.nq new file mode 100644 index 00000000..b6f2b5e8 --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1028-JSON/output.nq @@ -0,0 +1,12 @@ + . + . + "BE" . + "EU" . + "50.901389" . + "4.484444" . + . + . + "GB" . + "EU" . + "51.51334" . + "-0.08901" . diff --git a/src/test/resources/test-cases/RMLTC1028-JSON/venue.json b/src/test/resources/test-cases/RMLTC1028-JSON/venue.json new file mode 100644 index 00000000..defa5e98 --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1028-JSON/venue.json @@ -0,0 +1,22 @@ +{ + "venue": [ + { + "latitude": "50.901389", + "longitude": "4.484444", + "location": { + "continent": "EU", + "country": "BE", + "city": "Brussels" + } + }, + { + "latitude": "51.51334", + "longitude": "-0.08901", + "location": { + "continent": "EU", + "country": "GB", + "city": "London" + } + } + ] +} \ No newline at end of file diff --git a/src/test/resources/test-cases/RMLTC1028b-JSON/mapping.ttl b/src/test/resources/test-cases/RMLTC1028b-JSON/mapping.ttl new file mode 100644 index 00000000..012791c7 --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1028b-JSON/mapping.ttl @@ -0,0 +1,58 @@ +@prefix rml: . +@prefix rr: . +@prefix ql: . +@prefix ex: . +@base . + +<#LogicalSource> rml:source "venue.json"; + rml:referenceFormulation ql:JSONPath ; + rml:iterator "$.venue[*]" . + +<#LogicalSource2> rml:source "venue.json"; + rml:referenceFormulation ql:JSONPath ; + rml:iterator "$.venue[*]" . + +<#VenueMapping> a rr:TriplesMap; + rml:logicalSource <#LogicalSource> ; + rr:subjectMap + [ + rr:template "http://loc.example.com/city/{location.city}"; + rr:class ex:City; + ]; + rr:predicateObjectMap [ + rr:predicate ex:latlong; + rr:objectMap [ + rr:parentTriplesMap <#LocationMapping_JSON> + ] + ]; + rr:predicateObjectMap [ + rr:predicate ex:countryCode; + rr:objectMap [ + rml:reference "location.country" + ] + ]; + rr:predicateObjectMap [ + rr:predicate ex:onContinent; + rr:objectMap [ + rml:reference "location.continent" + ] + ]. + +<#LocationMapping_JSON> a rr:TriplesMap; + rml:logicalSource <#LogicalSource2> ; + rr:subjectMap [ + rr:template "http://loc.example.com/latlong/{latitude},{longitude}" + ]; + rr:predicateObjectMap [ + rr:predicate ex:lat; + rr:objectMap [ + rml:reference "latitude" + ] + ]; + + rr:predicateObjectMap [ + rr:predicate ex:long; + rr:objectMap [ + rml:reference "longitude" + ] + ]. \ No newline at end of file diff --git a/src/test/resources/test-cases/RMLTC1028b-JSON/output.nq b/src/test/resources/test-cases/RMLTC1028b-JSON/output.nq new file mode 100644 index 00000000..09baddf4 --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1028b-JSON/output.nq @@ -0,0 +1,14 @@ + . + . + . + "BE" . + "EU" . + "50.901389" . + "4.484444" . + . + . + . + "GB" . + "EU" . + "51.51334" . + "-0.08901" . diff --git a/src/test/resources/test-cases/RMLTC1028b-JSON/venue.json b/src/test/resources/test-cases/RMLTC1028b-JSON/venue.json new file mode 100644 index 00000000..defa5e98 --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1028b-JSON/venue.json @@ -0,0 +1,22 @@ +{ + "venue": [ + { + "latitude": "50.901389", + "longitude": "4.484444", + "location": { + "continent": "EU", + "country": "BE", + "city": "Brussels" + } + }, + { + "latitude": "51.51334", + "longitude": "-0.08901", + "location": { + "continent": "EU", + "country": "GB", + "city": "London" + } + } + ] +} \ No newline at end of file From 2f128a86bf00c83ca32515b666c47f596edf0791 Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 12:22:29 +0100 Subject: [PATCH 014/609] Updated org.eclipse.jetty and com.github.tomakehurst to compatible version. --- CHANGELOG.md | 2 ++ pom.xml | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3265ae8..f3a609dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Clarified Readme for quick start - No cartesian product when referring to the same logical source (see [issue 28](https://github.com/RMLio/rmlmapper-java/issues/28)]) - If you still want the cartesian product, update your mappings to refer to different logical sources with the same contents +- Upgrade jetty-server and jetty-security to 9.4.44.v20210927 +- Upgrade wiremock-jre8 to 2.32.0 ### Changed diff --git a/pom.xml b/pom.xml index 712dcad5..5ad28510 100644 --- a/pom.xml +++ b/pom.xml @@ -211,20 +211,20 @@ org.eclipse.jetty jetty-server - 9.4.17.v20190418 + 9.4.44.v20210927 test org.eclipse.jetty jetty-security - 9.4.17.v20190418 + 9.4.44.v20210927 test com.github.tomakehurst wiremock-jre8 - 2.23.2 + 2.32.0 test From a7269f72974d2ec3088a3343e52d89dbc695d696 Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 13:41:29 +0100 Subject: [PATCH 015/609] ch.qos.logback upgraded ch.qos.logback from 1.2.3 -> 1.2.10 --- CHANGELOG.md | 3 ++- pom.xml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3a609dd..dd667d39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - No cartesian product when referring to the same logical source (see [issue 28](https://github.com/RMLio/rmlmapper-java/issues/28)]) - If you still want the cartesian product, update your mappings to refer to different logical sources with the same contents - Upgrade jetty-server and jetty-security to 9.4.44.v20210927 -- Upgrade wiremock-jre8 to 2.32.0 +- Upgrade wiremock-jre8 to 2.32.0 +- Upgrade ch.qos.logback to 1.2.10 ### Changed diff --git a/pom.xml b/pom.xml index 5ad28510..d332dea4 100644 --- a/pom.xml +++ b/pom.xml @@ -125,7 +125,7 @@ ch.qos.logback logback-classic - 1.2.3 + 1.2.10 commons-lang From a8513096a5fdc04b5c73bf5e75b01e5598ba65d5 Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 13:52:53 +0100 Subject: [PATCH 016/609] commons-cli upgraded from 1.4 -> 1.5.0 --- CHANGELOG.md | 1 + pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd667d39..d45f38a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Upgrade jetty-server and jetty-security to 9.4.44.v20210927 - Upgrade wiremock-jre8 to 2.32.0 - Upgrade ch.qos.logback to 1.2.10 +- Upgrade commons-cli to 1.5.0 ### Changed diff --git a/pom.xml b/pom.xml index d332dea4..0b7f95dc 100644 --- a/pom.xml +++ b/pom.xml @@ -135,7 +135,7 @@ commons-cli commons-cli - 1.4 + 1.5.0 org.eclipse.rdf4j From f392126a3f442ccdd3653b9ea8231d8ffff4cd7d Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 14:04:27 +0100 Subject: [PATCH 017/609] com.jayway.jsonpath upgraded from 2.6.0 -> 2.7.0 --- CHANGELOG.md | 1 + pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d45f38a5..d1cef24c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Upgrade wiremock-jre8 to 2.32.0 - Upgrade ch.qos.logback to 1.2.10 - Upgrade commons-cli to 1.5.0 +- Upgrade com.jayway.jsonpath to 2.7.0 ### Changed diff --git a/pom.xml b/pom.xml index 0b7f95dc..cf311828 100644 --- a/pom.xml +++ b/pom.xml @@ -157,7 +157,7 @@ com.jayway.jsonpath json-path - 2.6.0 + 2.7.0 javax.xml.parsers From ddfacb46df0b51725c176a63c33cd20ec7d515f6 Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 14:14:32 +0100 Subject: [PATCH 018/609] ch.vorburger.mariaDB4j upgraded from 2.4.0 -> 2.5.3 --- CHANGELOG.md | 1 + pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1cef24c..6a3c31ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Upgrade ch.qos.logback to 1.2.10 - Upgrade commons-cli to 1.5.0 - Upgrade com.jayway.jsonpath to 2.7.0 +- Upgrade ch.vorburger.mariaDB4j to 2.5.3 ### Changed diff --git a/pom.xml b/pom.xml index cf311828..7157599a 100644 --- a/pom.xml +++ b/pom.xml @@ -172,7 +172,7 @@ ch.vorburger.mariaDB4j mariaDB4j - 2.4.0 + 2.5.3 test From 5baca6c1907bbfbe8a89a8bdf534ece61615e4aa Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 14:47:25 +0100 Subject: [PATCH 019/609] com.microsoft.sqlserver upgraded from 7.2.2.jre8 -> 9.4.0.jre8 --- CHANGELOG.md | 1 + pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a3c31ad..f47ba712 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Upgrade commons-cli to 1.5.0 - Upgrade com.jayway.jsonpath to 2.7.0 - Upgrade ch.vorburger.mariaDB4j to 2.5.3 +- Upgrade com.microsoft.sqlserver to 9.4.0.jre8 ### Changed diff --git a/pom.xml b/pom.xml index 7157599a..852a23d7 100644 --- a/pom.xml +++ b/pom.xml @@ -184,7 +184,7 @@ com.microsoft.sqlserver mssql-jdbc - 7.2.2.jre8 + 9.4.0.jre8 test From 5a6b97db59d5d9b7a6828657196abac3a67476c8 Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 15:21:47 +0100 Subject: [PATCH 020/609] com.fasterxml.jackson.core upgraded to 2.12.1 -> 2.13.1 --- CHANGELOG.md | 1 + pom.xml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f47ba712..352d40b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Upgrade com.jayway.jsonpath to 2.7.0 - Upgrade ch.vorburger.mariaDB4j to 2.5.3 - Upgrade com.microsoft.sqlserver to 9.4.0.jre8 +- Upgrade com.fasterxml.jackson.core to 2.13.1 ### Changed diff --git a/pom.xml b/pom.xml index 852a23d7..eda66975 100644 --- a/pom.xml +++ b/pom.xml @@ -196,17 +196,17 @@ com.fasterxml.jackson.core jackson-core - 2.12.1 + 2.13.1 com.fasterxml.jackson.core jackson-databind - 2.12.1 + 2.13.1 com.fasterxml.jackson.core jackson-annotations - 2.12.1 + 2.13.1 org.eclipse.jetty From c00c727545dcafd5736dc4a33b3cb39cac7098e2 Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 15:30:13 +0100 Subject: [PATCH 021/609] org.jsoup upgrade from 1.14.2 -> 1.14.3 --- CHANGELOG.md | 1 + pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 352d40b6..e2de6509 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Upgrade ch.vorburger.mariaDB4j to 2.5.3 - Upgrade com.microsoft.sqlserver to 9.4.0.jre8 - Upgrade com.fasterxml.jackson.core to 2.13.1 +- Upgrade org.jsoup to 1.14.3 ### Changed diff --git a/pom.xml b/pom.xml index eda66975..6fc753b2 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ org.jsoup jsoup - 1.14.2 + 1.14.3 From ebbdcd54b138f9ebeac575675ec2370019815a69 Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 15:46:42 +0100 Subject: [PATCH 022/609] org.apache.poi upgraded to 4.1.0 -> 5.0.0 --- CHANGELOG.md | 1 + pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2de6509..78424149 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Upgrade com.microsoft.sqlserver to 9.4.0.jre8 - Upgrade com.fasterxml.jackson.core to 2.13.1 - Upgrade org.jsoup to 1.14.3 +- Upgrade org.apache.poi to 5.0.0 ### Changed diff --git a/pom.xml b/pom.xml index 6fc753b2..19a430ff 100644 --- a/pom.xml +++ b/pom.xml @@ -281,7 +281,7 @@ org.apache.poi poi-ooxml - 4.1.0 + 5.0.0 From 8d7d8cb45095d09eedd7e59f4ed395f42757bef8 Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 15:54:42 +0100 Subject: [PATCH 023/609] org.apache.jena upgraded from 3.8.0 -> 4.3.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 19a430ff..9924fd38 100644 --- a/pom.xml +++ b/pom.xml @@ -231,7 +231,7 @@ org.apache.jena apache-jena-libs pom - 3.8.0 + 4.3.2 test From ecbeb6c16c1b70dc4959ea472e94271528148331 Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 16:03:23 +0100 Subject: [PATCH 024/609] com.microsoft.sqlserver upgrade from 9.4.0.jre8 -> 10.2.0.jre8 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9924fd38..d300aaf5 100644 --- a/pom.xml +++ b/pom.xml @@ -184,7 +184,7 @@ com.microsoft.sqlserver mssql-jdbc - 9.4.0.jre8 + 10.2.0.jre8 test From 712ed9ae22d0d7d7d1b5e86abbc6b08eb3224a57 Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 16:12:31 +0100 Subject: [PATCH 025/609] org.eclipse.rdf4j upgrade from 2.5.5 -> 3.7.4 --- CHANGELOG.md | 22 ++++++++++++---------- pom.xml | 2 +- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78424149..81154620 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,16 +16,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Clarified Readme for quick start - No cartesian product when referring to the same logical source (see [issue 28](https://github.com/RMLio/rmlmapper-java/issues/28)]) - If you still want the cartesian product, update your mappings to refer to different logical sources with the same contents -- Upgrade jetty-server and jetty-security to 9.4.44.v20210927 -- Upgrade wiremock-jre8 to 2.32.0 -- Upgrade ch.qos.logback to 1.2.10 -- Upgrade commons-cli to 1.5.0 -- Upgrade com.jayway.jsonpath to 2.7.0 -- Upgrade ch.vorburger.mariaDB4j to 2.5.3 -- Upgrade com.microsoft.sqlserver to 9.4.0.jre8 -- Upgrade com.fasterxml.jackson.core to 2.13.1 -- Upgrade org.jsoup to 1.14.3 -- Upgrade org.apache.poi to 5.0.0 +- Upgraded jetty-server and jetty-security to 9.4.44.v20210927 +- Upgraded wiremock-jre8 to 2.32.0 +- Upgraded ch.qos.logback to 1.2.10 +- Upgraded commons-cli to 1.5.0 +- Upgraded com.jayway.jsonpath to 2.7.0 +- Upgraded ch.vorburger.mariaDB4j to 2.5.3 +- Upgraded com.microsoft.sqlserver to 10.2.0.jre8 +- Upgraded com.fasterxml.jackson.core to 2.13.1 +- Upgraded org.jsoup to 1.14.3 +- Upgraded org.apache.poi to 5.0.0 +- Upgraded org.apache.jena to 4.3.2 +- Upgraded org.eclipse.rdf4j to 3.7.4 ### Changed diff --git a/pom.xml b/pom.xml index d300aaf5..afb4fa0d 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ org.eclipse.rdf4j rdf4j-runtime - 2.5.5 + 3.7.4 junit From e275c0faced9410ff48dca002c51b553b0a2c3e1 Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 17:18:46 +0100 Subject: [PATCH 026/609] rollback org.apache.jena to 3.8.0 rollback org.eclipse.rdf4j to 2.5.5 --- CHANGELOG.md | 3 +-- pom.xml | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81154620..f2b602fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,8 +26,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Upgraded com.fasterxml.jackson.core to 2.13.1 - Upgraded org.jsoup to 1.14.3 - Upgraded org.apache.poi to 5.0.0 -- Upgraded org.apache.jena to 4.3.2 -- Upgraded org.eclipse.rdf4j to 3.7.4 + ### Changed diff --git a/pom.xml b/pom.xml index afb4fa0d..19a430ff 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ org.eclipse.rdf4j rdf4j-runtime - 3.7.4 + 2.5.5 junit @@ -184,7 +184,7 @@ com.microsoft.sqlserver mssql-jdbc - 10.2.0.jre8 + 9.4.0.jre8 test @@ -231,7 +231,7 @@ org.apache.jena apache-jena-libs pom - 4.3.2 + 3.8.0 test From 5e3cf7f711b555b13a6ae0b85f13b3c6dfdc7e2a Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 17:54:32 +0100 Subject: [PATCH 027/609] TestCore: enable debug logs with VERBOSE env var. Tests will output logs on debug level when the environment variable VERBOSE was set prior their execution. --- CHANGELOG.md | 1 + src/test/java/be/ugent/rml/TestCore.java | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2b602fa..4d451161 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## Unreleased +- TestCore: enable debug logs when VERBOSE env variable is set (see [issue 230](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/230)) ### Added diff --git a/src/test/java/be/ugent/rml/TestCore.java b/src/test/java/be/ugent/rml/TestCore.java index f131ad0d..b227db4e 100644 --- a/src/test/java/be/ugent/rml/TestCore.java +++ b/src/test/java/be/ugent/rml/TestCore.java @@ -11,6 +11,7 @@ import be.ugent.rml.target.TargetFactory; import be.ugent.rml.term.NamedNode; import be.ugent.rml.term.Term; +import ch.qos.logback.classic.Level; import org.eclipse.rdf4j.rio.RDFFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,15 +32,22 @@ public abstract class TestCore { - final Logger logger = LoggerFactory.getLogger(this.getClass()); final String DEFAULT_BASE_IRI = "http://example.com/base/"; - + final ch.qos.logback.classic.Logger logger = (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME); // Mapping options to be applied by the MappingConformer protected static Map mappingOptions = new HashMap<>(); + TestCore(){ + if(System.getenv("VERBOSE") != null){ + logger.setLevel(Level.DEBUG); + }; + } + + /** * Note: the created Executor will run in best effort mode */ + Executor createExecutor(String mapPath) throws Exception { return createExecutor(mapPath, new ArrayList<>(), null, BEST_EFFORT); } From 4c451bb59d3602bbc57757bd501945590a394f89 Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Thu, 17 Feb 2022 16:56:30 +0100 Subject: [PATCH 028/609] added vocabulary to support ldes generation --- src/main/resources/functions_idlab.ttl | 41 +++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/src/main/resources/functions_idlab.ttl b/src/main/resources/functions_idlab.ttl index ba6ec694..8b334879 100644 --- a/src/main/resources/functions_idlab.ttl +++ b/src/main/resources/functions_idlab.ttl @@ -200,6 +200,17 @@ idlab-fn:normalizeDateTime lib:class "IDLabFunctions" ; lib:method "normalizeDateTimeWithLang" ] . +idlab-fn:generateUniqueIRI + a fno:Function; + fno:name "generateUniqueIRI"; + rdfs:label "generateUniqueIRI"; + dcterms:description "Generates a unique IRI by watching the given property value" + fno:expects ( idlab-fn:iri idlab-fn:watchedProperty idlab-fn:unique idlab-fn:_state); + fno:returns ( idlab-fn:_str ); + lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; + lib:class "IDLabFunctions" ; + lib:method "generateUniqueIRI" ] . + grel:valueParam a fno:Parameter ; @@ -357,4 +368,32 @@ idlab-fn:_lang fno:name "string representing a BCP 47 language tag" ; rdfs:label "string representing a BCP 47 language tag" ; fno:type xsd:string ; - fno:predicate idlab-fn:lang . \ No newline at end of file + fno:predicate idlab-fn:lang . + +idlab-fn:iri + a fno:Parameter; + fno:name "String representation of an IRI"; + rdfs:label "String representation of an IRI"; + fno:type xsd:string; + fno:predicate idlab-fn:iri . + +idlab-fn:watchedProperty + a fno:Parameter; + fno:name "Property field of the data record to be watched"; + rdfs:label "Property field of the data record to be watched"; + fno:type xsd:string ; + fno:predicate idlab-fn:watchedProperty. + +idlab-fn:unique + a fno:Parameter; + fno:name "Indicates if a field contains unique values"; + rdfs:label "Indicates if a field contains unique values"; + fno:type xsd:boolean ; + fno:predicate idlab-fn:unique. + +idlab-fn:_state + a fno:Parameter; + fno:name "string representing the name of the state file"; + rdfs:label "string representing the name of the state file"; + fno:type xsd:string ; + fno:predicate idlab-fn:state . \ No newline at end of file From 00eb4cf1c635e7bc644c58901c11cbfe80048ba5 Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Sat, 19 Feb 2022 13:14:18 +0100 Subject: [PATCH 029/609] implemented unique iri generation for ldes --- src/main/java/be/ugent/rml/NAMESPACES.java | 1 + .../rml/functions/lib/IDLabFunctions.java | 74 ++++++++++++++++++- 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/src/main/java/be/ugent/rml/NAMESPACES.java b/src/main/java/be/ugent/rml/NAMESPACES.java index 13c9a3f2..eccec4c9 100644 --- a/src/main/java/be/ugent/rml/NAMESPACES.java +++ b/src/main/java/be/ugent/rml/NAMESPACES.java @@ -10,6 +10,7 @@ public class NAMESPACES { public static final String D2RQ = "http://www.wiwiss.fu-berlin.de/suhl/bizer/D2RQ/0.1#"; public static final String SD = "http://www.w3.org/ns/sparql-service-description#"; public static final String FNO_S = "https://w3id.org/function/ontology#"; + public static final String LDES = "https://w3id.org/ldes#"; @Deprecated public static final String FNO = "http://w3id.org/function/ontology#"; @Deprecated diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index 227cf83d..94f8fec8 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -13,17 +13,21 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; +import java.io.*; +import java.nio.file.Files; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLEncoder; +import java.nio.file.Path; +import java.nio.file.Paths; import java.text.SimpleDateFormat; import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; public class IDLabFunctions { @@ -288,6 +292,70 @@ public static String normalizeDateTime(String dateTimeStr, String pattern) { + public static String generateUniqueIRI(String template, String watchedValue, boolean isUnique, String stateDirPathStr) { + //TODO: move this to the parameter of the function? (might get too bloated in RML definitions) + int m_buckets = 10; + if (isUnique){ + return template; + } + int templateHash = template.hashCode(); + String hexBucketFileName= Integer.toHexString(templateHash % m_buckets); + String hexKey = Integer.toHexString(templateHash); + String hexPairs = String.format("%s:%s", hexKey, watchedValue); + + + Path stateFilePath = Paths.get(String.format("%s/%s.log", stateDirPathStr, hexBucketFileName)); + + List outputPairs = new ArrayList<>(); + + File stateFile = stateFilePath.toFile(); + AtomicBoolean found = new AtomicBoolean(false); + AtomicBoolean isDifferent = new AtomicBoolean(true); + + try{ + + if (Files.notExists(stateFilePath) ) { + Files.createDirectories(stateFilePath); + }; + + try (BufferedReader reader = new BufferedReader(new FileReader(stateFile))) { + + outputPairs = reader.lines().map(s -> { + String[] strings = s.split(":"); + String key = strings[0]; + String val = strings[1]; + if (hexKey.equals(key)) { + isDifferent.set(!val.equals(watchedValue)); + if (isDifferent.get()){ + val = watchedValue; + } + found.set(true); + } + return String.format("%s:%s", key,val); + }).collect(Collectors.toList()); + + } + + if (!found.get()) { + outputPairs.add(hexPairs); + } + + + } catch (IOException e){ + e.printStackTrace(); + } + + String output = null; + if (isDifferent.get()){ + OffsetDateTime now = OffsetDateTime.now(); + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; + output = String.format("%s#%s", template, formatter.format(now) ); + } + + return output; + } + + // TODO check whether this is the right place for this public static String jsonize(Object s) throws JsonProcessingException { ObjectWriter ow = new ObjectMapper().writer().withDefaultPrettyPrinter(); From 9e786596e0ee277c190f4e99dfa46c23da6670f0 Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Sat, 19 Feb 2022 16:44:05 +0100 Subject: [PATCH 030/609] added state updates --- .../rml/functions/lib/IDLabFunctions.java | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index 94f8fec8..c81b8c03 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -306,19 +306,22 @@ public static String generateUniqueIRI(String template, String watchedValue, boo Path stateFilePath = Paths.get(String.format("%s/%s.log", stateDirPathStr, hexBucketFileName)); - List outputPairs = new ArrayList<>(); + System.out.println(String.format("State file path for ldes is: %s", stateFilePath)); + + List outputPairs; - File stateFile = stateFilePath.toFile(); AtomicBoolean found = new AtomicBoolean(false); AtomicBoolean isDifferent = new AtomicBoolean(true); + String output = null; try{ + Files.createDirectories(Paths.get(stateDirPathStr)); if (Files.notExists(stateFilePath) ) { - Files.createDirectories(stateFilePath); + Files.createFile(stateFilePath); }; - try (BufferedReader reader = new BufferedReader(new FileReader(stateFile))) { + try (BufferedReader reader = new BufferedReader(new FileReader(stateFilePath.toFile()))) { outputPairs = reader.lines().map(s -> { String[] strings = s.split(":"); @@ -339,18 +342,23 @@ public static String generateUniqueIRI(String template, String watchedValue, boo if (!found.get()) { outputPairs.add(hexPairs); } - + try (BufferedWriter writer= new BufferedWriter(new FileWriter(stateFilePath.toFile()))){ + for(String line : outputPairs){ + writer.write(line); + writer.newLine(); + } + } + if (isDifferent.get()){ + OffsetDateTime now = OffsetDateTime.now(); + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; + output = String.format("%s#%s", template, formatter.format(now) ); + } } catch (IOException e){ e.printStackTrace(); + output = null; } - String output = null; - if (isDifferent.get()){ - OffsetDateTime now = OffsetDateTime.now(); - DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; - output = String.format("%s#%s", template, formatter.format(now) ); - } return output; } From b7047244b09a2382781ec7571a5a8af7849e3940 Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Sun, 20 Feb 2022 18:33:35 +0100 Subject: [PATCH 031/609] added unit tests --- .../rml/functions/lib/IDLabFunctionsTest.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java b/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java index 1e8c9cb7..79a1b10e 100644 --- a/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java +++ b/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java @@ -1,6 +1,7 @@ package be.ugent.rml.functions.lib; import org.hamcrest.CoreMatchers; +import org.junit.BeforeClass; import org.junit.Test; import java.util.ArrayList; @@ -219,4 +220,43 @@ public void normalizeDateTime() { assertEquals("2020-04-01T09:04:00", IDLabFunctions.normalizeDateTime(input3, format3)); } + public static class LDESGenerationTests{ + + private static final String STATE_DIRECTORY = "/tmp/test-state"; + + + @Test + public void skipGenerateUniqueIRI(){ + String template = "http://example.com/sensor1/"; + String value = "5"; + boolean isUnique = false; + + String generated_iri = IDLabFunctions.generateUniqueIRI(template, value, isUnique, STATE_DIRECTORY); + assertNull(generated_iri); + } + + + @Test + public void generateUniqueIRI(){ + String template = "http://example.com/sensor2/"; + String value = "5"; + boolean isUnique = true; + + String generated_iri = IDLabFunctions.generateUniqueIRI(template, value, isUnique, STATE_DIRECTORY); + assertEquals(generated_iri, template); + + } + + @Test + public void generateUniqueIRIWithDate(){ + + String template = "http://example.com/sensor2/"; + String value = "5"; + boolean isUnique = false; + + String generated_iri = IDLabFunctions.generateUniqueIRI(template, value, isUnique, STATE_DIRECTORY); + assertEquals(generated_iri, template); + } + } + } From 6a3b8ec2fec4792d77bce0df8b141b5db3adb788 Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Mon, 21 Feb 2022 13:56:54 +0100 Subject: [PATCH 032/609] added test case for ldes generation --- src/main/resources/functions_idlab.ttl | 6 +-- .../java/be/ugent/rml/Mapper_LDES_Test.java | 29 +++++++++++ .../web-of-things/ldes/generation/data1.csv | 4 ++ .../web-of-things/ldes/generation/data2.csv | 3 ++ .../web-of-things/ldes/generation/mapping.ttl | 50 +++++++++++++++++++ .../web-of-things/ldes/generation/output.nq | 0 6 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 src/test/java/be/ugent/rml/Mapper_LDES_Test.java create mode 100644 src/test/resources/web-of-things/ldes/generation/data1.csv create mode 100644 src/test/resources/web-of-things/ldes/generation/data2.csv create mode 100644 src/test/resources/web-of-things/ldes/generation/mapping.ttl create mode 100644 src/test/resources/web-of-things/ldes/generation/output.nq diff --git a/src/main/resources/functions_idlab.ttl b/src/main/resources/functions_idlab.ttl index 8b334879..10b011ff 100644 --- a/src/main/resources/functions_idlab.ttl +++ b/src/main/resources/functions_idlab.ttl @@ -204,9 +204,9 @@ idlab-fn:generateUniqueIRI a fno:Function; fno:name "generateUniqueIRI"; rdfs:label "generateUniqueIRI"; - dcterms:description "Generates a unique IRI by watching the given property value" - fno:expects ( idlab-fn:iri idlab-fn:watchedProperty idlab-fn:unique idlab-fn:_state); - fno:returns ( idlab-fn:_str ); + dcterms:description "Generates a unique IRI by watching the given property value"; + fno:expects ( idlab-fn:iri idlab-fn:watchedProperty idlab-fn:unique idlab-fn:_state ); + fno:returns ( idlab-fn:_stringOut ); lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; lib:class "IDLabFunctions" ; lib:method "generateUniqueIRI" ] . diff --git a/src/test/java/be/ugent/rml/Mapper_LDES_Test.java b/src/test/java/be/ugent/rml/Mapper_LDES_Test.java new file mode 100644 index 00000000..52f9a060 --- /dev/null +++ b/src/test/java/be/ugent/rml/Mapper_LDES_Test.java @@ -0,0 +1,29 @@ +package be.ugent.rml; + +import be.ugent.rml.functions.FunctionLoader; +import be.ugent.rml.store.QuadStore; +import be.ugent.rml.store.RDF4JStore; +import org.eclipse.rdf4j.rio.RDFFormat; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; + +public class Mapper_LDES_Test extends TestCore { + + private static FunctionLoader LOADER; + + @BeforeClass + public static void setups() throws Exception { + QuadStore functionDescriptionTriples = new RDF4JStore(); + functionDescriptionTriples.read(Utils.getInputStreamFromFile(new File("./src/main/resources/functions_idlab.ttl")), null, RDFFormat.TURTLE); + LOADER = new FunctionLoader(functionDescriptionTriples); + + } + + @Test + public void evaluate_basic_LDES () throws Exception { + Executor executor = this.createExecutor("./web-of-things/ldes/generation/mapping.ttl", LOADER); + doMapping(executor, "./web-of-things/ldes/generation/output.nq"); + } +} diff --git a/src/test/resources/web-of-things/ldes/generation/data1.csv b/src/test/resources/web-of-things/ldes/generation/data1.csv new file mode 100644 index 00000000..29143cd0 --- /dev/null +++ b/src/test/resources/web-of-things/ldes/generation/data1.csv @@ -0,0 +1,4 @@ +id,value +sensor1,3 +sensor2,3 +sensor3,4 \ No newline at end of file diff --git a/src/test/resources/web-of-things/ldes/generation/data2.csv b/src/test/resources/web-of-things/ldes/generation/data2.csv new file mode 100644 index 00000000..0ef3eb59 --- /dev/null +++ b/src/test/resources/web-of-things/ldes/generation/data2.csv @@ -0,0 +1,3 @@ +id,value +sensor2,10 +sensor3,4 diff --git a/src/test/resources/web-of-things/ldes/generation/mapping.ttl b/src/test/resources/web-of-things/ldes/generation/mapping.ttl new file mode 100644 index 00000000..7e906479 --- /dev/null +++ b/src/test/resources/web-of-things/ldes/generation/mapping.ttl @@ -0,0 +1,50 @@ +@prefix rr: . +@prefix ex: . +@prefix rdfs: . +@prefix xsd: . +@prefix ql: . +@prefix rml: . +@prefix rdf: . +@prefix fnml: . +@prefix fno: . +@prefix dcterms: . +@prefix idlab-fn: . +@base . + + + a rr:TriplesMap; + + rml:logicalSource [ + rml:source "data1.csv"; + rml:referenceFormulation ql:CSV + ]; + + rr:subjectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant idlab-fn:generateUniqueIRI ] ] ; + # IRI template + rr:predicateObjectMap [ + rr:predicate idlab-fn:iri ; + rr:objectMap [ rr:template "http://ex.org/{id}/" ] + ]; + # Properties to watch, can be one or multiple + # Note that the template does not need to match with these properties! + rr:predicateObjectMap [ + rr:predicate idlab-fn:watchedProperty ; + rr:objectMap [ rml:reference "value" ] + ]; + # Flag to indicate if the properties are unique on their own + rr:predicateObjectMap [ + rr:predicate idlab-fn:unique ; + rr:objectMap [ rr:constant "true"; rr:termType xsd:boolean; ] + ]; + # Directory path where the state for this function is/will be stored + rr:predicateObjectMap [ + rr:predicate idlab-fn:state ; + rr:objectMap [ rr:constant "/tmp/test-case"; rr:termType xsd:string; ] + ]; + ]; + ]; + . diff --git a/src/test/resources/web-of-things/ldes/generation/output.nq b/src/test/resources/web-of-things/ldes/generation/output.nq new file mode 100644 index 00000000..e69de29b From 52c840b66281bf424d41c846fcd5b0d7c1ab5111 Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Mon, 21 Feb 2022 15:08:25 +0100 Subject: [PATCH 033/609] Fixed function not found exception --- .../rml/functions/lib/IDLabFunctions.java | 2 +- src/main/resources/functions_idlab.ttl | 18 +++++++++--------- .../web-of-things/ldes/generation/mapping.ttl | 5 +++++ 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index c81b8c03..19f7b933 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -292,7 +292,7 @@ public static String normalizeDateTime(String dateTimeStr, String pattern) { - public static String generateUniqueIRI(String template, String watchedValue, boolean isUnique, String stateDirPathStr) { + public static String generateUniqueIRI(String template, String watchedValue, Boolean isUnique, String stateDirPathStr) { //TODO: move this to the parameter of the function? (might get too bloated in RML definitions) int m_buckets = 10; if (isUnique){ diff --git a/src/main/resources/functions_idlab.ttl b/src/main/resources/functions_idlab.ttl index 10b011ff..6e161b1b 100644 --- a/src/main/resources/functions_idlab.ttl +++ b/src/main/resources/functions_idlab.ttl @@ -201,12 +201,12 @@ idlab-fn:normalizeDateTime lib:method "normalizeDateTimeWithLang" ] . idlab-fn:generateUniqueIRI - a fno:Function; - fno:name "generateUniqueIRI"; - rdfs:label "generateUniqueIRI"; - dcterms:description "Generates a unique IRI by watching the given property value"; - fno:expects ( idlab-fn:iri idlab-fn:watchedProperty idlab-fn:unique idlab-fn:_state ); - fno:returns ( idlab-fn:_stringOut ); + a fno:Function ; + fno:name "generateUniqueIRI" ; + rdfs:label "generateUniqueIRI" ; + dcterms:description "Generates a unique IRI by watching the given property value" ; + fno:expects ( idlab-fn:_iri idlab-fn:_watchedProperty idlab-fn:_unique idlab-fn:_state ) ; + fno:returns ( idlab-fn:_stringOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; lib:class "IDLabFunctions" ; lib:method "generateUniqueIRI" ] . @@ -370,21 +370,21 @@ idlab-fn:_lang fno:type xsd:string ; fno:predicate idlab-fn:lang . -idlab-fn:iri +idlab-fn:_iri a fno:Parameter; fno:name "String representation of an IRI"; rdfs:label "String representation of an IRI"; fno:type xsd:string; fno:predicate idlab-fn:iri . -idlab-fn:watchedProperty +idlab-fn:_watchedProperty a fno:Parameter; fno:name "Property field of the data record to be watched"; rdfs:label "Property field of the data record to be watched"; fno:type xsd:string ; fno:predicate idlab-fn:watchedProperty. -idlab-fn:unique +idlab-fn:_unique a fno:Parameter; fno:name "Indicates if a field contains unique values"; rdfs:label "Indicates if a field contains unique values"; diff --git a/src/test/resources/web-of-things/ldes/generation/mapping.ttl b/src/test/resources/web-of-things/ldes/generation/mapping.ttl index 7e906479..5a917c7d 100644 --- a/src/test/resources/web-of-things/ldes/generation/mapping.ttl +++ b/src/test/resources/web-of-things/ldes/generation/mapping.ttl @@ -47,4 +47,9 @@ ]; ]; ]; + + rr:predicateObjectMap [ + rr:predicate ex:value; + rr:objectMap [ rml:reference "value" ]; + ]; . From 60710354320c972256b45e38e34710c91c649f3e Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Mon, 21 Feb 2022 22:55:01 +0100 Subject: [PATCH 034/609] added integration tests --- .../rml/functions/lib/IDLabFunctions.java | 1 - .../java/be/ugent/rml/Mapper_LDES_Test.java | 37 ++++++++++++- .../ldes/generation/basic/mapping.ttl | 55 +++++++++++++++++++ .../ldes/generation/basic/output.nq | 3 + .../ldes/generation/partial/mapping.ttl | 55 +++++++++++++++++++ .../ldes/generation/partial/mapping2.ttl | 55 +++++++++++++++++++ .../ldes/generation/repeat/mapping.ttl | 55 +++++++++++++++++++ .../ldes/generation/{ => repeat}/output.nq | 0 8 files changed, 257 insertions(+), 4 deletions(-) create mode 100644 src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl create mode 100644 src/test/resources/web-of-things/ldes/generation/basic/output.nq create mode 100644 src/test/resources/web-of-things/ldes/generation/partial/mapping.ttl create mode 100644 src/test/resources/web-of-things/ldes/generation/partial/mapping2.ttl create mode 100644 src/test/resources/web-of-things/ldes/generation/repeat/mapping.ttl rename src/test/resources/web-of-things/ldes/generation/{ => repeat}/output.nq (100%) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index 19f7b933..48f8f8aa 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -306,7 +306,6 @@ public static String generateUniqueIRI(String template, String watchedValue, Boo Path stateFilePath = Paths.get(String.format("%s/%s.log", stateDirPathStr, hexBucketFileName)); - System.out.println(String.format("State file path for ldes is: %s", stateFilePath)); List outputPairs; diff --git a/src/test/java/be/ugent/rml/Mapper_LDES_Test.java b/src/test/java/be/ugent/rml/Mapper_LDES_Test.java index 52f9a060..5d8bb160 100644 --- a/src/test/java/be/ugent/rml/Mapper_LDES_Test.java +++ b/src/test/java/be/ugent/rml/Mapper_LDES_Test.java @@ -3,15 +3,25 @@ import be.ugent.rml.functions.FunctionLoader; import be.ugent.rml.store.QuadStore; import be.ugent.rml.store.RDF4JStore; +import be.ugent.rml.term.NamedNode; +import org.apache.commons.io.FileUtils; import org.eclipse.rdf4j.rio.RDFFormat; +import org.junit.After; import org.junit.BeforeClass; import org.junit.Test; +import static org.junit.Assert.*; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; public class Mapper_LDES_Test extends TestCore { private static FunctionLoader LOADER; + @After + public void cleanUp() throws IOException { + FileUtils.deleteDirectory(new File("/tmp/ldes-test")); + } @BeforeClass public static void setups() throws Exception { @@ -22,8 +32,29 @@ public static void setups() throws Exception { } @Test - public void evaluate_basic_LDES () throws Exception { - Executor executor = this.createExecutor("./web-of-things/ldes/generation/mapping.ttl", LOADER); - doMapping(executor, "./web-of-things/ldes/generation/output.nq"); + public void evaluate_unique_LDES () throws Exception { + Executor executor = this.createExecutor("./web-of-things/ldes/generation/basic/mapping.ttl", LOADER); + doMapping(executor, "./web-of-things/ldes/generation/basic/output.nq"); } + + @Test + public void evaluate_repeat_LDES() throws Exception { + Executor executor = this.createExecutor("./web-of-things/ldes/generation/repeat/mapping.ttl", LOADER); + executor.executeV5(null).get(new NamedNode("rmlmapper://default.store")); + executor = this.createExecutor("./web-of-things/ldes/generation/repeat/mapping.ttl", LOADER); + doMapping(executor, "./web-of-things/ldes/generation/repeat/output.nq"); + } + + @Test + public void evaluate_partial_repeat_LDES() throws Exception { + Executor executor = this.createExecutor("./web-of-things/ldes/generation/partial/mapping.ttl", LOADER); + QuadStore result = executor.executeV5(null).get(new NamedNode("rmlmapper://default.store")); + executor = this.createExecutor("./web-of-things/ldes/generation/partial/mapping2.ttl", LOADER); + QuadStore result_second = executor.executeV5(null).get(new NamedNode("rmlmapper://default.store")); + assertEquals(3, result.size()); + assertEquals(1, result_second.size()); + + } + + } diff --git a/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl b/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl new file mode 100644 index 00000000..781e48a5 --- /dev/null +++ b/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl @@ -0,0 +1,55 @@ +@prefix rr: . +@prefix ex: . +@prefix rdfs: . +@prefix xsd: . +@prefix ql: . +@prefix rml: . +@prefix rdf: . +@prefix fnml: . +@prefix fno: . +@prefix dcterms: . +@prefix idlab-fn: . +@base . + + + a rr:TriplesMap; + + rml:logicalSource [ + rml:source "../data1.csv"; + rml:referenceFormulation ql:CSV + ]; + + rr:subjectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant idlab-fn:generateUniqueIRI ] ] ; + # IRI template + rr:predicateObjectMap [ + rr:predicate idlab-fn:iri ; + rr:objectMap [ rr:template "http://ex.org/{id}/" ] + ]; + # Properties to watch, can be one or multiple + # Note that the template does not need to match with these properties! + rr:predicateObjectMap [ + rr:predicate idlab-fn:watchedProperty ; + rr:objectMap [ rml:reference "value" ] + ]; + # Flag to indicate if the properties are unique on their own + rr:predicateObjectMap [ + rr:predicate idlab-fn:unique ; + rr:objectMap [ rr:constant "true"; rr:termType xsd:boolean; ] + ]; + # Directory path where the state for this function is/will be stored + rr:predicateObjectMap [ + rr:predicate idlab-fn:state ; + rr:objectMap [ rr:constant "/tmp/ldes-test/test-case-basic"; rr:termType xsd:string; ] + ]; + ]; + ]; + + rr:predicateObjectMap [ + rr:predicate ex:value; + rr:objectMap [ rml:reference "value" ]; + ]; + . diff --git a/src/test/resources/web-of-things/ldes/generation/basic/output.nq b/src/test/resources/web-of-things/ldes/generation/basic/output.nq new file mode 100644 index 00000000..9e6b8ddd --- /dev/null +++ b/src/test/resources/web-of-things/ldes/generation/basic/output.nq @@ -0,0 +1,3 @@ + "3". + "3". + "4". diff --git a/src/test/resources/web-of-things/ldes/generation/partial/mapping.ttl b/src/test/resources/web-of-things/ldes/generation/partial/mapping.ttl new file mode 100644 index 00000000..27a2bc9b --- /dev/null +++ b/src/test/resources/web-of-things/ldes/generation/partial/mapping.ttl @@ -0,0 +1,55 @@ +@prefix rr: . +@prefix ex: . +@prefix rdfs: . +@prefix xsd: . +@prefix ql: . +@prefix rml: . +@prefix rdf: . +@prefix fnml: . +@prefix fno: . +@prefix dcterms: . +@prefix idlab-fn: . +@base . + + + a rr:TriplesMap; + + rml:logicalSource [ + rml:source "../data1.csv"; + rml:referenceFormulation ql:CSV + ]; + + rr:subjectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant idlab-fn:generateUniqueIRI ] ] ; + # IRI template + rr:predicateObjectMap [ + rr:predicate idlab-fn:iri ; + rr:objectMap [ rr:template "http://ex.org/{id}/" ] + ]; + # Properties to watch, can be one or multiple + # Note that the template does not need to match with these properties! + rr:predicateObjectMap [ + rr:predicate idlab-fn:watchedProperty ; + rr:objectMap [ rml:reference "value" ] + ]; + # Flag to indicate if the properties are unique on their own + rr:predicateObjectMap [ + rr:predicate idlab-fn:unique ; + rr:objectMap [ rr:constant "false"; rr:termType xsd:boolean; ] + ]; + # Directory path where the state for this function is/will be stored + rr:predicateObjectMap [ + rr:predicate idlab-fn:state ; + rr:objectMap [ rr:constant "/tmp/ldes-test/partial-test-case"; rr:termType xsd:string; ] + ]; + ]; + ]; + + rr:predicateObjectMap [ + rr:predicate ex:value; + rr:objectMap [ rml:reference "value" ]; + ]; + . diff --git a/src/test/resources/web-of-things/ldes/generation/partial/mapping2.ttl b/src/test/resources/web-of-things/ldes/generation/partial/mapping2.ttl new file mode 100644 index 00000000..0b3d257d --- /dev/null +++ b/src/test/resources/web-of-things/ldes/generation/partial/mapping2.ttl @@ -0,0 +1,55 @@ +@prefix rr: . +@prefix ex: . +@prefix rdfs: . +@prefix xsd: . +@prefix ql: . +@prefix rml: . +@prefix rdf: . +@prefix fnml: . +@prefix fno: . +@prefix dcterms: . +@prefix idlab-fn: . +@base . + + + a rr:TriplesMap; + + rml:logicalSource [ + rml:source "../data2.csv"; + rml:referenceFormulation ql:CSV + ]; + + rr:subjectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant idlab-fn:generateUniqueIRI ] ] ; + # IRI template + rr:predicateObjectMap [ + rr:predicate idlab-fn:iri ; + rr:objectMap [ rr:template "http://ex.org/{id}/" ] + ]; + # Properties to watch, can be one or multiple + # Note that the template does not need to match with these properties! + rr:predicateObjectMap [ + rr:predicate idlab-fn:watchedProperty ; + rr:objectMap [ rml:reference "value" ] + ]; + # Flag to indicate if the properties are unique on their own + rr:predicateObjectMap [ + rr:predicate idlab-fn:unique ; + rr:objectMap [ rr:constant "false"; rr:termType xsd:boolean; ] + ]; + # Directory path where the state for this function is/will be stored + rr:predicateObjectMap [ + rr:predicate idlab-fn:state ; + rr:objectMap [ rr:constant "/tmp/ldes-test/partial-test-case"; rr:termType xsd:string; ] + ]; + ]; + ]; + + rr:predicateObjectMap [ + rr:predicate ex:value; + rr:objectMap [ rml:reference "value" ]; + ]; + . diff --git a/src/test/resources/web-of-things/ldes/generation/repeat/mapping.ttl b/src/test/resources/web-of-things/ldes/generation/repeat/mapping.ttl new file mode 100644 index 00000000..1cca254f --- /dev/null +++ b/src/test/resources/web-of-things/ldes/generation/repeat/mapping.ttl @@ -0,0 +1,55 @@ +@prefix rr: . +@prefix ex: . +@prefix rdfs: . +@prefix xsd: . +@prefix ql: . +@prefix rml: . +@prefix rdf: . +@prefix fnml: . +@prefix fno: . +@prefix dcterms: . +@prefix idlab-fn: . +@base . + + + a rr:TriplesMap; + + rml:logicalSource [ + rml:source "../data1.csv"; + rml:referenceFormulation ql:CSV + ]; + + rr:subjectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant idlab-fn:generateUniqueIRI ] ] ; + # IRI template + rr:predicateObjectMap [ + rr:predicate idlab-fn:iri ; + rr:objectMap [ rr:template "http://ex.org/{id}/" ] + ]; + # Properties to watch, can be one or multiple + # Note that the template does not need to match with these properties! + rr:predicateObjectMap [ + rr:predicate idlab-fn:watchedProperty ; + rr:objectMap [ rml:reference "value" ] + ]; + # Flag to indicate if the properties are unique on their own + rr:predicateObjectMap [ + rr:predicate idlab-fn:unique ; + rr:objectMap [ rr:constant "false"; rr:termType xsd:boolean; ] + ]; + # Directory path where the state for this function is/will be stored + rr:predicateObjectMap [ + rr:predicate idlab-fn:state ; + rr:objectMap [ rr:constant "/tmp/ldes-test/repeat-test-case"; rr:termType xsd:string; ] + ]; + ]; + ]; + + rr:predicateObjectMap [ + rr:predicate ex:value; + rr:objectMap [ rml:reference "value" ]; + ]; + . diff --git a/src/test/resources/web-of-things/ldes/generation/output.nq b/src/test/resources/web-of-things/ldes/generation/repeat/output.nq similarity index 100% rename from src/test/resources/web-of-things/ldes/generation/output.nq rename to src/test/resources/web-of-things/ldes/generation/repeat/output.nq From 98d92d48a0ca380d282b2353e2b9309e74d07153 Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Wed, 23 Feb 2022 10:39:39 +0100 Subject: [PATCH 035/609] updated iri generation to watch multiple properties --- .../rml/functions/lib/IDLabFunctions.java | 122 ++++++++++++++---- .../ldes/generation/basic/mapping.ttl | 6 +- .../ldes/generation/basic/output.nq | 6 +- .../web-of-things/ldes/generation/data1.csv | 8 +- .../web-of-things/ldes/generation/data2.csv | 6 +- .../ldes/generation/partial/mapping.ttl | 6 +- .../ldes/generation/partial/mapping2.ttl | 6 +- .../ldes/generation/repeat/mapping.ttl | 6 +- 8 files changed, 116 insertions(+), 50 deletions(-) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index 48f8f8aa..3959f07d 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -291,26 +291,86 @@ public static String normalizeDateTime(String dateTimeStr, String pattern) { } + /** + * Returns a HashMap containing the key,value pairs of the watched properties from the given + * template string of the form "key1=val1&key2=val2&...." + * + * @param watchedValueTemplate Input string template representing the key,value pairs of + * the form "key1=val1&key2=val2&..." + * @return A map containing the parsed pairs of the properties which are being watched. + */ + private static Map parseWatchedPropertyTemplate(String watchedValueTemplate){ + Map result = new HashMap<>(); - public static String generateUniqueIRI(String template, String watchedValue, Boolean isUnique, String stateDirPathStr) { - //TODO: move this to the parameter of the function? (might get too bloated in RML definitions) - int m_buckets = 10; - if (isUnique){ - return template; + Arrays.stream(watchedValueTemplate.split("&")) + .map(s -> s.split("=")) + .filter(sArr -> sArr.length == 2) + .forEach(sArr -> result.put(sArr[0], sArr[1])); + + return result; + } + + private static String updateStatePropertyRecord(Map watchedMap, + String hexKey, AtomicBoolean found, AtomicBoolean isDifferent, + String storedStateRecord) { + int split_idx = storedStateRecord.indexOf(':'); + String storedHexKey = storedStateRecord.substring(0, split_idx); + if (!storedHexKey.equals(hexKey)){ + return storedStateRecord; } + + found.set(true); + + List propertyValPairs = + Arrays.stream(storedStateRecord.substring(split_idx+1) + .split("&")) + .map(str -> { + String[] propVal = str.split("="); + String property = propVal[0]; + String storeVal = propVal[1]; + String watchedVal = watchedMap.getOrDefault(property, null); + + if(!watchedVal.equals(storeVal)){ + isDifferent.set(true); + storeVal = watchedVal; + } + + return String.format("%s=%s", property, storeVal); + }) + .collect(Collectors.toList()); + + return String.format("%s:%s", storedHexKey, String.join("&", propertyValPairs)); + } + + /** + * Handles the case if the isUniqueIRI flag is not set for the method {@link #generateUniqueIRI(String, String, Boolean, String)}. + * The generation of the IRI depends on the value of the watched properties. + * If any of the watched properties changes in value or gets dropped, a unique IRI will be + * generated. Otherwise, null String will be returned. + * In order to check if the watched properties have changed, a file state is written to keep track of + * previously seen property values. + * A unique IRI will be generated from the provided "template" string by appending the current + * date timestamp. + * + * @param template The template string used to generate unique IRI by appending current date timestamp + * @param watchedValueTemplate The template string containing the key-value pairs of properties being watched + * @param stateDirPathStr String representation of the directory path in which the state of the function + * will be stored + * @param m_buckets The number of bucket files to be created + * @return A unique IRI will be generated from the provided "template" string by appending the current + * date timestamp if possible. Otherwise, null is returned + */ + private static String handleNonUniqueIRI(String template, String watchedValueTemplate, String stateDirPathStr, int m_buckets) { + Map watchedMap = parseWatchedPropertyTemplate(watchedValueTemplate); + int templateHash = template.hashCode(); String hexBucketFileName= Integer.toHexString(templateHash % m_buckets); String hexKey = Integer.toHexString(templateHash); - String hexPairs = String.format("%s:%s", hexKey, watchedValue); - Path stateFilePath = Paths.get(String.format("%s/%s.log", stateDirPathStr, hexBucketFileName)); - - List outputPairs; - AtomicBoolean found = new AtomicBoolean(false); - AtomicBoolean isDifferent = new AtomicBoolean(true); + AtomicBoolean isDifferent = new AtomicBoolean(false); String output = null; try{ @@ -321,33 +381,25 @@ public static String generateUniqueIRI(String template, String watchedValue, Boo }; try (BufferedReader reader = new BufferedReader(new FileReader(stateFilePath.toFile()))) { - - outputPairs = reader.lines().map(s -> { - String[] strings = s.split(":"); - String key = strings[0]; - String val = strings[1]; - if (hexKey.equals(key)) { - isDifferent.set(!val.equals(watchedValue)); - if (isDifferent.get()){ - val = watchedValue; - } - found.set(true); - } - return String.format("%s:%s", key,val); - }).collect(Collectors.toList()); - + outputPairs = reader.lines() + .map(s -> updateStatePropertyRecord(watchedMap, hexKey, found, isDifferent, s)) + .collect(Collectors.toList()); } if (!found.get()) { + String hexPairs = String.format("%s:%s", hexKey, watchedValueTemplate); outputPairs.add(hexPairs); } + System.out.println(Arrays.toString(outputPairs.toArray())); + try (BufferedWriter writer= new BufferedWriter(new FileWriter(stateFilePath.toFile()))){ for(String line : outputPairs){ writer.write(line); writer.newLine(); } } - if (isDifferent.get()){ + + if (isDifferent.get() || !found.get()){ OffsetDateTime now = OffsetDateTime.now(); DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; output = String.format("%s#%s", template, formatter.format(now) ); @@ -355,14 +407,28 @@ public static String generateUniqueIRI(String template, String watchedValue, Boo } catch (IOException e){ e.printStackTrace(); - output = null; } + return output; + } + + + public static String generateUniqueIRI(String template, String watchedValueTemplate, Boolean isUnique, String stateDirPathStr) { + //TODO: move this to the parameter of the function? (might get too bloated in RML definitions) + int m_buckets = 10; + + if (isUnique){ + return template; + } + + String output = handleNonUniqueIRI(template, watchedValueTemplate, stateDirPathStr, m_buckets); + return output; } + // TODO check whether this is the right place for this public static String jsonize(Object s) throws JsonProcessingException { ObjectWriter ow = new ObjectMapper().writer().withDefaultPrettyPrinter(); diff --git a/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl b/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl index 781e48a5..add4ad33 100644 --- a/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl +++ b/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl @@ -33,7 +33,7 @@ # Note that the template does not need to match with these properties! rr:predicateObjectMap [ rr:predicate idlab-fn:watchedProperty ; - rr:objectMap [ rml:reference "value" ] + rr:objectMap [ rr:template "pressure={pressure}&temperature={temperature}" ] ]; # Flag to indicate if the properties are unique on their own rr:predicateObjectMap [ @@ -49,7 +49,7 @@ ]; rr:predicateObjectMap [ - rr:predicate ex:value; - rr:objectMap [ rml:reference "value" ]; + rr:predicate ex:temp; + rr:objectMap [ rml:reference "temperature" ]; ]; . diff --git a/src/test/resources/web-of-things/ldes/generation/basic/output.nq b/src/test/resources/web-of-things/ldes/generation/basic/output.nq index 9e6b8ddd..4fd0afca 100644 --- a/src/test/resources/web-of-things/ldes/generation/basic/output.nq +++ b/src/test/resources/web-of-things/ldes/generation/basic/output.nq @@ -1,3 +1,3 @@ - "3". - "3". - "4". + "3". + "3". + "4". diff --git a/src/test/resources/web-of-things/ldes/generation/data1.csv b/src/test/resources/web-of-things/ldes/generation/data1.csv index 29143cd0..5d3c4c89 100644 --- a/src/test/resources/web-of-things/ldes/generation/data1.csv +++ b/src/test/resources/web-of-things/ldes/generation/data1.csv @@ -1,4 +1,4 @@ -id,value -sensor1,3 -sensor2,3 -sensor3,4 \ No newline at end of file +id,temperature,pressure +sensor1,3,10 +sensor2,3,10 +sensor3,4,10 \ No newline at end of file diff --git a/src/test/resources/web-of-things/ldes/generation/data2.csv b/src/test/resources/web-of-things/ldes/generation/data2.csv index 0ef3eb59..a2cfdd29 100644 --- a/src/test/resources/web-of-things/ldes/generation/data2.csv +++ b/src/test/resources/web-of-things/ldes/generation/data2.csv @@ -1,3 +1,3 @@ -id,value -sensor2,10 -sensor3,4 +id,temperature,pressure +sensor2,10,10 +sensor3,4,10 diff --git a/src/test/resources/web-of-things/ldes/generation/partial/mapping.ttl b/src/test/resources/web-of-things/ldes/generation/partial/mapping.ttl index 27a2bc9b..6373a85f 100644 --- a/src/test/resources/web-of-things/ldes/generation/partial/mapping.ttl +++ b/src/test/resources/web-of-things/ldes/generation/partial/mapping.ttl @@ -33,7 +33,7 @@ # Note that the template does not need to match with these properties! rr:predicateObjectMap [ rr:predicate idlab-fn:watchedProperty ; - rr:objectMap [ rml:reference "value" ] + rr:objectMap [ rr:template "temperature={temperature}&pressure={pressure}" ] ]; # Flag to indicate if the properties are unique on their own rr:predicateObjectMap [ @@ -49,7 +49,7 @@ ]; rr:predicateObjectMap [ - rr:predicate ex:value; - rr:objectMap [ rml:reference "value" ]; + rr:predicate ex:temp; + rr:objectMap [ rml:reference "temperature" ]; ]; . diff --git a/src/test/resources/web-of-things/ldes/generation/partial/mapping2.ttl b/src/test/resources/web-of-things/ldes/generation/partial/mapping2.ttl index 0b3d257d..4216bdeb 100644 --- a/src/test/resources/web-of-things/ldes/generation/partial/mapping2.ttl +++ b/src/test/resources/web-of-things/ldes/generation/partial/mapping2.ttl @@ -33,7 +33,7 @@ # Note that the template does not need to match with these properties! rr:predicateObjectMap [ rr:predicate idlab-fn:watchedProperty ; - rr:objectMap [ rml:reference "value" ] + rr:objectMap [ rr:template "pressure={pressure}&temperature={temperature}" ] ]; # Flag to indicate if the properties are unique on their own rr:predicateObjectMap [ @@ -49,7 +49,7 @@ ]; rr:predicateObjectMap [ - rr:predicate ex:value; - rr:objectMap [ rml:reference "value" ]; + rr:predicate ex:temp; + rr:objectMap [ rml:reference "temperature" ]; ]; . diff --git a/src/test/resources/web-of-things/ldes/generation/repeat/mapping.ttl b/src/test/resources/web-of-things/ldes/generation/repeat/mapping.ttl index 1cca254f..f46783aa 100644 --- a/src/test/resources/web-of-things/ldes/generation/repeat/mapping.ttl +++ b/src/test/resources/web-of-things/ldes/generation/repeat/mapping.ttl @@ -33,7 +33,7 @@ # Note that the template does not need to match with these properties! rr:predicateObjectMap [ rr:predicate idlab-fn:watchedProperty ; - rr:objectMap [ rml:reference "value" ] + rr:objectMap [ rml:template "temperature={temperature}&pressure={pressure}" ] ]; # Flag to indicate if the properties are unique on their own rr:predicateObjectMap [ @@ -49,7 +49,7 @@ ]; rr:predicateObjectMap [ - rr:predicate ex:value; - rr:objectMap [ rml:reference "value" ]; + rr:predicate ex:temperature; + rr:objectMap [ rml:reference "temperature" ]; ]; . From 72cba4efa53fa0d2968bad03bd7d3bf267c55a14 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Wed, 23 Feb 2022 14:00:06 +0100 Subject: [PATCH 036/609] CHANGELOG: LDES unique IRIs --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d451161..70df71f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - Add support for LDES Logical Target +- Add support for generating unique reproducible IRIs for LDES ### Fixed From 624aa581a8dfc7a7d8dac3ed77ae16c8904e9053 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Wed, 23 Feb 2022 14:07:23 +0100 Subject: [PATCH 037/609] NAMESPACES: only define LDES once --- src/main/java/be/ugent/rml/NAMESPACES.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/be/ugent/rml/NAMESPACES.java b/src/main/java/be/ugent/rml/NAMESPACES.java index eccec4c9..13c9a3f2 100644 --- a/src/main/java/be/ugent/rml/NAMESPACES.java +++ b/src/main/java/be/ugent/rml/NAMESPACES.java @@ -10,7 +10,6 @@ public class NAMESPACES { public static final String D2RQ = "http://www.wiwiss.fu-berlin.de/suhl/bizer/D2RQ/0.1#"; public static final String SD = "http://www.w3.org/ns/sparql-service-description#"; public static final String FNO_S = "https://w3id.org/function/ontology#"; - public static final String LDES = "https://w3id.org/ldes#"; @Deprecated public static final String FNO = "http://w3id.org/function/ontology#"; @Deprecated From 01923044f6a77f6b8ac93089b718ace1d9746dda Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Wed, 23 Feb 2022 18:44:40 +0100 Subject: [PATCH 038/609] implemented isUnique=true flag path of ldes iri generation --- .../rml/functions/lib/IDLabFunctions.java | 73 +++++++++++-------- .../ldes/generation/basic/mapping.ttl | 6 -- 2 files changed, 43 insertions(+), 36 deletions(-) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index 3959f07d..aeecc766 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -312,7 +312,17 @@ private static Map parseWatchedPropertyTemplate(String watchedVa private static String updateStatePropertyRecord(Map watchedMap, String hexKey, AtomicBoolean found, AtomicBoolean isDifferent, + boolean isUnique, String storedStateRecord) { + //No need to check the properties, we just need to know if the + //given hexKey has already been seen in the state file. + if (isUnique){ + found.set(storedStateRecord.equals(hexKey)); + isDifferent.set(found.get()); + return storedStateRecord; + } + + int split_idx = storedStateRecord.indexOf(':'); String storedHexKey = storedStateRecord.substring(0, split_idx); if (!storedHexKey.equals(hexKey)){ @@ -342,8 +352,14 @@ private static String updateStatePropertyRecord(Map watchedMap, return String.format("%s:%s", storedHexKey, String.join("&", propertyValPairs)); } + + private static Path getStateFilePath(String stateDirPathStr, int m_buckets, int templateHash) { + String hexBucketFileName= Integer.toHexString(templateHash % m_buckets); + return Paths.get(String.format("%s/%s.log", stateDirPathStr, hexBucketFileName)); + } + + /** - * Handles the case if the isUniqueIRI flag is not set for the method {@link #generateUniqueIRI(String, String, Boolean, String)}. * The generation of the IRI depends on the value of the watched properties. * If any of the watched properties changes in value or gets dropped, a unique IRI will be * generated. Otherwise, null String will be returned. @@ -354,21 +370,26 @@ private static String updateStatePropertyRecord(Map watchedMap, * * @param template The template string used to generate unique IRI by appending current date timestamp * @param watchedValueTemplate The template string containing the key-value pairs of properties being watched + * @param isUnique A flag to indicate if the given template already creates unique IRI * @param stateDirPathStr String representation of the directory path in which the state of the function * will be stored - * @param m_buckets The number of bucket files to be created * @return A unique IRI will be generated from the provided "template" string by appending the current * date timestamp if possible. Otherwise, null is returned */ - private static String handleNonUniqueIRI(String template, String watchedValueTemplate, String stateDirPathStr, int m_buckets) { + public static String generateUniqueIRI(String template, String watchedValueTemplate, Boolean isUnique, String stateDirPathStr) { + //TODO: move this to the parameter of the function? (might get too bloated in RML definitions) + int m_buckets = 10; + + // null check just in case idlab-fn:_watchedProperty is not provided in the mapping file + watchedValueTemplate = (watchedValueTemplate == null)? "" : watchedValueTemplate; Map watchedMap = parseWatchedPropertyTemplate(watchedValueTemplate); int templateHash = template.hashCode(); - String hexBucketFileName= Integer.toHexString(templateHash % m_buckets); + Path stateFilePath = getStateFilePath(stateDirPathStr, m_buckets, templateHash); + String hexKey = Integer.toHexString(templateHash); - Path stateFilePath = Paths.get(String.format("%s/%s.log", stateDirPathStr, hexBucketFileName)); - List outputPairs; + List outputRecords; AtomicBoolean found = new AtomicBoolean(false); AtomicBoolean isDifferent = new AtomicBoolean(false); String output = null; @@ -380,29 +401,36 @@ private static String handleNonUniqueIRI(String template, String watchedValueTem Files.createFile(stateFilePath); }; + // Get and update the state records which will be rewritten back to the state file by overwriting try (BufferedReader reader = new BufferedReader(new FileReader(stateFilePath.toFile()))) { - outputPairs = reader.lines() - .map(s -> updateStatePropertyRecord(watchedMap, hexKey, found, isDifferent, s)) + outputRecords = reader.lines() + .map(s -> updateStatePropertyRecord(watchedMap, hexKey, found, isDifferent, isUnique, s)) .collect(Collectors.toList()); } if (!found.get()) { - String hexPairs = String.format("%s:%s", hexKey, watchedValueTemplate); - outputPairs.add(hexPairs); + String hexedStateRecord = (isUnique)? hexKey:String.format("%s:%s", hexKey, watchedValueTemplate); + outputRecords.add(hexedStateRecord); } - System.out.println(Arrays.toString(outputPairs.toArray())); try (BufferedWriter writer= new BufferedWriter(new FileWriter(stateFilePath.toFile()))){ - for(String line : outputPairs){ + for(String line : outputRecords){ writer.write(line); writer.newLine(); } } + + // The unique IRI will be generated if there's any differences found with the corresponding record in + // the state file if (isDifferent.get() || !found.get()){ - OffsetDateTime now = OffsetDateTime.now(); - DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; - output = String.format("%s#%s", template, formatter.format(now) ); + if (isUnique) { + output= template; + }else { + OffsetDateTime now = OffsetDateTime.now(); + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; + output = String.format("%s#%s", template, formatter.format(now)); + } } } catch (IOException e){ @@ -413,21 +441,6 @@ private static String handleNonUniqueIRI(String template, String watchedValueTem } - public static String generateUniqueIRI(String template, String watchedValueTemplate, Boolean isUnique, String stateDirPathStr) { - //TODO: move this to the parameter of the function? (might get too bloated in RML definitions) - int m_buckets = 10; - - if (isUnique){ - return template; - } - - String output = handleNonUniqueIRI(template, watchedValueTemplate, stateDirPathStr, m_buckets); - - - return output; - } - - // TODO check whether this is the right place for this public static String jsonize(Object s) throws JsonProcessingException { diff --git a/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl b/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl index add4ad33..3a207445 100644 --- a/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl +++ b/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl @@ -29,12 +29,6 @@ rr:predicate idlab-fn:iri ; rr:objectMap [ rr:template "http://ex.org/{id}/" ] ]; - # Properties to watch, can be one or multiple - # Note that the template does not need to match with these properties! - rr:predicateObjectMap [ - rr:predicate idlab-fn:watchedProperty ; - rr:objectMap [ rr:template "pressure={pressure}&temperature={temperature}" ] - ]; # Flag to indicate if the properties are unique on their own rr:predicateObjectMap [ rr:predicate idlab-fn:unique ; From 659e7bea6e0a46582faaa337a2a5dd20f45435fb Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Thu, 24 Feb 2022 09:34:37 +0100 Subject: [PATCH 039/609] update unit test cases --- .../rml/functions/lib/IDLabFunctionsTest.java | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java b/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java index 79a1b10e..d7ab69fb 100644 --- a/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java +++ b/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java @@ -1,9 +1,13 @@ package be.ugent.rml.functions.lib; +import org.apache.commons.io.FileUtils; import org.hamcrest.CoreMatchers; +import org.junit.After; import org.junit.BeforeClass; import org.junit.Test; +import java.io.IOException; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -225,12 +229,18 @@ public static class LDESGenerationTests{ private static final String STATE_DIRECTORY = "/tmp/test-state"; + @After + public void cleanUp() throws IOException { + FileUtils.deleteDirectory(Paths.get(STATE_DIRECTORY).toFile()); + } + @Test public void skipGenerateUniqueIRI(){ String template = "http://example.com/sensor1/"; - String value = "5"; + String value = "pressure=5"; boolean isUnique = false; + IDLabFunctions.generateUniqueIRI(template, value, isUnique, STATE_DIRECTORY); String generated_iri = IDLabFunctions.generateUniqueIRI(template, value, isUnique, STATE_DIRECTORY); assertNull(generated_iri); } @@ -239,7 +249,7 @@ public void skipGenerateUniqueIRI(){ @Test public void generateUniqueIRI(){ String template = "http://example.com/sensor2/"; - String value = "5"; + String value = "pressure=5"; boolean isUnique = true; String generated_iri = IDLabFunctions.generateUniqueIRI(template, value, isUnique, STATE_DIRECTORY); @@ -251,11 +261,12 @@ public void generateUniqueIRI(){ public void generateUniqueIRIWithDate(){ String template = "http://example.com/sensor2/"; - String value = "5"; + String value = "pressure=5"; boolean isUnique = false; String generated_iri = IDLabFunctions.generateUniqueIRI(template, value, isUnique, STATE_DIRECTORY); - assertEquals(generated_iri, template); + assertNotNull(generated_iri); + assertTrue(generated_iri.contains(template)); } } From 6834bda66810e90b1db79780e3c089736176ac26 Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Thu, 24 Feb 2022 11:30:40 +0100 Subject: [PATCH 040/609] fixed isUnique=true generation of iri --- src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index aeecc766..65bf8af5 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -318,7 +318,6 @@ private static String updateStatePropertyRecord(Map watchedMap, //given hexKey has already been seen in the state file. if (isUnique){ found.set(storedStateRecord.equals(hexKey)); - isDifferent.set(found.get()); return storedStateRecord; } From 511eee60985d3cb2028bfda860b638a2428dd863 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Thu, 24 Feb 2022 12:10:49 +0100 Subject: [PATCH 041/609] IDLabFunctions: fix IRI generation for unique IRIs --- src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index 65bf8af5..e34bd9ad 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -317,7 +317,9 @@ private static String updateStatePropertyRecord(Map watchedMap, //No need to check the properties, we just need to know if the //given hexKey has already been seen in the state file. if (isUnique){ - found.set(storedStateRecord.equals(hexKey)); + if (storedStateRecord.equals(hexKey)) { + found.set(true); + } return storedStateRecord; } @@ -423,6 +425,7 @@ public static String generateUniqueIRI(String template, String watchedValueTempl // The unique IRI will be generated if there's any differences found with the corresponding record in // the state file if (isDifferent.get() || !found.get()){ + logger.debug("Update found! Generating IRI..."); if (isUnique) { output= template; }else { From 43e5491b1e0cf28b3c00850b314d2c1111e96bb9 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Tue, 1 Mar 2022 11:05:31 +0100 Subject: [PATCH 042/609] IDLabFunctions: handle missing data for LDES diff check --- .../ugent/rml/functions/lib/IDLabFunctions.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index e34bd9ad..486875b3 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -323,7 +323,6 @@ private static String updateStatePropertyRecord(Map watchedMap, return storedStateRecord; } - int split_idx = storedStateRecord.indexOf(':'); String storedHexKey = storedStateRecord.substring(0, split_idx); if (!storedHexKey.equals(hexKey)){ @@ -336,12 +335,20 @@ private static String updateStatePropertyRecord(Map watchedMap, Arrays.stream(storedStateRecord.substring(split_idx+1) .split("&")) .map(str -> { - String[] propVal = str.split("="); + String[] propVal = str.split("="); String property = propVal[0]; - String storeVal = propVal[1]; + String storeVal = "NULL"; + // Data may not provide a given property value so try and fail gracefully + try { + storeVal = propVal[1]; + } + catch (IndexOutOfBoundsException e) { + + } + String watchedVal = watchedMap.getOrDefault(property, null); - if(!watchedVal.equals(storeVal)){ + if(watchedVal != null && !watchedVal.equals(storeVal)){ isDifferent.set(true); storeVal = watchedVal; } From edef19ecf6ef26bd9a4b5af47f27e59cbaa17366 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Thu, 3 Mar 2022 15:51:32 +0100 Subject: [PATCH 043/609] IDLabFunctions: handle edge case --- src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index 486875b3..8092de77 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -324,6 +324,10 @@ private static String updateStatePropertyRecord(Map watchedMap, } int split_idx = storedStateRecord.indexOf(':'); + // Ignore if no split is possible + if (split_idx == -1) { + return storedStateRecord; + } String storedHexKey = storedStateRecord.substring(0, split_idx); if (!storedHexKey.equals(hexKey)){ return storedStateRecord; @@ -432,7 +436,6 @@ public static String generateUniqueIRI(String template, String watchedValueTempl // The unique IRI will be generated if there's any differences found with the corresponding record in // the state file if (isDifferent.get() || !found.get()){ - logger.debug("Update found! Generating IRI..."); if (isUnique) { output= template; }else { From 94a53db2c7dcebabfd283780536ffa53b7a4c344 Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Thu, 3 Mar 2022 11:54:37 +0100 Subject: [PATCH 044/609] refactored to use in-memory states --- CHANGELOG.md | 4 + src/main/java/be/ugent/rml/cli/Main.java | 2 + .../rml/functions/lib/IDLabFunctions.java | 158 ++++++++++++++++-- .../java/be/ugent/rml/Mapper_LDES_Test.java | 4 + .../rml/functions/lib/IDLabFunctionsTest.java | 2 +- 5 files changed, 153 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70df71f6..9f05d125 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add support for LDES Logical Target - Add support for generating unique reproducible IRIs for LDES +### Changed + +- Write LDES state to disk when mapping execution is complete. + ### Fixed - Clarified Readme for quick start diff --git a/src/main/java/be/ugent/rml/cli/Main.java b/src/main/java/be/ugent/rml/cli/Main.java index 7b542fc4..fd4e228e 100644 --- a/src/main/java/be/ugent/rml/cli/Main.java +++ b/src/main/java/be/ugent/rml/cli/Main.java @@ -386,6 +386,8 @@ public static void main(String[] args, String basePath) { String outputFile = getPriorityOptionValue(outputfileOption, lineArgs, configFile); result.copyNameSpaces(rmlStore); + IDLabFunctions.saveState(); + writeOutputTargets(targets, rmlStore, basePath, outputFile, outputFormat); } catch (Exception e) { logger.error(e.getMessage()); diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index 8092de77..acd4c7eb 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -14,6 +14,7 @@ import org.slf4j.LoggerFactory; import java.io.*; +import java.nio.Buffer; import java.nio.file.Files; import java.net.HttpURLConnection; import java.net.URL; @@ -28,10 +29,13 @@ import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +import java.util.stream.Stream; public class IDLabFunctions { private static final Logger logger = LoggerFactory.getLogger(IDLabFunctions.class); + private static Map> LDES_FILE_STATE_MAP = new HashMap<>(); + private static Path STATE_DIR = null; public static boolean stringContainsOtherString(String str, String otherStr, String delimiter) { String[] split = str.split(delimiter); @@ -299,7 +303,7 @@ public static String normalizeDateTime(String dateTimeStr, String pattern) { * the form "key1=val1&key2=val2&..." * @return A map containing the parsed pairs of the properties which are being watched. */ - private static Map parseWatchedPropertyTemplate(String watchedValueTemplate){ + private static Map parsePropertyValueTemplate(String watchedValueTemplate){ Map result = new HashMap<>(); Arrays.stream(watchedValueTemplate.split("&")) @@ -388,13 +392,13 @@ private static Path getStateFilePath(String stateDirPathStr, int m_buckets, int * @return A unique IRI will be generated from the provided "template" string by appending the current * date timestamp if possible. Otherwise, null is returned */ - public static String generateUniqueIRI(String template, String watchedValueTemplate, Boolean isUnique, String stateDirPathStr) { + public static String generateUniqueIRIOLD(String template, String watchedValueTemplate, Boolean isUnique, String stateDirPathStr) { //TODO: move this to the parameter of the function? (might get too bloated in RML definitions) int m_buckets = 10; // null check just in case idlab-fn:_watchedProperty is not provided in the mapping file watchedValueTemplate = (watchedValueTemplate == null)? "" : watchedValueTemplate; - Map watchedMap = parseWatchedPropertyTemplate(watchedValueTemplate); + Map watchedMap = parsePropertyValueTemplate(watchedValueTemplate); int templateHash = template.hashCode(); Path stateFilePath = getStateFilePath(stateDirPathStr, m_buckets, templateHash); @@ -411,7 +415,7 @@ public static String generateUniqueIRI(String template, String watchedValueTempl Files.createDirectories(Paths.get(stateDirPathStr)); if (Files.notExists(stateFilePath) ) { Files.createFile(stateFilePath); - }; + } // Get and update the state records which will be rewritten back to the state file by overwriting try (BufferedReader reader = new BufferedReader(new FileReader(stateFilePath.toFile()))) { @@ -432,18 +436,7 @@ public static String generateUniqueIRI(String template, String watchedValueTempl } } - - // The unique IRI will be generated if there's any differences found with the corresponding record in - // the state file - if (isDifferent.get() || !found.get()){ - if (isUnique) { - output= template; - }else { - OffsetDateTime now = OffsetDateTime.now(); - DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; - output = String.format("%s#%s", template, formatter.format(now)); - } - } + output = getOutput(template, isUnique, found, isDifferent); } catch (IOException e){ e.printStackTrace(); @@ -452,6 +445,139 @@ public static String generateUniqueIRI(String template, String watchedValueTempl return output; } + private static String getOutput(String template, Boolean isUnique, AtomicBoolean found, AtomicBoolean isDifferent) { + // The unique IRI will be generated if there's any differences found with the corresponding record in + // the state file + String output = null; + if (isDifferent.get() || !found.get()){ + logger.debug("Update found! Generating IRI..."); + if (isUnique) { + output = template; + }else { + OffsetDateTime now = OffsetDateTime.now(); + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; + output = String.format("%s#%s", template, formatter.format(now)); + } + } + return output; + } + + private static void initStateFile(Path stateDirPath){ + try (Stream statePaths = Files.find(stateDirPath, 1, (filePath, fileAttr) -> fileAttr.isRegularFile())){ + + Stream readers = statePaths.map(path -> { + try { + return new AbstractMap.SimpleImmutableEntry(path, new BufferedReader(new FileReader(path.toFile()))); + } catch (FileNotFoundException e) { + return null; + } + }).filter(Objects::nonNull); + + + readers.forEach( entry -> { + try (BufferedReader bReader = (BufferedReader) entry.getValue()){ + String fileName = ((Path) entry.getKey()).toString(); + bReader.lines().forEach( + line -> { + String[] pairs = line.split(":"); + String key = pairs[0]; + String val = null; + try { + val = pairs[1]; + }catch (IndexOutOfBoundsException e) { + e.printStackTrace(); + } + LDES_FILE_STATE_MAP.computeIfAbsent(fileName, k -> new HashMap<>()).put(key, val); + } + ); + } catch (IOException e) { + e.printStackTrace(); + } + + }); + + }catch (IOException e){ + e.printStackTrace(); + } + } + + public static void saveState() throws IOException { + if (Objects.nonNull(STATE_DIR)){ + + Files.createDirectories(STATE_DIR); + for (Map.Entry> entry: LDES_FILE_STATE_MAP.entrySet()){ + + Path file = Paths.get(entry.getKey()); + Map storedMap = entry.getValue(); + + if (Files.notExists(file)){ + Files.createFile(file); + } + + + try( BufferedWriter writer = new BufferedWriter(new FileWriter(file.toFile()))){ + for( Map.Entry record: storedMap.entrySet()){ + String line = String.format("%s:%s", record.getKey(), record.getValue()); + writer.write(line); + writer.newLine(); + } + } + + } + } + + } + + public static void resetState(){ + LDES_FILE_STATE_MAP = new HashMap<>(); + STATE_DIR = null; + } + public static String generateUniqueIRI(String template, String watchedValueTemplate, Boolean isUnique, String stateDirPathStr) { + if (STATE_DIR == null){ + STATE_DIR = Paths.get(stateDirPathStr); + initStateFile(STATE_DIR); + } + int m_buckets = 10; + + // null check just in case idlab-fn:_watchedProperty is not provided in the mapping file + watchedValueTemplate = (watchedValueTemplate == null)? "" : watchedValueTemplate; + Map watchedMap = parsePropertyValueTemplate(watchedValueTemplate); + + int templateHash = template.hashCode(); + String stateFilePathStr = getStateFilePath(stateDirPathStr, m_buckets, templateHash).toString(); + + String hexKey = Integer.toHexString(templateHash); + + AtomicBoolean found = new AtomicBoolean(false); + AtomicBoolean isDifferent = new AtomicBoolean(false); + + Map keyValMap = LDES_FILE_STATE_MAP.computeIfAbsent(stateFilePathStr, f -> new HashMap<>()); + if (keyValMap.containsKey(hexKey)){ + found.set(true); + String storedProp = keyValMap.get(hexKey); + Map storedPropMap = parsePropertyValueTemplate(storedProp); + for (Map.Entry kv : watchedMap.entrySet()){ + String prop = kv.getKey(); + String val = kv.getValue(); + + String storedVal = storedPropMap.getOrDefault(prop, null); + if (!val.equals(storedVal)){ + isDifferent.set(true); + } + } + + } + keyValMap.put(hexKey, watchedValueTemplate); + + + + + + + return getOutput(template,isUnique, found, isDifferent); + + } + // TODO check whether this is the right place for this diff --git a/src/test/java/be/ugent/rml/Mapper_LDES_Test.java b/src/test/java/be/ugent/rml/Mapper_LDES_Test.java index 5d8bb160..92d2ca10 100644 --- a/src/test/java/be/ugent/rml/Mapper_LDES_Test.java +++ b/src/test/java/be/ugent/rml/Mapper_LDES_Test.java @@ -1,6 +1,7 @@ package be.ugent.rml; import be.ugent.rml.functions.FunctionLoader; +import be.ugent.rml.functions.lib.IDLabFunctions; import be.ugent.rml.store.QuadStore; import be.ugent.rml.store.RDF4JStore; import be.ugent.rml.term.NamedNode; @@ -20,6 +21,7 @@ public class Mapper_LDES_Test extends TestCore { private static FunctionLoader LOADER; @After public void cleanUp() throws IOException { + IDLabFunctions.resetState(); FileUtils.deleteDirectory(new File("/tmp/ldes-test")); } @@ -41,6 +43,7 @@ public void evaluate_unique_LDES () throws Exception { public void evaluate_repeat_LDES() throws Exception { Executor executor = this.createExecutor("./web-of-things/ldes/generation/repeat/mapping.ttl", LOADER); executor.executeV5(null).get(new NamedNode("rmlmapper://default.store")); + IDLabFunctions.saveState(); executor = this.createExecutor("./web-of-things/ldes/generation/repeat/mapping.ttl", LOADER); doMapping(executor, "./web-of-things/ldes/generation/repeat/output.nq"); } @@ -49,6 +52,7 @@ public void evaluate_repeat_LDES() throws Exception { public void evaluate_partial_repeat_LDES() throws Exception { Executor executor = this.createExecutor("./web-of-things/ldes/generation/partial/mapping.ttl", LOADER); QuadStore result = executor.executeV5(null).get(new NamedNode("rmlmapper://default.store")); + IDLabFunctions.saveState(); executor = this.createExecutor("./web-of-things/ldes/generation/partial/mapping2.ttl", LOADER); QuadStore result_second = executor.executeV5(null).get(new NamedNode("rmlmapper://default.store")); assertEquals(3, result.size()); diff --git a/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java b/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java index d7ab69fb..8d0c6e03 100644 --- a/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java +++ b/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java @@ -253,7 +253,7 @@ public void generateUniqueIRI(){ boolean isUnique = true; String generated_iri = IDLabFunctions.generateUniqueIRI(template, value, isUnique, STATE_DIRECTORY); - assertEquals(generated_iri, template); + assertEquals(template, generated_iri); } From 893c9837d2148d1b8d3200796decee98105922c7 Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Mon, 7 Mar 2022 10:20:03 +0100 Subject: [PATCH 045/609] handle null string case for property parsing --- .../be/ugent/rml/functions/lib/IDLabFunctions.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index acd4c7eb..8e91f2a9 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -306,10 +306,12 @@ public static String normalizeDateTime(String dateTimeStr, String pattern) { private static Map parsePropertyValueTemplate(String watchedValueTemplate){ Map result = new HashMap<>(); - Arrays.stream(watchedValueTemplate.split("&")) - .map(s -> s.split("=")) - .filter(sArr -> sArr.length == 2) - .forEach(sArr -> result.put(sArr[0], sArr[1])); + if (watchedValueTemplate != null || watchedValueTemplate.length() > 0) { + Arrays.stream(watchedValueTemplate.split("&")) + .map(s -> s.split("=")) + .filter(sArr -> sArr.length == 2) + .forEach(sArr -> result.put(sArr[0], sArr[1])); + } return result; } From 15b20a976ebb089243506cce25287d81ac04c453 Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Mon, 7 Mar 2022 11:12:14 +0100 Subject: [PATCH 046/609] fix isUnique=True case of iri generation --- .../rml/functions/lib/IDLabFunctions.java | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index 8e91f2a9..d56889f9 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -303,11 +303,12 @@ public static String normalizeDateTime(String dateTimeStr, String pattern) { * the form "key1=val1&key2=val2&..." * @return A map containing the parsed pairs of the properties which are being watched. */ - private static Map parsePropertyValueTemplate(String watchedValueTemplate){ + private static Map parsePropertyValueTemplate(Optional watchedValueTemplate){ Map result = new HashMap<>(); + String watchedVal = watchedValueTemplate.orElse(""); - if (watchedValueTemplate != null || watchedValueTemplate.length() > 0) { - Arrays.stream(watchedValueTemplate.split("&")) + if (watchedVal.length() > 0) { + Arrays.stream(watchedVal.split("&")) .map(s -> s.split("=")) .filter(sArr -> sArr.length == 2) .forEach(sArr -> result.put(sArr[0], sArr[1])); @@ -398,9 +399,7 @@ public static String generateUniqueIRIOLD(String template, String watchedValueTe //TODO: move this to the parameter of the function? (might get too bloated in RML definitions) int m_buckets = 10; - // null check just in case idlab-fn:_watchedProperty is not provided in the mapping file - watchedValueTemplate = (watchedValueTemplate == null)? "" : watchedValueTemplate; - Map watchedMap = parsePropertyValueTemplate(watchedValueTemplate); + Map watchedMap = parsePropertyValueTemplate(Optional.ofNullable(watchedValueTemplate)); int templateHash = template.hashCode(); Path stateFilePath = getStateFilePath(stateDirPathStr, m_buckets, templateHash); @@ -519,7 +518,11 @@ public static void saveState() throws IOException { try( BufferedWriter writer = new BufferedWriter(new FileWriter(file.toFile()))){ for( Map.Entry record: storedMap.entrySet()){ - String line = String.format("%s:%s", record.getKey(), record.getValue()); + String line = record.getKey(); + if (record.getValue() != null) { + line = String.format("%s:%s", record.getKey(), record.getValue()); + } + writer.write(line); writer.newLine(); } @@ -542,8 +545,9 @@ public static String generateUniqueIRI(String template, String watchedValueTempl int m_buckets = 10; // null check just in case idlab-fn:_watchedProperty is not provided in the mapping file - watchedValueTemplate = (watchedValueTemplate == null)? "" : watchedValueTemplate; - Map watchedMap = parsePropertyValueTemplate(watchedValueTemplate); + + Optional watchedValueOption = Optional.ofNullable(watchedValueTemplate); + Map watchedMap = parsePropertyValueTemplate(watchedValueOption); int templateHash = template.hashCode(); String stateFilePathStr = getStateFilePath(stateDirPathStr, m_buckets, templateHash).toString(); @@ -557,7 +561,7 @@ public static String generateUniqueIRI(String template, String watchedValueTempl if (keyValMap.containsKey(hexKey)){ found.set(true); String storedProp = keyValMap.get(hexKey); - Map storedPropMap = parsePropertyValueTemplate(storedProp); + Map storedPropMap = parsePropertyValueTemplate(Optional.ofNullable(storedProp)); for (Map.Entry kv : watchedMap.entrySet()){ String prop = kv.getKey(); String val = kv.getValue(); From cd9b5198ae3c9ec6a5440a7791950509bab62dd5 Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Mon, 7 Mar 2022 14:05:40 +0100 Subject: [PATCH 047/609] fixed tests --- src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java b/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java index 8d0c6e03..82dd0d3b 100644 --- a/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java +++ b/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java @@ -231,6 +231,7 @@ public static class LDESGenerationTests{ @After public void cleanUp() throws IOException { + IDLabFunctions.resetState(); FileUtils.deleteDirectory(Paths.get(STATE_DIRECTORY).toFile()); } From e4cc725edb3ee0aa12e64ed78ebb8b961235d3a0 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Tue, 8 Mar 2022 08:23:45 +0100 Subject: [PATCH 048/609] resources: functions_grel: use xsd:integer xsd:number does not exist, use xsd:integer instead. Fixes https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/234 --- CHANGELOG.md | 2 ++ src/main/resources/functions_grel.ttl | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f05d125..b44391fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Upgraded com.fasterxml.jackson.core to 2.13.1 - Upgraded org.jsoup to 1.14.3 - Upgraded org.apache.poi to 5.0.0 +- Resources: functions_grel: use xsd:integer (see [issue 234](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/234)) + ### Changed diff --git a/src/main/resources/functions_grel.ttl b/src/main/resources/functions_grel.ttl index 9a4bb572..7ce38d0f 100644 --- a/src/main/resources/functions_grel.ttl +++ b/src/main/resources/functions_grel.ttl @@ -412,7 +412,7 @@ grel:output_number fno:name "number" ; rdfs:label "number" ; fno:predicate grel:o_number ; - fno:type xsd:number . + fno:type xsd:integer . #TOSTRING From 8539a030ff9bc877b423d6425a061da552b95461 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Wed, 9 Mar 2022 14:45:03 +0100 Subject: [PATCH 049/609] RecordsFactory: print error if reference formulation is unknown --- CHANGELOG.md | 3 +-- src/main/java/be/ugent/rml/records/RecordsFactory.java | 7 +++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b44391fa..74cba1b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,8 +33,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Upgraded org.jsoup to 1.14.3 - Upgraded org.apache.poi to 5.0.0 - Resources: functions_grel: use xsd:integer (see [issue 234](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/234)) - - +- Print error if referenceformulation is unsupported ### Changed diff --git a/src/main/java/be/ugent/rml/records/RecordsFactory.java b/src/main/java/be/ugent/rml/records/RecordsFactory.java index ec01b39d..83fc9a9f 100644 --- a/src/main/java/be/ugent/rml/records/RecordsFactory.java +++ b/src/main/java/be/ugent/rml/records/RecordsFactory.java @@ -6,8 +6,11 @@ import be.ugent.rml.access.AccessFactory; import be.ugent.rml.store.Quad; import be.ugent.rml.store.QuadStore; +import be.ugent.rml.target.TargetFactory; import be.ugent.rml.term.NamedNode; import be.ugent.rml.term.Term; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; @@ -23,6 +26,7 @@ public class RecordsFactory { private Map>>> recordCache; private AccessFactory accessFactory; private Map referenceFormulationRecordFactoryMap; + private static final Logger logger = LoggerFactory.getLogger(RecordsFactory.class); public RecordsFactory(String basePath) { accessFactory = new AccessFactory(basePath); @@ -133,6 +137,9 @@ private List getRecords(Access access, Term logicalSource, String refere if (records == null) { try { // Select the Record Factory based on the reference formulation. + if (!referenceFormulationRecordFactoryMap.containsKey(referenceFormulation)) { + logger.error("Referenceformulation " + referenceFormulation + " is unsupported!"); + } ReferenceFormulationRecordFactory factory = referenceFormulationRecordFactoryMap.get(referenceFormulation); records = factory.getRecords(access, logicalSource, rmlStore); From 207edf831fc8d6493cd499df415151edd70251d8 Mon Sep 17 00:00:00 2001 From: tihabils Date: Thu, 10 Mar 2022 16:05:08 +0100 Subject: [PATCH 050/609] Changed to (deprecated) simple-odf fork. --- pom.xml | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index 19a430ff..64c9b124 100644 --- a/pom.xml +++ b/pom.xml @@ -283,18 +283,11 @@ poi-ooxml 5.0.0 - + - - org.apache.odftoolkit + org.odftoolkit simple-odf - 0.8.2-incubating + 0.9.0 tools @@ -302,6 +295,26 @@ + + + + + + + + + + + + + + From f40afa2334331a60944a4c6f6f979f0347fc3c68 Mon Sep 17 00:00:00 2001 From: tihabils Date: Thu, 10 Mar 2022 16:24:04 +0100 Subject: [PATCH 051/609] clean up of pom file. --- pom.xml | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/pom.xml b/pom.xml index 64c9b124..f76f2f0e 100644 --- a/pom.xml +++ b/pom.xml @@ -288,33 +288,7 @@ org.odftoolkit simple-odf 0.9.0 - - - tools - com.sun - - - - - - - - - - - - - - - - From 64c93e6a693cce36c70cb5d9c39d9329702a8633 Mon Sep 17 00:00:00 2001 From: tihabils Date: Thu, 10 Mar 2022 17:18:52 +0100 Subject: [PATCH 052/609] added CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74cba1b4..78f1cc12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased - TestCore: enable debug logs when VERBOSE env variable is set (see [issue 230](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/230)) +- Switched to fork of ODFtoolkit (see [issue 237](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/237)) ### Added From f80c6fb52d6838291b929c1c3ab77821cdcc66ad Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Fri, 11 Mar 2022 10:45:00 +0000 Subject: [PATCH 053/609] CHANGELOG: release v5.0.0 --- CHANGELOG.md | 3 +++ buildNumber.properties | 4 ++-- pom.xml | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78f1cc12..43eaf512 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## Unreleased + +## [5.0.0] - 2022-03-11 - TestCore: enable debug logs when VERBOSE env variable is set (see [issue 230](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/230)) - Switched to fork of ODFtoolkit (see [issue 237](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/237)) @@ -505,6 +507,7 @@ and [169](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/169)) - support for accessing remote files (via HTTP GET) - basic support for functions +[5.0.0]: https://github.com/RMLio/rmlmapper-java/compare/v4.15.0...v5.0.0 [4.15.0]: https://github.com/RMLio/rmlmapper-java/compare/v4.14.3...v4.15.0 [4.14.3]: https://github.com/RMLio/rmlmapper-java/compare/v4.14.2...v4.14.3 [4.14.2]: https://github.com/RMLio/rmlmapper-java/compare/v4.14.1...v4.14.2 diff --git a/buildNumber.properties b/buildNumber.properties index ad7e6c73..c5b85fe0 100644 --- a/buildNumber.properties +++ b/buildNumber.properties @@ -1,3 +1,3 @@ #maven.buildNumber.plugin properties file -#Tue Feb 01 10:31:54 GMT 2022 -buildNumber0=360 +#Fri Mar 11 10:44:24 GMT 2022 +buildNumber0=361 diff --git a/pom.xml b/pom.xml index f76f2f0e..4b20349a 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ be.ugent.rml rmlmapper RMLMapper - 4.15.0 + 5.0.0 The RMLMapper executes RML rules to generate high quality Linked Data from multiple originally (semi-)structured data sources. From 60c4020f53c3714e30b6a6df7bfbd97cea622ba5 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Fri, 11 Mar 2022 11:52:29 +0100 Subject: [PATCH 054/609] pom: upgrade to postgresql 42.3.3 Security fix --- CHANGELOG.md | 3 +++ pom.xml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43eaf512..20c7a577 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +### Fixed +- Upgrade postgresql JDBC driver to 42.3.3 + ## [5.0.0] - 2022-03-11 - TestCore: enable debug logs when VERBOSE env variable is set (see [issue 230](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/230)) - Switched to fork of ODFtoolkit (see [issue 237](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/237)) diff --git a/pom.xml b/pom.xml index 4b20349a..dc1425d6 100644 --- a/pom.xml +++ b/pom.xml @@ -178,7 +178,7 @@ org.postgresql postgresql - 42.3.2 + 42.3.3 From 26595ca863be039196bf67bcf6a7d3df1f6f8f3a Mon Sep 17 00:00:00 2001 From: Els de Vleeschauwer Date: Thu, 17 Mar 2022 10:20:33 +0100 Subject: [PATCH 055/609] functions: lib: IDLabFunctions: functions lookup and lookupWithDelimiter Implementation of a function that looks for the first occurence of a certain value in a column of a csv file, in order to return a value from a different column in the same row. It will be used in the AI4Fl project to align ID's between different source systems. --- .../rml/functions/lib/IDLabFunctions.java | 183 ++++++++++++------ src/main/resources/functions_idlab.ttl | 45 ++++- .../resources/rml-fno-test-cases/classB.csv | 3 + 3 files changed, 174 insertions(+), 57 deletions(-) create mode 100644 src/test/resources/rml-fno-test-cases/classB.csv diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index d56889f9..5145c025 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -8,13 +8,15 @@ import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.PathNotFoundException; +import com.opencsv.CSVParserBuilder; +import com.opencsv.exceptions.CsvValidationException; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; -import java.nio.Buffer; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.net.HttpURLConnection; import java.net.URL; @@ -31,12 +33,23 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import com.opencsv.CSVReader; +import com.opencsv.CSVParser; +import com.opencsv.CSVReaderBuilder; +import com.opencsv.enums.CSVReaderNullFieldIndicator; + public class IDLabFunctions { private static final Logger logger = LoggerFactory.getLogger(IDLabFunctions.class); private static Map> LDES_FILE_STATE_MAP = new HashMap<>(); private static Path STATE_DIR = null; + // used by the lookup function + private static Map LOOKUP_STATE_MAP = new HashMap(); + private static String LOOKUP_STATE_INPUTFILE = null; + private static Integer LOOKUP_STATE_FROM_COLUMN = -1; + private static Integer LOOKUP_STATE_TO_COLUMN = -1; + public static boolean stringContainsOtherString(String str, String otherStr, String delimiter) { String[] split = str.split(delimiter); List list = Arrays.asList(split); @@ -104,6 +117,7 @@ public static String decide(String input, String expected, String result) { /** * Tests whether a given input is null or not. + * * @param input String to evaluate * @return whether it's null or not */ @@ -162,13 +176,14 @@ public static String toUpperCaseURL(String test) { /** * Tests whether a certain number is in a certain range. * Everything is optional: - * - function returns false when testNumber is null - * - function only tests from constraint when to is null - * - function only tests to constraint when from is null - * - function returns true when from and to are null. + * - function returns false when testNumber is null + * - function only tests from constraint when to is null + * - function only tests to constraint when from is null + * - function returns true when from and to are null. + * * @param testNumber The number put under the test. Optional (function returns false when is null) - * @param from The number from where (inclusive) - * @param to The number until where (exclusive) + * @param from The number from where (inclusive) + * @param to The number until where (exclusive) * @return whether it's in range or not */ public static boolean inRange(Double testNumber, Double from, Double to) { @@ -190,6 +205,7 @@ public static boolean inRange(Double testNumber, Double from, Double to) { /** * Convert a string to its slugified equivalent. + * * @param str The String to slugify * @return the slugified string. Returns null if the input was also null. */ @@ -303,7 +319,7 @@ public static String normalizeDateTime(String dateTimeStr, String pattern) { * the form "key1=val1&key2=val2&..." * @return A map containing the parsed pairs of the properties which are being watched. */ - private static Map parsePropertyValueTemplate(Optional watchedValueTemplate){ + private static Map parsePropertyValueTemplate(Optional watchedValueTemplate) { Map result = new HashMap<>(); String watchedVal = watchedValueTemplate.orElse(""); @@ -323,7 +339,7 @@ private static String updateStatePropertyRecord(Map watchedMap, String storedStateRecord) { //No need to check the properties, we just need to know if the //given hexKey has already been seen in the state file. - if (isUnique){ + if (isUnique) { if (storedStateRecord.equals(hexKey)) { found.set(true); } @@ -336,14 +352,14 @@ private static String updateStatePropertyRecord(Map watchedMap, return storedStateRecord; } String storedHexKey = storedStateRecord.substring(0, split_idx); - if (!storedHexKey.equals(hexKey)){ + if (!storedHexKey.equals(hexKey)) { return storedStateRecord; } found.set(true); List propertyValPairs = - Arrays.stream(storedStateRecord.substring(split_idx+1) + Arrays.stream(storedStateRecord.substring(split_idx + 1) .split("&")) .map(str -> { String[] propVal = str.split("="); @@ -352,19 +368,18 @@ private static String updateStatePropertyRecord(Map watchedMap, // Data may not provide a given property value so try and fail gracefully try { storeVal = propVal[1]; - } - catch (IndexOutOfBoundsException e) { + } catch (IndexOutOfBoundsException e) { } String watchedVal = watchedMap.getOrDefault(property, null); - if(watchedVal != null && !watchedVal.equals(storeVal)){ + if (watchedVal != null && !watchedVal.equals(storeVal)) { isDifferent.set(true); storeVal = watchedVal; } - return String.format("%s=%s", property, storeVal); + return String.format("%s=%s", property, storeVal); }) .collect(Collectors.toList()); @@ -373,7 +388,7 @@ private static String updateStatePropertyRecord(Map watchedMap, private static Path getStateFilePath(String stateDirPathStr, int m_buckets, int templateHash) { - String hexBucketFileName= Integer.toHexString(templateHash % m_buckets); + String hexBucketFileName = Integer.toHexString(templateHash % m_buckets); return Paths.get(String.format("%s/%s.log", stateDirPathStr, hexBucketFileName)); } @@ -387,11 +402,11 @@ private static Path getStateFilePath(String stateDirPathStr, int m_buckets, int * A unique IRI will be generated from the provided "template" string by appending the current * date timestamp. * - * @param template The template string used to generate unique IRI by appending current date timestamp + * @param template The template string used to generate unique IRI by appending current date timestamp * @param watchedValueTemplate The template string containing the key-value pairs of properties being watched - * @param isUnique A flag to indicate if the given template already creates unique IRI - * @param stateDirPathStr String representation of the directory path in which the state of the function - * will be stored + * @param isUnique A flag to indicate if the given template already creates unique IRI + * @param stateDirPathStr String representation of the directory path in which the state of the function + * will be stored * @return A unique IRI will be generated from the provided "template" string by appending the current * date timestamp if possible. Otherwise, null is returned */ @@ -399,7 +414,7 @@ public static String generateUniqueIRIOLD(String template, String watchedValueTe //TODO: move this to the parameter of the function? (might get too bloated in RML definitions) int m_buckets = 10; - Map watchedMap = parsePropertyValueTemplate(Optional.ofNullable(watchedValueTemplate)); + Map watchedMap = parsePropertyValueTemplate(Optional.ofNullable(watchedValueTemplate)); int templateHash = template.hashCode(); Path stateFilePath = getStateFilePath(stateDirPathStr, m_buckets, templateHash); @@ -411,10 +426,10 @@ public static String generateUniqueIRIOLD(String template, String watchedValueTe AtomicBoolean isDifferent = new AtomicBoolean(false); String output = null; - try{ + try { Files.createDirectories(Paths.get(stateDirPathStr)); - if (Files.notExists(stateFilePath) ) { + if (Files.notExists(stateFilePath)) { Files.createFile(stateFilePath); } @@ -426,12 +441,12 @@ public static String generateUniqueIRIOLD(String template, String watchedValueTe } if (!found.get()) { - String hexedStateRecord = (isUnique)? hexKey:String.format("%s:%s", hexKey, watchedValueTemplate); + String hexedStateRecord = (isUnique) ? hexKey : String.format("%s:%s", hexKey, watchedValueTemplate); outputRecords.add(hexedStateRecord); } - try (BufferedWriter writer= new BufferedWriter(new FileWriter(stateFilePath.toFile()))){ - for(String line : outputRecords){ + try (BufferedWriter writer = new BufferedWriter(new FileWriter(stateFilePath.toFile()))) { + for (String line : outputRecords) { writer.write(line); writer.newLine(); } @@ -439,7 +454,7 @@ public static String generateUniqueIRIOLD(String template, String watchedValueTe output = getOutput(template, isUnique, found, isDifferent); - } catch (IOException e){ + } catch (IOException e) { e.printStackTrace(); } @@ -450,11 +465,11 @@ private static String getOutput(String template, Boolean isUnique, AtomicBoolean // The unique IRI will be generated if there's any differences found with the corresponding record in // the state file String output = null; - if (isDifferent.get() || !found.get()){ + if (isDifferent.get() || !found.get()) { logger.debug("Update found! Generating IRI..."); if (isUnique) { output = template; - }else { + } else { OffsetDateTime now = OffsetDateTime.now(); DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; output = String.format("%s#%s", template, formatter.format(now)); @@ -463,20 +478,20 @@ private static String getOutput(String template, Boolean isUnique, AtomicBoolean return output; } - private static void initStateFile(Path stateDirPath){ - try (Stream statePaths = Files.find(stateDirPath, 1, (filePath, fileAttr) -> fileAttr.isRegularFile())){ + private static void initStateFile(Path stateDirPath) { + try (Stream statePaths = Files.find(stateDirPath, 1, (filePath, fileAttr) -> fileAttr.isRegularFile())) { Stream readers = statePaths.map(path -> { try { - return new AbstractMap.SimpleImmutableEntry(path, new BufferedReader(new FileReader(path.toFile()))); + return new AbstractMap.SimpleImmutableEntry(path, new BufferedReader(new FileReader(path.toFile()))); } catch (FileNotFoundException e) { return null; } }).filter(Objects::nonNull); - readers.forEach( entry -> { - try (BufferedReader bReader = (BufferedReader) entry.getValue()){ + readers.forEach(entry -> { + try (BufferedReader bReader = (BufferedReader) entry.getValue()) { String fileName = ((Path) entry.getKey()).toString(); bReader.lines().forEach( line -> { @@ -484,8 +499,8 @@ private static void initStateFile(Path stateDirPath){ String key = pairs[0]; String val = null; try { - val = pairs[1]; - }catch (IndexOutOfBoundsException e) { + val = pairs[1]; + } catch (IndexOutOfBoundsException e) { e.printStackTrace(); } LDES_FILE_STATE_MAP.computeIfAbsent(fileName, k -> new HashMap<>()).put(key, val); @@ -497,27 +512,27 @@ private static void initStateFile(Path stateDirPath){ }); - }catch (IOException e){ + } catch (IOException e) { e.printStackTrace(); } } public static void saveState() throws IOException { - if (Objects.nonNull(STATE_DIR)){ + if (Objects.nonNull(STATE_DIR)) { Files.createDirectories(STATE_DIR); - for (Map.Entry> entry: LDES_FILE_STATE_MAP.entrySet()){ + for (Map.Entry> entry : LDES_FILE_STATE_MAP.entrySet()) { Path file = Paths.get(entry.getKey()); Map storedMap = entry.getValue(); - if (Files.notExists(file)){ + if (Files.notExists(file)) { Files.createFile(file); } - try( BufferedWriter writer = new BufferedWriter(new FileWriter(file.toFile()))){ - for( Map.Entry record: storedMap.entrySet()){ + try (BufferedWriter writer = new BufferedWriter(new FileWriter(file.toFile()))) { + for (Map.Entry record : storedMap.entrySet()) { String line = record.getKey(); if (record.getValue() != null) { line = String.format("%s:%s", record.getKey(), record.getValue()); @@ -533,12 +548,13 @@ public static void saveState() throws IOException { } - public static void resetState(){ + public static void resetState() { LDES_FILE_STATE_MAP = new HashMap<>(); STATE_DIR = null; } + public static String generateUniqueIRI(String template, String watchedValueTemplate, Boolean isUnique, String stateDirPathStr) { - if (STATE_DIR == null){ + if (STATE_DIR == null) { STATE_DIR = Paths.get(stateDirPathStr); initStateFile(STATE_DIR); } @@ -547,7 +563,7 @@ public static String generateUniqueIRI(String template, String watchedValueTempl // null check just in case idlab-fn:_watchedProperty is not provided in the mapping file Optional watchedValueOption = Optional.ofNullable(watchedValueTemplate); - Map watchedMap = parsePropertyValueTemplate(watchedValueOption); + Map watchedMap = parsePropertyValueTemplate(watchedValueOption); int templateHash = template.hashCode(); String stateFilePathStr = getStateFilePath(stateDirPathStr, m_buckets, templateHash).toString(); @@ -557,17 +573,17 @@ public static String generateUniqueIRI(String template, String watchedValueTempl AtomicBoolean found = new AtomicBoolean(false); AtomicBoolean isDifferent = new AtomicBoolean(false); - Map keyValMap = LDES_FILE_STATE_MAP.computeIfAbsent(stateFilePathStr, f -> new HashMap<>()); - if (keyValMap.containsKey(hexKey)){ + Map keyValMap = LDES_FILE_STATE_MAP.computeIfAbsent(stateFilePathStr, f -> new HashMap<>()); + if (keyValMap.containsKey(hexKey)) { found.set(true); String storedProp = keyValMap.get(hexKey); Map storedPropMap = parsePropertyValueTemplate(Optional.ofNullable(storedProp)); - for (Map.Entry kv : watchedMap.entrySet()){ + for (Map.Entry kv : watchedMap.entrySet()) { String prop = kv.getKey(); String val = kv.getValue(); String storedVal = storedPropMap.getOrDefault(prop, null); - if (!val.equals(storedVal)){ + if (!val.equals(storedVal)) { isDifferent.set(true); } } @@ -576,19 +592,74 @@ public static String generateUniqueIRI(String template, String watchedValueTempl keyValMap.put(hexKey, watchedValueTemplate); - - - - - return getOutput(template,isUnique, found, isDifferent); + return getOutput(template, isUnique, found, isDifferent); } - // TODO check whether this is the right place for this public static String jsonize(Object s) throws JsonProcessingException { ObjectWriter ow = new ObjectMapper().writer().withDefaultPrettyPrinter(); return ow.writeValueAsString(s); } -} + + /** + * It is a function that looks for the first occurence of a certain value in a column of a csv file, + * in order to return a value from a different column in the same row. + * + * @param searchString The string which needs to be found + * @param inputFile The path of a csv file in which the searchString needs to be found. + * @param fromColumn The index of the column which should contain the search string, numbering starts with 0 + * @param toColumn The index of the column which should contain the return value, numbering starts with 0 + * @param delimiter The delimiter used in the csv file, default is ',' + * @return String found in the toColumn on the first row containing the searchString in the fromColumn. If a column + * index is out of range or if the searchString is not found, a message is logged and null is returned + */ + + public static String lookupWithDelimiter(String searchString, String inputFile, Integer fromColumn, Integer toColumn, String delimiter) throws CsvValidationException, IOException, CsvValidationException { + + String result = null; + //check if the LOOKUP_STATE_MAP contains the right values + if (LOOKUP_STATE_INPUTFILE != inputFile || LOOKUP_STATE_FROM_COLUMN != fromColumn || LOOKUP_STATE_TO_COLUMN != toColumn) { + LOOKUP_STATE_INPUTFILE = inputFile; + LOOKUP_STATE_FROM_COLUMN = fromColumn; + LOOKUP_STATE_TO_COLUMN = toColumn; + //empty the hashmap + LOOKUP_STATE_MAP.clear(); + //load inputfile into hashmap + InputStream inputStream = new FileInputStream(new File(inputFile)); + CSVParser parser = new CSVParserBuilder() + .withSeparator(delimiter.charAt(0)) //not passing a delimiter in fno results delimiter = null + .withIgnoreQuotations(true) + .build(); + CSVReader reader = new CSVReaderBuilder(new InputStreamReader(inputStream, StandardCharsets.UTF_8)) + .withFieldAsNull(CSVReaderNullFieldIndicator.EMPTY_SEPARATORS) + .withCSVParser(parser) + .build(); + String[] nextLine = reader.readNext(); + if (fromColumn < 0 || toColumn < 0 || fromColumn >= nextLine.length || toColumn >= nextLine.length) { + logger.error("Column index out of boundries; inputFile: \"{}\", fromColumn: \"{}\", toColumn: \"{}\"", inputFile, fromColumn, toColumn); + return result; + } + while (nextLine != null) { + // only save first occurrence in hashmap + if (!LOOKUP_STATE_MAP.containsKey(nextLine[fromColumn])) { + LOOKUP_STATE_MAP.put(nextLine[fromColumn], nextLine[toColumn]); + } + nextLine = reader.readNext(); + ; + } + reader.close(); + } + result = LOOKUP_STATE_MAP.get(searchString); + + if (result == null) { + logger.error("The searchString is not found; searchString: \"{}\", inputFile: \"{}\", fromColumn: \"{}\"", searchString, inputFile, fromColumn); + } + return result; + } + + public static String lookup(String searchString, String inputFile, Integer fromColumn, Integer toColumn) throws CsvValidationException, IOException, CsvValidationException { + return lookupWithDelimiter(searchString, inputFile, fromColumn, toColumn, ","); + } +} \ No newline at end of file diff --git a/src/main/resources/functions_idlab.ttl b/src/main/resources/functions_idlab.ttl index 6e161b1b..1d58b309 100644 --- a/src/main/resources/functions_idlab.ttl +++ b/src/main/resources/functions_idlab.ttl @@ -396,4 +396,47 @@ idlab-fn:_state fno:name "string representing the name of the state file"; rdfs:label "string representing the name of the state file"; fno:type xsd:string ; - fno:predicate idlab-fn:state . \ No newline at end of file + fno:predicate idlab-fn:state . + +idlab-fn:lookup + a fno:Function ; + fno:name "lookup" ; + rdfs:label "lookup" ; + dcterms:description "Looks for the first appearance of the search string in the fromColumn of a csv file (supposing default delimiter ','), and returns the value of the toColumn on the same row."; + fno:expects ( idlab-fn:_str idlab-fn:_inputFile idlab-fn:_fromColumn idlab-fn:_toColumn) ; + fno:returns ( idlab-fn:_stringOut ) ; + lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; + lib:class "IDLabFunctions" ; + lib:method "lookup" ] . + +idlab-fn:lookupWithDelimiter + a fno:Function ; + fno:name "lookup" ; + rdfs:label "lookup" ; + dcterms:description "Looks for the first appearance of the search string in the fromColumn of a csv file, and returns the value of the toColumn on the same row."; + fno:expects ( idlab-fn:_str idlab-fn:_inputFile idlab-fn:_fromColumn idlab-fn:_toColumn idlab-fn:_delimiter) ; + fno:returns ( idlab-fn:_stringOut ) ; + lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; + lib:class "IDLabFunctions" ; + lib:method "lookupWithDelimiter" ] . + +idlab-fn:_inputFile + a fno:Parameter; + fno:name "string representing the path to an input file"; + rdfs:label "string representing the path to an input file"; + fno:type xsd:string ; + fno:predicate idlab-fn:inputFile . + +idlab-fn:_fromColumn + a fno:Parameter; + fno:name "index of the fromColumn"; + rdfs:label "index of the fromColumn"; + fno:type xsd:integer ; + fno:predicate idlab-fn:fromColumn . + +idlab-fn:_toColumn + a fno:Parameter; + fno:name "index of the toColumn"; + rdfs:label "index of the toColumn"; + fno:type xsd:integer ; + fno:predicate idlab-fn:toColumn . diff --git a/src/test/resources/rml-fno-test-cases/classB.csv b/src/test/resources/rml-fno-test-cases/classB.csv new file mode 100644 index 00000000..74718ff9 --- /dev/null +++ b/src/test/resources/rml-fno-test-cases/classB.csv @@ -0,0 +1,3 @@ +Id;Name +A;"Class A" +B;Class B From 0a4b3f5492de1f96a39493ba0128fdf1734f2523 Mon Sep 17 00:00:00 2001 From: Els de Vleeschauwer Date: Thu, 17 Mar 2022 10:26:54 +0100 Subject: [PATCH 056/609] functions: lib: IDLabFunctionsTest: unit tests for lookup functions Units tests for the functions lookup and lookupWithDelimiter --- .../rml/functions/lib/IDLabFunctionsTest.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java b/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java index 82dd0d3b..02e1b0b7 100644 --- a/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java +++ b/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java @@ -1,5 +1,6 @@ package be.ugent.rml.functions.lib; +import com.opencsv.exceptions.CsvValidationException; import org.apache.commons.io.FileUtils; import org.hamcrest.CoreMatchers; import org.junit.After; @@ -271,4 +272,29 @@ public void generateUniqueIRIWithDate(){ } } + @Test + public void lookup() throws CsvValidationException, IOException { + String searchString = "A"; + String inputFile = "src/test/resources/rml-fno-test-cases/class.csv"; + Integer fromColumn = 0; + Integer toColumn = 1; + assertEquals("Class A", IDLabFunctions.lookup(searchString, inputFile, fromColumn, toColumn)); + + String delimiter = ","; + assertEquals("Class A", IDLabFunctions.lookupWithDelimiter(searchString, inputFile, fromColumn, toColumn, delimiter)); + + searchString = "Class B"; + assertEquals(null, IDLabFunctions.lookup(searchString, inputFile, fromColumn, toColumn)); + + searchString = "Class B"; + fromColumn = 2; + assertEquals(null, IDLabFunctions.lookup(searchString, inputFile, fromColumn, toColumn)); + + searchString = "B"; + fromColumn = 0; + inputFile = "src/test/resources/rml-fno-test-cases/classB.csv"; + delimiter = ";"; + assertEquals("Class B", IDLabFunctions.lookupWithDelimiter(searchString, inputFile, fromColumn, toColumn, delimiter)); + } + } From b6a3e8368751656d5a0d69b6abfd744e7f729b3d Mon Sep 17 00:00:00 2001 From: Els de Vleeschauwer Date: Thu, 17 Mar 2022 13:57:10 +0100 Subject: [PATCH 057/609] CHANGELOG: added lookupFunction Added lookupFunction to the CHANGELOG.md --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20c7a577..75965ee3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +## Added + +- IDLabFunctions: added functions lookup and lookupWithDelimiter +- IDLabFunctionsTest: added unit tests for lookup functions + ### Fixed - Upgrade postgresql JDBC driver to 42.3.3 From cd3ca22d17184a59c20a2a4769d5271c68e2c4c0 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Thu, 13 Jan 2022 09:45:11 +0100 Subject: [PATCH 058/609] CI: mirror development when releasing --- .gitlab-ci.yml | 2 +- CHANGELOG.md | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e6698b76..c0b41e5a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -161,7 +161,7 @@ Create Release: git commit -m "${COMMIT_MESSAGE}" # Update the repository and make sure to skip the pipeline create for this commit - git push origin "${CI_DEFAULT_BRANCH}" -o ci.skip + git push origin "${CI_DEFAULT_BRANCH}" # Push new tags and trigger the pipeline since we're deploying git push --tags origin "${CI_DEFAULT_BRANCH}" diff --git a/CHANGELOG.md b/CHANGELOG.md index f73e2538..47ec992f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +### Fixed +- Mirror development branch to Github when releasing new versions. + ## [4.14.0] - 2022-01-13 ### Added From 4e62b7ddaa414046f6dbcabc2012f428626fa6c0 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Thu, 13 Jan 2022 10:04:16 +0100 Subject: [PATCH 059/609] CI: fix Github Release bugs --- .gitlab-ci.yml | 3 +++ CHANGELOG.md | 1 + get-changes.sh | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c0b41e5a..9317b66e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -208,6 +208,9 @@ Github Release: # Set the displayed user with the commits that are about to be made - git config --global user.email "${GIT_USER_EMAIL:-$GITLAB_USER_EMAIL}" - git config --global user.name "${GIT_USER_NAME:-$GITLAB_USER_NAME}" + + # Install dependencies + - apk add github-cli bash script: # Build fat jar for Github Release - mvn install -DskipTests=true diff --git a/CHANGELOG.md b/CHANGELOG.md index 47ec992f..5a4dddf8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed - Mirror development branch to Github when releasing new versions. +- Github Release stage works now properly. ## [4.14.0] - 2022-01-13 diff --git a/get-changes.sh b/get-changes.sh index e5c4baa9..88ab054e 100755 --- a/get-changes.sh +++ b/get-changes.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash FOUND_CHANGES=false cat CHANGELOG.md | while read line; do From 3517aeec8936aca46985e49c4a140a39cd442661 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Thu, 13 Jan 2022 10:09:04 +0100 Subject: [PATCH 060/609] CI: Create tag after bumping version --- .gitlab-ci.yml | 6 +++--- CHANGELOG.md | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9317b66e..1d56b306 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -142,9 +142,6 @@ Create Release: # Go to the new directory - cd "${CI_COMMIT_SHA}" - # Create git tag - - git tag "v$RELEASE_TAG_NAME" - # Add all generated files to Git - git add . - |- @@ -160,6 +157,9 @@ Create Release: echo "Commit message: ${COMMIT_MESSAGE}" git commit -m "${COMMIT_MESSAGE}" + # Create git tag + git tag "v$RELEASE_TAG_NAME" + # Update the repository and make sure to skip the pipeline create for this commit git push origin "${CI_DEFAULT_BRANCH}" # Push new tags and trigger the pipeline since we're deploying diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a4dddf8..10ffd43f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed - Mirror development branch to Github when releasing new versions. - Github Release stage works now properly. +- Build newer version during deployment. ## [4.14.0] - 2022-01-13 From cebdb7e90ffd04cae86ad07c02dea69d41aab088 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Thu, 13 Jan 2022 09:16:01 +0000 Subject: [PATCH 061/609] CHANGELOG: release v4.14.1 --- CHANGELOG.md | 3 +++ pom.xml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10ffd43f..ce08cafd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +## [4.14.1] - 2022-01-13 + ### Fixed - Mirror development branch to Github when releasing new versions. - Github Release stage works now properly. @@ -440,6 +442,7 @@ and [169](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/169)) - support for accessing remote files (via HTTP GET) - basic support for functions +[4.14.1]: https://github.com/RMLio/rmlmapper-java/compare/v4.14.0...v4.14.1 [4.14.0]: https://github.com/RMLio/rmlmapper-java/compare/v4.13.0...v4.14.0 [4.13.0]: https://github.com/RMLio/rmlmapper-java/compare/v4.12.0...v4.13.0 [4.12.0]: https://github.com/RMLio/rmlmapper-java/compare/v4.11.0...v4.12.0 diff --git a/pom.xml b/pom.xml index b4e567b3..461beb94 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ be.ugent.rml rmlmapper RMLMapper - 4.14.0 + 4.14.1 The RMLMapper executes RML rules to generate high quality Linked Data from multiple originally (semi-)structured data sources. From 9b8c381284b4d6494f28b7a8c6c193ce23f3fbef Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Thu, 13 Jan 2022 10:30:55 +0100 Subject: [PATCH 062/609] CI: correctly install github-cli --- .gitlab-ci.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1d56b306..95fb77d0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -210,8 +210,14 @@ Github Release: - git config --global user.name "${GIT_USER_NAME:-$GITLAB_USER_NAME}" # Install dependencies - - apk add github-cli bash + - apt install -y apt-transport-https + - curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg + - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null + - apt update + - apt install gh script: + # Login into Github + - gh auth login -s repo --with-token < $GITHUB_ACCESS_TOKEN_FILE # Build fat jar for Github Release - mvn install -DskipTests=true # Create Github Release @@ -219,8 +225,8 @@ Github Release: - CHANGES=$(./get-changes.sh) - echo "Creating Github release $TAG with changes\n$CHANGES" - gh release create "$TAG" -n "$CHANGES" -t "$TAG" target/*.jar - only: - - tags +# only: +# - tags # Create a Maven Central release on new tags # Thanks to https://1337codersblog.wordpress.com/2019/09/18/automatical-artifact-deployment-to-maven-central-using-gitlab-ci-part-1/ From 7962a2a7eb3d689798ca0530dfabcf558f229b7c Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Thu, 13 Jan 2022 10:50:47 +0100 Subject: [PATCH 063/609] CI: use R2RML test cases upstream --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 95fb77d0..32fab909 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -259,7 +259,7 @@ R2RML Test Report: # Clone the repository via HTTPS inside a new directory - git clone "https://${GITLAB_USERNAME}:${GITLAB_TOKEN_WEBSITE}@${CI_SERVER_HOST}/rml/doc/rmlio-website.git" - - git clone "https://github.com/DylanVanAssche/r2rml-test-cases-support" # TODO: change this when patch is merged + - git clone "https://github.com/kg-construct/r2rml-test-cases-support" # Set the displayed user with the commits that are about to be made - git config --global user.email "${GIT_USER_EMAIL:-$GITLAB_USER_EMAIL}" From fdc2829328a87d8a3250edc547b58690ba2c7230 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Thu, 13 Jan 2022 11:13:08 +0100 Subject: [PATCH 064/609] CI: refresh debian repos before installing --- .gitlab-ci.yml | 5 +++-- CHANGELOG.md | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 32fab909..e20ae193 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -210,6 +210,7 @@ Github Release: - git config --global user.name "${GIT_USER_NAME:-$GITLAB_USER_NAME}" # Install dependencies + - apt update - apt install -y apt-transport-https - curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null @@ -225,8 +226,8 @@ Github Release: - CHANGES=$(./get-changes.sh) - echo "Creating Github release $TAG with changes\n$CHANGES" - gh release create "$TAG" -n "$CHANGES" -t "$TAG" target/*.jar -# only: -# - tags + only: + - tags # Create a Maven Central release on new tags # Thanks to https://1337codersblog.wordpress.com/2019/09/18/automatical-artifact-deployment-to-maven-central-using-gitlab-ci-part-1/ diff --git a/CHANGELOG.md b/CHANGELOG.md index ce08cafd..546b12f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +### Fixed +- Maven Central automatic builds fixed. +- Upstream R2RML test cases. +- Correctly install Github CLI in CI. + ## [4.14.1] - 2022-01-13 ### Fixed From f6c806f9b048c5bb729f3af3af13bf2b83dc4bf2 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Thu, 13 Jan 2022 12:49:46 +0000 Subject: [PATCH 065/609] CHANGELOG: release v4.14.2 --- CHANGELOG.md | 3 +++ pom.xml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 546b12f2..3289ec69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +## [4.14.2] - 2022-01-13 + ### Fixed - Maven Central automatic builds fixed. - Upstream R2RML test cases. @@ -447,6 +449,7 @@ and [169](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/169)) - support for accessing remote files (via HTTP GET) - basic support for functions +[4.14.2]: https://github.com/RMLio/rmlmapper-java/compare/v4.14.1...v4.14.2 [4.14.1]: https://github.com/RMLio/rmlmapper-java/compare/v4.14.0...v4.14.1 [4.14.0]: https://github.com/RMLio/rmlmapper-java/compare/v4.13.0...v4.14.0 [4.13.0]: https://github.com/RMLio/rmlmapper-java/compare/v4.12.0...v4.13.0 diff --git a/pom.xml b/pom.xml index 461beb94..174df8c5 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ be.ugent.rml rmlmapper RMLMapper - 4.14.1 + 4.14.2 The RMLMapper executes RML rules to generate high quality Linked Data from multiple originally (semi-)structured data sources. From 5ff8b96cca286e75c83f3db2731a64edee9a3957 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Thu, 13 Jan 2022 14:48:16 +0100 Subject: [PATCH 066/609] CI: Github CLI needs to specify repo --- .gitlab-ci.yml | 2 +- CHANGELOG.md | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e20ae193..7ec673de 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -225,7 +225,7 @@ Github Release: - TAG=$(git tag -l "v*" --sort=-creatordate | head -n1) - CHANGES=$(./get-changes.sh) - echo "Creating Github release $TAG with changes\n$CHANGES" - - gh release create "$TAG" -n "$CHANGES" -t "$TAG" target/*.jar + - gh release create "$TAG" -n "$CHANGES" -t "$TAG" -R "github.com/RMLio/rmlmapper-java" target/*.jar only: - tags diff --git a/CHANGELOG.md b/CHANGELOG.md index 3289ec69..5f8fdd1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +### Fixed +- Github CLI needs to specify a repo to create a release. + ## [4.14.2] - 2022-01-13 ### Fixed From b730e8f547336324650f21e0626ea0174bb5bf47 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Thu, 13 Jan 2022 13:49:57 +0000 Subject: [PATCH 067/609] CHANGELOG: release v4.14.3 --- CHANGELOG.md | 3 +++ pom.xml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f8fdd1d..356bccac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +## [4.14.3] - 2022-01-13 + ### Fixed - Github CLI needs to specify a repo to create a release. @@ -452,6 +454,7 @@ and [169](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/169)) - support for accessing remote files (via HTTP GET) - basic support for functions +[4.14.3]: https://github.com/RMLio/rmlmapper-java/compare/v4.14.2...v4.14.3 [4.14.2]: https://github.com/RMLio/rmlmapper-java/compare/v4.14.1...v4.14.2 [4.14.1]: https://github.com/RMLio/rmlmapper-java/compare/v4.14.0...v4.14.1 [4.14.0]: https://github.com/RMLio/rmlmapper-java/compare/v4.13.0...v4.14.0 diff --git a/pom.xml b/pom.xml index 174df8c5..ca87e2f3 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ be.ugent.rml rmlmapper RMLMapper - 4.14.2 + 4.14.3 The RMLMapper executes RML rules to generate high quality Linked Data from multiple originally (semi-)structured data sources. From 98a70ef1e171f10745278beefdcba8878e7a0297 Mon Sep 17 00:00:00 2001 From: Winnie De Ridder Date: Mon, 17 Jan 2022 12:47:28 +0000 Subject: [PATCH 068/609] records/CSVRecord: rewrite with OpenCSV library --- CHANGELOG.md | 1 + README.md | 2 +- pom.xml | 6 +- .../java/be/ugent/rml/access/RDBAccess.java | 25 ++- .../java/be/ugent/rml/records/CSVRecord.java | 8 +- .../ugent/rml/records/CSVRecordFactory.java | 59 +++--- src/main/java/be/ugent/rml/records/CSVW.java | 83 +++++--- .../java/be/ugent/rml/Mapper_CSV_Test.java | 14 +- .../test-cases/RMLTC1022-CSV/mapping.ttl | 27 +++ .../missing-column-names.csv | 0 .../test-cases/RMLTC1022-CSV/output.nq | 1 + .../test-cases/RMLTC1025-CSV/mapping.ttl | 192 ++++++++++++++++-- .../onderwijsniveaus.csv | 0 .../test-cases/RMLTC1025-CSV/output.nq | 3 +- .../test-cases/RMLTC1025_CSV/mapping.ttl | 175 ---------------- .../test-cases/RMLTC1025_CSV/output.nq | 2 - .../test-cases/RMLTC1030-CSV/mapping.ttl | 51 +++++ .../test-cases/RMLTC1030-CSV/output.nq | 3 + .../test-cases/RMLTC1030-CSV/sparseInput.csv | 3 + .../test-cases/RMLTC1031-CSV/mapping.ttl | 51 +++++ .../test-cases/RMLTC1031-CSV/output.nq | 4 + .../test-cases/RMLTC1031-CSV/sparseInput.csv | 3 + 22 files changed, 435 insertions(+), 278 deletions(-) create mode 100644 src/test/resources/test-cases/RMLTC1022-CSV/mapping.ttl rename src/test/resources/test-cases/{RMLTC1025-CSV => RMLTC1022-CSV}/missing-column-names.csv (100%) create mode 100644 src/test/resources/test-cases/RMLTC1022-CSV/output.nq rename src/test/resources/test-cases/{RMLTC1025_CSV => RMLTC1025-CSV}/onderwijsniveaus.csv (100%) delete mode 100644 src/test/resources/test-cases/RMLTC1025_CSV/mapping.ttl delete mode 100644 src/test/resources/test-cases/RMLTC1025_CSV/output.nq create mode 100644 src/test/resources/test-cases/RMLTC1030-CSV/mapping.ttl create mode 100644 src/test/resources/test-cases/RMLTC1030-CSV/output.nq create mode 100644 src/test/resources/test-cases/RMLTC1030-CSV/sparseInput.csv create mode 100644 src/test/resources/test-cases/RMLTC1031-CSV/mapping.ttl create mode 100644 src/test/resources/test-cases/RMLTC1031-CSV/output.nq create mode 100644 src/test/resources/test-cases/RMLTC1031-CSV/sparseInput.csv diff --git a/CHANGELOG.md b/CHANGELOG.md index 356bccac..94221623 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - CSVW: Add support for csvw:null (see [issue 217](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/217)) - FnO: improve error messages for incomplete descriptions (see [issue 222](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/222)) - CSVW: Ignore nulls instead of removing (see [issue 224](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/224)) +- CSVRecordFactory: Replace Apache CSV library with OpenCSV to be able to differentiate between empty string and null (see [issue 140](https://github.com/RMLio/rmlmapper-java/issues/140)) ### Fixed - Update RELEASE.md with up-to-date notes for releasing new versions (see [issue 218](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/218)). diff --git a/README.md b/README.md index dfc2768c..52484def 100644 --- a/README.md +++ b/README.md @@ -275,7 +275,7 @@ Make sure you have [Docker](https://www.docker.com) running. |:---------------------------------------:|--------------------------------------------------------------------| | ch.qos.logback logback-classic | Eclipse Public License 1.0 & GNU Lesser General Public License 2.1 | | commons-cli commons-lang | Apache License 2.0 | -| org.apache.commons commons-csv | Apache License 2.0 | +| com.opencsv opencsv | Apache License 2.0 | | commons-cli commons-cli | Apache License 2.0 | | org.eclipse.rdf4j rdf4j-runtime | Eclipse Public License 1.0 | | junit junit | Eclipse Public License 1.0 | diff --git a/pom.xml b/pom.xml index ca87e2f3..cf893b3b 100644 --- a/pom.xml +++ b/pom.xml @@ -274,9 +274,9 @@ - org.apache.commons - commons-csv - 1.9.0 + com.opencsv + opencsv + 5.5.2 org.apache.poi diff --git a/src/main/java/be/ugent/rml/access/RDBAccess.java b/src/main/java/be/ugent/rml/access/RDBAccess.java index 896e3b64..4af8d351 100644 --- a/src/main/java/be/ugent/rml/access/RDBAccess.java +++ b/src/main/java/be/ugent/rml/access/RDBAccess.java @@ -1,19 +1,24 @@ package be.ugent.rml.access; import be.ugent.rml.NAMESPACES; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVPrinter; +import com.opencsv.CSVWriter; import org.w3c.dom.Document; import org.w3c.dom.Element; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.transform.*; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; -import java.io.*; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.StringWriter; import java.sql.*; -import java.util.*; +import java.util.HashMap; +import java.util.Map; import static be.ugent.rml.Utils.getHashOfString; @@ -180,10 +185,8 @@ private InputStream getCSVInputStream(ResultSet rs) throws SQLException { try { // Differentiate null and "" - CSVFormat format = CSVFormat.DEFAULT.withHeader(getCSVHeader(rsmd, columnCount)) - .withNullString("@@@@NULL@@@@"); - CSVPrinter printer = new CSVPrinter(writer, format); - printer.printRecords(); + CSVWriter csvWriter = new CSVWriter(writer); + csvWriter.writeNext(getCSVHeader(rsmd, columnCount)); // Extract data from result set while (rs.next()) { @@ -213,10 +216,10 @@ private InputStream getCSVInputStream(ResultSet rs) throws SQLException { // Add CSV row to CSVPrinter. // non-varargs call - printer.printRecord((Object[]) csvRow); + csvWriter.writeNext(csvRow); filledInDataTypes = true; } - printer.close(); + csvWriter.close(); } catch (IOException e) { e.printStackTrace(); } diff --git a/src/main/java/be/ugent/rml/records/CSVRecord.java b/src/main/java/be/ugent/rml/records/CSVRecord.java index a0edb6e7..890d10af 100644 --- a/src/main/java/be/ugent/rml/records/CSVRecord.java +++ b/src/main/java/be/ugent/rml/records/CSVRecord.java @@ -1,6 +1,7 @@ package be.ugent.rml.records; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -14,8 +15,11 @@ public class CSVRecord extends Record { private Map data; private Map datatypes; - CSVRecord(Map data, Map datatypes) { - this.data = data; + CSVRecord(String[] header, String[] data, Map datatypes) { + this.data = new HashMap<>(); + for(int i = 0; i < header.length; i += 1){ + this.data.put(header[i], data[i]); + } this.datatypes = datatypes; } diff --git a/src/main/java/be/ugent/rml/records/CSVRecordFactory.java b/src/main/java/be/ugent/rml/records/CSVRecordFactory.java index 8c6ce213..ac88ac31 100644 --- a/src/main/java/be/ugent/rml/records/CSVRecordFactory.java +++ b/src/main/java/be/ugent/rml/records/CSVRecordFactory.java @@ -7,27 +7,28 @@ import be.ugent.rml.term.Literal; import be.ugent.rml.term.NamedNode; import be.ugent.rml.term.Term; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVParser; +import com.opencsv.CSVReader; +import com.opencsv.CSVReaderBuilder; +import com.opencsv.enums.CSVReaderNullFieldIndicator; +import com.opencsv.exceptions.CsvException; +import org.apache.commons.io.FilenameUtils; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.xssf.usermodel.XSSFWorkbook; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.commons.io.FilenameUtils; import org.odftoolkit.simple.Document; import org.odftoolkit.simple.SpreadsheetDocument; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.sql.SQLException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import java.util.stream.Stream; /** * This class is a record factory that creates CSV records. @@ -134,35 +135,31 @@ private List getRecordsForODT(Access access) throws Exception { * @throws IOException */ private List getRecordsForCSV(Access access, CSVW csvw) throws IOException, SQLException, ClassNotFoundException { - CSVParser parser; - // Check if we are dealing with CSVW. - if (csvw != null) { - parser = csvw.getCSVParser(); - } else { - // RDBs fall under this. - CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader().withSkipHeaderRecord(false).withNullString("@@@@NULL@@@@").withAllowMissingColumnNames(true); - InputStream inputStream = access.getInputStream(); - - try { - parser = CSVParser.parse(inputStream, StandardCharsets.UTF_8, csvFormat); - } catch (IllegalArgumentException e) { - logger.debug("Could not parse CSV inputstream", e); - parser = null; + try { + // Check if we are dealing with CSVW. + if (csvw == null) { + // RDBs fall under this + InputStream inputStream = access.getInputStream(); + CSVReader reader = new CSVReaderBuilder(new InputStreamReader(inputStream, StandardCharsets.UTF_8)) + .withSkipLines(0) + .withFieldAsNull(CSVReaderNullFieldIndicator.EMPTY_SEPARATORS) + .build(); + List records = reader.readAll(); + final String[] header = records.get(0); + return records.subList(1, records.size()).stream() + // throw away empty records + .filter(r -> r.length != 0 && !(r.length == 1 && r[0] == null)) + .map(record -> new CSVRecord(header, record, access.getDataTypes())) + .collect(Collectors.toList()); + } else { + return csvw.getRecords(access); } - } - if (parser != null) { - Stream myEntries = parser.getRecords().stream() - .map(record -> new CSVRecord(record.toMap(), access.getDataTypes())); - if(csvw != null){ - myEntries = myEntries.map(csvw::replaceNulls); - } - return myEntries - .collect(Collectors.toList()); - } else { + } catch (IllegalArgumentException | CsvException e) { // We still return an empty list of records when a parser is not found. // This is to support certain use cases with RDBs where queries might not be valid, // but you don't want the RMLMapper to crash. + logger.debug("Could not parse CSV inputstream", e); return new ArrayList<>(); } } diff --git a/src/main/java/be/ugent/rml/records/CSVW.java b/src/main/java/be/ugent/rml/records/CSVW.java index acdc43d0..97d723c0 100644 --- a/src/main/java/be/ugent/rml/records/CSVW.java +++ b/src/main/java/be/ugent/rml/records/CSVW.java @@ -2,28 +2,33 @@ import be.ugent.rml.NAMESPACES; import be.ugent.rml.Utils; +import be.ugent.rml.access.Access; import be.ugent.rml.store.QuadStore; import be.ugent.rml.term.NamedNode; import be.ugent.rml.term.Term; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVParser; +import com.opencsv.CSVParserBuilder; +import com.opencsv.CSVReaderBuilder; +import com.opencsv.enums.CSVReaderNullFieldIndicator; +import com.opencsv.exceptions.CsvException; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Collections; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * This class has as main goal to create a CSVParser for a Logical Source with CSVW. */ class CSVW { - private CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader().withSkipHeaderRecord(false).withAllowMissingColumnNames(true); + private com.opencsv.CSVParserBuilder csvParser = new CSVParserBuilder().withIgnoreLeadingWhiteSpace(true); private Charset csvCharset = StandardCharsets.UTF_8; private InputStream inputStream; @@ -31,6 +36,8 @@ class CSVW { private Term dialect; private Term logicalSource; private List nulls; + private boolean skipHeader = false; + private String commentPrefix = "#"; CSVW(InputStream inputStream, QuadStore rmlStore, Term logicalSource) { this.rmlStore = rmlStore; @@ -40,13 +47,33 @@ class CSVW { setOptions(); } + + /** - * This method returns a CSVParser. - * @return a CSVParser. - * @throws IOException + * Read the records from the given Access + * @param access The access containing the records + * @return The list of records in the Access */ - CSVParser getCSVParser() throws IOException { - return CSVParser.parse(inputStream, csvCharset, csvFormat); + List getRecords(Access access) throws IOException, CsvException { + int skipLines = this.skipHeader ? 1 : 0; + List records = new CSVReaderBuilder(new InputStreamReader(inputStream, csvCharset)) + .withCSVParser(this.csvParser.build()) + .withSkipLines(skipLines) + .withFieldAsNull(CSVReaderNullFieldIndicator.EMPTY_SEPARATORS) + .build() + .readAll(); + String[] header = records.get(0); + Stream readRecords = records.subList(1, records.size()) + .stream() + // throw away empty records + .filter(r -> r.length != 0 && !(r.length == 1 && r[0] == null)); + if(this.getTrim()){ // trim each record value + readRecords = readRecords.map(r -> Arrays.stream(r).map(String::trim).toArray(String[]::new)); + } + return readRecords + .map(record -> new CSVRecord(header, record, access.getDataTypes())) + .map(this::replaceNulls) + .collect(Collectors.toList()); } /** @@ -67,23 +94,22 @@ private void setOptions() { if (!dialectTerms.isEmpty()) { this.dialect = dialectTerms.get(0); - // TODO implement rest of https://www.w3.org/TR/tabular-metadata/#dialect-descriptions // TODO implement CSVW Schema class to add header types - this.csvFormat = this.csvFormat - // commentPrefix - .withCommentMarker(getCommentPrefix()) + this.csvParser = this.csvParser + // commentPrefix TODO + // .withComment(getCommentPrefix()) // delimiter - .withDelimiter(getDelimiter()) + .withSeparator(getDelimiter()) // doubleQuote - .withEscape(getEscapeCharacter()) - // header - .withSkipHeaderRecord(getSkipHeaderRecord()) + .withEscapeChar(getEscapeCharacter()) + // header TODO + //.(getSkipHeaderRecord()) // headerRowCount // lineTerminators // trim // TODO Commons CSV doesn't support start or end trimming - .withTrim(getTrim()) + .withIgnoreLeadingWhiteSpace(getTrim()) // skipBlankRows // skipColumns // skipInitialSpace @@ -91,9 +117,10 @@ private void setOptions() { // @id // @type // withQuoteChar - .withQuote(getQuoteCharacter()) + .withQuoteChar(getQuoteCharacter()) ; - + this.skipHeader = getSkipHeaderRecord(); + this.commentPrefix = getCommentPrefix(); // Encoding String encoding = getValueFromTerm("encoding"); @@ -122,13 +149,13 @@ private String getValueFromTerm(String term) { * This method determines the comment prefix. * @return the comment prefix. */ - private Character getCommentPrefix() { + private String getCommentPrefix() { String output = getValueFromTerm("commentPrefix"); if (output == null) { - return this.csvFormat.getCommentMarker(); + return this.commentPrefix; } else { - return output.toCharArray()[0]; + return output; } } @@ -140,7 +167,7 @@ private boolean getSkipHeaderRecord() { String output = getValueFromTerm("header"); if (output == null) { - return this.csvFormat.getSkipHeaderRecord(); + return this.skipHeader; } else { return output.equals("true"); } @@ -154,7 +181,7 @@ private boolean getTrim() { String output = getValueFromTerm("trim"); if (output == null) { - return this.csvFormat.getTrim(); + return this.csvParser.isIgnoreLeadingWhiteSpace(); } else { return output.equals("true"); } @@ -168,7 +195,7 @@ private Character getDelimiter() { String output = getValueFromTerm("delimiter"); if (output == null) { - return this.csvFormat.getDelimiter(); + return this.csvParser.getSeparator(); } else { return output.toCharArray()[0]; } @@ -182,7 +209,7 @@ private Character getEscapeCharacter() { String output = getValueFromTerm("doubleQuote"); if (output == null) { - return this.csvFormat.getEscapeCharacter(); + return this.csvParser.getEscapeChar(); } else { return output.equals("true") ? '\\' : '"'; } @@ -196,7 +223,7 @@ private Character getQuoteCharacter() { String output = getValueFromTerm("quoteChar"); if (output == null) { - return this.csvFormat.getQuoteCharacter(); + return this.csvParser.getQuoteChar(); } else { return output.toCharArray()[0]; } diff --git a/src/test/java/be/ugent/rml/Mapper_CSV_Test.java b/src/test/java/be/ugent/rml/Mapper_CSV_Test.java index 64c18562..14950a67 100644 --- a/src/test/java/be/ugent/rml/Mapper_CSV_Test.java +++ b/src/test/java/be/ugent/rml/Mapper_CSV_Test.java @@ -286,16 +286,26 @@ public void evaluate_1021_CSV() { @Test public void evaluate_1022_CSV() { - doMapping("test-cases/RMLTC1025-CSV/mapping.ttl", "test-cases/RMLTC1025-CSV/output.nq"); + doMapping("test-cases/RMLTC1022-CSV/mapping.ttl", "test-cases/RMLTC1022-CSV/output.nq"); } @Test public void evaluate_1025_CSV() { - doMapping("test-cases/RMLTC1025_CSV/mapping.ttl", "test-cases/RMLTC1025_CSV/output.nq"); + doMapping("test-cases/RMLTC1025-CSV/mapping.ttl", "test-cases/RMLTC1025-CSV/output.nq"); } @Test public void evaluate_1027_CSV() { doMapping("test-cases/RMLTC1027-CSV/mapping.ttl", "test-cases/RMLTC1027-CSV/output.nq"); } + + @Test + public void evaluate_1030_CSV() { + doMapping("test-cases/RMLTC1030-CSV/mapping.ttl", "test-cases/RMLTC1030-CSV/output.nq"); + } + + @Test + public void evaluate_1031_CSV() { + doMapping("test-cases/RMLTC1031-CSV/mapping.ttl", "test-cases/RMLTC1031-CSV/output.nq"); + } } diff --git a/src/test/resources/test-cases/RMLTC1022-CSV/mapping.ttl b/src/test/resources/test-cases/RMLTC1022-CSV/mapping.ttl new file mode 100644 index 00000000..ab884ed4 --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1022-CSV/mapping.ttl @@ -0,0 +1,27 @@ +@prefix rr: . +@prefix rml: . +@prefix rdf: . +@prefix rdfs: . +@prefix ql: . +@prefix map: . +@base . + + a rr:TriplesMap; + rml:logicalSource [ a rml:LogicalSource; + rml:source "missing-column-names.csv"; + rml:referenceFormulation ql:CSV; + ]; + rr:subjectMap [ a rr:SubjectMap; + rr:template "http://example.com/{ColumnA}"; + ]; + rr:predicateObjectMap [ + rr:objectMap [ a rr:ObjectMap; + rr:template "http://example.com/{ColumnC}"; + rr:termType rr:IRI; + ]; + rr:predicateMap [ a rr:PredicateMap; + rr:constant ; + ]; + ]; +. + diff --git a/src/test/resources/test-cases/RMLTC1025-CSV/missing-column-names.csv b/src/test/resources/test-cases/RMLTC1022-CSV/missing-column-names.csv similarity index 100% rename from src/test/resources/test-cases/RMLTC1025-CSV/missing-column-names.csv rename to src/test/resources/test-cases/RMLTC1022-CSV/missing-column-names.csv diff --git a/src/test/resources/test-cases/RMLTC1022-CSV/output.nq b/src/test/resources/test-cases/RMLTC1022-CSV/output.nq new file mode 100644 index 00000000..cac5eca5 --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1022-CSV/output.nq @@ -0,0 +1 @@ + . diff --git a/src/test/resources/test-cases/RMLTC1025-CSV/mapping.ttl b/src/test/resources/test-cases/RMLTC1025-CSV/mapping.ttl index ab884ed4..e3f1285e 100644 --- a/src/test/resources/test-cases/RMLTC1025-CSV/mapping.ttl +++ b/src/test/resources/test-cases/RMLTC1025-CSV/mapping.ttl @@ -1,27 +1,175 @@ @prefix rr: . -@prefix rml: . @prefix rdf: . @prefix rdfs: . +@prefix fnml: . +@prefix fno: . +@prefix rml: . @prefix ql: . -@prefix map: . -@base . - - a rr:TriplesMap; - rml:logicalSource [ a rml:LogicalSource; - rml:source "missing-column-names.csv"; - rml:referenceFormulation ql:CSV; - ]; - rr:subjectMap [ a rr:SubjectMap; - rr:template "http://example.com/{ColumnA}"; - ]; - rr:predicateObjectMap [ - rr:objectMap [ a rr:ObjectMap; - rr:template "http://example.com/{ColumnC}"; - rr:termType rr:IRI; - ]; - rr:predicateMap [ a rr:PredicateMap; - rr:constant ; - ]; - ]; -. +@prefix : . +@prefix idlab-fn: . +@prefix grel: . + +:map_testcase rml:logicalSource :source. +:source a rml:LogicalSource; + rml:source "onderwijsniveaus.csv"; + rml:referenceFormulation ql:CSV. +:map_testcase a rr:TriplesMap; + rdfs:label "testcase". +:s a rr:SubjectMap. +:map_testcase rr:subjectMap :s. +:s a fnml:FunctionTermMap; + rr:termType rr:BlankNode; + fnml:functionValue :fn_condition. + +# condition +:fn_condition rml:logicalSource :source; + rr:predicateObjectMap :pomexec_condition. +:pomexec_condition rr:predicateMap :pmexec_condition. +:pmexec_condition rr:constant fno:executes. + +:pomexec_condition rr:objectMap :omexec_condition. +:omexec_condition rr:constant idlab-fn:trueCondition; + rr:termType rr:IRI. + +:fn_condition rr:predicateObjectMap :pom_condition. +:pom_condition a rr:PredicateObjectMap; + rr:predicateMap :pm_condition. +:pm_condition a rr:PredicateMap; + rr:constant idlab-fn:strBoolean. + +:pom_condition rr:objectMap :om_condition. +:om_condition a rr:ObjectMap, fnml:FunctionTermMap; + fnml:functionValue :fn_bool_and. + +# function: grel:boolean_and +:fn_bool_and rml:logicalSource :source; + rr:predicateObjectMap :pomexec_and. +:pomexec_and rr:predicateMap :pmexec_and. +:pmexec_and rr:constant fno:executes. + +:pomexec_and rr:objectMap :omexec_and. +:omexec_and rr:constant grel:boolean_and; + rr:termType rr:IRI. + +# First parameter for boolean_and : Slug1 idlab-fn:equal to _scheme +:fn_bool_and rr:predicateObjectMap :pom_and_1. +:pom_and_1 a rr:PredicateObjectMap; + rr:predicateMap :pm_and_1. +:pm_and_1 a rr:PredicateMap; + rr:constant grel:param_rep_b. + +:pom_and_1 rr:objectMap :om_and_1. +:om_and_1 a rr:ObjectMap, fnml:FunctionTermMap; + rr:datatype xsd:boolean; + fnml:functionValue :fn_eq_par1. + +# idlab-fn:equal +:fn_eq_par1 rml:logicalSource :source; + rr:predicateObjectMap :pomexec_eq_par1. +:pomexec_eq_par1 rr:predicateMap :pmexec_eq_par1. +:pmexec_eq_par1 rr:constant fno:executes. + +:pomexec_eq_par1 rr:objectMap :omexec_eq_par1. +:omexec_eq_par1 rr:constant idlab-fn:equal; + rr:termType rr:IRI. + +# idlab-fn:equal valueParamter +:fn_eq_par1 rr:predicateObjectMap :pom_eq_par1_par1. +:pom_eq_par1_par1 a rr:PredicateObjectMap; + rr:predicateMap :pm_eq_par1_par1. +:pm_eq_par1_par1 a rr:PredicateMap; + rr:constant grel:valueParameter. +:pom_eq_par1_par1 rr:objectMap :om_eq_par1_par1. +:om_eq_par1_par1 a rr:ObjectMap; + rml:reference "Slug1"; + rr:termType rr:Literal. + +# idlab-fn:equal valueParameter2 +:fn_eq_par1 rr:predicateObjectMap :pom_eq_par1_par2. +:pom_eq_par1_par2 a rr:PredicateObjectMap; + rr:predicateMap :pm_eq_par1_par2. +:pm_eq_par1_par2 a rr:PredicateMap; + rr:constant grel:valueParameter2. +:pom_eq_par1_par2 rr:objectMap :om_eq_par1_par2. +:om_eq_par1_par2 a rr:ObjectMap; + rr:constant "_scheme"; + rr:termType rr:Literal. + +# Second parameter for boolean_and : Slug1 idlab-fn:equal to _scheme +:fn_bool_and rr:predicateObjectMap :pom_and_2. +:pom_and_2 a rr:PredicateObjectMap; + rr:predicateMap :pm_and_2. +:pm_and_2 a rr:PredicateMap; + rr:constant grel:param_rep_b. + +:pom_and_2 rr:objectMap :om_and_2. +:om_and_2 a rr:ObjectMap, fnml:FunctionTermMap; + rr:datatype xsd:boolean; + fnml:functionValue :fn_eq_par2. + +# idlab-fn:equal +:fn_eq_par2 rml:logicalSource :source; + rr:predicateObjectMap :pomexec_eq_par2. +:pomexec_eq_par2 rr:predicateMap :pmexec_eq_par2. +:pmexec_eq_par2 rr:constant fno:executes. +:pomexec_eq_par2 rr:objectMap :omexec_eq_par2. + +:omexec_eq_par2 rr:constant idlab-fn:equal; + rr:termType rr:IRI. + +:fn_eq_par2 rr:predicateObjectMap :pom_eq_par2_par1. + +# idlab-fn:equal valueParamter +:pom_eq_par2_par1 a rr:PredicateObjectMap; + rr:predicateMap :pm_eq_par2_par1. +:pm_eq_par2_par1 a rr:PredicateMap; + rr:constant grel:valueParameter. +:pom_eq_par2_par1 rr:objectMap :om_eq_par2_par1. +:om_eq_par2_par1 a rr:ObjectMap; + rml:reference "Slug1"; + rr:termType rr:Literal. + +# idlab-fn:equal valueParameter2 +:fn_eq_par2 rr:predicateObjectMap :pom_eq_par2_par2. +:pom_eq_par2_par2 a rr:PredicateObjectMap; + rr:predicateMap :pm_eq_par2_par2. +:pm_eq_par2_par2 a rr:PredicateMap; + rr:constant grel:valueParameter2. +:pom_eq_par2_par2 rr:objectMap :om_eq_par2_par2. +:om_eq_par2_par2 a rr:ObjectMap; + rr:constant "_scheme"; + rr:termType rr:Literal. + +:fn_condition rr:predicateObjectMap :pom_blank. +:pom_blank a rr:PredicateObjectMap; + rr:predicateMap :pm_blank. +:pm_blank a rr:PredicateMap; + rr:constant idlab-fn:str. +:pom_blank rr:objectMap :om_blank. +:om_blank a rr:ObjectMap; + rr:termType rr:BlankNode. + +# po: [a, skos:ConceptScheme] +:pom_0 a rr:PredicateObjectMap. +:map_testcase rr:predicateObjectMap :pom_0. +:pm_0 a rr:PredicateMap. +:pom_0 rr:predicateMap :pm_0. +:pm_0 rr:constant rdf:type. + +:pom_0 rr:objectMap :om_0. +:om_0 a rr:ObjectMap; + rr:constant skos:ConceptScheme; + rr:termType rr:IRI. + +# po: [skos:prefLabel, "test2", nl~lang] +:pom_1 a rr:PredicateObjectMap. +:map_testcase rr:predicateObjectMap :pom_1. +:pm_1 a rr:PredicateMap. +:pom_1 rr:predicateMap :pm_1. +:pm_1 rr:constant skos:prefLabel. +:pom_1 rr:objectMap :om_1. +:om_1 a rr:ObjectMap; + rr:constant "test2"; + rr:termType rr:Literal; + rr:language "nl". \ No newline at end of file diff --git a/src/test/resources/test-cases/RMLTC1025_CSV/onderwijsniveaus.csv b/src/test/resources/test-cases/RMLTC1025-CSV/onderwijsniveaus.csv similarity index 100% rename from src/test/resources/test-cases/RMLTC1025_CSV/onderwijsniveaus.csv rename to src/test/resources/test-cases/RMLTC1025-CSV/onderwijsniveaus.csv diff --git a/src/test/resources/test-cases/RMLTC1025-CSV/output.nq b/src/test/resources/test-cases/RMLTC1025-CSV/output.nq index cac5eca5..65a6fcb7 100644 --- a/src/test/resources/test-cases/RMLTC1025-CSV/output.nq +++ b/src/test/resources/test-cases/RMLTC1025-CSV/output.nq @@ -1 +1,2 @@ - . +_:0 . +_:0 "test2"@nl. \ No newline at end of file diff --git a/src/test/resources/test-cases/RMLTC1025_CSV/mapping.ttl b/src/test/resources/test-cases/RMLTC1025_CSV/mapping.ttl deleted file mode 100644 index e3f1285e..00000000 --- a/src/test/resources/test-cases/RMLTC1025_CSV/mapping.ttl +++ /dev/null @@ -1,175 +0,0 @@ -@prefix rr: . -@prefix rdf: . -@prefix rdfs: . -@prefix fnml: . -@prefix fno: . -@prefix rml: . -@prefix ql: . -@prefix : . -@prefix idlab-fn: . -@prefix grel: . - -:map_testcase rml:logicalSource :source. -:source a rml:LogicalSource; - rml:source "onderwijsniveaus.csv"; - rml:referenceFormulation ql:CSV. -:map_testcase a rr:TriplesMap; - rdfs:label "testcase". -:s a rr:SubjectMap. -:map_testcase rr:subjectMap :s. -:s a fnml:FunctionTermMap; - rr:termType rr:BlankNode; - fnml:functionValue :fn_condition. - -# condition -:fn_condition rml:logicalSource :source; - rr:predicateObjectMap :pomexec_condition. -:pomexec_condition rr:predicateMap :pmexec_condition. -:pmexec_condition rr:constant fno:executes. - -:pomexec_condition rr:objectMap :omexec_condition. -:omexec_condition rr:constant idlab-fn:trueCondition; - rr:termType rr:IRI. - -:fn_condition rr:predicateObjectMap :pom_condition. -:pom_condition a rr:PredicateObjectMap; - rr:predicateMap :pm_condition. -:pm_condition a rr:PredicateMap; - rr:constant idlab-fn:strBoolean. - -:pom_condition rr:objectMap :om_condition. -:om_condition a rr:ObjectMap, fnml:FunctionTermMap; - fnml:functionValue :fn_bool_and. - -# function: grel:boolean_and -:fn_bool_and rml:logicalSource :source; - rr:predicateObjectMap :pomexec_and. -:pomexec_and rr:predicateMap :pmexec_and. -:pmexec_and rr:constant fno:executes. - -:pomexec_and rr:objectMap :omexec_and. -:omexec_and rr:constant grel:boolean_and; - rr:termType rr:IRI. - -# First parameter for boolean_and : Slug1 idlab-fn:equal to _scheme -:fn_bool_and rr:predicateObjectMap :pom_and_1. -:pom_and_1 a rr:PredicateObjectMap; - rr:predicateMap :pm_and_1. -:pm_and_1 a rr:PredicateMap; - rr:constant grel:param_rep_b. - -:pom_and_1 rr:objectMap :om_and_1. -:om_and_1 a rr:ObjectMap, fnml:FunctionTermMap; - rr:datatype xsd:boolean; - fnml:functionValue :fn_eq_par1. - -# idlab-fn:equal -:fn_eq_par1 rml:logicalSource :source; - rr:predicateObjectMap :pomexec_eq_par1. -:pomexec_eq_par1 rr:predicateMap :pmexec_eq_par1. -:pmexec_eq_par1 rr:constant fno:executes. - -:pomexec_eq_par1 rr:objectMap :omexec_eq_par1. -:omexec_eq_par1 rr:constant idlab-fn:equal; - rr:termType rr:IRI. - -# idlab-fn:equal valueParamter -:fn_eq_par1 rr:predicateObjectMap :pom_eq_par1_par1. -:pom_eq_par1_par1 a rr:PredicateObjectMap; - rr:predicateMap :pm_eq_par1_par1. -:pm_eq_par1_par1 a rr:PredicateMap; - rr:constant grel:valueParameter. -:pom_eq_par1_par1 rr:objectMap :om_eq_par1_par1. -:om_eq_par1_par1 a rr:ObjectMap; - rml:reference "Slug1"; - rr:termType rr:Literal. - -# idlab-fn:equal valueParameter2 -:fn_eq_par1 rr:predicateObjectMap :pom_eq_par1_par2. -:pom_eq_par1_par2 a rr:PredicateObjectMap; - rr:predicateMap :pm_eq_par1_par2. -:pm_eq_par1_par2 a rr:PredicateMap; - rr:constant grel:valueParameter2. -:pom_eq_par1_par2 rr:objectMap :om_eq_par1_par2. -:om_eq_par1_par2 a rr:ObjectMap; - rr:constant "_scheme"; - rr:termType rr:Literal. - -# Second parameter for boolean_and : Slug1 idlab-fn:equal to _scheme -:fn_bool_and rr:predicateObjectMap :pom_and_2. -:pom_and_2 a rr:PredicateObjectMap; - rr:predicateMap :pm_and_2. -:pm_and_2 a rr:PredicateMap; - rr:constant grel:param_rep_b. - -:pom_and_2 rr:objectMap :om_and_2. -:om_and_2 a rr:ObjectMap, fnml:FunctionTermMap; - rr:datatype xsd:boolean; - fnml:functionValue :fn_eq_par2. - -# idlab-fn:equal -:fn_eq_par2 rml:logicalSource :source; - rr:predicateObjectMap :pomexec_eq_par2. -:pomexec_eq_par2 rr:predicateMap :pmexec_eq_par2. -:pmexec_eq_par2 rr:constant fno:executes. -:pomexec_eq_par2 rr:objectMap :omexec_eq_par2. - -:omexec_eq_par2 rr:constant idlab-fn:equal; - rr:termType rr:IRI. - -:fn_eq_par2 rr:predicateObjectMap :pom_eq_par2_par1. - -# idlab-fn:equal valueParamter -:pom_eq_par2_par1 a rr:PredicateObjectMap; - rr:predicateMap :pm_eq_par2_par1. -:pm_eq_par2_par1 a rr:PredicateMap; - rr:constant grel:valueParameter. -:pom_eq_par2_par1 rr:objectMap :om_eq_par2_par1. -:om_eq_par2_par1 a rr:ObjectMap; - rml:reference "Slug1"; - rr:termType rr:Literal. - -# idlab-fn:equal valueParameter2 -:fn_eq_par2 rr:predicateObjectMap :pom_eq_par2_par2. -:pom_eq_par2_par2 a rr:PredicateObjectMap; - rr:predicateMap :pm_eq_par2_par2. -:pm_eq_par2_par2 a rr:PredicateMap; - rr:constant grel:valueParameter2. -:pom_eq_par2_par2 rr:objectMap :om_eq_par2_par2. -:om_eq_par2_par2 a rr:ObjectMap; - rr:constant "_scheme"; - rr:termType rr:Literal. - -:fn_condition rr:predicateObjectMap :pom_blank. -:pom_blank a rr:PredicateObjectMap; - rr:predicateMap :pm_blank. -:pm_blank a rr:PredicateMap; - rr:constant idlab-fn:str. -:pom_blank rr:objectMap :om_blank. -:om_blank a rr:ObjectMap; - rr:termType rr:BlankNode. - -# po: [a, skos:ConceptScheme] -:pom_0 a rr:PredicateObjectMap. -:map_testcase rr:predicateObjectMap :pom_0. -:pm_0 a rr:PredicateMap. -:pom_0 rr:predicateMap :pm_0. -:pm_0 rr:constant rdf:type. - -:pom_0 rr:objectMap :om_0. -:om_0 a rr:ObjectMap; - rr:constant skos:ConceptScheme; - rr:termType rr:IRI. - -# po: [skos:prefLabel, "test2", nl~lang] -:pom_1 a rr:PredicateObjectMap. -:map_testcase rr:predicateObjectMap :pom_1. -:pm_1 a rr:PredicateMap. -:pom_1 rr:predicateMap :pm_1. -:pm_1 rr:constant skos:prefLabel. - -:pom_1 rr:objectMap :om_1. -:om_1 a rr:ObjectMap; - rr:constant "test2"; - rr:termType rr:Literal; - rr:language "nl". \ No newline at end of file diff --git a/src/test/resources/test-cases/RMLTC1025_CSV/output.nq b/src/test/resources/test-cases/RMLTC1025_CSV/output.nq deleted file mode 100644 index 65a6fcb7..00000000 --- a/src/test/resources/test-cases/RMLTC1025_CSV/output.nq +++ /dev/null @@ -1,2 +0,0 @@ -_:0 . -_:0 "test2"@nl. \ No newline at end of file diff --git a/src/test/resources/test-cases/RMLTC1030-CSV/mapping.ttl b/src/test/resources/test-cases/RMLTC1030-CSV/mapping.ttl new file mode 100644 index 00000000..ff284953 --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1030-CSV/mapping.ttl @@ -0,0 +1,51 @@ +@prefix rr: . +@prefix rdf: . +@prefix rdfs: . +@prefix fnml: . +@prefix fno: . +@prefix d2rq: . +@prefix void: . +@prefix dc: . +@prefix foaf: . +@prefix rml: . +@prefix ql: . +@prefix : . +@prefix ex: . + +:rules_000 rdf:type void:Dataset ; + void:exampleResource :map_test_000 . + +:map_test_000 rml:logicalSource :source_000 ; + rdf:type rr:TriplesMap ; + rdfs:label "test" ; + rr:subjectMap :s_000 ; + rr:predicateObjectMap :pom_000, :pom_001 . + +:source_000 rdf:type rml:LogicalSource ; + rml:source "sparseInput.csv" ; + rml:referenceFormulation ql:CSV . + +:s_000 rdf:type rr:SubjectMap ; + rr:template "http://example.com/{A}" . + +:pom_000 rdf:type rr:PredicateObjectMap ; + rr:predicateMap :pm_000 ; + rr:objectMap :om_000 . + +:pm_000 rdf:type rr:PredicateMap ; + rr:constant ex:A . + +:om_000 rdf:type rr:ObjectMap ; + rml:reference "A" ; + rr:termType rr:Literal . + +:pom_001 rdf:type rr:PredicateObjectMap ; + rr:predicateMap :pm_001 ; + rr:objectMap :om_001 . + +:pm_001 rdf:type rr:PredicateMap ; + rr:constant ex:B . + +:om_001 rdf:type rr:ObjectMap ; + rml:reference "B" ; + rr:termType rr:Literal . \ No newline at end of file diff --git a/src/test/resources/test-cases/RMLTC1030-CSV/output.nq b/src/test/resources/test-cases/RMLTC1030-CSV/output.nq new file mode 100644 index 00000000..5d37d55a --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1030-CSV/output.nq @@ -0,0 +1,3 @@ + "1" . + "4" . + "5" . \ No newline at end of file diff --git a/src/test/resources/test-cases/RMLTC1030-CSV/sparseInput.csv b/src/test/resources/test-cases/RMLTC1030-CSV/sparseInput.csv new file mode 100644 index 00000000..3e7cde5f --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1030-CSV/sparseInput.csv @@ -0,0 +1,3 @@ +A,B,C +1,,3 +4,5,6 \ No newline at end of file diff --git a/src/test/resources/test-cases/RMLTC1031-CSV/mapping.ttl b/src/test/resources/test-cases/RMLTC1031-CSV/mapping.ttl new file mode 100644 index 00000000..ff284953 --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1031-CSV/mapping.ttl @@ -0,0 +1,51 @@ +@prefix rr: . +@prefix rdf: . +@prefix rdfs: . +@prefix fnml: . +@prefix fno: . +@prefix d2rq: . +@prefix void: . +@prefix dc: . +@prefix foaf: . +@prefix rml: . +@prefix ql: . +@prefix : . +@prefix ex: . + +:rules_000 rdf:type void:Dataset ; + void:exampleResource :map_test_000 . + +:map_test_000 rml:logicalSource :source_000 ; + rdf:type rr:TriplesMap ; + rdfs:label "test" ; + rr:subjectMap :s_000 ; + rr:predicateObjectMap :pom_000, :pom_001 . + +:source_000 rdf:type rml:LogicalSource ; + rml:source "sparseInput.csv" ; + rml:referenceFormulation ql:CSV . + +:s_000 rdf:type rr:SubjectMap ; + rr:template "http://example.com/{A}" . + +:pom_000 rdf:type rr:PredicateObjectMap ; + rr:predicateMap :pm_000 ; + rr:objectMap :om_000 . + +:pm_000 rdf:type rr:PredicateMap ; + rr:constant ex:A . + +:om_000 rdf:type rr:ObjectMap ; + rml:reference "A" ; + rr:termType rr:Literal . + +:pom_001 rdf:type rr:PredicateObjectMap ; + rr:predicateMap :pm_001 ; + rr:objectMap :om_001 . + +:pm_001 rdf:type rr:PredicateMap ; + rr:constant ex:B . + +:om_001 rdf:type rr:ObjectMap ; + rml:reference "B" ; + rr:termType rr:Literal . \ No newline at end of file diff --git a/src/test/resources/test-cases/RMLTC1031-CSV/output.nq b/src/test/resources/test-cases/RMLTC1031-CSV/output.nq new file mode 100644 index 00000000..3236e38d --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1031-CSV/output.nq @@ -0,0 +1,4 @@ + "1" . + "" . + "4" . + "5" . \ No newline at end of file diff --git a/src/test/resources/test-cases/RMLTC1031-CSV/sparseInput.csv b/src/test/resources/test-cases/RMLTC1031-CSV/sparseInput.csv new file mode 100644 index 00000000..a244f6ea --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1031-CSV/sparseInput.csv @@ -0,0 +1,3 @@ +A,B,C +1,"",3 +4,5,6 \ No newline at end of file From 0fc3267348e0ea59d687023cb384109e55e97df9 Mon Sep 17 00:00:00 2001 From: winniederidder Date: Tue, 7 Dec 2021 17:21:48 +0100 Subject: [PATCH 069/609] JSONRecordFactory: add support for JSONL files JSONL files are regular files where each line is a valid JSON object. --- src/main/java/be/ugent/rml/access/Access.java | 2 ++ .../be/ugent/rml/access/LocalFileAccess.java | 7 +++++ .../be/ugent/rml/records/IteratorFormat.java | 6 ++++- .../ugent/rml/records/JSONRecordFactory.java | 20 ++++++++++++++ .../java/be/ugent/rml/Mapper_JSON_Test.java | 5 ++++ .../test-cases/RMLTC1027-JSONL/data.jsonl | 3 +++ .../test-cases/RMLTC1027-JSONL/mapping.ttl | 27 +++++++++++++++++++ .../test-cases/RMLTC1027-JSONL/output.nq | 2 ++ 8 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 src/test/resources/test-cases/RMLTC1027-JSONL/data.jsonl create mode 100644 src/test/resources/test-cases/RMLTC1027-JSONL/mapping.ttl create mode 100644 src/test/resources/test-cases/RMLTC1027-JSONL/output.nq diff --git a/src/main/java/be/ugent/rml/access/Access.java b/src/main/java/be/ugent/rml/access/Access.java index 690ed7af..db9a1bf5 100644 --- a/src/main/java/be/ugent/rml/access/Access.java +++ b/src/main/java/be/ugent/rml/access/Access.java @@ -24,4 +24,6 @@ public interface Access { * @return map of datatypes. */ Map getDataTypes(); + + String getContentType(); } diff --git a/src/main/java/be/ugent/rml/access/LocalFileAccess.java b/src/main/java/be/ugent/rml/access/LocalFileAccess.java index 00eda2c6..f64e9007 100644 --- a/src/main/java/be/ugent/rml/access/LocalFileAccess.java +++ b/src/main/java/be/ugent/rml/access/LocalFileAccess.java @@ -8,6 +8,7 @@ import static be.ugent.rml.Utils.getHashOfString; import static be.ugent.rml.Utils.getInputStreamFromFile; import static org.apache.commons.io.FileUtils.getFile; +import static org.apache.commons.io.FilenameUtils.getExtension; /** * This class represents access to a local file. @@ -43,6 +44,7 @@ public InputStream getInputStream() throws IOException { return getInputStreamFromFile(file); } + /** * This methods returns the datatypes of the file. * This method always returns null, because the datatypes can't be determined from a local file for the moment. @@ -88,4 +90,9 @@ public String getBasePath() { public String toString() { return this.path; } + + @Override + public String getContentType() { + return getExtension(this.path); + } } diff --git a/src/main/java/be/ugent/rml/records/IteratorFormat.java b/src/main/java/be/ugent/rml/records/IteratorFormat.java index e7518c95..e820bdfa 100644 --- a/src/main/java/be/ugent/rml/records/IteratorFormat.java +++ b/src/main/java/be/ugent/rml/records/IteratorFormat.java @@ -39,7 +39,7 @@ public List getRecords(Access access, Term logicalSource, QuadStore rmlS if (! documentMap.containsKey(access)) { logger.debug("No document found for {}. Creating new one", access); InputStream stream = access.getInputStream(); - documentMap.put(access, getDocumentFromStream(stream)); + documentMap.put(access, getDocumentFromStream(stream, access.getContentType())); } List iterators = Utils.getObjectsFromQuads(rmlStore.getQuads(logicalSource, new NamedNode(NAMESPACES.RML + "iterator"), null)); @@ -73,4 +73,8 @@ public List getRecords(Access access, Term logicalSource, QuadStore rmlS * @throws IOException */ abstract DocumentClass getDocumentFromStream(InputStream stream) throws IOException; + + DocumentClass getDocumentFromStream(InputStream stream, String contentType) throws IOException { + return getDocumentFromStream(stream); + } } diff --git a/src/main/java/be/ugent/rml/records/JSONRecordFactory.java b/src/main/java/be/ugent/rml/records/JSONRecordFactory.java index 4d5e84ba..27210471 100644 --- a/src/main/java/be/ugent/rml/records/JSONRecordFactory.java +++ b/src/main/java/be/ugent/rml/records/JSONRecordFactory.java @@ -1,9 +1,12 @@ package be.ugent.rml.records; import com.jayway.jsonpath.*; +import com.jayway.jsonpath.spi.json.JsonProvider; +import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.util.ArrayList; import java.util.List; @@ -51,4 +54,21 @@ List getRecordsFromDocument(Object document, String iterator) { Object getDocumentFromStream(InputStream stream) throws IOException { return Configuration.defaultConfiguration().jsonProvider().parse(stream, "utf-8"); } + + @Override + Object getDocumentFromStream(InputStream stream, String contentType) throws IOException { + if(contentType.toLowerCase().equals("jsonl")){ + JsonProvider provider = Configuration.defaultConfiguration().jsonProvider(); + BufferedReader lineReader = new BufferedReader(new InputStreamReader(stream)); + Object items = provider.createArray(); + int index = 0; + while (lineReader.ready()){ + provider.setArrayIndex(items, index, provider.parse(lineReader.readLine())); + index += 1; + } + return items; + } else { + return getDocumentFromStream(stream); + } + } } diff --git a/src/test/java/be/ugent/rml/Mapper_JSON_Test.java b/src/test/java/be/ugent/rml/Mapper_JSON_Test.java index f8aa3d58..73a34814 100644 --- a/src/test/java/be/ugent/rml/Mapper_JSON_Test.java +++ b/src/test/java/be/ugent/rml/Mapper_JSON_Test.java @@ -269,4 +269,9 @@ public void evaluate_1024_JSON() { public void evaluate_1027_JSON() { doMapping("./test-cases/RMLTC1027-JSON/mapping.ttl", "./test-cases/RMLTC1027-JSON/output.nq"); } + + @Test + public void evaluate_1027_JSONL() { + doMapping("./test-cases/RMLTC1027-JSONL/mapping.ttl", "./test-cases/RMLTC1027-JSONL/output.nq"); + } } diff --git a/src/test/resources/test-cases/RMLTC1027-JSONL/data.jsonl b/src/test/resources/test-cases/RMLTC1027-JSONL/data.jsonl new file mode 100644 index 00000000..73393ccf --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1027-JSONL/data.jsonl @@ -0,0 +1,3 @@ +{"ID": 10,"Name": "Venus"} +{"ID": 11,"Name": null} +{"ID": 12,"Name": "Serena"} diff --git a/src/test/resources/test-cases/RMLTC1027-JSONL/mapping.ttl b/src/test/resources/test-cases/RMLTC1027-JSONL/mapping.ttl new file mode 100644 index 00000000..d1ef9c00 --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1027-JSONL/mapping.ttl @@ -0,0 +1,27 @@ +@prefix rr: . +@prefix foaf: . +@prefix ex: . +@prefix xsd: . +@prefix rml: . +@prefix ql: . +@base . + + + a rr:TriplesMap; + + rml:logicalSource [ + rml:source "data.jsonl"; + rml:referenceFormulation ql:JSONPath; + rml:iterator "$[*]"; + ]; + + rr:subjectMap [ + rr:template "http://example.com/{ID}/{Name}" + ]; + + rr:predicateObjectMap [ + rr:predicate foaf:name; + rr:objectMap [ + rml:reference "Name" + ] + ]. diff --git a/src/test/resources/test-cases/RMLTC1027-JSONL/output.nq b/src/test/resources/test-cases/RMLTC1027-JSONL/output.nq new file mode 100644 index 00000000..2b2b83df --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1027-JSONL/output.nq @@ -0,0 +1,2 @@ + "Venus" . + "Serena" . From 9ccadad904308ecc0400732474b70a1d12d2337a Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Mon, 17 Jan 2022 14:24:58 +0100 Subject: [PATCH 070/609] ci: bump buildnumber as well --- .gitlab-ci.yml | 5 +++++ CHANGELOG.md | 3 +++ 2 files changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7ec673de..6d2dd83a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -115,6 +115,8 @@ Create Release: name: alpine/git:${GIT_VERSION} stage: release before_script: + # Dependencies + - apk add maven java-jdk # Clone the repository via HTTPS inside a new directory - git clone "https://${GITLAB_USERNAME}:${GITLAB_TOKEN_RMLMAPPER}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" "${CI_COMMIT_SHA}" @@ -135,9 +137,12 @@ Create Release: - mv pom_updated.xml pom.xml # Update changelog. Changefrog does not like vX.X.X so drop 'v' - changefrog -n "$RELEASE_TAG_NAME" + # Build a Jar + - mvn install -DskipTests=true # Stage changes for commit to master - cp -u pom.xml "${CI_COMMIT_SHA}/pom.xml" - cp -u CHANGELOG.md "${CI_COMMIT_SHA}/CHANGELOG.md" + - cp -u buildNumber.properties "${CI_COMMIT_SHA}/buildNumber.properties" after_script: # Go to the new directory - cd "${CI_COMMIT_SHA}" diff --git a/CHANGELOG.md b/CHANGELOG.md index 94221623..1a0f1b00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +### Fixed +- Bump buildnumber during release (see [issue 227](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/227)) + ## [4.14.3] - 2022-01-13 ### Fixed From be18ec71546e3d70b6b87f618c3c9f6e920781ff Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Mon, 17 Jan 2022 14:31:13 +0100 Subject: [PATCH 071/609] CHANGELOG: fix changelog --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a0f1b00..51b78d14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed - Github CLI needs to specify a repo to create a release. +### Changed +- CSVRecordFactory: Replace Apache CSV library with OpenCSV to be able to differentiate between empty string and null (see [issue 140](https://github.com/RMLio/rmlmapper-java/issues/140)) + +### Added +- JSONRecordFactory: support JSONL files (see [issue 221](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/221)). + ## [4.14.2] - 2022-01-13 ### Fixed @@ -39,7 +45,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - CSVW: Add support for csvw:null (see [issue 217](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/217)) - FnO: improve error messages for incomplete descriptions (see [issue 222](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/222)) - CSVW: Ignore nulls instead of removing (see [issue 224](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/224)) -- CSVRecordFactory: Replace Apache CSV library with OpenCSV to be able to differentiate between empty string and null (see [issue 140](https://github.com/RMLio/rmlmapper-java/issues/140)) ### Fixed - Update RELEASE.md with up-to-date notes for releasing new versions (see [issue 218](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/218)). From 680aaaab3d23d81f6e2732963d00e2862d4a74e4 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Thu, 20 Jan 2022 08:56:42 +0100 Subject: [PATCH 072/609] records/xpath: always know xml: prefix According to W3C, the xml: prefix is always assumed to be known, even if not declared in the document --- CHANGELOG.md | 1 + .../be/ugent/rml/records/xpath/NamespaceResolver.java | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51b78d14..b1e9b21e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed - Bump buildnumber during release (see [issue 227](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/227)) +- Always include xml: prefix (see [issue 144](https://github.com/RMLio/rmlmapper-java/issues/144)) ## [4.14.3] - 2022-01-13 diff --git a/src/main/java/be/ugent/rml/records/xpath/NamespaceResolver.java b/src/main/java/be/ugent/rml/records/xpath/NamespaceResolver.java index d414666b..68c45b05 100644 --- a/src/main/java/be/ugent/rml/records/xpath/NamespaceResolver.java +++ b/src/main/java/be/ugent/rml/records/xpath/NamespaceResolver.java @@ -21,9 +21,14 @@ public NamespaceResolver(Document document) { //The lookup for the namespace uris is delegated to the stored document. // TODO performance: verify check whether caching this would improve performance public String getNamespaceURI(String prefix) { - if (prefix.equals(XMLConstants.DEFAULT_NS_PREFIX)) { + if (prefix.equals(XMLConstants.DEFAULT_NS_PREFIX) || prefix.equals(XMLConstants.NULL_NS_URI)) { return sourceDocument.lookupNamespaceURI(null); - } else { + } + // xml: prefix is assumed to be known according to W3C: https://www.w3.org/TR/xml-names/ + else if (prefix.equals(XMLConstants.XML_NS_PREFIX)) { + return XMLConstants.XML_NS_URI; + } + else { return sourceDocument.lookupNamespaceURI(prefix); } } From dc8f1c78998c392e6dce5a908b78e623afc7334d Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Thu, 20 Jan 2022 09:15:09 +0100 Subject: [PATCH 073/609] Mapper_XML_Test: add RMLTC1032-XML Verify that we can process XML where the xml: prefix is not declared but used. According to W3C, this prefix is always known and should not cause errors. --- .../java/be/ugent/rml/Mapper_XML_Test.java | 5 ++++ .../RMLTC1032-XML/RML_demo_data.xml | 8 ++++++ .../test-cases/RMLTC1032-XML/mapping.ttl | 28 +++++++++++++++++++ .../test-cases/RMLTC1032-XML/output.nq | 2 ++ 4 files changed, 43 insertions(+) create mode 100644 src/test/resources/test-cases/RMLTC1032-XML/RML_demo_data.xml create mode 100644 src/test/resources/test-cases/RMLTC1032-XML/mapping.ttl create mode 100644 src/test/resources/test-cases/RMLTC1032-XML/output.nq diff --git a/src/test/java/be/ugent/rml/Mapper_XML_Test.java b/src/test/java/be/ugent/rml/Mapper_XML_Test.java index 95b1aa33..0681cb75 100644 --- a/src/test/java/be/ugent/rml/Mapper_XML_Test.java +++ b/src/test/java/be/ugent/rml/Mapper_XML_Test.java @@ -217,4 +217,9 @@ public void evaluate_1026_XML() { public void evaluate_1027_XML() { doMapping("./test-cases/RMLTC1027-XML/mapping.ttl", "./test-cases/RMLTC1027-XML/output.nq"); } + + @Test + public void evaluate_1032_XML() { + doMapping("./test-cases/RMLTC1032-XML/mapping.ttl", "./test-cases/RMLTC1032-XML/output.nq"); + } } diff --git a/src/test/resources/test-cases/RMLTC1032-XML/RML_demo_data.xml b/src/test/resources/test-cases/RMLTC1032-XML/RML_demo_data.xml new file mode 100644 index 00000000..89b3de11 --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1032-XML/RML_demo_data.xml @@ -0,0 +1,8 @@ + + + + Zapiski iz mertvogo doma + + diff --git a/src/test/resources/test-cases/RMLTC1032-XML/mapping.ttl b/src/test/resources/test-cases/RMLTC1032-XML/mapping.ttl new file mode 100644 index 00000000..91ecdc6c --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1032-XML/mapping.ttl @@ -0,0 +1,28 @@ +@prefix bf: . +@prefix ex: . +@prefix rml: . +@prefix ql: . + +ex:ExampleMap a rr:TriplesMap; + rml:logicalSource [ + rml:source "RML_demo_data.xml"; + rml:referenceFormulation ql:XPath; + rml:iterator "/rdf:RDF/rdf:Description" + ]; + + rr:subjectMap [ + rml:reference "@rdf:about"; + rr:class bf:Work + ]; + + rr:predicateObjectMap [ + rr:predicate bf:title; + rr:objectMap [ + rml:reference "rdaw:P10223"; + rr:termType rr:Literal; + rml:languageMap [ + rml:reference "rdaw:P10223/@xml:lang" + ] + ] + ]. + diff --git a/src/test/resources/test-cases/RMLTC1032-XML/output.nq b/src/test/resources/test-cases/RMLTC1032-XML/output.nq new file mode 100644 index 00000000..b6b283f2 --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1032-XML/output.nq @@ -0,0 +1,2 @@ + . + "Zapiski iz mertvogo doma"@ru. From 6dbd9bbdfdd90ec3a473fae3312b55dfbb7c2d57 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Thu, 20 Jan 2022 09:20:51 +0100 Subject: [PATCH 074/609] CHANGELOG: fix entries Some entries are not released yet --- CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1e9b21e..d395c195 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,17 +11,17 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Bump buildnumber during release (see [issue 227](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/227)) - Always include xml: prefix (see [issue 144](https://github.com/RMLio/rmlmapper-java/issues/144)) -## [4.14.3] - 2022-01-13 - -### Fixed -- Github CLI needs to specify a repo to create a release. - ### Changed - CSVRecordFactory: Replace Apache CSV library with OpenCSV to be able to differentiate between empty string and null (see [issue 140](https://github.com/RMLio/rmlmapper-java/issues/140)) ### Added - JSONRecordFactory: support JSONL files (see [issue 221](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/221)). +## [4.14.3] - 2022-01-13 + +### Fixed +- Github CLI needs to specify a repo to create a release. + ## [4.14.2] - 2022-01-13 ### Fixed From c5d1cc8e60c19cb79fc07537448c026c437eb2b6 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Fri, 21 Jan 2022 09:13:22 +0100 Subject: [PATCH 075/609] CI: use Gitlab's internal mirror feature --- .gitlab-ci.yml | 28 ---------------------------- CHANGELOG.md | 1 + 2 files changed, 1 insertion(+), 28 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6d2dd83a..537a301e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -334,31 +334,3 @@ R2RML Test Report: fi only: - tags - -############################### -# # -# Automated Mirroring # -# # -############################### - -# Sync internal Gitlab master & development branches and tags with public Github repo -# Make sure you configure the following CI variable: GITHUB_ACCESS_TOKEN -Github: - stage: mirror - image: - entrypoint: [''] - name: alpine/git:${GIT_VERSION} - before_script: - # Set the displayed user with the commits that are about to be made - - git config --global user.email "${GIT_USER_EMAIL:-$GITLAB_USER_EMAIL}" - - git config --global user.name "${GIT_USER_NAME:-$GITLAB_USER_NAME}" - script: - - echo "Pushing to Github $CI_COMMIT_BRANCH branch" - - git remote set-url origin "https://${GITHUB_ACCESS_TOKEN}@github.com/RMLio/rmlmapper-java.git" - - git checkout "$CI_COMMIT_BRANCH" - # Use --force to make sure we're in sync - - git push --force origin "$CI_COMMIT_BRANCH" - - git push --force --tags origin "$CI_COMMIT_BRANCH" - only: - - master - - development diff --git a/CHANGELOG.md b/CHANGELOG.md index d395c195..5b2de6fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - CSVRecordFactory: Replace Apache CSV library with OpenCSV to be able to differentiate between empty string and null (see [issue 140](https://github.com/RMLio/rmlmapper-java/issues/140)) +- CI: use Gitlab's own mirror feature instead of mirroring in a CI job. ### Added - JSONRecordFactory: support JSONL files (see [issue 221](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/221)). From a4b3380030f1c112559907f4a78747ed511f7908 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Fri, 21 Jan 2022 14:19:10 +0100 Subject: [PATCH 076/609] CI: enforce changelog updates --- .gitlab-ci.yml | 6 ++++++ CHANGELOG.md | 1 + 2 files changed, 7 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 537a301e..7a0f2f95 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,12 @@ stages: - release - deploy +include: + # Make sure the CHANGELOG is always updated + - project: 'rml/util/ci-templates' + ref: main + file: 'CHANGELOG.gitlab-ci.yml' + variables: # This will suppress any download for dependencies and plugins or upload messages which would clutter the console log. # `showDateTime` will show the passed time in milliseconds. diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b2de6fc..01a05dce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - CSVRecordFactory: Replace Apache CSV library with OpenCSV to be able to differentiate between empty string and null (see [issue 140](https://github.com/RMLio/rmlmapper-java/issues/140)) - CI: use Gitlab's own mirror feature instead of mirroring in a CI job. +- CI: enforce CHANGELOG updates. ### Added - JSONRecordFactory: support JSONL files (see [issue 221](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/221)). From 01ad76e8c78e10d33ce2a8a50a5205e926571f68 Mon Sep 17 00:00:00 2001 From: Gerald Haesendonck Date: Tue, 25 Jan 2022 14:49:10 +0000 Subject: [PATCH 077/609] IDLabFunctions: use correct date pattern for normalizeDate normalizeDate outputs the minutes of a date instead of the months. Use 'MMMM' instead of 'mm' when normalizing dates. --- CHANGELOG.md | 2 + .../rml/functions/lib/IDLabFunctions.java | 92 ++++++++++++++++--- src/main/resources/functions_idlab.ttl | 65 +++++++++++++ .../ugent/rml/Custom_RML_FnO_Mapper_Test.java | 32 +++++++ .../rml/functions/lib/IDLabFunctionsTest.java | 55 +++++++++++ .../rml-fno-test-cases/RMLFNOTCF009/data.csv | 3 + .../RMLFNOTCF009/mapping.ttl | 47 ++++++++++ .../RMLFNOTCF009/output.ttl | 2 + .../rml-fno-test-cases/RMLFNOTCF010/data.csv | 4 + .../RMLFNOTCF010/mapping.ttl | 51 ++++++++++ .../RMLFNOTCF010/output.ttl | 3 + .../rml-fno-test-cases/RMLFNOTCF011/data.csv | 3 + .../RMLFNOTCF011/mapping.ttl | 47 ++++++++++ .../RMLFNOTCF011/output.ttl | 2 + .../rml-fno-test-cases/RMLFNOTCF012/data.csv | 3 + .../RMLFNOTCF012/mapping.ttl | 52 +++++++++++ .../RMLFNOTCF012/output.ttl | 2 + 17 files changed, 450 insertions(+), 15 deletions(-) create mode 100644 src/test/resources/rml-fno-test-cases/RMLFNOTCF009/data.csv create mode 100644 src/test/resources/rml-fno-test-cases/RMLFNOTCF009/mapping.ttl create mode 100644 src/test/resources/rml-fno-test-cases/RMLFNOTCF009/output.ttl create mode 100644 src/test/resources/rml-fno-test-cases/RMLFNOTCF010/data.csv create mode 100644 src/test/resources/rml-fno-test-cases/RMLFNOTCF010/mapping.ttl create mode 100644 src/test/resources/rml-fno-test-cases/RMLFNOTCF010/output.ttl create mode 100644 src/test/resources/rml-fno-test-cases/RMLFNOTCF011/data.csv create mode 100644 src/test/resources/rml-fno-test-cases/RMLFNOTCF011/mapping.ttl create mode 100644 src/test/resources/rml-fno-test-cases/RMLFNOTCF011/output.ttl create mode 100644 src/test/resources/rml-fno-test-cases/RMLFNOTCF012/data.csv create mode 100644 src/test/resources/rml-fno-test-cases/RMLFNOTCF012/mapping.ttl create mode 100644 src/test/resources/rml-fno-test-cases/RMLFNOTCF012/output.ttl diff --git a/CHANGELOG.md b/CHANGELOG.md index 01a05dce..62b418da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,11 +10,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed - Bump buildnumber during release (see [issue 227](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/227)) - Always include xml: prefix (see [issue 144](https://github.com/RMLio/rmlmapper-java/issues/144)) +- `normalizeDate` function uses wrong pattern (see [issue 228](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/228)) ### Changed - CSVRecordFactory: Replace Apache CSV library with OpenCSV to be able to differentiate between empty string and null (see [issue 140](https://github.com/RMLio/rmlmapper-java/issues/140)) - CI: use Gitlab's own mirror feature instead of mirroring in a CI job. - CI: enforce CHANGELOG updates. +- Added functions to normalize dates and datetimes. ### Added - JSONRecordFactory: support JSONL files (see [issue 221](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/221)). diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index cc26b971..227cf83d 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -19,9 +19,10 @@ import java.net.HttpURLConnection; import java.net.URL; import java.net.URLEncoder; -import java.text.DateFormat; -import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.*; public class IDLabFunctions { @@ -205,27 +206,88 @@ public static boolean booleanMatch(String valueParameter, String regexParameter) return valueParameter.matches(regexParameter); } + /////////////////////////////// + // Date formatting functions // + /////////////////////////////// // TODO check whether this is the right place for this + /** - * Returns `s` as a normalized xsd:date string, using `f` as current date form. + * Returns a given date(time) string as an ISO-8601 formatted date(time) string. * - * @param s string - * @param f format - * @return a normalized xsd:date string + * @param dateStr Input string representing a parsable date or dateTime, e.g. "01 April 22" + * @param pattern DateTime format pattern used to parse the given dateStr as defined in {@link DateTimeFormatter}, e.g. "dd LLLL uu" + * @param language The language of dateStr, as defined in {@link Locale} + * @param includeTime If true, include the time part in the output. + * @return A normalized date string in the ISO-8601 format uuuu-MM-dd (xs:date) or uuuu-MM-ddTHH:mm:ss, + * or null if parsing the input fails. */ - public static String normalizeDate(String s, String f) { - DateFormat format = new SimpleDateFormat(f); - Date date = null; + private static String normalizeDateTimeStr(String dateStr, String pattern, String language, boolean includeTime) { try { - date = format.parse(s); - } catch (ParseException e) { - e.printStackTrace(); - return s; + Locale locale = new Locale(language); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern, locale); + if (includeTime) { + LocalDateTime dateTime = LocalDateTime.parse(dateStr, formatter); + return dateTime.format(DateTimeFormatter.ISO_DATE_TIME); + } else { + LocalDate date = LocalDate.parse(dateStr, formatter); + return date.toString(); + } + } catch (Throwable e) { + logger.error("{}; format pattern: \"{}\", input: \"{}\", language: \"{}\"", e.getMessage(), pattern, dateStr, language); + return null; } - DateFormat xsdDateFormat = new SimpleDateFormat("yyyy-mm-dd"); - return xsdDateFormat.format(date); } + /** + * Returns `dateStr` as a normalized xs:date string, using `pattern` as current date form. + * + * @param dateStr Input string representing a parsable date, e.g. "01 April 22" + * @param pattern Date format pattern used to parse the given dateStr as defined in {@link DateTimeFormatter}, e.g. "dd LLLL uu" + * @param language The language of dateStr, as defined in {@link Locale} + * @return A normalized date string in the ISO-8601 format uuuu-MM-dd (xs:date), or null if parsing the input fails. + */ + public static String normalizeDateWithLang(String dateStr, String pattern, String language) { + return normalizeDateTimeStr(dateStr, pattern, language, false); + } + + /** + * Returns `dateStr` as a normalized xs:date string, using `pattern` as current date form. It uses the language of + * the current locale of the JVM to parse certain input strings, like names of months. + * + * @param dateStr Input string representing a parsable date, e.g. "01 April 22" + * @param pattern Date format pattern used to parse the given dateStr as defined in {@link SimpleDateFormat}, e.g. "dd LLL y" + * @return A normalized date string in the ISO-8601 format uuuu-MM-dd (xs:date), or null if parsing the input fails. + */ + public static String normalizeDate(String dateStr, String pattern) { + return normalizeDateWithLang(dateStr, pattern, Locale.getDefault().getLanguage()); + } + + /** + * Returns `dateTimeStr` as a normalized xs:date string, using `pattern` as current datetime form. + * + * @param dateTimeStr Input string representing a parsable datetime, e.g. "01 April 22T11:44:00" + * @param pattern Date format pattern used to parse the given dateStr as defined in {@link DateTimeFormatter}, e.g. "dd LLLL uuTHH:mm:ss" + * @param language The language of dateStr, as defined in {@link Locale} + * @return A normalized date string in the ISO-8601 format uuuu-MM-ddTHH:mm:ss (xs:datetime), or null if parsing the input fails. + */ + public static String normalizeDateTimeWithLang(String dateTimeStr, String pattern, String language) { + return normalizeDateTimeStr(dateTimeStr, pattern, language, true); + } + + /** + * Returns `dateTimeStr` as a normalized xs:date string, using `pattern` as current datetime form. + * It uses the language of the current locale of the JVM to parse certain input strings, like names of months. + * + * @param dateTimeStr Input string representing a parsable datetime, e.g. "01 April 22T11:44:00" + * @param pattern Date format pattern used to parse the given dateStr as defined in {@link DateTimeFormatter}, e.g. "dd LLLL uuTHH:mm:ss" + * @return A normalized date string in the ISO-8601 format uuuu-MM-ddTHH:mm:ss (xs:datetime), or null if parsing the input fails. + */ + public static String normalizeDateTime(String dateTimeStr, String pattern) { + return normalizeDateTimeWithLang(dateTimeStr, pattern, Locale.getDefault().getLanguage()); + } + + + // TODO check whether this is the right place for this public static String jsonize(Object s) throws JsonProcessingException { ObjectWriter ow = new ObjectMapper().writer().withDefaultPrettyPrinter(); diff --git a/src/main/resources/functions_idlab.ttl b/src/main/resources/functions_idlab.ttl index e20f541a..ba6ec694 100644 --- a/src/main/resources/functions_idlab.ttl +++ b/src/main/resources/functions_idlab.ttl @@ -156,6 +156,50 @@ idlab-fn:slugify lib:class "IDLabFunctions" ; lib:method "slugify" ] . +idlab-fn:normalizeDate + a fno:Function ; + fno:name "normalizeDate" ; + rdfs:label "normalizeDate" ; + dcterms:description "Parses the input as a date and returns it as a string in the ISO-8601 format uuuu-MM-dd. It uses the current language to parse month or day names." ; + fno:expects ( idlab-fn:_strDate idlab-fn:_pattern ) ; + fno:returns ( idlab-fn:_stringOut ) ; + lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; + lib:class "IDLabFunctions" ; + lib:method "normalizeDate" ] . + + idlab-fn:normalizeDateWithLang + a fno:Function ; + fno:name "normalizeDateWithLang" ; + rdfs:label "normalizeDateWithLang" ; + dcterms:description "Parses the input as a date and returns it as a string in the ISO-8601 format uuuu-MM-dd. It uses the given language to parse month or day names." ; + fno:expects ( idlab-fn:_strDate idlab-fn:_pattern idlab-fn:_lang ) ; + fno:returns ( idlab-fn:_stringOut ) ; + lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; + lib:class "IDLabFunctions" ; + lib:method "normalizeDateWithLang" ] . + +idlab-fn:normalizeDateTime + a fno:Function ; + fno:name "normalizeDateTime" ; + rdfs:label "normalizeDateTime" ; + dcterms:description "Parses the input as a datetime and returns it as a string in the ISO-8601 format uuuu-MM-ddTHH:mm:ss. It uses the current language to parse month or day names." ; + fno:expects ( idlab-fn:_strDate idlab-fn:_pattern ) ; + fno:returns ( idlab-fn:_stringOut ) ; + lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; + lib:class "IDLabFunctions" ; + lib:method "normalizeDateTime" ] . + + idlab-fn:normalizeDateTimeWithLang + a fno:Function ; + fno:name "normalizeDateTimeWithLang" ; + rdfs:label "normalizeDateTimeWithLang" ; + dcterms:description "Parses the input as a datetime and returns it as a string in the ISO-8601 format uuuu-MM-ddTHH:mm:ss. It uses the given language to parse month or day names." ; + fno:expects ( idlab-fn:_strDate idlab-fn:_pattern idlab-fn:_lang ) ; + fno:returns ( idlab-fn:_stringOut ) ; + lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; + lib:class "IDLabFunctions" ; + lib:method "normalizeDateTimeWithLang" ] . + grel:valueParam a fno:Parameter ; @@ -293,3 +337,24 @@ idlab-fn:_boolOut rdfs:label "output boolean" ; fno:type xsd:boolean ; fno:predicate idlab-fn:o_boolOut . + +idlab-fn:_strDate + a fno:Parameter ; + fno:name "string parsable to a date" ; + rdfs:label "string parsable to a date" ; + fno:type xsd:string ; + fno:predicate idlab-fn:strDate . + +idlab-fn:_pattern + a fno:Parameter ; + fno:name "string representing a date pattern" ; + rdfs:label "string representing a date pattern" ; + fno:type xsd:string ; + fno:predicate idlab-fn:pattern . + +idlab-fn:_lang + a fno:Parameter ; + fno:name "string representing a BCP 47 language tag" ; + rdfs:label "string representing a BCP 47 language tag" ; + fno:type xsd:string ; + fno:predicate idlab-fn:lang . \ No newline at end of file diff --git a/src/test/java/be/ugent/rml/Custom_RML_FnO_Mapper_Test.java b/src/test/java/be/ugent/rml/Custom_RML_FnO_Mapper_Test.java index 3bd92f0d..7e5d6877 100644 --- a/src/test/java/be/ugent/rml/Custom_RML_FnO_Mapper_Test.java +++ b/src/test/java/be/ugent/rml/Custom_RML_FnO_Mapper_Test.java @@ -151,6 +151,38 @@ public void evaluate_idlab_F008() { doMapping("./rml-fno-test-cases/RMLFNOTCF008/mapping.rml.ttl", "./rml-fno-test-cases/RMLFNOTCF008/output.nq"); } + /** + * Tests whether the function idlab-fn:normalizeDate is supported correctly by the mapper + */ + @Test + public void evaluate_idlab_F009() { + doMapping("./rml-fno-test-cases/RMLFNOTCF009/mapping.ttl", "./rml-fno-test-cases/RMLFNOTCF009/output.ttl"); + } + + /** + * Tests whether the function idlab-fn:normalizeDateWithLang is supported correctly by the mapper + */ + @Test + public void evaluate_idlab_F010() { + doMapping("./rml-fno-test-cases/RMLFNOTCF010/mapping.ttl", "./rml-fno-test-cases/RMLFNOTCF010/output.ttl"); + } + + /** + * Tests whether the function idlab-fn:normalizeDateTime is supported correctly by the mapper + */ + @Test + public void evaluate_idlab_F011() { + doMapping("./rml-fno-test-cases/RMLFNOTCF011/mapping.ttl", "./rml-fno-test-cases/RMLFNOTCF011/output.ttl"); + } + + /** + * Tests whether the function idlab-fn:normalizeDateTimeWithLang is supported correctly by the mapper + */ + @Test + public void evaluate_idlab_F012() { + doMapping("./rml-fno-test-cases/RMLFNOTCF012/mapping.ttl", "./rml-fno-test-cases/RMLFNOTCF012/output.ttl"); + } + /** * Tests whether the function millisecondsToInstant can be loaded and is supported correctly by the mapper */ diff --git a/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java b/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java index 3f5d6af9..1e8c9cb7 100644 --- a/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java +++ b/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java @@ -164,4 +164,59 @@ public void slugify() { String result = IDLabFunctions.slugify("Ben De Mééster"); assertEquals("ben-de-meester", result); } + + @Test + public void normalizeDateWithLang() { + String input1 = "20220121"; + String format1 = "yyyyMMdd"; + assertEquals("2022-01-21", IDLabFunctions.normalizeDateWithLang(input1, format1, "en")); + + String input2 = "01 April 22"; + // String format2 = "dd LLLL uu"; // This does not work on Java 8! + String format2 = "dd MMMM uu"; + assertEquals("2022-04-01", IDLabFunctions.normalizeDateWithLang(input2, format2, "en")); + + assertNull(IDLabFunctions.normalizeDateWithLang("rubbish", "yodelahiti", "en")); + + // will fail because "April" is no French + assertNull(IDLabFunctions.normalizeDateWithLang(input2, format2, "fr")); + + String input3 = "01-avr.-22"; // yes, French abbreviations need a '.' ! + String format3 = "dd-MMM-yy"; + assertEquals("2022-04-01", IDLabFunctions.normalizeDateWithLang(input3, format3, "fr")); + } + + @Test + public void normalizeDate() { + String input1 = "20220121"; + String format1 = "yyyyMMdd"; + assertEquals("2022-01-21", IDLabFunctions.normalizeDate(input1, format1)); + + assertNull(IDLabFunctions.normalizeDate("rubbish", "yodelahiti")); + + } + + @Test + public void normalizeDateTimeWithLang() { + String input1 = "20220121 7 14 33"; + String format1 = "yyyyMMdd H m s"; + assertEquals("2022-01-21T07:14:33", IDLabFunctions.normalizeDateTimeWithLang(input1, format1, "en")); + } + + @Test + public void normalizeDateTime() { + String input1 = "20200521 17 14 33"; + String format1 = "yyyyMMdd H m s"; + assertEquals("2020-05-21T17:14:33", IDLabFunctions.normalizeDateTime(input1, format1)); + + // 20220124T09:36:04,yyyyMMdd'THH:mm:ss + String input2 = "20220124T09:36:04"; + String format2 = "yyyyMMdd'T'HH:mm:ss"; + assertEquals("2022-01-24T09:36:04", IDLabFunctions.normalizeDateTime(input2, format2)); + + String input3 = "01-Apr-20 9u4"; + String format3 = "dd-MMM-yy H'u'm"; + assertEquals("2020-04-01T09:04:00", IDLabFunctions.normalizeDateTime(input3, format3)); + + } } diff --git a/src/test/resources/rml-fno-test-cases/RMLFNOTCF009/data.csv b/src/test/resources/rml-fno-test-cases/RMLFNOTCF009/data.csv new file mode 100644 index 00000000..6a8ef9ce --- /dev/null +++ b/src/test/resources/rml-fno-test-cases/RMLFNOTCF009/data.csv @@ -0,0 +1,3 @@ +ID,INPUT,PATTERN +0,20220124,yyyyMMdd +1,01-Apr-20,dd-MMM-yy \ No newline at end of file diff --git a/src/test/resources/rml-fno-test-cases/RMLFNOTCF009/mapping.ttl b/src/test/resources/rml-fno-test-cases/RMLFNOTCF009/mapping.ttl new file mode 100644 index 00000000..c8e1720e --- /dev/null +++ b/src/test/resources/rml-fno-test-cases/RMLFNOTCF009/mapping.ttl @@ -0,0 +1,47 @@ +@prefix rr: . +@prefix ex: . +@prefix xsd: . +@prefix rml: . +@prefix ql: . +@prefix fnml: . +@prefix fno: . +@prefix grel: . +@prefix idlab-fn: . + +@base . + + + a rr:TriplesMap; + + rml:logicalSource [ + rml:source "data.csv"; + rml:referenceFormulation ql:CSV + ]; + + rr:subjectMap [ + rr:template "http://example.com/{ID}" + ]; + + rr:predicateObjectMap [ + + rr:predicate ex:createdAt; + + rr:objectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant idlab-fn:normalizeDate ] + ]; + + rr:predicateObjectMap [ + rr:predicate idlab-fn:strDate ; + rr:objectMap [ rml:reference "INPUT" ] + ]; + + rr:predicateObjectMap [ + rr:predicate idlab-fn:pattern ; + rr:objectMap [ rml:reference "PATTERN" ] + ]; + ] + ] + ]. \ No newline at end of file diff --git a/src/test/resources/rml-fno-test-cases/RMLFNOTCF009/output.ttl b/src/test/resources/rml-fno-test-cases/RMLFNOTCF009/output.ttl new file mode 100644 index 00000000..2ba884aa --- /dev/null +++ b/src/test/resources/rml-fno-test-cases/RMLFNOTCF009/output.ttl @@ -0,0 +1,2 @@ + "2022-01-24" . + "2020-04-01" . \ No newline at end of file diff --git a/src/test/resources/rml-fno-test-cases/RMLFNOTCF010/data.csv b/src/test/resources/rml-fno-test-cases/RMLFNOTCF010/data.csv new file mode 100644 index 00000000..0108bb5a --- /dev/null +++ b/src/test/resources/rml-fno-test-cases/RMLFNOTCF010/data.csv @@ -0,0 +1,4 @@ +ID,INPUT,PATTERN,LANG +0,20220124,yyyyMMdd,en +1,01-Apr-20,dd-MMM-yy,en +2,01-avr.-20,dd-MMM-yy,fr \ No newline at end of file diff --git a/src/test/resources/rml-fno-test-cases/RMLFNOTCF010/mapping.ttl b/src/test/resources/rml-fno-test-cases/RMLFNOTCF010/mapping.ttl new file mode 100644 index 00000000..8f79ebc2 --- /dev/null +++ b/src/test/resources/rml-fno-test-cases/RMLFNOTCF010/mapping.ttl @@ -0,0 +1,51 @@ +@prefix rr: . +@prefix ex: . +@prefix xsd: . +@prefix rml: . +@prefix ql: . +@prefix fnml: . +@prefix fno: . +@prefix grel: . +@prefix idlab-fn: . + +@base . + + + a rr:TriplesMap; + + rml:logicalSource [ + rml:source "data.csv"; + rml:referenceFormulation ql:CSV + ]; + + rr:subjectMap [ + rr:template "http://example.com/{ID}" + ]; + + rr:predicateObjectMap [ + + rr:predicate ex:createdAt; + + rr:objectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant idlab-fn:normalizeDateWithLang ] + ]; + + rr:predicateObjectMap [ + rr:predicate idlab-fn:strDate ; + rr:objectMap [ rml:reference "INPUT" ] + ]; + + rr:predicateObjectMap [ + rr:predicate idlab-fn:pattern ; + rr:objectMap [ rml:reference "PATTERN" ] + ]; + rr:predicateObjectMap [ + rr:predicate idlab-fn:lang ; + rr:objectMap [ rml:reference "LANG" ] + ]; + ] + ] + ]. \ No newline at end of file diff --git a/src/test/resources/rml-fno-test-cases/RMLFNOTCF010/output.ttl b/src/test/resources/rml-fno-test-cases/RMLFNOTCF010/output.ttl new file mode 100644 index 00000000..163074de --- /dev/null +++ b/src/test/resources/rml-fno-test-cases/RMLFNOTCF010/output.ttl @@ -0,0 +1,3 @@ + "2022-01-24" . + "2020-04-01" . + "2020-04-01" . diff --git a/src/test/resources/rml-fno-test-cases/RMLFNOTCF011/data.csv b/src/test/resources/rml-fno-test-cases/RMLFNOTCF011/data.csv new file mode 100644 index 00000000..81a38f94 --- /dev/null +++ b/src/test/resources/rml-fno-test-cases/RMLFNOTCF011/data.csv @@ -0,0 +1,3 @@ +ID,INPUT,PATTERN +0,20220124T09:36:04,"yyyyMMdd'T'HH:mm:ss" +1,01-Apr-20 9u4,"dd-MMM-yy H'u'm" \ No newline at end of file diff --git a/src/test/resources/rml-fno-test-cases/RMLFNOTCF011/mapping.ttl b/src/test/resources/rml-fno-test-cases/RMLFNOTCF011/mapping.ttl new file mode 100644 index 00000000..80ff8c45 --- /dev/null +++ b/src/test/resources/rml-fno-test-cases/RMLFNOTCF011/mapping.ttl @@ -0,0 +1,47 @@ +@prefix rr: . +@prefix ex: . +@prefix xsd: . +@prefix rml: . +@prefix ql: . +@prefix fnml: . +@prefix fno: . +@prefix grel: . +@prefix idlab-fn: . + +@base . + + + a rr:TriplesMap; + + rml:logicalSource [ + rml:source "data.csv"; + rml:referenceFormulation ql:CSV + ]; + + rr:subjectMap [ + rr:template "http://example.com/{ID}" + ]; + + rr:predicateObjectMap [ + + rr:predicate ex:createdAt; + + rr:objectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant idlab-fn:normalizeDateTime ] + ]; + + rr:predicateObjectMap [ + rr:predicate idlab-fn:strDate ; + rr:objectMap [ rml:reference "INPUT" ] + ]; + + rr:predicateObjectMap [ + rr:predicate idlab-fn:pattern ; + rr:objectMap [ rml:reference "PATTERN" ] + ]; + ] + ] + ]. \ No newline at end of file diff --git a/src/test/resources/rml-fno-test-cases/RMLFNOTCF011/output.ttl b/src/test/resources/rml-fno-test-cases/RMLFNOTCF011/output.ttl new file mode 100644 index 00000000..f90fc317 --- /dev/null +++ b/src/test/resources/rml-fno-test-cases/RMLFNOTCF011/output.ttl @@ -0,0 +1,2 @@ + "2022-01-24T09:36:04" . + "2020-04-01T09:04:00" . \ No newline at end of file diff --git a/src/test/resources/rml-fno-test-cases/RMLFNOTCF012/data.csv b/src/test/resources/rml-fno-test-cases/RMLFNOTCF012/data.csv new file mode 100644 index 00000000..7bb323f7 --- /dev/null +++ b/src/test/resources/rml-fno-test-cases/RMLFNOTCF012/data.csv @@ -0,0 +1,3 @@ +ID,INPUT,PATTERN,LANG +0,20220124T09:36:04,"yyyyMMdd'T'HH:mm:ss",nl +1,01-avr.-20 9u4,"dd-MMM-yy H'u'm",fr \ No newline at end of file diff --git a/src/test/resources/rml-fno-test-cases/RMLFNOTCF012/mapping.ttl b/src/test/resources/rml-fno-test-cases/RMLFNOTCF012/mapping.ttl new file mode 100644 index 00000000..36c7c9f1 --- /dev/null +++ b/src/test/resources/rml-fno-test-cases/RMLFNOTCF012/mapping.ttl @@ -0,0 +1,52 @@ +@prefix rr: . +@prefix ex: . +@prefix xsd: . +@prefix rml: . +@prefix ql: . +@prefix fnml: . +@prefix fno: . +@prefix grel: . +@prefix idlab-fn: . + +@base . + + + a rr:TriplesMap; + + rml:logicalSource [ + rml:source "data.csv"; + rml:referenceFormulation ql:CSV + ]; + + rr:subjectMap [ + rr:template "http://example.com/{ID}" + ]; + + rr:predicateObjectMap [ + + rr:predicate ex:createdAt; + + rr:objectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant idlab-fn:normalizeDateTimeWithLang ] + ]; + + rr:predicateObjectMap [ + rr:predicate idlab-fn:strDate ; + rr:objectMap [ rml:reference "INPUT" ] + ]; + + rr:predicateObjectMap [ + rr:predicate idlab-fn:pattern ; + rr:objectMap [ rml:reference "PATTERN" ] + ]; + + rr:predicateObjectMap [ + rr:predicate idlab-fn:lang ; + rr:objectMap [ rml:reference "LANG" ] + ]; + ] + ] + ]. \ No newline at end of file diff --git a/src/test/resources/rml-fno-test-cases/RMLFNOTCF012/output.ttl b/src/test/resources/rml-fno-test-cases/RMLFNOTCF012/output.ttl new file mode 100644 index 00000000..f90fc317 --- /dev/null +++ b/src/test/resources/rml-fno-test-cases/RMLFNOTCF012/output.ttl @@ -0,0 +1,2 @@ + "2022-01-24T09:36:04" . + "2020-04-01T09:04:00" . \ No newline at end of file From ea2bc3281665e63a1b3395446695d5b5d48bdb4d Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Wed, 26 Jan 2022 11:07:07 +0100 Subject: [PATCH 078/609] CI: only publish -all.jar releases on GH --- .gitlab-ci.yml | 2 +- CHANGELOG.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7a0f2f95..7616741e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -236,7 +236,7 @@ Github Release: - TAG=$(git tag -l "v*" --sort=-creatordate | head -n1) - CHANGES=$(./get-changes.sh) - echo "Creating Github release $TAG with changes\n$CHANGES" - - gh release create "$TAG" -n "$CHANGES" -t "$TAG" -R "github.com/RMLio/rmlmapper-java" target/*.jar + - gh release create "$TAG" -n "$CHANGES" -t "$TAG" -R "github.com/RMLio/rmlmapper-java" target/*-all.jar only: - tags diff --git a/CHANGELOG.md b/CHANGELOG.md index 62b418da..651b8125 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Bump buildnumber during release (see [issue 227](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/227)) - Always include xml: prefix (see [issue 144](https://github.com/RMLio/rmlmapper-java/issues/144)) - `normalizeDate` function uses wrong pattern (see [issue 228](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/228)) +- CI: only publish `-all.jar` releases on Github (see [issue 145](https://github.com/RMLio/rmlmapper-java/issues/145)) ### Changed - CSVRecordFactory: Replace Apache CSV library with OpenCSV to be able to differentiate between empty string and null (see [issue 140](https://github.com/RMLio/rmlmapper-java/issues/140)) From a0b053c266a892e428761d2ae7afb23c3d7c3b09 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Tue, 1 Feb 2022 10:34:22 +0100 Subject: [PATCH 079/609] CI: use Github Release template --- .gitlab-ci.yml | 36 ++++-------------------------------- build-release.sh | 4 ++++ 2 files changed, 8 insertions(+), 32 deletions(-) create mode 100755 build-release.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7616741e..debd5f8d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,10 @@ include: - project: 'rml/util/ci-templates' ref: main file: 'CHANGELOG.gitlab-ci.yml' + # Make a Github Release on new tags + - project: 'rml/util/ci-templates' + ref: main + file: 'Github-Release.gitlab-ci.yml' variables: # This will suppress any download for dependencies and plugins or upload messages which would clutter the console log. @@ -208,38 +212,6 @@ Docker Hub: only: - tags -# Create Github Releases on new tags -Github Release: - stage: deploy - image: maven:3.5.0-jdk-8 - before_script: - # Clone the repository via HTTPS inside a new directory - - git clone "https://${GITLAB_USERNAME}:${GITLAB_TOKEN_RMLMAPPER}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" "${CI_COMMIT_SHA}" - - # Set the displayed user with the commits that are about to be made - - git config --global user.email "${GIT_USER_EMAIL:-$GITLAB_USER_EMAIL}" - - git config --global user.name "${GIT_USER_NAME:-$GITLAB_USER_NAME}" - - # Install dependencies - - apt update - - apt install -y apt-transport-https - - curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg - - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null - - apt update - - apt install gh - script: - # Login into Github - - gh auth login -s repo --with-token < $GITHUB_ACCESS_TOKEN_FILE - # Build fat jar for Github Release - - mvn install -DskipTests=true - # Create Github Release - - TAG=$(git tag -l "v*" --sort=-creatordate | head -n1) - - CHANGES=$(./get-changes.sh) - - echo "Creating Github release $TAG with changes\n$CHANGES" - - gh release create "$TAG" -n "$CHANGES" -t "$TAG" -R "github.com/RMLio/rmlmapper-java" target/*-all.jar - only: - - tags - # Create a Maven Central release on new tags # Thanks to https://1337codersblog.wordpress.com/2019/09/18/automatical-artifact-deployment-to-maven-central-using-gitlab-ci-part-1/ # Define the following CI variables: diff --git a/build-release.sh b/build-release.sh new file mode 100755 index 00000000..d0d49673 --- /dev/null +++ b/build-release.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +apk add java-jre-headless maven java-jdk +mvn install -DskipTests=true From cba3e255245e3f19c51ea9c9f5f273e953b46a1a Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Tue, 1 Feb 2022 10:35:49 +0100 Subject: [PATCH 080/609] CI: use Docker Hub template --- .gitlab-ci.yml | 29 ++++------------------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index debd5f8d..b3c8a40b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,10 @@ include: - project: 'rml/util/ci-templates' ref: main file: 'Github-Release.gitlab-ci.yml' + # Push a Docker Image to Docker Hub on new tags + - project: 'rml/util/ci-templates' + ref: main + file: 'Docker-Hub.gitlab-ci.yml' variables: # This will suppress any download for dependencies and plugins or upload messages which would clutter the console log. @@ -187,31 +191,6 @@ Create Release: fi when: manual -# Push Docker image of the latest master branch to Docker Hub -# Updates 'latest' and the last git tag that is available -# Only triggered when a deployment is done from development -> master (manual) -Docker Hub: - stage: deploy - image: docker:latest - variables: - DOCKER_IMAGE_NAME: "rmlio/rmlmapper-java" - services: - - docker:19.03.12-dind - before_script: - - apk add git - - docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD" docker.io - script: - # Docker 'latest' tag - - echo "Deploying image $DOCKER_IMAGE_NAME":latest - - docker build -t "$DOCKER_IMAGE_NAME":latest . - - docker push "$DOCKER_IMAGE_NAME":latest - # Get latest git tag and push it as well - - echo "Deploying image $DOCKER_IMAGE_NAME":$(git tag -l "v*" --sort=-creatordate | head -n1) - - docker build -t "$DOCKER_IMAGE_NAME":$(git tag -l "v*" --sort=-creatordate | head -n1) . - - docker push "$DOCKER_IMAGE_NAME":$(git tag -l "v*" --sort=-creatordate | head -n1) - only: - - tags - # Create a Maven Central release on new tags # Thanks to https://1337codersblog.wordpress.com/2019/09/18/automatical-artifact-deployment-to-maven-central-using-gitlab-ci-part-1/ # Define the following CI variables: From 7058ede5be3d4243e64daac4b84396dd818ccfc3 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Tue, 1 Feb 2022 10:39:17 +0100 Subject: [PATCH 081/609] CI: Use Maven Central template --- .gitlab-ci.yml | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b3c8a40b..efe1765d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,6 +17,10 @@ include: - project: 'rml/util/ci-templates' ref: main file: 'Docker-Hub.gitlab-ci.yml' + # Push a build to Maven Central on new tags + - project: 'rml/util/ci-templates' + ref: main + file: 'Maven-Central.gitlab-ci.yml' variables: # This will suppress any download for dependencies and plugins or upload messages which would clutter the console log. @@ -191,24 +195,6 @@ Create Release: fi when: manual -# Create a Maven Central release on new tags -# Thanks to https://1337codersblog.wordpress.com/2019/09/18/automatical-artifact-deployment-to-maven-central-using-gitlab-ci-part-1/ -# Define the following CI variables: -# - MAVEN_REPO_USER: Maven user for deployment -# - MAVEN_REPO_PASSWORD: Maven password for deployment -# - GPG_SECRET_KEY: file with your secret key to sign things -# You can get these credentials by creating an user access token on oss.sonatype.org > Profile > User Token -Maven Central: - stage: deploy - image: maven:3.5.0-jdk-8 - before_script: - - gpg --version - - gpg --import --batch --yes $GPG_SECRET_KEY - script: - - mvn $MAVEN_CLI_OPTS clean deploy -P release -DskipTests=true - only: - - tags - # Generate R2RML test report and send MR to rml.io website # Push options: https://docs.gitlab.com/ee/user/project/push_options.html R2RML Test Report: From b74b4b7804c8ae64e2f419503969425859ac3d47 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Tue, 1 Feb 2022 10:39:59 +0100 Subject: [PATCH 082/609] CHANGELOG: use CI templates --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 651b8125..5fc41f7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Always include xml: prefix (see [issue 144](https://github.com/RMLio/rmlmapper-java/issues/144)) - `normalizeDate` function uses wrong pattern (see [issue 228](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/228)) - CI: only publish `-all.jar` releases on Github (see [issue 145](https://github.com/RMLio/rmlmapper-java/issues/145)) +- CI: make use of templates ### Changed - CSVRecordFactory: Replace Apache CSV library with OpenCSV to be able to differentiate between empty string and null (see [issue 140](https://github.com/RMLio/rmlmapper-java/issues/140)) From 55a127bbc5ee5c7c451d756398b7d12f616f7a22 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Tue, 1 Feb 2022 10:32:23 +0000 Subject: [PATCH 083/609] CHANGELOG: release v4.15.0 --- CHANGELOG.md | 3 +++ buildNumber.properties | 4 ++-- pom.xml | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fc41f7d..17527c75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +## [4.15.0] - 2022-02-01 + ### Fixed - Bump buildnumber during release (see [issue 227](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/227)) - Always include xml: prefix (see [issue 144](https://github.com/RMLio/rmlmapper-java/issues/144)) @@ -470,6 +472,7 @@ and [169](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/169)) - support for accessing remote files (via HTTP GET) - basic support for functions +[4.15.0]: https://github.com/RMLio/rmlmapper-java/compare/v4.14.3...v4.15.0 [4.14.3]: https://github.com/RMLio/rmlmapper-java/compare/v4.14.2...v4.14.3 [4.14.2]: https://github.com/RMLio/rmlmapper-java/compare/v4.14.1...v4.14.2 [4.14.1]: https://github.com/RMLio/rmlmapper-java/compare/v4.14.0...v4.14.1 diff --git a/buildNumber.properties b/buildNumber.properties index 4dd61134..ad7e6c73 100644 --- a/buildNumber.properties +++ b/buildNumber.properties @@ -1,3 +1,3 @@ #maven.buildNumber.plugin properties file -#Mon Nov 22 14:13:55 CET 2021 -buildNumber0=359 +#Tue Feb 01 10:31:54 GMT 2022 +buildNumber0=360 diff --git a/pom.xml b/pom.xml index cf893b3b..bbb7d810 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ be.ugent.rml rmlmapper RMLMapper - 4.14.3 + 4.15.0 The RMLMapper executes RML rules to generate high quality Linked Data from multiple originally (semi-)structured data sources. From ca1ef327cc3cd7d89b2553d10bcf5e35ecd9c587 Mon Sep 17 00:00:00 2001 From: Ben De Meester Date: Thu, 3 Feb 2022 13:02:26 +0000 Subject: [PATCH 084/609] README: add quick start guide Make it clear for people how to easily start using the RMLMapper. --- CHANGELOG.md | 4 ++++ README.md | 18 +++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17527c75..d19f5e8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +### Fixed + +- Clarified Readme for quick start + ## [4.15.0] - 2022-02-01 ### Fixed diff --git a/README.md b/README.md index 52484def..272d49de 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,9 @@ The RMLMapper execute RML rules to generate Linked Data. It is a Java library, which is available via the command line ([API docs online](https://javadoc.io/doc/be.ugent.rml/rmlmapper)). The RMLMapper loads all data in memory, so be aware when working with big datasets. +Want to get started quickly? Check out [Releases](#releases) on where to find the latest CLI build as a jar, +and see [Usage](#cli) on how to use the commandline interface! + ## Table of contents - [Features](#features) @@ -21,13 +24,14 @@ The RMLMapper loads all data in memory, so be aware when working with big datase - [Generating metadata](#generating-metadata) - [Testing](#testing) - [RDBs](#rdbs) -- [Deploy on Central Repository](#deploy-on-central-repository) - [Dependencies](#dependencies) - [Commercial Support](#commercial-support) - [Remarks](#remarks) + - [Typed spreadsheet files](#typed-spreadsheet-files) - [XML file parsing performance](#xml-file-parsing-performance) - [Language tag support](#language-tag-support) - [Duplicate removal and serialization format](#duplicate-removal-and-serialization-format) + - [I have a question! Where can I get help?](#i-have-a-question-where-can-i-get-help) - [Documentation](#documentation) - [UML Diagrams](#uml-diagrams) @@ -67,11 +71,19 @@ The RMLMapper loads all data in memory, so be aware when working with big datase - TPF servers ## Releases -The standalone jar file for every release can be found on the release's page on GitHub. + +The standalone jar file (that has a [commandline interface](#cli)) for every release can be found on the release's page on GitHub. You can find the latest release [here](https://github.com/RMLio/rmlmapper-java/releases/latest). +This is the recommended way to get started with RMLMapper. +Do you want to build from source yourself? Check [Build](#build). ## Build -The RMLMapper is build using Maven: `mvn install`. +The RMLMapper is build using Maven. +As it is also tested against Oracle (check [here](#accessing-oracle-database) for details), +it needs a specific set-up to run all tests. +That's why we recommend to build without testing: `mvn install -DskipTests=true`. +If you want, you can install with tests, and just skip the Oracle tests: `mvn test -Dtest=!Mapper_OracleDB_Test`. + A standalone jar can be found in `/target`. Two jars are found in `/target`: a slim jar without bundled dependencies, and a standalone jar (suffixed with `-all.jar`) with all dependencies bundled. From 701dd679d86108a87ad304f924f1c07612394bd0 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Tue, 8 Feb 2022 16:52:14 +0100 Subject: [PATCH 085/609] pom: upgrade postgresql JDBC driver to 42.3.2 Fixes https://github.com/RMLio/rmlmapper-java/issues/146 --- CHANGELOG.md | 4 ++++ pom.xml | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d19f5e8a..ada1946f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Clarified Readme for quick start +### Changed + +- Upgrade postgresql JDBC driver to 42.3.2 (see [issue 146](https://github.com/RMLio/rmlmapper-java/issues/146)) + ## [4.15.0] - 2022-02-01 ### Fixed diff --git a/pom.xml b/pom.xml index bbb7d810..712dcad5 100644 --- a/pom.xml +++ b/pom.xml @@ -176,9 +176,9 @@ test - postgresql + org.postgresql postgresql - 9.1-901-1.jdbc4 + 42.3.2 From 925e63b4acb9bc55da4c6fb1b5b5e6a5814ad83f Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Sun, 20 Feb 2022 13:54:37 +0100 Subject: [PATCH 086/609] Executor: expose rmlStore Necessary for unittests --- src/main/java/be/ugent/rml/Executor.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/be/ugent/rml/Executor.java b/src/main/java/be/ugent/rml/Executor.java index f1586d2a..2ebbabab 100644 --- a/src/main/java/be/ugent/rml/Executor.java +++ b/src/main/java/be/ugent/rml/Executor.java @@ -464,4 +464,8 @@ public static String getNewBlankNodeID() { public List getTriplesMaps() { return initializer.getTriplesMaps(); } + + public QuadStore getRMLStore() { + return this.rmlStore; + } } \ No newline at end of file From 7b55d2112b562e835f6f9304974a6966afdde193 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Sun, 20 Feb 2022 13:55:02 +0100 Subject: [PATCH 087/609] NAMESPACES: add LDES and TREE --- src/main/java/be/ugent/rml/NAMESPACES.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/be/ugent/rml/NAMESPACES.java b/src/main/java/be/ugent/rml/NAMESPACES.java index dcf4e63f..13c9a3f2 100644 --- a/src/main/java/be/ugent/rml/NAMESPACES.java +++ b/src/main/java/be/ugent/rml/NAMESPACES.java @@ -30,4 +30,6 @@ public class NAMESPACES { public static final String RMLT = "http://semweb.mmlab.be/ns/rml-target#"; public static final String FORMATS = "http://www.w3.org/ns/formats/"; public static final String COMP = "http://semweb.mmlab.be/ns/rml-compression#"; + public static final String LDES = "https://w3id.org/ldes#"; + public static final String TREE = "https://w3id.org/tree#"; } From a28c58163261b5db77df0ca764374d09bd315c4e Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Sun, 20 Feb 2022 13:55:50 +0100 Subject: [PATCH 088/609] target: add LDES Logical Target An LDES Logical Target is a regular RML Logical Target with only some metadata added to encapsulate the exported RDF as an LDES. The LDES ecosystem such as the LDES Server and bucketizers can directly use this even stream thanks to this metadata. --- .../be/ugent/rml/target/LocalFileTarget.java | 13 +- .../rml/target/SPARQLEndpointTarget.java | 12 +- src/main/java/be/ugent/rml/target/Target.java | 8 ++ .../be/ugent/rml/target/TargetFactory.java | 112 +++++++++++++++++- 4 files changed, 138 insertions(+), 7 deletions(-) diff --git a/src/main/java/be/ugent/rml/target/LocalFileTarget.java b/src/main/java/be/ugent/rml/target/LocalFileTarget.java index 7fbacea3..e864e84b 100644 --- a/src/main/java/be/ugent/rml/target/LocalFileTarget.java +++ b/src/main/java/be/ugent/rml/target/LocalFileTarget.java @@ -1,6 +1,8 @@ package be.ugent.rml.target; import be.ugent.rml.access.COMPRESSION; +import be.ugent.rml.store.Quad; +import be.ugent.rml.term.Term; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -8,6 +10,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.util.List; import java.util.zip.GZIPOutputStream; import static org.apache.commons.io.FileUtils.getFile; @@ -21,6 +24,7 @@ public class LocalFileTarget implements Target { private String basePath; private String serializationFormat; private String compression; + private List metadata; private OutputStream outputStream; private static final Logger logger = LoggerFactory.getLogger(LocalFileTarget.class); @@ -30,12 +34,14 @@ public class LocalFileTarget implements Target { * @param basePath the used base path. * @param serializationFormat serialization format to use. * @param compression compression to apply. + * @param metadata additional metadata to add when writing. */ - public LocalFileTarget(String path, String basePath, String serializationFormat, String compression) { + public LocalFileTarget(String path, String basePath, String serializationFormat, String compression, List metadata) { this.path = path; this.basePath = basePath; this.serializationFormat = serializationFormat; this.compression = compression; + this.metadata = metadata; } /** @@ -119,4 +125,9 @@ public void close() { logger.error("Failed to close target: " + e); } } + + @Override + public List getMetadata() { + return this.metadata; + } } \ No newline at end of file diff --git a/src/main/java/be/ugent/rml/target/SPARQLEndpointTarget.java b/src/main/java/be/ugent/rml/target/SPARQLEndpointTarget.java index 50d52e3f..cf1fa789 100644 --- a/src/main/java/be/ugent/rml/target/SPARQLEndpointTarget.java +++ b/src/main/java/be/ugent/rml/target/SPARQLEndpointTarget.java @@ -1,5 +1,7 @@ package be.ugent.rml.target; +import be.ugent.rml.store.Quad; +import be.ugent.rml.term.Term; import org.apache.http.client.HttpResponseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -8,10 +10,12 @@ import java.net.HttpURLConnection; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.util.List; public class SPARQLEndpointTarget implements Target { private final String url; + private final List metadata; private final File tempFile; private static final Logger logger = LoggerFactory.getLogger(SPARQLEndpointTarget.class); private static final int BUFFER_SIZE = 8192; @@ -21,8 +25,9 @@ public class SPARQLEndpointTarget implements Target { * This constructor takes an URL of the SPARQL endpoint as argument. * @param url URL of the SPARQL endpoint */ - public SPARQLEndpointTarget(String url) throws IOException { + public SPARQLEndpointTarget(String url, List metadata) throws IOException { this.url = url; + this.metadata = metadata; this.tempFile = File.createTempFile("rmlmapper-", ".nt"); this.tempFile.deleteOnExit(); } @@ -120,4 +125,9 @@ public void close() { logger.error("Failed to close target: " + e); } } + + @Override + public List getMetadata() { + return this.metadata; + } } \ No newline at end of file diff --git a/src/main/java/be/ugent/rml/target/Target.java b/src/main/java/be/ugent/rml/target/Target.java index 35cb0dc8..7945af6f 100644 --- a/src/main/java/be/ugent/rml/target/Target.java +++ b/src/main/java/be/ugent/rml/target/Target.java @@ -1,7 +1,10 @@ package be.ugent.rml.target; +import be.ugent.rml.store.Quad; + import java.io.IOException; import java.io.OutputStream; +import java.util.List; /** * This interface represents the target of a knowledge graph. @@ -26,4 +29,9 @@ public interface Target { * This method closes the target. */ void close(); + + /** + * This method returns the metadata associated with the target. + */ + List getMetadata(); } diff --git a/src/main/java/be/ugent/rml/target/TargetFactory.java b/src/main/java/be/ugent/rml/target/TargetFactory.java index bd32cfe8..eb227cd1 100644 --- a/src/main/java/be/ugent/rml/target/TargetFactory.java +++ b/src/main/java/be/ugent/rml/target/TargetFactory.java @@ -2,15 +2,19 @@ import be.ugent.rml.NAMESPACES; import be.ugent.rml.Utils; +import be.ugent.rml.store.Quad; import be.ugent.rml.store.QuadStore; import be.ugent.rml.term.Literal; import be.ugent.rml.term.NamedNode; import be.ugent.rml.term.Term; import org.apache.commons.lang3.NotImplementedException; +import org.apache.poi.ss.formula.functions.Na; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; public class TargetFactory { @@ -30,11 +34,14 @@ public TargetFactory(String basePath) { * This method returns a Target instance based on the RML rules in rmlStore. * @param logicalTarget the Logical Target for which the Target needs to be created. * @param rmlStore a QuadStore with RML rules. + * @param outputStore a QuadStore with the RDF triples to write to the target. */ - public Target getTarget(Term logicalTarget, QuadStore rmlStore) throws NotImplementedException, IOException { + public Target getTarget(Term logicalTarget, QuadStore rmlStore, QuadStore outputStore) throws NotImplementedException, IOException { Target target = null; String serializationFormat = "nquads"; String compression = null; + boolean isLDES = false; + List metadata = new ArrayList<>(); // Old Logical Source reference is supported for Logical Targets as well for backwards compatibility if (logicalTarget instanceof Literal) { @@ -55,7 +62,7 @@ else if (location.endsWith(".jsonld")) { else { throw new NotImplementedException("Serialization format for " + location + " not implemented!"); } - target = new LocalFileTarget(location, this.basePath, serializationFormat, null); + target = new LocalFileTarget(location, this.basePath, serializationFormat, null, metadata); return target; } @@ -106,6 +113,101 @@ else if (location.endsWith(".jsonld")) { logger.debug("Compression disabled"); } + // Detect LDES EventStreamTarget + List types = Utils.getObjectsFromQuads(rmlStore.getQuads(logicalTarget, + new NamedNode(NAMESPACES.RDF + "type"), null)); + for (Term type: types) { + // Target has LDES features, read them + if (type.getValue().equals(NAMESPACES.LDES + "EventStreamTarget")) { + logger.debug("Found LDES EventStreamTarget"); + isLDES = true; + Term iri; + Term retention_iri; + Term eventstream_iri; + Term versionOfPathObj = null; + Term timestampPathObj = null; + Term retentionPolicyType; + Term retentionPolicyAmount; + + // Required LDES IRI + try { + iri = Utils.getObjectsFromQuads(rmlStore.getQuads(logicalTarget, + new NamedNode(NAMESPACES.LDES + "baseIRI"), null)).get(0); + retention_iri = new NamedNode(iri.getValue() + "#retention"); + eventstream_iri = new NamedNode(iri.getValue() + "#eventstream"); + logger.debug("LDES base IRI: " + iri.getValue()); + } + catch (IndexOutOfBoundsException e) { + throw new IllegalArgumentException("No base IRI specified for LDES!"); + } + + // Required SHACL shape + try { + Term shape = Utils.getObjectsFromQuads(rmlStore.getQuads(logicalTarget, + new NamedNode(NAMESPACES.TREE + "shape"), null)).get(0); + logger.debug("SHACL shape: " + shape.getValue()); + + // TODO: Handle embedded SHACL shapes in RML mapping rules as well. + metadata.add(new Quad(eventstream_iri, new NamedNode(NAMESPACES.TREE + "shape"), shape)); + } + catch (IndexOutOfBoundsException e) { + logger.error("No SHACL shape specified for LDES!"); + } + + // LDES tree:view + metadata.add(new Quad(eventstream_iri, new NamedNode(NAMESPACES.RDF + "type"), + new NamedNode(NAMESPACES.LDES + "EventStream"))); + List subjects = new ArrayList(new HashSet(Utils.getSubjectsFromQuads(outputStore.getQuads(null, null, null)))); + for (Term s: subjects) { + metadata.add(new Quad(eventstream_iri, new NamedNode(NAMESPACES.TREE + "member"), s)); + } + metadata.add(new Quad(eventstream_iri, new NamedNode(NAMESPACES.TREE + "view"), iri)); + + // Optional versionOf path + try { + versionOfPathObj = Utils.getObjectsFromQuads(rmlStore.getQuads(logicalTarget, + new NamedNode(NAMESPACES.LDES + "versionOfPath"), null)).get(0); + } + catch (IndexOutOfBoundsException e) { + logger.debug("No versionOfPath found"); + } + + // Optional timestamp path + try { + timestampPathObj = Utils.getObjectsFromQuads(rmlStore.getQuads(logicalTarget, + new NamedNode(NAMESPACES.LDES + "timestampPath"), null)).get(0); + } + catch (IndexOutOfBoundsException e) { + logger.debug("No timestampPath found"); + } + + // Optional retention policy + try { + Term retentionPolicy = Utils.getObjectsFromQuads(rmlStore.getQuads(logicalTarget, + new NamedNode(NAMESPACES.LDES + "retentionPolicy"), null)).get(0); + // TODO parse retention policies + throw new NotImplementedException("Parsing LDES retention policies not implemented yet!"); + } + catch (IndexOutOfBoundsException e) { + logger.debug("No retention policy specified, default to latest only"); + retentionPolicyType = new NamedNode(NAMESPACES.LDES + "LatestVersionSubject"); + retentionPolicyAmount = new Literal("1"); + } + + metadata.add(new Quad(iri, new NamedNode(NAMESPACES.LDES + "retentionPolicy"), retention_iri)); + metadata.add(new Quad(retention_iri, new NamedNode(NAMESPACES.RDF + "type"), retentionPolicyType)); + metadata.add(new Quad(retention_iri, new NamedNode(NAMESPACES.LDES + "amount"), retentionPolicyAmount)); + if (versionOfPathObj != null) { + metadata.add(new Quad(retention_iri, new NamedNode(NAMESPACES.LDES + "versionOfPath"), versionOfPathObj)); + } + if (timestampPathObj != null) { + metadata.add(new Quad(retention_iri, new NamedNode(NAMESPACES.LDES + "timestampPath"), timestampPathObj)); + } + + break; + } + } + // Build target if (!targets.isEmpty()) { Term t = targets.get(0); @@ -123,7 +225,7 @@ else if (location.endsWith(".jsonld")) { new NamedNode(NAMESPACES.VOID + "dataDump"), null)).get(0).getValue(); location = location.replace("file://", ""); // Local file starts with file:// logger.debug("VoID datadump location: " + location); - target = new LocalFileTarget(location, this.basePath, serializationFormat, compression); + target = new LocalFileTarget(location, this.basePath, serializationFormat, compression, metadata); break; } case NAMESPACES.DCAT + "Dataset": { // DCAT Dataset @@ -132,7 +234,7 @@ else if (location.endsWith(".jsonld")) { new NamedNode(NAMESPACES.DCAT + "dataDump"), null)).get(0).getValue(); location = location.replace("file://", ""); // Local file starts with file:// logger.debug("DCAT datadump location: " + location); - target = new LocalFileTarget(location, this.basePath, serializationFormat, compression); + target = new LocalFileTarget(location, this.basePath, serializationFormat, compression, metadata); break; } case NAMESPACES.SD + "Service": { // SPARQL Service @@ -150,7 +252,7 @@ else if (location.endsWith(".jsonld")) { } // Try to instantiate a SPARQL endpoint - target = new SPARQLEndpointTarget(endpoint); + target = new SPARQLEndpointTarget(endpoint, metadata); break; } default: { From 4a9bbd7124786ad10ef865cb6f0059f140835416 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Sun, 20 Feb 2022 13:58:48 +0100 Subject: [PATCH 089/609] test: add LDES Logical Target tests --- src/test/java/be/ugent/rml/Mapper_Target.java | 32 + .../java/be/ugent/rml/Mapper_WoT_Test.java | 21 + src/test/java/be/ugent/rml/TestCore.java | 10 + .../web-of-things/ldes/defaults/bluebike.ttl | 1142 +++++++++ .../web-of-things/ldes/defaults/mapping.ttl | 159 ++ .../ldes/defaults/out-default.nq | 0 .../ldes/defaults/out-local-file.nq | 1073 ++++++++ .../web-of-things/ldes/paths/mapping.ttl | 161 ++ .../web-of-things/ldes/paths/out-default.nq | 0 .../ldes/paths/out-local-file.nq | 1075 ++++++++ .../ldes/private-security-data.ttl | 2 + .../web-of-things/ldes/stations.jsonld | 2178 +++++++++++++++++ 12 files changed, 5853 insertions(+) create mode 100644 src/test/resources/web-of-things/ldes/defaults/bluebike.ttl create mode 100644 src/test/resources/web-of-things/ldes/defaults/mapping.ttl create mode 100644 src/test/resources/web-of-things/ldes/defaults/out-default.nq create mode 100644 src/test/resources/web-of-things/ldes/defaults/out-local-file.nq create mode 100644 src/test/resources/web-of-things/ldes/paths/mapping.ttl create mode 100644 src/test/resources/web-of-things/ldes/paths/out-default.nq create mode 100644 src/test/resources/web-of-things/ldes/paths/out-local-file.nq create mode 100644 src/test/resources/web-of-things/ldes/private-security-data.ttl create mode 100644 src/test/resources/web-of-things/ldes/stations.jsonld diff --git a/src/test/java/be/ugent/rml/Mapper_Target.java b/src/test/java/be/ugent/rml/Mapper_Target.java index 51b19161..b0f572ad 100644 --- a/src/test/java/be/ugent/rml/Mapper_Target.java +++ b/src/test/java/be/ugent/rml/Mapper_Target.java @@ -86,6 +86,38 @@ public void evaluate_local_file_target_dcat() throws Exception { webApi.stop(0); } + @Test + public void evaluate_ldes_default_target_dcat() throws Exception { + // Create Web API + HttpServer webApi = HttpServer.create(new InetSocketAddress(8000), 0); + webApi.createContext("/bluebike", new Mapper_WoT_Test.BlueBikeStationHandler()); + webApi.setExecutor(null); // creates a default executor + webApi.start(); + + HashMap outPaths = new HashMap(); + outPaths.put(new NamedNode("http://example.com/rules/#TargetDump"), "web-of-things/ldes/defaults/out-local-file.nq"); + outPaths.put(new NamedNode("rmlmapper://default.store"), "web-of-things/ldes/defaults/out-default.nq"); + doMapping("web-of-things/ldes/defaults/mapping.ttl", outPaths, "./web-of-things/ldes/private-security-data.ttl"); + + webApi.stop(0); + } + + @Test + public void evaluate_ldes_paths_target_dcat() throws Exception { + // Create Web API + HttpServer webApi = HttpServer.create(new InetSocketAddress(8000), 0); + webApi.createContext("/bluebike", new Mapper_WoT_Test.BlueBikeStationHandler()); + webApi.setExecutor(null); // creates a default executor + webApi.start(); + + HashMap outPaths = new HashMap(); + outPaths.put(new NamedNode("http://example.com/rules/#TargetDump"), "web-of-things/ldes/paths/out-local-file.nq"); + outPaths.put(new NamedNode("rmlmapper://default.store"), "web-of-things/ldes/paths/out-default.nq"); + doMapping("web-of-things/ldes/paths/mapping.ttl", outPaths, "./web-of-things/ldes/private-security-data.ttl"); + + webApi.stop(0); + } + @Test public void evaluate_nquads_serialization() throws Exception { // Create Web API diff --git a/src/test/java/be/ugent/rml/Mapper_WoT_Test.java b/src/test/java/be/ugent/rml/Mapper_WoT_Test.java index d80d9e5f..78c7ba0c 100644 --- a/src/test/java/be/ugent/rml/Mapper_WoT_Test.java +++ b/src/test/java/be/ugent/rml/Mapper_WoT_Test.java @@ -81,6 +81,27 @@ public void handle(HttpExchange t) throws IOException { } } + static class BlueBikeStationHandler implements HttpHandler { + @Override + public void handle(HttpExchange t) throws IOException { + String response = "couldn't load iRail stations JSON file"; + try { + response = Utils.fileToString(Utils.getFile("./web-of-things/ldes/stations.jsonld")); + } catch (IOException ex) { + ex.printStackTrace(); + } + List contentType = new ArrayList<>(); + contentType.add("application/json"); + + // Return stations if not redirected + t.getResponseHeaders().put("Content-Type", contentType); + t.sendResponseHeaders(200, response.length()); + OutputStream os = t.getResponseBody(); + os.write(response.getBytes()); + os.close(); + } + } + static class IRailStationHandler1 implements HttpHandler { @Override public void handle(HttpExchange t) throws IOException { diff --git a/src/test/java/be/ugent/rml/TestCore.java b/src/test/java/be/ugent/rml/TestCore.java index b8c48a53..f131ad0d 100644 --- a/src/test/java/be/ugent/rml/TestCore.java +++ b/src/test/java/be/ugent/rml/TestCore.java @@ -7,6 +7,8 @@ import be.ugent.rml.store.Quad; import be.ugent.rml.store.QuadStore; import be.ugent.rml.store.QuadStoreFactory; +import be.ugent.rml.target.Target; +import be.ugent.rml.target.TargetFactory; import be.ugent.rml.term.NamedNode; import be.ugent.rml.term.Term; import org.eclipse.rdf4j.rio.RDFFormat; @@ -251,7 +253,9 @@ void doMapping(Executor executor, String outPath) throws Exception { */ void doMapping(Executor executor, HashMap outPaths) throws Exception { logger.debug("Comparing target outputs"); + TargetFactory targetFactory = new TargetFactory("http://example.org/rules/"); HashMap results = executor.executeV5(null); + for (Map.Entry entry: outPaths.entrySet()) { Term target = entry.getKey(); String outPath = entry.getValue(); @@ -260,6 +264,12 @@ void doMapping(Executor executor, HashMap outPaths) throws Excepti logger.debug("\tSize:" + results.get(target).size()); results.get(target).removeDuplicates(); + // Targets may have additional metadata that needs to be included such as LDES encapsulation + if (!target.getValue().equals("rmlmapper://default.store")) { + Target t = targetFactory.getTarget(target, executor.getRMLStore(), results.get(target)); + results.get(target).addQuads(t.getMetadata()); + } + compareStores(filePathToStore(outPath), results.get(target)); } } diff --git a/src/test/resources/web-of-things/ldes/defaults/bluebike.ttl b/src/test/resources/web-of-things/ldes/defaults/bluebike.ttl new file mode 100644 index 00000000..f451af12 --- /dev/null +++ b/src/test/resources/web-of-things/ldes/defaults/bluebike.ttl @@ -0,0 +1,1142 @@ + + a ; + 11; + 11; + 0; + ; + "2021-10-12T06:20:08.208Z"^^; + ; + "50.942901611328"^^; + "4.038559913635"^^; + "Aalst" . + + a ; + 6; + 6; + 0; + ; + "2021-10-12T06:20:44.789Z"^^; + ; + "51.092498779297"^^; + "3.449820041656"^^; + "Aalter" . + + a ; + 9; + 14; + 5; + ; + "2021-10-12T06:20:12.927Z"^^; + ; + "50.984699249268"^^; + "4.824250221252"^^; + "Aarschot" . + + a ; + 51; + 60; + 9; + ; + "2021-10-12T06:20:11.221Z"^^; + ; + "51.199600219727"^^; + "4.431789875031"^^; + "Antwerpen-Berchem" . + + a + ; + 31; + 43; + 12; + ; + "2021-10-12T06:20:06.338Z"^^; + ; + "51.217498779297"^^; + "4.421050071716"^^; + "Antwerpen-Centraal" . + + a + ; + 5; + 5; + 0; + ; + "2021-10-12T06:21:00.798Z"^^; + ; + "51.245498657227"^^; + "4.424610137939"^^; + "Antwerpen-Luchtbal" . + + + a ; + 7; + 7; + 0; + ; + "2021-10-12T06:21:01.494Z"^^; + ; + "51.2617"^^; + "4.42755"^^; + "Antwerpen-Noorderdokken" . + + a ; + 8; + 8; + 0; + ; + "2021-10-12T06:20:51.829Z"^^; + ; + "51.198001861572"^^; + "4.390810012817"^^; + "Antwerpen-Zuid" . + + a ; + 7; + 10; + 3; + ; + "2021-10-12T06:20:09.208Z"^^; + ; + "50.906700134277"^^; + "4.207960128784"^^; + "Asse" . + + a ; + 5; + 5; + 0; + ; + "2021-10-12T06:20:09.940Z"^^; + ; + "50.455600738525"^^; + "3.944180011749"^^; + "Bergen" . + + a ; + 10; + 11; + 1; + ; + "2021-10-12T06:20:55.643Z"^^; + ; + "51.208499908447"^^; + "4.260260105133"^^; + "Beveren" . + + a ; + 6; + 7; + 1; + ; + "2021-10-12T06:21:14.201Z"^^; + ; + "51.3119"^^; + "3.1338"^^; + "Blankenberge" . + + + a ; + 5; + 5; + 0; + ; + "2021-10-12T06:20:44.070Z"^^; + ; + "51.16849899292"^^; + "4.494080066681"^^; + "Boechout P+R Capenberg" . + + a ; + 6; + 6; + 0; + ; + "2021-10-12T06:21:05.822Z"^^; + ; + "51.165508"^^; + "4.746427"^^; + "Bouwel" . + + a ; + 34; + 63; + 29; + ; + "2021-10-12T06:20:10.484Z"^^; + ; + "51.196998596191"^^; + "3.218539953232"^^; + "Brugge" . + + a ; + 17; + 27; + 10; + ; + "2021-10-12T06:20:54.646Z"^^; + ; + "51.196800231934"^^; + "3.216079950333"^^; + "Brugge (Kamgebouw)" . + + a ; + 13; + 14; + 1; + ; + "2021-10-12T06:20:07.205Z"^^; + ; + "50.844898223877"^^; + "4.35586977005"^^; + "Brussel-Centraal" . + + a ; + 4; + 6; + 2; + ; + "2021-10-12T06:20:13.845Z"^^; + ; + "50.838199615479"^^; + "4.372889995575"^^; + "Brussel-Luxemburg" . + + a ; + 4; + 6; + 2; + ; + "2021-10-12T06:21:19.556Z"^^; + ; + "50.835529468767"^^; + "4.333756181866"^^; + "Brussel-Zuid" . + + a ; + 27; + 28; + 1; + ; + "2021-10-12T06:20:14.572Z"^^; + ; + "50.86009979248"^^; + "4.360489845276"^^; + "Brussel Noord" . + + a ; + 8; + 8; + 0; + ; + "2021-10-12T06:21:13.294Z"^^; + ; + "51.077376"^^; + "2.601611"^^; + "De Panne" . + + a ; + 4; + 62; + 58; + ; + "2021-10-12T06:20:15.386Z"^^; + ; + "50.978199005127"^^; + "3.534009933472"^^; + "Deinze" . + + a ; + 9; + 11; + 2; + ; + "2021-10-12T06:20:45.522Z"^^; + ; + "50.982398986816"^^; + "3.525919914246"^^; + "Deinze Leiespiegel" . + + a ; + 22; + 27; + 5; + ; + "2021-10-12T06:20:16.315Z"^^; + ; + "51.023498535156"^^; + "4.101920127869"^^; + "Dendermonde" . + + a ; + 4; + 4; + 0; + ; + "2021-10-12T06:21:02.217Z"^^; + ; + "50.889099121094"^^; + "4.447710037231"^^; + "Diegem" . + + a ; + 10; + 12; + 2; + ; + "2021-10-12T06:20:17.089Z"^^; + ; + "50.993000030518"^^; + "5.051129817963"^^; + "Diest" . + + a ; + 1; + 11; + 10; + ; + "2021-10-12T06:20:34.600Z"^^; + ; + "51.180801391602"^^; + "3.575040102005"^^; + "Eeklo" . + + a + ; + 3; + 4; + 1; + ; + "2021-10-12T06:20:59.366Z"^^; + ; + "51.284198760986"^^; + "4.43501996994"^^; + "Ekeren (perron 1)" . + + a + ; + 7; + 7; + 0; + ; + "2021-10-12T06:21:00.100Z"^^; + ; + "51.281398773193"^^; + "4.434390068054"^^; + "Ekeren (perron 2)" . + + a ; + 7; + 8; + 1; + ; + "2021-10-12T06:21:11.779Z"^^; + ; + "51.462176"^^; + "4.451532"^^; + "Essen" . + + a ; + 7; + 7; + 0; + ; + "2021-10-12T06:20:57.946Z"^^; + ; + "50.929401397705"^^; + "3.64013004303"^^; + "Gavere-Asper" . + + a ; + 4; + 4; + 0; + ; + "2021-10-12T06:20:58.653Z"^^; + ; + "50.930500030518"^^; + "3.654690027237"^^; + "Gavere mobipunt" . + + a ; + 8; + 15; + 7; + ; + "2021-10-12T06:20:35.295Z"^^; + ; + "51.168800354004"^^; + "4.989160060883"^^; + "Geel" . + + a ; + 5; + 6; + 1; + ; + "2021-10-12T06:20:17.608Z"^^; + ; + "50.967098236084"^^; + "5.496369838715"^^; + "Genk" . + + a ; + 31; + 41; + 10; + ; + "2021-10-12T06:20:18.137Z"^^; + ; + "51.055999755859"^^; + "3.740159988403"^^; + "Gent-Dampoort" . + + + a ; + 16; + 26; + 10; + ; + "2021-10-12T06:20:53.258Z"^^; + ; + "51.035999298096"^^; + "3.711899995804"^^; + "Gent-Sint-Pieters (M. Hendrikaplein)" . + + + a ; + 52; + 70; + 18; + ; + "2021-10-12T06:20:18.962Z"^^; + ; + "51.034900665283"^^; + "3.709640026093"^^; + "Gent-Sint-Pieters (St.-Denijslaan)" . + + a ; + 2; + 2; + 0; + ; + "2021-10-12T06:20:39.838Z"^^; + ; + "50.770500183105"^^; + "3.872560024261"^^; + "Geraardsbergen " . + + a ; + 7; + 8; + 1; + ; + "2021-10-12T06:21:06.544Z"^^; + ; + "50.966578"^^; + "4.613315"^^; + "Haacht" . + + a ; + 7; + 12; + 5; + ; + "2021-10-12T06:20:19.671Z"^^; + ; + "50.7333984375"^^; + "4.239940166473"^^; + "Halle" . + + a ; + 8; + 9; + 1; + ; + "2021-10-12T06:20:57.256Z"^^; + ; + "50.855701446533"^^; + "3.31374001503"^^; + "Harelbeke" . + + a ; + 35; + 59; + 24; + ; + "2021-10-12T06:20:20.529Z"^^; + ; + "50.930999755859"^^; + "5.32742023468"^^; + "Hasselt" . + + a ; + 11; + 12; + 1; + ; + "2021-10-12T06:20:48.339Z"^^; + ; + "51.365100860596"^^; + "4.460559844971"^^; + "Heide" . + + a ; + 5; + 5; + 0; + ; + "2021-10-12T06:21:15.934Z"^^; + ; + "51.334168"^^; + "3.239668"^^; + "Heist" . + + a ; + 12; + 12; + 0; + ; + "2021-10-12T06:20:36.712Z"^^; + ; + "51.074199676514"^^; + "4.708779811859"^^; + "Heist-op-den-Berg" . + + a ; + 11; + 12; + 1; + ; + "2021-10-12T06:20:21.475Z"^^; + ; + "51.180999755859"^^; + "4.828859806061"^^; + "Herentals" . + + a ; + 8; + 8; + 0; + ; + "2021-10-12T06:21:03.654Z"^^; + ; + "51.03833"^^; + "5.281699"^^; + "Heusden" . + + a ; + 7; + 9; + 2; + ; + "2021-10-12T06:20:56.370Z"^^; + ; + "50.862598419189"^^; + "4.695439815521"^^; + "Heverlee " . + + a ; + 7; + 7; + 0; + ; + "2021-10-12T06:21:10.807Z"^^; + ; + "51.1541195"^^; + "4.464733"^^; + "Hove" . + + a ; + 5; + 10; + 5; + ; + "2021-10-12T06:20:41.845Z"^^; + ; + "50.848400115967"^^; + "2.876270055771"^^; + "Ieper" . + + a ; + 6; + 8; + 2; + ; + "2021-10-12T06:20:52.530Z"^^; + ; + "50.9208984375"^^; + "3.214819908142"^^; + "Izegem" . + + a ; + 7; + 8; + 1; + ; + "2021-10-12T06:21:16.629Z"^^; + ; + "51.34018"^^; + "3.28454"^^; + "Knokke" . + + a ; + 5; + 7; + 2; + ; + "2021-10-12T06:20:49.729Z"^^; + ; + "51.134300231934"^^; + "4.476119995117"^^; + "Kontich-Lint" . + + a ; + 28; + 47; + 19; + ; + "2021-10-12T06:20:21.952Z"^^; + ; + "50.823799133301"^^; + "3.263269901276"^^; + "Kortrijk" . + + a ; + 8; + 8; + 0; + ; + "2021-10-12T06:21:18.066Z"^^; + ; + "51.064208"^^; + "3.575647"^^; + "Landegem" . + + a ; + 7; + 8; + 1; + ; + "2021-10-12T06:21:12.535Z"^^; + ; + "50.74829"^^; + "5.079402"^^; + "Landen" . + + + a ; + 16; + 24; + 8; + ; + "2021-10-12T06:20:53.955Z"^^; + ; + "50.882499694824"^^; + "4.716639995575"^^; + "Leuven (Kop van Kessel-Lo)" . + + a + ; + 57; + 86; + 29; + ; + "2021-10-12T06:20:22.664Z"^^; + ; + "50.87979888916"^^; + "4.715939998627"^^; + "Leuven (Tiensevest)" . + + a ; + 21; + 26; + 5; + ; + "2021-10-12T06:20:23.383Z"^^; + ; + "51.135799407959"^^; + "4.56152009964"^^; + "Lier" . + + + a ; + 8; + 8; + 0; + ; + "2021-10-12T06:20:43.366Z"^^; + ; + "51.125301361084"^^; + "4.573339939117"^^; + "Lier P+R De Mol" . + + a ; + 7; + 7; + 0; + ; + "2021-10-12T06:20:42.586Z"^^; + ; + "51.128299713135"^^; + "4.583220005035"^^; + "Lier Veemarkt" . + + a ; + 13; + 17; + 4; + ; + "2021-10-12T06:20:24.095Z"^^; + ; + "51.107398986816"^^; + "3.986469984055"^^; + "Lokeren" . + + a ; + 6; + 6; + 0; + ; + "2021-10-12T06:21:10.091Z"^^; + ; + "51.21177"^^; + "5.312579"^^; + "Lommel" . + + a ; + 0; + 4; + 4; + ; + "2021-10-12T06:20:24.831Z"^^; + ; + "50.625801086426"^^; + "5.565639972687"^^; + "Luik" . + + a ; + 47; + 76; + 29; + ; + "2021-10-12T06:20:25.537Z"^^; + ; + "51.0178"^^; + "4.48299"^^; + "Mechelen" . + + + a ; + 17; + 20; + 3; + ; + "2021-10-12T06:20:38.128Z"^^; + ; + "51.030200958252"^^; + "4.489570140839"^^; + "Mechelen-Nekkerspoel" . + + a ; + 6; + 7; + 1; + ; + "2021-10-12T06:20:49.040Z"^^; + ; + "51.029300689697"^^; + "4.484320163727"^^; + "Mechelen Veemarkt" . + + a ; + 8; + 8; + 0; + ; + "2021-10-12T06:21:05.094Z"^^; + ; + "50.932948204381"^^; + "4.329820851913"^^; + "Meise" . + + a ; + 10; + 15; + 5; + ; + "2021-10-12T06:20:26.232Z"^^; + ; + "51.190498352051"^^; + "5.115129947662"^^; + "Mol" . + + a ; + 7; + 7; + 0; + ; + "2021-10-12T06:21:20.277Z"^^; + ; + "51.18119"^^; + "5.112421"^^; + "Mol - De Nete" . + + a ; + 4; + 6; + 2; + ; + "2021-10-12T06:20:26.934Z"^^; + ; + "51.171199798584"^^; + "4.455999851227"^^; + "Mortsel-Oude-God" . + + a ; + 6; + 6; + 0; + ; + "2021-10-12T06:20:27.637Z"^^; + ; + "50.468601226807"^^; + "4.862239837646"^^; + "Namen" . + + a ; + 7; + 10; + 3; + ; + "2021-10-12T06:20:36.006Z"^^; + ; + "50.839500427246"^^; + "4.026179790497"^^; + "Ninove" . + + a ; + 10; + 10; + 0; + ; + "2021-10-12T06:20:47.633Z"^^; + ; + "51.35710144043"^^; + "4.632410049438"^^; + "Noorderkempen" . + + a ; + 53; + 69; + 16; + ; + "2021-10-12T06:20:28.362Z"^^; + ; + "51.228298187256"^^; + "2.925649881363"^^; + "Oostende" . + + a ; + 0; + 2; + 2; + ; + "2021-10-12T06:20:39.151Z"^^; + ; + "50.851299285889"^^; + "3.601880073547"^^; + "Oudenaarde " . + + a ; + 7; + 7; + 0; + ; + "2021-10-12T06:20:46.915Z"^^; + ; + "50.854499816895"^^; + "2.735719919205"^^; + "Poperinge" . + + a ; + 14; + 27; + 13; + ; + "2021-10-12T06:20:29.065Z"^^; + ; + "50.949001312256"^^; + "3.130480051041"^^; + "Roeselare" . + + a + ; + 9; + 9; + 0; + ; + "2021-10-12T06:21:17.346Z"^^; + ; + "50.747786"^^; + "4.361626"^^; + "Sint-Genesius-Rode" . + + a ; + 15; + 30; + 15; + ; + "2021-10-12T06:20:29.783Z"^^; + ; + "51.171398162842"^^; + "4.143700122833"^^; + "Sint-Niklaas" . + + a ; + 9; + 9; + 0; + ; + "2021-10-12T06:20:30.506Z"^^; + ; + "50.817401885986"^^; + "5.176330089569"^^; + "Sint-Truiden" . + + a ; + 5; + 5; + 0; + ; + "2021-10-12T06:21:07.957Z"^^; + ; + "51.125867"^^; + "4.221333"^^; + "Temse" . + + a ; + 7; + 7; + 0; + ; + "2021-10-12T06:20:50.446Z"^^; + ; + "50.990600585938"^^; + "3.329819917679"^^; + "Tielt" . + + a ; + 5; + 13; + 8; + ; + "2021-10-12T06:20:46.217Z"^^; + ; + "50.808200836182"^^; + "4.92561006546"^^; + "Tienen " . + + a ; + 8; + 8; + 0; + ; + "2021-10-12T06:20:37.434Z"^^; + ; + "50.78450012207"^^; + "5.473400115967"^^; + "Tongeren" . + + a ; + 3; + 4; + 1; + ; + "2021-10-12T06:20:31.284Z"^^; + ; + "51.064800262451"^^; + "3.105509996414"^^; + "Torhout" . + + a ; + 9; + 13; + 4; + ; + "2021-10-12T06:20:32.245Z"^^; + ; + "51.322101593018"^^; + "4.937900066376"^^; + "Turnhout" . + + a ; + 20; + 40; + 20; + ; + "2021-10-12T06:20:33.067Z"^^; + ; + "50.924701690674"^^; + "4.433000087738"^^; + "Vilvoorde" . + + + a ; + 6; + 7; + 1; + ; + "2021-10-12T06:21:18.772Z"^^; + ; + "50.923187"^^; + "4.413591"^^; + "Vilvoorde Hoppin De Kassei" . + + a ; + 12; + 16; + 4; + ; + "2021-10-12T06:20:33.881Z"^^; + ; + "50.891799926758"^^; + "3.424420118332"^^; + "Waregem" . + + a + ; + 4; + 4; + 0; + ; + "2021-10-12T06:21:02.941Z"^^; + ; + "50.881999969482"^^; + "3.426719903946"^^; + "Waregem Expo P+R" . + + a ; + 8; + 8; + 0; + ; + "2021-10-12T06:21:09.366Z"^^; + ; + "50.810926"^^; + "3.182521"^^; + "Wevelgem" . + + a ; + 8; + 8; + 0; + ; + "2021-10-12T06:20:51.139Z"^^; + ; + "50.885200500488"^^; + "4.470870018005"^^; + "Zaventem" . + + a ; + 2; + 2; + 0; + ; + "2021-10-12T06:21:14.998Z"^^; + ; + "51.32651"^^; + "3.195476"^^; + "Zeebrugge-dorp" . + + a ; + 6; + 7; + 1; + ; + "2021-10-12T06:21:08.664Z"^^; + ; + "51.073108"^^; + "4.041885"^^; + "Zele" . + + a ; + 8; + 8; + 0; + ; + "2021-10-12T06:21:04.377Z"^^; + ; + "51.033617"^^; + "5.329922"^^; + "Zolder" . + + a ; + 7; + 8; + 1; + ; + "2021-10-12T06:21:07.245Z"^^; + ; + "50.86948"^^; + "3.814341"^^; + "Zottegem " . + + + . + + a ; + , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , + , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , , + , + , , + , , + , , + , ; + . + + a ; + "1" . diff --git a/src/test/resources/web-of-things/ldes/defaults/mapping.ttl b/src/test/resources/web-of-things/ldes/defaults/mapping.ttl new file mode 100644 index 00000000..8b1c6f8e --- /dev/null +++ b/src/test/resources/web-of-things/ldes/defaults/mapping.ttl @@ -0,0 +1,159 @@ +# +# RML mapping rules for https://datapiloten.be/bluebike/availabilities.geojson +# (c) Dylan Van Assche (2021) +# IDLab - Ghent University - imec +# +@prefix rr: . +@prefix ex: . +@prefix rdfs: . +@prefix xsd: . +@prefix rml: . +@prefix rmlt: . +@prefix comp: . +@prefix formats: . +@prefix ql: . +@prefix rdf: . +@prefix geo: . +@prefix fnml: . +@prefix fno: . +@prefix grel: . +@prefix dcterms: . +@prefix tree: . +@prefix ldes: . +@prefix purl: . +@prefix td: . +@prefix htv: . +@prefix hctl: . +@base . + +<#TargetDump> a rmlt:LogicalTarget, ldes:EventStreamTarget; + rmlt:target [ a void:Dataset; + void:dataDump ; + ]; + rmlt:serialization formats:N-Triples; + ldes:baseIRI ; + tree:shape ; +. + +<#WoTWebResource> a td:PropertyAffordance; + td:hasForm [ + # URL and content type + hctl:hasTarget "http://localhost:8000/bluebike"; + hctl:forContentType "application/json"; + # Read only + hctl:hasOperationType td:readproperty ; + # Set HTTP method and headers + htv:methodName "GET"; + htv:headers ([ + htv:fieldName "User-Agent"; + htv:fieldValue "RML Processor"; + ]); + ]; +. + +<#WoTWebAPI> a td:Thing; + td:hasPropertyAffordance <#WoTWebResource>; +. + +<#StationsTriplesMap> a rr:TriplesMap; + rml:logicalSource [ + rml:source <#WoTWebResource>; + rml:referenceFormulation ql:JSONPath; + rml:iterator "$.features[*]" + ]; + + # Unique IRI generation: $stationId#$generatedAtTime + rr:subjectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant grel:array_join ] + ]; + rr:predicateObjectMap [ + rr:predicate grel:p_array_a ; + rr:objectMap [ rml:reference "properties.@id" ] + ]; + rr:predicateObjectMap [ + rr:predicate grel:p_array_a ; + rr:objectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant grel:array_join ] + ]; + + rr:predicateObjectMap [ + rr:predicate grel:p_array_a ; + rr:objectMap [ rr:constant "#" ] + ]; + + rr:predicateObjectMap [ + rr:predicate grel:p_array_a ; + rr:objectMap [ rml:reference "generatedAtTime" ] + ]; + ]; + ]; + ]; + ]; + rr:termType rr:IRI; + rml:logicalTarget <#TargetDump>; + ]; + + # rdf:type + rr:predicateObjectMap [ + rr:predicate rdf:type ; + rr:objectMap [ rml:reference "properties.@type"; rr:termType rr:IRI ] + ]; + + # BlueBike station: name + rr:predicateObjectMap [ + rr:predicate rdfs:label ; + rr:objectMap [ rml:reference "properties.name"; rr:datatype xsd:string; ] + ]; + + # BlueBike station: available bicycles + rr:predicateObjectMap [ + rr:predicate ex:bikesAvailable ; + rr:objectMap [ rml:reference "properties.bikes_available"; rr:datatype xsd:integer; ] + ]; + + # BlueBike station: maximum capacity + rr:predicateObjectMap [ + rr:predicate ex:capacity ; + rr:objectMap [ rml:reference "properties.capacity"; rr:datatype xsd:integer; ] + ]; + + # BlueBike station: available docks + rr:predicateObjectMap [ + rr:predicate ex:docksAvailable ; + rr:objectMap [ rml:reference "properties.docks_available"; rr:datatype xsd:integer; ] + ]; + + # Nearby NMBS station + rr:predicateObjectMap [ + rr:predicate ex:nearby ; + rr:objectMap [ rml:reference "properties.nearby"; rr:termType rr:IRI; ] + ]; + + # Geo location + rr:predicateObjectMap [ + rr:predicate geo:latitude ; + rr:objectMap [ rml:reference "properties.latitude"; rr:datatype xsd:float; ] + ]; + + rr:predicateObjectMap [ + rr:predicate geo:longitude ; + rr:objectMap [ rml:reference "properties.longitude"; rr:datatype xsd:float; ] + ]; + + # Versioning + rr:predicateObjectMap [ + rr:predicate dcterms:created ; + rr:objectMap [ rml:reference "generatedAtTime"; rr:datatype xsd:dateTime; ] + ]; + + rr:predicateObjectMap [ + rr:predicate dcterms:isVersionOf ; + rr:objectMap [ rml:reference "properties.@id"; rr:termType rr:IRI; ] + ]; +. diff --git a/src/test/resources/web-of-things/ldes/defaults/out-default.nq b/src/test/resources/web-of-things/ldes/defaults/out-default.nq new file mode 100644 index 00000000..e69de29b diff --git a/src/test/resources/web-of-things/ldes/defaults/out-local-file.nq b/src/test/resources/web-of-things/ldes/defaults/out-local-file.nq new file mode 100644 index 00000000..7f1c789e --- /dev/null +++ b/src/test/resources/web-of-things/ldes/defaults/out-local-file.nq @@ -0,0 +1,1073 @@ + "11"^^ . + "11"^^ . + "0"^^ . + . + "2021-10-12T06:20:08.208Z"^^ . + . + . + "50.942901611328"^^ . + "4.038559913635"^^ . + "Aalst" . + "6"^^ . + "6"^^ . + "0"^^ . + . + "2021-10-12T06:20:44.789Z"^^ . + . + . + "51.092498779297"^^ . + "3.449820041656"^^ . + "Aalter" . + "9"^^ . + "14"^^ . + "5"^^ . + . + "2021-10-12T06:20:12.927Z"^^ . + . + . + "50.984699249268"^^ . + "4.824250221252"^^ . + "Aarschot" . + "51"^^ . + "60"^^ . + "9"^^ . + . + "2021-10-12T06:20:11.221Z"^^ . + . + . + "51.199600219727"^^ . + "4.431789875031"^^ . + "Antwerpen-Berchem" . + "31"^^ . + "43"^^ . + "12"^^ . + . + "2021-10-12T06:20:06.338Z"^^ . + . + . + "51.217498779297"^^ . + "4.421050071716"^^ . + "Antwerpen-Centraal" . + "5"^^ . + "5"^^ . + "0"^^ . + . + "2021-10-12T06:21:00.798Z"^^ . + . + . + "51.245498657227"^^ . + "4.424610137939"^^ . + "Antwerpen-Luchtbal" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:21:01.494Z"^^ . + . + . + "51.2617"^^ . + "4.42755"^^ . + "Antwerpen-Noorderdokken" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:20:51.829Z"^^ . + . + . + "51.198001861572"^^ . + "4.390810012817"^^ . + "Antwerpen-Zuid" . + "7"^^ . + "10"^^ . + "3"^^ . + . + "2021-10-12T06:20:09.208Z"^^ . + . + . + "50.906700134277"^^ . + "4.207960128784"^^ . + "Asse" . + "5"^^ . + "5"^^ . + "0"^^ . + . + "2021-10-12T06:20:09.940Z"^^ . + . + . + "50.455600738525"^^ . + "3.944180011749"^^ . + "Bergen" . + "10"^^ . + "11"^^ . + "1"^^ . + . + "2021-10-12T06:20:55.643Z"^^ . + . + . + "51.208499908447"^^ . + "4.260260105133"^^ . + "Beveren" . + "6"^^ . + "7"^^ . + "1"^^ . + . + "2021-10-12T06:21:14.201Z"^^ . + . + . + "51.3119"^^ . + "3.1338"^^ . + "Blankenberge" . + "5"^^ . + "5"^^ . + "0"^^ . + . + "2021-10-12T06:20:44.070Z"^^ . + . + . + "51.16849899292"^^ . + "4.494080066681"^^ . + "Boechout P+R Capenberg" . + "6"^^ . + "6"^^ . + "0"^^ . + . + "2021-10-12T06:21:05.822Z"^^ . + . + . + "51.165508"^^ . + "4.746427"^^ . + "Bouwel" . + "34"^^ . + "63"^^ . + "29"^^ . + . + "2021-10-12T06:20:10.484Z"^^ . + . + . + "51.196998596191"^^ . + "3.218539953232"^^ . + "Brugge" . + "17"^^ . + "27"^^ . + "10"^^ . + . + "2021-10-12T06:20:54.646Z"^^ . + . + . + "51.196800231934"^^ . + "3.216079950333"^^ . + "Brugge (Kamgebouw)" . + "13"^^ . + "14"^^ . + "1"^^ . + . + "2021-10-12T06:20:07.205Z"^^ . + . + . + "50.844898223877"^^ . + "4.35586977005"^^ . + "Brussel-Centraal" . + "4"^^ . + "6"^^ . + "2"^^ . + . + "2021-10-12T06:20:13.845Z"^^ . + . + . + "50.838199615479"^^ . + "4.372889995575"^^ . + "Brussel-Luxemburg" . + "4"^^ . + "6"^^ . + "2"^^ . + . + "2021-10-12T06:21:19.556Z"^^ . + . + . + "50.835529468767"^^ . + "4.333756181866"^^ . + "Brussel-Zuid" . + "27"^^ . + "28"^^ . + "1"^^ . + . + "2021-10-12T06:20:14.572Z"^^ . + . + . + "50.86009979248"^^ . + "4.360489845276"^^ . + "Brussel Noord" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:13.294Z"^^ . + . + . + "51.077376"^^ . + "2.601611"^^ . + "De Panne" . + "4"^^ . + "62"^^ . + "58"^^ . + . + "2021-10-12T06:20:15.386Z"^^ . + . + . + "50.978199005127"^^ . + "3.534009933472"^^ . + "Deinze" . + "9"^^ . + "11"^^ . + "2"^^ . + . + "2021-10-12T06:20:45.522Z"^^ . + . + . + "50.982398986816"^^ . + "3.525919914246"^^ . + "Deinze Leiespiegel" . + "22"^^ . + "27"^^ . + "5"^^ . + . + "2021-10-12T06:20:16.315Z"^^ . + . + . + "51.023498535156"^^ . + "4.101920127869"^^ . + "Dendermonde" . + "4"^^ . + "4"^^ . + "0"^^ . + . + "2021-10-12T06:21:02.217Z"^^ . + . + . + "50.889099121094"^^ . + "4.447710037231"^^ . + "Diegem" . + "10"^^ . + "12"^^ . + "2"^^ . + . + "2021-10-12T06:20:17.089Z"^^ . + . + . + "50.993000030518"^^ . + "5.051129817963"^^ . + "Diest" . + "1"^^ . + "11"^^ . + "10"^^ . + . + "2021-10-12T06:20:34.600Z"^^ . + . + . + "51.180801391602"^^ . + "3.575040102005"^^ . + "Eeklo" . + "3"^^ . + "4"^^ . + "1"^^ . + . + "2021-10-12T06:20:59.366Z"^^ . + . + . + "51.284198760986"^^ . + "4.43501996994"^^ . + "Ekeren (perron 1)" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:21:00.100Z"^^ . + . + . + "51.281398773193"^^ . + "4.434390068054"^^ . + "Ekeren (perron 2)" . + "7"^^ . + "8"^^ . + "1"^^ . + . + "2021-10-12T06:21:11.779Z"^^ . + . + . + "51.462176"^^ . + "4.451532"^^ . + "Essen" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:20:57.946Z"^^ . + . + . + "50.929401397705"^^ . + "3.64013004303"^^ . + "Gavere-Asper" . + "4"^^ . + "4"^^ . + "0"^^ . + . + "2021-10-12T06:20:58.653Z"^^ . + . + . + "50.930500030518"^^ . + "3.654690027237"^^ . + "Gavere mobipunt" . + "8"^^ . + "15"^^ . + "7"^^ . + . + "2021-10-12T06:20:35.295Z"^^ . + . + . + "51.168800354004"^^ . + "4.989160060883"^^ . + "Geel" . + "5"^^ . + "6"^^ . + "1"^^ . + . + "2021-10-12T06:20:17.608Z"^^ . + . + . + "50.967098236084"^^ . + "5.496369838715"^^ . + "Genk" . + "31"^^ . + "41"^^ . + "10"^^ . + . + "2021-10-12T06:20:18.137Z"^^ . + . + . + "51.055999755859"^^ . + "3.740159988403"^^ . + "Gent-Dampoort" . + "16"^^ . + "26"^^ . + "10"^^ . + . + "2021-10-12T06:20:53.258Z"^^ . + . + . + "51.035999298096"^^ . + "3.711899995804"^^ . + "Gent-Sint-Pieters (M. Hendrikaplein)" . + "52"^^ . + "70"^^ . + "18"^^ . + . + "2021-10-12T06:20:18.962Z"^^ . + . + . + "51.034900665283"^^ . + "3.709640026093"^^ . + "Gent-Sint-Pieters (St.-Denijslaan)" . + "2"^^ . + "2"^^ . + "0"^^ . + . + "2021-10-12T06:20:39.838Z"^^ . + . + . + "50.770500183105"^^ . + "3.872560024261"^^ . + "Geraardsbergen " . + "7"^^ . + "8"^^ . + "1"^^ . + . + "2021-10-12T06:21:06.544Z"^^ . + . + . + "50.966578"^^ . + "4.613315"^^ . + "Haacht" . + "7"^^ . + "12"^^ . + "5"^^ . + . + "2021-10-12T06:20:19.671Z"^^ . + . + . + "50.7333984375"^^ . + "4.239940166473"^^ . + "Halle" . + "8"^^ . + "9"^^ . + "1"^^ . + . + "2021-10-12T06:20:57.256Z"^^ . + . + . + "50.855701446533"^^ . + "3.31374001503"^^ . + "Harelbeke" . + "35"^^ . + "59"^^ . + "24"^^ . + . + "2021-10-12T06:20:20.529Z"^^ . + . + . + "50.930999755859"^^ . + "5.32742023468"^^ . + "Hasselt" . + "11"^^ . + "12"^^ . + "1"^^ . + . + "2021-10-12T06:20:48.339Z"^^ . + . + . + "51.365100860596"^^ . + "4.460559844971"^^ . + "Heide" . + "5"^^ . + "5"^^ . + "0"^^ . + . + "2021-10-12T06:21:15.934Z"^^ . + . + . + "51.334168"^^ . + "3.239668"^^ . + "Heist" . + "12"^^ . + "12"^^ . + "0"^^ . + . + "2021-10-12T06:20:36.712Z"^^ . + . + . + "51.074199676514"^^ . + "4.708779811859"^^ . + "Heist-op-den-Berg" . + "11"^^ . + "12"^^ . + "1"^^ . + . + "2021-10-12T06:20:21.475Z"^^ . + . + . + "51.180999755859"^^ . + "4.828859806061"^^ . + "Herentals" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:03.654Z"^^ . + . + . + "51.03833"^^ . + "5.281699"^^ . + "Heusden" . + "7"^^ . + "9"^^ . + "2"^^ . + . + "2021-10-12T06:20:56.370Z"^^ . + . + . + "50.862598419189"^^ . + "4.695439815521"^^ . + "Heverlee " . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:21:10.807Z"^^ . + . + . + "51.1541195"^^ . + "4.464733"^^ . + "Hove" . + "5"^^ . + "10"^^ . + "5"^^ . + . + "2021-10-12T06:20:41.845Z"^^ . + . + . + "50.848400115967"^^ . + "2.876270055771"^^ . + "Ieper" . + "6"^^ . + "8"^^ . + "2"^^ . + . + "2021-10-12T06:20:52.530Z"^^ . + . + . + "50.9208984375"^^ . + "3.214819908142"^^ . + "Izegem" . + "7"^^ . + "8"^^ . + "1"^^ . + . + "2021-10-12T06:21:16.629Z"^^ . + . + . + "51.34018"^^ . + "3.28454"^^ . + "Knokke" . + "5"^^ . + "7"^^ . + "2"^^ . + . + "2021-10-12T06:20:49.729Z"^^ . + . + . + "51.134300231934"^^ . + "4.476119995117"^^ . + "Kontich-Lint" . + "28"^^ . + "47"^^ . + "19"^^ . + . + "2021-10-12T06:20:21.952Z"^^ . + . + . + "50.823799133301"^^ . + "3.263269901276"^^ . + "Kortrijk" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:18.066Z"^^ . + . + . + "51.064208"^^ . + "3.575647"^^ . + "Landegem" . + "7"^^ . + "8"^^ . + "1"^^ . + . + "2021-10-12T06:21:12.535Z"^^ . + . + . + "50.74829"^^ . + "5.079402"^^ . + "Landen" . + "16"^^ . + "24"^^ . + "8"^^ . + . + "2021-10-12T06:20:53.955Z"^^ . + . + . + "50.882499694824"^^ . + "4.716639995575"^^ . + "Leuven (Kop van Kessel-Lo)" . + "57"^^ . + "86"^^ . + "29"^^ . + . + "2021-10-12T06:20:22.664Z"^^ . + . + . + "50.87979888916"^^ . + "4.715939998627"^^ . + "Leuven (Tiensevest)" . + "21"^^ . + "26"^^ . + "5"^^ . + . + "2021-10-12T06:20:23.383Z"^^ . + . + . + "51.135799407959"^^ . + "4.56152009964"^^ . + "Lier" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:20:43.366Z"^^ . + . + . + "51.125301361084"^^ . + "4.573339939117"^^ . + "Lier P+R De Mol" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:20:42.586Z"^^ . + . + . + "51.128299713135"^^ . + "4.583220005035"^^ . + "Lier Veemarkt" . + "13"^^ . + "17"^^ . + "4"^^ . + . + "2021-10-12T06:20:24.095Z"^^ . + . + . + "51.107398986816"^^ . + "3.986469984055"^^ . + "Lokeren" . + "6"^^ . + "6"^^ . + "0"^^ . + . + "2021-10-12T06:21:10.091Z"^^ . + . + . + "51.21177"^^ . + "5.312579"^^ . + "Lommel" . + "0"^^ . + "4"^^ . + "4"^^ . + . + "2021-10-12T06:20:24.831Z"^^ . + . + . + "50.625801086426"^^ . + "5.565639972687"^^ . + "Luik" . + "47"^^ . + "76"^^ . + "29"^^ . + . + "2021-10-12T06:20:25.537Z"^^ . + . + . + "51.0178"^^ . + "4.48299"^^ . + "Mechelen" . + "17"^^ . + "20"^^ . + "3"^^ . + . + "2021-10-12T06:20:38.128Z"^^ . + . + . + "51.030200958252"^^ . + "4.489570140839"^^ . + "Mechelen-Nekkerspoel" . + "6"^^ . + "7"^^ . + "1"^^ . + . + "2021-10-12T06:20:49.040Z"^^ . + . + . + "51.029300689697"^^ . + "4.484320163727"^^ . + "Mechelen Veemarkt" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:05.094Z"^^ . + . + . + "50.932948204381"^^ . + "4.329820851913"^^ . + "Meise" . + "10"^^ . + "15"^^ . + "5"^^ . + . + "2021-10-12T06:20:26.232Z"^^ . + . + . + "51.190498352051"^^ . + "5.115129947662"^^ . + "Mol" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:21:20.277Z"^^ . + . + . + "51.18119"^^ . + "5.112421"^^ . + "Mol - De Nete" . + "4"^^ . + "6"^^ . + "2"^^ . + . + "2021-10-12T06:20:26.934Z"^^ . + . + . + "51.171199798584"^^ . + "4.455999851227"^^ . + "Mortsel-Oude-God" . + "6"^^ . + "6"^^ . + "0"^^ . + . + "2021-10-12T06:20:27.637Z"^^ . + . + . + "50.468601226807"^^ . + "4.862239837646"^^ . + "Namen" . + "7"^^ . + "10"^^ . + "3"^^ . + . + "2021-10-12T06:20:36.006Z"^^ . + . + . + "50.839500427246"^^ . + "4.026179790497"^^ . + "Ninove" . + "10"^^ . + "10"^^ . + "0"^^ . + . + "2021-10-12T06:20:47.633Z"^^ . + . + . + "51.35710144043"^^ . + "4.632410049438"^^ . + "Noorderkempen" . + "53"^^ . + "69"^^ . + "16"^^ . + . + "2021-10-12T06:20:28.362Z"^^ . + . + . + "51.228298187256"^^ . + "2.925649881363"^^ . + "Oostende" . + "0"^^ . + "2"^^ . + "2"^^ . + . + "2021-10-12T06:20:39.151Z"^^ . + . + . + "50.851299285889"^^ . + "3.601880073547"^^ . + "Oudenaarde " . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:20:46.915Z"^^ . + . + . + "50.854499816895"^^ . + "2.735719919205"^^ . + "Poperinge" . + "14"^^ . + "27"^^ . + "13"^^ . + . + "2021-10-12T06:20:29.065Z"^^ . + . + . + "50.949001312256"^^ . + "3.130480051041"^^ . + "Roeselare" . + "9"^^ . + "9"^^ . + "0"^^ . + . + "2021-10-12T06:21:17.346Z"^^ . + . + . + "50.747786"^^ . + "4.361626"^^ . + "Sint-Genesius-Rode" . + "15"^^ . + "30"^^ . + "15"^^ . + . + "2021-10-12T06:20:29.783Z"^^ . + . + . + "51.171398162842"^^ . + "4.143700122833"^^ . + "Sint-Niklaas" . + "9"^^ . + "9"^^ . + "0"^^ . + . + "2021-10-12T06:20:30.506Z"^^ . + . + . + "50.817401885986"^^ . + "5.176330089569"^^ . + "Sint-Truiden" . + "5"^^ . + "5"^^ . + "0"^^ . + . + "2021-10-12T06:21:07.957Z"^^ . + . + . + "51.125867"^^ . + "4.221333"^^ . + "Temse" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:20:50.446Z"^^ . + . + . + "50.990600585938"^^ . + "3.329819917679"^^ . + "Tielt" . + "5"^^ . + "13"^^ . + "8"^^ . + . + "2021-10-12T06:20:46.217Z"^^ . + . + . + "50.808200836182"^^ . + "4.92561006546"^^ . + "Tienen " . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:20:37.434Z"^^ . + . + . + "50.78450012207"^^ . + "5.473400115967"^^ . + "Tongeren" . + "3"^^ . + "4"^^ . + "1"^^ . + . + "2021-10-12T06:20:31.284Z"^^ . + . + . + "51.064800262451"^^ . + "3.105509996414"^^ . + "Torhout" . + "9"^^ . + "13"^^ . + "4"^^ . + . + "2021-10-12T06:20:32.245Z"^^ . + . + . + "51.322101593018"^^ . + "4.937900066376"^^ . + "Turnhout" . + "20"^^ . + "40"^^ . + "20"^^ . + . + "2021-10-12T06:20:33.067Z"^^ . + . + . + "50.924701690674"^^ . + "4.433000087738"^^ . + "Vilvoorde" . + "6"^^ . + "7"^^ . + "1"^^ . + . + "2021-10-12T06:21:18.772Z"^^ . + . + . + "50.923187"^^ . + "4.413591"^^ . + "Vilvoorde Hoppin De Kassei" . + "12"^^ . + "16"^^ . + "4"^^ . + . + "2021-10-12T06:20:33.881Z"^^ . + . + . + "50.891799926758"^^ . + "3.424420118332"^^ . + "Waregem" . + "4"^^ . + "4"^^ . + "0"^^ . + . + "2021-10-12T06:21:02.941Z"^^ . + . + . + "50.881999969482"^^ . + "3.426719903946"^^ . + "Waregem Expo P+R" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:09.366Z"^^ . + . + . + "50.810926"^^ . + "3.182521"^^ . + "Wevelgem" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:20:51.139Z"^^ . + . + . + "50.885200500488"^^ . + "4.470870018005"^^ . + "Zaventem" . + "2"^^ . + "2"^^ . + "0"^^ . + . + "2021-10-12T06:21:14.998Z"^^ . + . + . + "51.32651"^^ . + "3.195476"^^ . + "Zeebrugge-dorp" . + "6"^^ . + "7"^^ . + "1"^^ . + . + "2021-10-12T06:21:08.664Z"^^ . + . + . + "51.073108"^^ . + "4.041885"^^ . + "Zele" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:04.377Z"^^ . + . + . + "51.033617"^^ . + "5.329922"^^ . + "Zolder" . + "7"^^ . + "8"^^ . + "1"^^ . + . + "2021-10-12T06:21:07.245Z"^^ . + . + . + "50.86948"^^ . + "3.814341"^^ . + "Zottegem " . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + "1" . diff --git a/src/test/resources/web-of-things/ldes/paths/mapping.ttl b/src/test/resources/web-of-things/ldes/paths/mapping.ttl new file mode 100644 index 00000000..603e4e45 --- /dev/null +++ b/src/test/resources/web-of-things/ldes/paths/mapping.ttl @@ -0,0 +1,161 @@ +# +# RML mapping rules for https://datapiloten.be/bluebike/availabilities.geojson +# (c) Dylan Van Assche (2021) +# IDLab - Ghent University - imec +# +@prefix rr: . +@prefix ex: . +@prefix rdfs: . +@prefix xsd: . +@prefix rml: . +@prefix rmlt: . +@prefix comp: . +@prefix formats: . +@prefix ql: . +@prefix rdf: . +@prefix geo: . +@prefix fnml: . +@prefix fno: . +@prefix grel: . +@prefix dcterms: . +@prefix tree: . +@prefix ldes: . +@prefix purl: . +@prefix td: . +@prefix htv: . +@prefix hctl: . +@base . + +<#TargetDump> a rmlt:LogicalTarget, ldes:EventStreamTarget; + rmlt:target [ a void:Dataset; + void:dataDump ; + ]; + rmlt:serialization formats:N-Triples; + ldes:baseIRI ; + ldes:timestampPath purl:created; + ldes:versionOfPath purl:isVersionOf; + tree:shape ; +. + +<#WoTWebResource> a td:PropertyAffordance; + td:hasForm [ + # URL and content type + hctl:hasTarget "http://localhost:8000/bluebike"; + hctl:forContentType "application/json"; + # Read only + hctl:hasOperationType td:readproperty ; + # Set HTTP method and headers + htv:methodName "GET"; + htv:headers ([ + htv:fieldName "User-Agent"; + htv:fieldValue "RML Processor"; + ]); + ]; +. + +<#WoTWebAPI> a td:Thing; + td:hasPropertyAffordance <#WoTWebResource>; +. + +<#StationsTriplesMap> a rr:TriplesMap; + rml:logicalSource [ + rml:source <#WoTWebResource>; + rml:referenceFormulation ql:JSONPath; + rml:iterator "$.features[*]" + ]; + + # Unique IRI generation: $stationId#$generatedAtTime + rr:subjectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant grel:array_join ] + ]; + rr:predicateObjectMap [ + rr:predicate grel:p_array_a ; + rr:objectMap [ rml:reference "properties.@id" ] + ]; + rr:predicateObjectMap [ + rr:predicate grel:p_array_a ; + rr:objectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant grel:array_join ] + ]; + + rr:predicateObjectMap [ + rr:predicate grel:p_array_a ; + rr:objectMap [ rr:constant "#" ] + ]; + + rr:predicateObjectMap [ + rr:predicate grel:p_array_a ; + rr:objectMap [ rml:reference "generatedAtTime" ] + ]; + ]; + ]; + ]; + ]; + rr:termType rr:IRI; + rml:logicalTarget <#TargetDump>; + ]; + + # rdf:type + rr:predicateObjectMap [ + rr:predicate rdf:type ; + rr:objectMap [ rml:reference "properties.@type"; rr:termType rr:IRI ] + ]; + + # BlueBike station: name + rr:predicateObjectMap [ + rr:predicate rdfs:label ; + rr:objectMap [ rml:reference "properties.name"; rr:datatype xsd:string; ] + ]; + + # BlueBike station: available bicycles + rr:predicateObjectMap [ + rr:predicate ex:bikesAvailable ; + rr:objectMap [ rml:reference "properties.bikes_available"; rr:datatype xsd:integer; ] + ]; + + # BlueBike station: maximum capacity + rr:predicateObjectMap [ + rr:predicate ex:capacity ; + rr:objectMap [ rml:reference "properties.capacity"; rr:datatype xsd:integer; ] + ]; + + # BlueBike station: available docks + rr:predicateObjectMap [ + rr:predicate ex:docksAvailable ; + rr:objectMap [ rml:reference "properties.docks_available"; rr:datatype xsd:integer; ] + ]; + + # Nearby NMBS station + rr:predicateObjectMap [ + rr:predicate ex:nearby ; + rr:objectMap [ rml:reference "properties.nearby"; rr:termType rr:IRI; ] + ]; + + # Geo location + rr:predicateObjectMap [ + rr:predicate geo:latitude ; + rr:objectMap [ rml:reference "properties.latitude"; rr:datatype xsd:float; ] + ]; + + rr:predicateObjectMap [ + rr:predicate geo:longitude ; + rr:objectMap [ rml:reference "properties.longitude"; rr:datatype xsd:float; ] + ]; + + # Versioning + rr:predicateObjectMap [ + rr:predicate dcterms:created ; + rr:objectMap [ rml:reference "generatedAtTime"; rr:datatype xsd:dateTime; ] + ]; + + rr:predicateObjectMap [ + rr:predicate dcterms:isVersionOf ; + rr:objectMap [ rml:reference "properties.@id"; rr:termType rr:IRI; ] + ]; +. diff --git a/src/test/resources/web-of-things/ldes/paths/out-default.nq b/src/test/resources/web-of-things/ldes/paths/out-default.nq new file mode 100644 index 00000000..e69de29b diff --git a/src/test/resources/web-of-things/ldes/paths/out-local-file.nq b/src/test/resources/web-of-things/ldes/paths/out-local-file.nq new file mode 100644 index 00000000..d00db8e4 --- /dev/null +++ b/src/test/resources/web-of-things/ldes/paths/out-local-file.nq @@ -0,0 +1,1075 @@ + "11"^^ . + "11"^^ . + "0"^^ . + . + "2021-10-12T06:20:08.208Z"^^ . + . + . + "50.942901611328"^^ . + "4.038559913635"^^ . + "Aalst" . + "6"^^ . + "6"^^ . + "0"^^ . + . + "2021-10-12T06:20:44.789Z"^^ . + . + . + "51.092498779297"^^ . + "3.449820041656"^^ . + "Aalter" . + "9"^^ . + "14"^^ . + "5"^^ . + . + "2021-10-12T06:20:12.927Z"^^ . + . + . + "50.984699249268"^^ . + "4.824250221252"^^ . + "Aarschot" . + "51"^^ . + "60"^^ . + "9"^^ . + . + "2021-10-12T06:20:11.221Z"^^ . + . + . + "51.199600219727"^^ . + "4.431789875031"^^ . + "Antwerpen-Berchem" . + "31"^^ . + "43"^^ . + "12"^^ . + . + "2021-10-12T06:20:06.338Z"^^ . + . + . + "51.217498779297"^^ . + "4.421050071716"^^ . + "Antwerpen-Centraal" . + "5"^^ . + "5"^^ . + "0"^^ . + . + "2021-10-12T06:21:00.798Z"^^ . + . + . + "51.245498657227"^^ . + "4.424610137939"^^ . + "Antwerpen-Luchtbal" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:21:01.494Z"^^ . + . + . + "51.2617"^^ . + "4.42755"^^ . + "Antwerpen-Noorderdokken" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:20:51.829Z"^^ . + . + . + "51.198001861572"^^ . + "4.390810012817"^^ . + "Antwerpen-Zuid" . + "7"^^ . + "10"^^ . + "3"^^ . + . + "2021-10-12T06:20:09.208Z"^^ . + . + . + "50.906700134277"^^ . + "4.207960128784"^^ . + "Asse" . + "5"^^ . + "5"^^ . + "0"^^ . + . + "2021-10-12T06:20:09.940Z"^^ . + . + . + "50.455600738525"^^ . + "3.944180011749"^^ . + "Bergen" . + "10"^^ . + "11"^^ . + "1"^^ . + . + "2021-10-12T06:20:55.643Z"^^ . + . + . + "51.208499908447"^^ . + "4.260260105133"^^ . + "Beveren" . + "6"^^ . + "7"^^ . + "1"^^ . + . + "2021-10-12T06:21:14.201Z"^^ . + . + . + "51.3119"^^ . + "3.1338"^^ . + "Blankenberge" . + "5"^^ . + "5"^^ . + "0"^^ . + . + "2021-10-12T06:20:44.070Z"^^ . + . + . + "51.16849899292"^^ . + "4.494080066681"^^ . + "Boechout P+R Capenberg" . + "6"^^ . + "6"^^ . + "0"^^ . + . + "2021-10-12T06:21:05.822Z"^^ . + . + . + "51.165508"^^ . + "4.746427"^^ . + "Bouwel" . + "34"^^ . + "63"^^ . + "29"^^ . + . + "2021-10-12T06:20:10.484Z"^^ . + . + . + "51.196998596191"^^ . + "3.218539953232"^^ . + "Brugge" . + "17"^^ . + "27"^^ . + "10"^^ . + . + "2021-10-12T06:20:54.646Z"^^ . + . + . + "51.196800231934"^^ . + "3.216079950333"^^ . + "Brugge (Kamgebouw)" . + "13"^^ . + "14"^^ . + "1"^^ . + . + "2021-10-12T06:20:07.205Z"^^ . + . + . + "50.844898223877"^^ . + "4.35586977005"^^ . + "Brussel-Centraal" . + "4"^^ . + "6"^^ . + "2"^^ . + . + "2021-10-12T06:20:13.845Z"^^ . + . + . + "50.838199615479"^^ . + "4.372889995575"^^ . + "Brussel-Luxemburg" . + "4"^^ . + "6"^^ . + "2"^^ . + . + "2021-10-12T06:21:19.556Z"^^ . + . + . + "50.835529468767"^^ . + "4.333756181866"^^ . + "Brussel-Zuid" . + "27"^^ . + "28"^^ . + "1"^^ . + . + "2021-10-12T06:20:14.572Z"^^ . + . + . + "50.86009979248"^^ . + "4.360489845276"^^ . + "Brussel Noord" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:13.294Z"^^ . + . + . + "51.077376"^^ . + "2.601611"^^ . + "De Panne" . + "4"^^ . + "62"^^ . + "58"^^ . + . + "2021-10-12T06:20:15.386Z"^^ . + . + . + "50.978199005127"^^ . + "3.534009933472"^^ . + "Deinze" . + "9"^^ . + "11"^^ . + "2"^^ . + . + "2021-10-12T06:20:45.522Z"^^ . + . + . + "50.982398986816"^^ . + "3.525919914246"^^ . + "Deinze Leiespiegel" . + "22"^^ . + "27"^^ . + "5"^^ . + . + "2021-10-12T06:20:16.315Z"^^ . + . + . + "51.023498535156"^^ . + "4.101920127869"^^ . + "Dendermonde" . + "4"^^ . + "4"^^ . + "0"^^ . + . + "2021-10-12T06:21:02.217Z"^^ . + . + . + "50.889099121094"^^ . + "4.447710037231"^^ . + "Diegem" . + "10"^^ . + "12"^^ . + "2"^^ . + . + "2021-10-12T06:20:17.089Z"^^ . + . + . + "50.993000030518"^^ . + "5.051129817963"^^ . + "Diest" . + "1"^^ . + "11"^^ . + "10"^^ . + . + "2021-10-12T06:20:34.600Z"^^ . + . + . + "51.180801391602"^^ . + "3.575040102005"^^ . + "Eeklo" . + "3"^^ . + "4"^^ . + "1"^^ . + . + "2021-10-12T06:20:59.366Z"^^ . + . + . + "51.284198760986"^^ . + "4.43501996994"^^ . + "Ekeren (perron 1)" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:21:00.100Z"^^ . + . + . + "51.281398773193"^^ . + "4.434390068054"^^ . + "Ekeren (perron 2)" . + "7"^^ . + "8"^^ . + "1"^^ . + . + "2021-10-12T06:21:11.779Z"^^ . + . + . + "51.462176"^^ . + "4.451532"^^ . + "Essen" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:20:57.946Z"^^ . + . + . + "50.929401397705"^^ . + "3.64013004303"^^ . + "Gavere-Asper" . + "4"^^ . + "4"^^ . + "0"^^ . + . + "2021-10-12T06:20:58.653Z"^^ . + . + . + "50.930500030518"^^ . + "3.654690027237"^^ . + "Gavere mobipunt" . + "8"^^ . + "15"^^ . + "7"^^ . + . + "2021-10-12T06:20:35.295Z"^^ . + . + . + "51.168800354004"^^ . + "4.989160060883"^^ . + "Geel" . + "5"^^ . + "6"^^ . + "1"^^ . + . + "2021-10-12T06:20:17.608Z"^^ . + . + . + "50.967098236084"^^ . + "5.496369838715"^^ . + "Genk" . + "31"^^ . + "41"^^ . + "10"^^ . + . + "2021-10-12T06:20:18.137Z"^^ . + . + . + "51.055999755859"^^ . + "3.740159988403"^^ . + "Gent-Dampoort" . + "16"^^ . + "26"^^ . + "10"^^ . + . + "2021-10-12T06:20:53.258Z"^^ . + . + . + "51.035999298096"^^ . + "3.711899995804"^^ . + "Gent-Sint-Pieters (M. Hendrikaplein)" . + "52"^^ . + "70"^^ . + "18"^^ . + . + "2021-10-12T06:20:18.962Z"^^ . + . + . + "51.034900665283"^^ . + "3.709640026093"^^ . + "Gent-Sint-Pieters (St.-Denijslaan)" . + "2"^^ . + "2"^^ . + "0"^^ . + . + "2021-10-12T06:20:39.838Z"^^ . + . + . + "50.770500183105"^^ . + "3.872560024261"^^ . + "Geraardsbergen " . + "7"^^ . + "8"^^ . + "1"^^ . + . + "2021-10-12T06:21:06.544Z"^^ . + . + . + "50.966578"^^ . + "4.613315"^^ . + "Haacht" . + "7"^^ . + "12"^^ . + "5"^^ . + . + "2021-10-12T06:20:19.671Z"^^ . + . + . + "50.7333984375"^^ . + "4.239940166473"^^ . + "Halle" . + "8"^^ . + "9"^^ . + "1"^^ . + . + "2021-10-12T06:20:57.256Z"^^ . + . + . + "50.855701446533"^^ . + "3.31374001503"^^ . + "Harelbeke" . + "35"^^ . + "59"^^ . + "24"^^ . + . + "2021-10-12T06:20:20.529Z"^^ . + . + . + "50.930999755859"^^ . + "5.32742023468"^^ . + "Hasselt" . + "11"^^ . + "12"^^ . + "1"^^ . + . + "2021-10-12T06:20:48.339Z"^^ . + . + . + "51.365100860596"^^ . + "4.460559844971"^^ . + "Heide" . + "5"^^ . + "5"^^ . + "0"^^ . + . + "2021-10-12T06:21:15.934Z"^^ . + . + . + "51.334168"^^ . + "3.239668"^^ . + "Heist" . + "12"^^ . + "12"^^ . + "0"^^ . + . + "2021-10-12T06:20:36.712Z"^^ . + . + . + "51.074199676514"^^ . + "4.708779811859"^^ . + "Heist-op-den-Berg" . + "11"^^ . + "12"^^ . + "1"^^ . + . + "2021-10-12T06:20:21.475Z"^^ . + . + . + "51.180999755859"^^ . + "4.828859806061"^^ . + "Herentals" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:03.654Z"^^ . + . + . + "51.03833"^^ . + "5.281699"^^ . + "Heusden" . + "7"^^ . + "9"^^ . + "2"^^ . + . + "2021-10-12T06:20:56.370Z"^^ . + . + . + "50.862598419189"^^ . + "4.695439815521"^^ . + "Heverlee " . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:21:10.807Z"^^ . + . + . + "51.1541195"^^ . + "4.464733"^^ . + "Hove" . + "5"^^ . + "10"^^ . + "5"^^ . + . + "2021-10-12T06:20:41.845Z"^^ . + . + . + "50.848400115967"^^ . + "2.876270055771"^^ . + "Ieper" . + "6"^^ . + "8"^^ . + "2"^^ . + . + "2021-10-12T06:20:52.530Z"^^ . + . + . + "50.9208984375"^^ . + "3.214819908142"^^ . + "Izegem" . + "7"^^ . + "8"^^ . + "1"^^ . + . + "2021-10-12T06:21:16.629Z"^^ . + . + . + "51.34018"^^ . + "3.28454"^^ . + "Knokke" . + "5"^^ . + "7"^^ . + "2"^^ . + . + "2021-10-12T06:20:49.729Z"^^ . + . + . + "51.134300231934"^^ . + "4.476119995117"^^ . + "Kontich-Lint" . + "28"^^ . + "47"^^ . + "19"^^ . + . + "2021-10-12T06:20:21.952Z"^^ . + . + . + "50.823799133301"^^ . + "3.263269901276"^^ . + "Kortrijk" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:18.066Z"^^ . + . + . + "51.064208"^^ . + "3.575647"^^ . + "Landegem" . + "7"^^ . + "8"^^ . + "1"^^ . + . + "2021-10-12T06:21:12.535Z"^^ . + . + . + "50.74829"^^ . + "5.079402"^^ . + "Landen" . + "16"^^ . + "24"^^ . + "8"^^ . + . + "2021-10-12T06:20:53.955Z"^^ . + . + . + "50.882499694824"^^ . + "4.716639995575"^^ . + "Leuven (Kop van Kessel-Lo)" . + "57"^^ . + "86"^^ . + "29"^^ . + . + "2021-10-12T06:20:22.664Z"^^ . + . + . + "50.87979888916"^^ . + "4.715939998627"^^ . + "Leuven (Tiensevest)" . + "21"^^ . + "26"^^ . + "5"^^ . + . + "2021-10-12T06:20:23.383Z"^^ . + . + . + "51.135799407959"^^ . + "4.56152009964"^^ . + "Lier" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:20:43.366Z"^^ . + . + . + "51.125301361084"^^ . + "4.573339939117"^^ . + "Lier P+R De Mol" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:20:42.586Z"^^ . + . + . + "51.128299713135"^^ . + "4.583220005035"^^ . + "Lier Veemarkt" . + "13"^^ . + "17"^^ . + "4"^^ . + . + "2021-10-12T06:20:24.095Z"^^ . + . + . + "51.107398986816"^^ . + "3.986469984055"^^ . + "Lokeren" . + "6"^^ . + "6"^^ . + "0"^^ . + . + "2021-10-12T06:21:10.091Z"^^ . + . + . + "51.21177"^^ . + "5.312579"^^ . + "Lommel" . + "0"^^ . + "4"^^ . + "4"^^ . + . + "2021-10-12T06:20:24.831Z"^^ . + . + . + "50.625801086426"^^ . + "5.565639972687"^^ . + "Luik" . + "47"^^ . + "76"^^ . + "29"^^ . + . + "2021-10-12T06:20:25.537Z"^^ . + . + . + "51.0178"^^ . + "4.48299"^^ . + "Mechelen" . + "17"^^ . + "20"^^ . + "3"^^ . + . + "2021-10-12T06:20:38.128Z"^^ . + . + . + "51.030200958252"^^ . + "4.489570140839"^^ . + "Mechelen-Nekkerspoel" . + "6"^^ . + "7"^^ . + "1"^^ . + . + "2021-10-12T06:20:49.040Z"^^ . + . + . + "51.029300689697"^^ . + "4.484320163727"^^ . + "Mechelen Veemarkt" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:05.094Z"^^ . + . + . + "50.932948204381"^^ . + "4.329820851913"^^ . + "Meise" . + "10"^^ . + "15"^^ . + "5"^^ . + . + "2021-10-12T06:20:26.232Z"^^ . + . + . + "51.190498352051"^^ . + "5.115129947662"^^ . + "Mol" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:21:20.277Z"^^ . + . + . + "51.18119"^^ . + "5.112421"^^ . + "Mol - De Nete" . + "4"^^ . + "6"^^ . + "2"^^ . + . + "2021-10-12T06:20:26.934Z"^^ . + . + . + "51.171199798584"^^ . + "4.455999851227"^^ . + "Mortsel-Oude-God" . + "6"^^ . + "6"^^ . + "0"^^ . + . + "2021-10-12T06:20:27.637Z"^^ . + . + . + "50.468601226807"^^ . + "4.862239837646"^^ . + "Namen" . + "7"^^ . + "10"^^ . + "3"^^ . + . + "2021-10-12T06:20:36.006Z"^^ . + . + . + "50.839500427246"^^ . + "4.026179790497"^^ . + "Ninove" . + "10"^^ . + "10"^^ . + "0"^^ . + . + "2021-10-12T06:20:47.633Z"^^ . + . + . + "51.35710144043"^^ . + "4.632410049438"^^ . + "Noorderkempen" . + "53"^^ . + "69"^^ . + "16"^^ . + . + "2021-10-12T06:20:28.362Z"^^ . + . + . + "51.228298187256"^^ . + "2.925649881363"^^ . + "Oostende" . + "0"^^ . + "2"^^ . + "2"^^ . + . + "2021-10-12T06:20:39.151Z"^^ . + . + . + "50.851299285889"^^ . + "3.601880073547"^^ . + "Oudenaarde " . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:20:46.915Z"^^ . + . + . + "50.854499816895"^^ . + "2.735719919205"^^ . + "Poperinge" . + "14"^^ . + "27"^^ . + "13"^^ . + . + "2021-10-12T06:20:29.065Z"^^ . + . + . + "50.949001312256"^^ . + "3.130480051041"^^ . + "Roeselare" . + "9"^^ . + "9"^^ . + "0"^^ . + . + "2021-10-12T06:21:17.346Z"^^ . + . + . + "50.747786"^^ . + "4.361626"^^ . + "Sint-Genesius-Rode" . + "15"^^ . + "30"^^ . + "15"^^ . + . + "2021-10-12T06:20:29.783Z"^^ . + . + . + "51.171398162842"^^ . + "4.143700122833"^^ . + "Sint-Niklaas" . + "9"^^ . + "9"^^ . + "0"^^ . + . + "2021-10-12T06:20:30.506Z"^^ . + . + . + "50.817401885986"^^ . + "5.176330089569"^^ . + "Sint-Truiden" . + "5"^^ . + "5"^^ . + "0"^^ . + . + "2021-10-12T06:21:07.957Z"^^ . + . + . + "51.125867"^^ . + "4.221333"^^ . + "Temse" . + "7"^^ . + "7"^^ . + "0"^^ . + . + "2021-10-12T06:20:50.446Z"^^ . + . + . + "50.990600585938"^^ . + "3.329819917679"^^ . + "Tielt" . + "5"^^ . + "13"^^ . + "8"^^ . + . + "2021-10-12T06:20:46.217Z"^^ . + . + . + "50.808200836182"^^ . + "4.92561006546"^^ . + "Tienen " . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:20:37.434Z"^^ . + . + . + "50.78450012207"^^ . + "5.473400115967"^^ . + "Tongeren" . + "3"^^ . + "4"^^ . + "1"^^ . + . + "2021-10-12T06:20:31.284Z"^^ . + . + . + "51.064800262451"^^ . + "3.105509996414"^^ . + "Torhout" . + "9"^^ . + "13"^^ . + "4"^^ . + . + "2021-10-12T06:20:32.245Z"^^ . + . + . + "51.322101593018"^^ . + "4.937900066376"^^ . + "Turnhout" . + "20"^^ . + "40"^^ . + "20"^^ . + . + "2021-10-12T06:20:33.067Z"^^ . + . + . + "50.924701690674"^^ . + "4.433000087738"^^ . + "Vilvoorde" . + "6"^^ . + "7"^^ . + "1"^^ . + . + "2021-10-12T06:21:18.772Z"^^ . + . + . + "50.923187"^^ . + "4.413591"^^ . + "Vilvoorde Hoppin De Kassei" . + "12"^^ . + "16"^^ . + "4"^^ . + . + "2021-10-12T06:20:33.881Z"^^ . + . + . + "50.891799926758"^^ . + "3.424420118332"^^ . + "Waregem" . + "4"^^ . + "4"^^ . + "0"^^ . + . + "2021-10-12T06:21:02.941Z"^^ . + . + . + "50.881999969482"^^ . + "3.426719903946"^^ . + "Waregem Expo P+R" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:09.366Z"^^ . + . + . + "50.810926"^^ . + "3.182521"^^ . + "Wevelgem" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:20:51.139Z"^^ . + . + . + "50.885200500488"^^ . + "4.470870018005"^^ . + "Zaventem" . + "2"^^ . + "2"^^ . + "0"^^ . + . + "2021-10-12T06:21:14.998Z"^^ . + . + . + "51.32651"^^ . + "3.195476"^^ . + "Zeebrugge-dorp" . + "6"^^ . + "7"^^ . + "1"^^ . + . + "2021-10-12T06:21:08.664Z"^^ . + . + . + "51.073108"^^ . + "4.041885"^^ . + "Zele" . + "8"^^ . + "8"^^ . + "0"^^ . + . + "2021-10-12T06:21:04.377Z"^^ . + . + . + "51.033617"^^ . + "5.329922"^^ . + "Zolder" . + "7"^^ . + "8"^^ . + "1"^^ . + . + "2021-10-12T06:21:07.245Z"^^ . + . + . + "50.86948"^^ . + "3.814341"^^ . + "Zottegem " . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + "1" . + . + . diff --git a/src/test/resources/web-of-things/ldes/private-security-data.ttl b/src/test/resources/web-of-things/ldes/private-security-data.ttl new file mode 100644 index 00000000..9da7b50c --- /dev/null +++ b/src/test/resources/web-of-things/ldes/private-security-data.ttl @@ -0,0 +1,2 @@ +@prefix idsa: . +@base . diff --git a/src/test/resources/web-of-things/ldes/stations.jsonld b/src/test/resources/web-of-things/ldes/stations.jsonld new file mode 100644 index 00000000..4931763e --- /dev/null +++ b/src/test/resources/web-of-things/ldes/stations.jsonld @@ -0,0 +1,2178 @@ +{ + "@context": { + "mv": "http://schema.mobivoc.org/", + "name": { + "@id": "http://xmlns.com/foaf/0.1/name", + "@type": "http://www.w3.org/2001/XMLSchema#string" + }, + "longitude": { + "@id": "http://www.w3.org/2003/01/geo/wgs84_pos#long", + "@type": "xsd:float" + }, + "latitude": { + "@id": "http://www.w3.org/2003/01/geo/wgs84_pos#lat", + "@type": "xsd:float" + }, + "alternative": { + "@id": "http://purl.org/dc/terms/alternative", + "@type": "http://www.w3.org/1999/02/22-rdf-syntax-ns#langString", + "@container": "@set" + }, + "xsd": "http://www.w3.org/2001/XMLSchema#", + "features": "@graph", + "properties": "@graph", + "generatedAtTime": { + "@id": "http://www.w3.org/ns/prov#generatedAtTime", + "@type": "xsd:date" + }, + "nearby": { + "@id": "http://www.geonames.org/ontology#nearby", + "@type": "@id" + }, + "bikes_available": { + "@id": "mv:capacity", + "@type": "http://www.w3.org/2001/XMLSchema#integer" + }, + "capacity": { + "@id": "mv:totalCapacity", + "@type": "http://www.w3.org/2001/XMLSchema#integer" + } + }, + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.421050071716, + 51.217498779297 + ] + }, + "generatedAtTime": "2021-10-12T06:20:06.338Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Antwerpen-Centraal", + "@type": "http://schema.org/ParkingFacility", + "name": "Antwerpen-Centraal", + "longitude": 4.421050071716, + "latitude": 51.217498779297, + "bikes_available": 31, + "capacity": 43, + "docks_available": 12, + "nearby": "http://irail.be/stations/NMBS/008821006" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.35586977005, + 50.844898223877 + ] + }, + "generatedAtTime": "2021-10-12T06:20:07.205Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Brussel-Centraal", + "@type": "http://schema.org/ParkingFacility", + "name": "Brussel-Centraal", + "longitude": 4.35586977005, + "latitude": 50.844898223877, + "bikes_available": 13, + "capacity": 14, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008813003" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.038559913635, + 50.942901611328 + ] + }, + "generatedAtTime": "2021-10-12T06:20:08.208Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Aalst", + "@type": "http://schema.org/ParkingFacility", + "name": "Aalst", + "longitude": 4.038559913635, + "latitude": 50.942901611328, + "bikes_available": 11, + "capacity": 11, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008895000" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.207960128784, + 50.906700134277 + ] + }, + "generatedAtTime": "2021-10-12T06:20:09.208Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Asse", + "@type": "http://schema.org/ParkingFacility", + "name": "Asse", + "longitude": 4.207960128784, + "latitude": 50.906700134277, + "bikes_available": 7, + "capacity": 10, + "docks_available": 3, + "nearby": "http://irail.be/stations/NMBS/008812070" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.944180011749, + 50.455600738525 + ] + }, + "generatedAtTime": "2021-10-12T06:20:09.940Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Bergen", + "@type": "http://schema.org/ParkingFacility", + "name": "Bergen", + "longitude": 3.944180011749, + "latitude": 50.455600738525, + "bikes_available": 5, + "capacity": 5, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008881000" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.218539953232, + 51.196998596191 + ] + }, + "generatedAtTime": "2021-10-12T06:20:10.484Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Brugge", + "@type": "http://schema.org/ParkingFacility", + "name": "Brugge", + "longitude": 3.218539953232, + "latitude": 51.196998596191, + "bikes_available": 34, + "capacity": 63, + "docks_available": 29, + "nearby": "http://irail.be/stations/NMBS/008891009" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.431789875031, + 51.199600219727 + ] + }, + "generatedAtTime": "2021-10-12T06:20:11.221Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Antwerpen-Berchem", + "@type": "http://schema.org/ParkingFacility", + "name": "Antwerpen-Berchem", + "longitude": 4.431789875031, + "latitude": 51.199600219727, + "bikes_available": 51, + "capacity": 60, + "docks_available": 9, + "nearby": "http://irail.be/stations/NMBS/008821121" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.824250221252, + 50.984699249268 + ] + }, + "generatedAtTime": "2021-10-12T06:20:12.927Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Aarschot", + "@type": "http://schema.org/ParkingFacility", + "name": "Aarschot", + "longitude": 4.824250221252, + "latitude": 50.984699249268, + "bikes_available": 9, + "capacity": 14, + "docks_available": 5, + "nearby": "http://irail.be/stations/NMBS/008833209" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.372889995575, + 50.838199615479 + ] + }, + "generatedAtTime": "2021-10-12T06:20:13.845Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Brussel-Luxemburg", + "@type": "http://schema.org/ParkingFacility", + "name": "Brussel-Luxemburg", + "longitude": 4.372889995575, + "latitude": 50.838199615479, + "bikes_available": 4, + "capacity": 6, + "docks_available": 2, + "nearby": "http://irail.be/stations/NMBS/008811304" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.360489845276, + 50.86009979248 + ] + }, + "generatedAtTime": "2021-10-12T06:20:14.572Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/BrusselNoord", + "@type": "http://schema.org/ParkingFacility", + "name": "Brussel Noord", + "longitude": 4.360489845276, + "latitude": 50.86009979248, + "bikes_available": 27, + "capacity": 28, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008812005" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.534009933472, + 50.978199005127 + ] + }, + "generatedAtTime": "2021-10-12T06:20:15.386Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Deinze", + "@type": "http://schema.org/ParkingFacility", + "name": "Deinze", + "longitude": 3.534009933472, + "latitude": 50.978199005127, + "bikes_available": 4, + "capacity": 62, + "docks_available": 58, + "nearby": "http://irail.be/stations/NMBS/008892106" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.101920127869, + 51.023498535156 + ] + }, + "generatedAtTime": "2021-10-12T06:20:16.315Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Dendermonde", + "@type": "http://schema.org/ParkingFacility", + "name": "Dendermonde", + "longitude": 4.101920127869, + "latitude": 51.023498535156, + "bikes_available": 22, + "capacity": 27, + "docks_available": 5, + "nearby": "http://irail.be/stations/NMBS/008893401" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.051129817963, + 50.993000030518 + ] + }, + "generatedAtTime": "2021-10-12T06:20:17.089Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Diest", + "@type": "http://schema.org/ParkingFacility", + "name": "Diest", + "longitude": 5.051129817963, + "latitude": 50.993000030518, + "bikes_available": 10, + "capacity": 12, + "docks_available": 2, + "nearby": "http://irail.be/stations/NMBS/008831401" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.496369838715, + 50.967098236084 + ] + }, + "generatedAtTime": "2021-10-12T06:20:17.608Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Genk", + "@type": "http://schema.org/ParkingFacility", + "name": "Genk", + "longitude": 5.496369838715, + "latitude": 50.967098236084, + "bikes_available": 5, + "capacity": 6, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008831765" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.740159988403, + 51.055999755859 + ] + }, + "generatedAtTime": "2021-10-12T06:20:18.137Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Gent-Dampoort", + "@type": "http://schema.org/ParkingFacility", + "name": "Gent-Dampoort", + "longitude": 3.740159988403, + "latitude": 51.055999755859, + "bikes_available": 31, + "capacity": 41, + "docks_available": 10, + "nearby": "http://irail.be/stations/NMBS/008893120" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.709640026093, + 51.034900665283 + ] + }, + "generatedAtTime": "2021-10-12T06:20:18.962Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Gent-Sint-Pieters(St.-Denijslaan)", + "@type": "http://schema.org/ParkingFacility", + "name": "Gent-Sint-Pieters (St.-Denijslaan)", + "longitude": 3.709640026093, + "latitude": 51.034900665283, + "bikes_available": 52, + "capacity": 70, + "docks_available": 18, + "nearby": "http://irail.be/stations/NMBS/008892007" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.239940166473, + 50.7333984375 + ] + }, + "generatedAtTime": "2021-10-12T06:20:19.671Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Halle", + "@type": "http://schema.org/ParkingFacility", + "name": "Halle", + "longitude": 4.239940166473, + "latitude": 50.7333984375, + "bikes_available": 7, + "capacity": 12, + "docks_available": 5, + "nearby": "http://irail.be/stations/NMBS/008814308" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.32742023468, + 50.930999755859 + ] + }, + "generatedAtTime": "2021-10-12T06:20:20.529Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Hasselt", + "@type": "http://schema.org/ParkingFacility", + "name": "Hasselt", + "longitude": 5.32742023468, + "latitude": 50.930999755859, + "bikes_available": 35, + "capacity": 59, + "docks_available": 24, + "nearby": "http://irail.be/stations/NMBS/008831005" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.828859806061, + 51.180999755859 + ] + }, + "generatedAtTime": "2021-10-12T06:20:21.475Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Herentals", + "@type": "http://schema.org/ParkingFacility", + "name": "Herentals", + "longitude": 4.828859806061, + "latitude": 51.180999755859, + "bikes_available": 11, + "capacity": 12, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008821717" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.263269901276, + 50.823799133301 + ] + }, + "generatedAtTime": "2021-10-12T06:20:21.952Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Kortrijk", + "@type": "http://schema.org/ParkingFacility", + "name": "Kortrijk", + "longitude": 3.263269901276, + "latitude": 50.823799133301, + "bikes_available": 28, + "capacity": 47, + "docks_available": 19, + "nearby": "http://irail.be/stations/NMBS/008896008" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.715939998627, + 50.87979888916 + ] + }, + "generatedAtTime": "2021-10-12T06:20:22.664Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Leuven(Tiensevest)", + "@type": "http://schema.org/ParkingFacility", + "name": "Leuven (Tiensevest)", + "longitude": 4.715939998627, + "latitude": 50.87979888916, + "bikes_available": 57, + "capacity": 86, + "docks_available": 29, + "nearby": "http://irail.be/stations/NMBS/008833001" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.56152009964, + 51.135799407959 + ] + }, + "generatedAtTime": "2021-10-12T06:20:23.383Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Lier", + "@type": "http://schema.org/ParkingFacility", + "name": "Lier", + "longitude": 4.56152009964, + "latitude": 51.135799407959, + "bikes_available": 21, + "capacity": 26, + "docks_available": 5, + "nearby": "http://irail.be/stations/NMBS/008821600" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.986469984055, + 51.107398986816 + ] + }, + "generatedAtTime": "2021-10-12T06:20:24.095Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Lokeren", + "@type": "http://schema.org/ParkingFacility", + "name": "Lokeren", + "longitude": 3.986469984055, + "latitude": 51.107398986816, + "bikes_available": 13, + "capacity": 17, + "docks_available": 4, + "nearby": "http://irail.be/stations/NMBS/008894201" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.565639972687, + 50.625801086426 + ] + }, + "generatedAtTime": "2021-10-12T06:20:24.831Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Luik", + "@type": "http://schema.org/ParkingFacility", + "name": "Luik", + "longitude": 5.565639972687, + "latitude": 50.625801086426, + "bikes_available": 0, + "capacity": 4, + "docks_available": 4, + "nearby": "http://irail.be/stations/NMBS/008841004" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.48299, + 51.0178 + ] + }, + "generatedAtTime": "2021-10-12T06:20:25.537Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Mechelen", + "@type": "http://schema.org/ParkingFacility", + "name": "Mechelen", + "longitude": 4.48299, + "latitude": 51.0178, + "bikes_available": 47, + "capacity": 76, + "docks_available": 29, + "nearby": "http://irail.be/stations/NMBS/008822004" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.115129947662, + 51.190498352051 + ] + }, + "generatedAtTime": "2021-10-12T06:20:26.232Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Mol", + "@type": "http://schema.org/ParkingFacility", + "name": "Mol", + "longitude": 5.115129947662, + "latitude": 51.190498352051, + "bikes_available": 10, + "capacity": 15, + "docks_available": 5, + "nearby": "http://irail.be/stations/NMBS/008832409" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.455999851227, + 51.171199798584 + ] + }, + "generatedAtTime": "2021-10-12T06:20:26.934Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Mortsel-Oude-God", + "@type": "http://schema.org/ParkingFacility", + "name": "Mortsel-Oude-God", + "longitude": 4.455999851227, + "latitude": 51.171199798584, + "bikes_available": 4, + "capacity": 6, + "docks_available": 2, + "nearby": "http://irail.be/stations/NMBS/008821238" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.862239837646, + 50.468601226807 + ] + }, + "generatedAtTime": "2021-10-12T06:20:27.637Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Namen", + "@type": "http://schema.org/ParkingFacility", + "name": "Namen", + "longitude": 4.862239837646, + "latitude": 50.468601226807, + "bikes_available": 6, + "capacity": 6, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008863008" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 2.925649881363, + 51.228298187256 + ] + }, + "generatedAtTime": "2021-10-12T06:20:28.362Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Oostende", + "@type": "http://schema.org/ParkingFacility", + "name": "Oostende", + "longitude": 2.925649881363, + "latitude": 51.228298187256, + "bikes_available": 53, + "capacity": 69, + "docks_available": 16, + "nearby": "http://irail.be/stations/NMBS/008891702" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.130480051041, + 50.949001312256 + ] + }, + "generatedAtTime": "2021-10-12T06:20:29.065Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Roeselare", + "@type": "http://schema.org/ParkingFacility", + "name": "Roeselare", + "longitude": 3.130480051041, + "latitude": 50.949001312256, + "bikes_available": 14, + "capacity": 27, + "docks_available": 13, + "nearby": "http://irail.be/stations/NMBS/008896800" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.143700122833, + 51.171398162842 + ] + }, + "generatedAtTime": "2021-10-12T06:20:29.783Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Sint-Niklaas", + "@type": "http://schema.org/ParkingFacility", + "name": "Sint-Niklaas", + "longitude": 4.143700122833, + "latitude": 51.171398162842, + "bikes_available": 15, + "capacity": 30, + "docks_available": 15, + "nearby": "http://irail.be/stations/NMBS/008894508" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.176330089569, + 50.817401885986 + ] + }, + "generatedAtTime": "2021-10-12T06:20:30.506Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Sint-Truiden", + "@type": "http://schema.org/ParkingFacility", + "name": "Sint-Truiden", + "longitude": 5.176330089569, + "latitude": 50.817401885986, + "bikes_available": 9, + "capacity": 9, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008831807" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.105509996414, + 51.064800262451 + ] + }, + "generatedAtTime": "2021-10-12T06:20:31.284Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Torhout", + "@type": "http://schema.org/ParkingFacility", + "name": "Torhout", + "longitude": 3.105509996414, + "latitude": 51.064800262451, + "bikes_available": 3, + "capacity": 4, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008891314" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.937900066376, + 51.322101593018 + ] + }, + "generatedAtTime": "2021-10-12T06:20:32.245Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Turnhout", + "@type": "http://schema.org/ParkingFacility", + "name": "Turnhout", + "longitude": 4.937900066376, + "latitude": 51.322101593018, + "bikes_available": 9, + "capacity": 13, + "docks_available": 4, + "nearby": "http://irail.be/stations/NMBS/008821907" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.433000087738, + 50.924701690674 + ] + }, + "generatedAtTime": "2021-10-12T06:20:33.067Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Vilvoorde", + "@type": "http://schema.org/ParkingFacility", + "name": "Vilvoorde", + "longitude": 4.433000087738, + "latitude": 50.924701690674, + "bikes_available": 20, + "capacity": 40, + "docks_available": 20, + "nearby": "http://irail.be/stations/NMBS/008811189" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.424420118332, + 50.891799926758 + ] + }, + "generatedAtTime": "2021-10-12T06:20:33.881Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Waregem", + "@type": "http://schema.org/ParkingFacility", + "name": "Waregem", + "longitude": 3.424420118332, + "latitude": 50.891799926758, + "bikes_available": 12, + "capacity": 16, + "docks_available": 4, + "nearby": "http://irail.be/stations/NMBS/008896149" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.575040102005, + 51.180801391602 + ] + }, + "generatedAtTime": "2021-10-12T06:20:34.600Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Eeklo", + "@type": "http://schema.org/ParkingFacility", + "name": "Eeklo", + "longitude": 3.575040102005, + "latitude": 51.180801391602, + "bikes_available": 1, + "capacity": 11, + "docks_available": 10, + "nearby": "http://irail.be/stations/NMBS/008893708" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.989160060883, + 51.168800354004 + ] + }, + "generatedAtTime": "2021-10-12T06:20:35.295Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Geel", + "@type": "http://schema.org/ParkingFacility", + "name": "Geel", + "longitude": 4.989160060883, + "latitude": 51.168800354004, + "bikes_available": 8, + "capacity": 15, + "docks_available": 7, + "nearby": "http://irail.be/stations/NMBS/008832433" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.026179790497, + 50.839500427246 + ] + }, + "generatedAtTime": "2021-10-12T06:20:36.006Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Ninove", + "@type": "http://schema.org/ParkingFacility", + "name": "Ninove", + "longitude": 4.026179790497, + "latitude": 50.839500427246, + "bikes_available": 7, + "capacity": 10, + "docks_available": 3, + "nearby": "http://irail.be/stations/NMBS/008895760" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.708779811859, + 51.074199676514 + ] + }, + "generatedAtTime": "2021-10-12T06:20:36.712Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Heist-op-den-Berg", + "@type": "http://schema.org/ParkingFacility", + "name": "Heist-op-den-Berg", + "longitude": 4.708779811859, + "latitude": 51.074199676514, + "bikes_available": 12, + "capacity": 12, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821832" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.473400115967, + 50.78450012207 + ] + }, + "generatedAtTime": "2021-10-12T06:20:37.434Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Tongeren", + "@type": "http://schema.org/ParkingFacility", + "name": "Tongeren", + "longitude": 5.473400115967, + "latitude": 50.78450012207, + "bikes_available": 8, + "capacity": 8, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008831310" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.489570140839, + 51.030200958252 + ] + }, + "generatedAtTime": "2021-10-12T06:20:38.128Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Mechelen-Nekkerspoel", + "@type": "http://schema.org/ParkingFacility", + "name": "Mechelen-Nekkerspoel", + "longitude": 4.489570140839, + "latitude": 51.030200958252, + "bikes_available": 17, + "capacity": 20, + "docks_available": 3, + "nearby": "http://irail.be/stations/NMBS/008822343" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.601880073547, + 50.851299285889 + ] + }, + "generatedAtTime": "2021-10-12T06:20:39.151Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Oudenaarde", + "@type": "http://schema.org/ParkingFacility", + "name": "Oudenaarde ", + "longitude": 3.601880073547, + "latitude": 50.851299285889, + "bikes_available": 0, + "capacity": 2, + "docks_available": 2, + "nearby": "http://irail.be/stations/NMBS/008892601" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.872560024261, + 50.770500183105 + ] + }, + "generatedAtTime": "2021-10-12T06:20:39.838Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Geraardsbergen", + "@type": "http://schema.org/ParkingFacility", + "name": "Geraardsbergen ", + "longitude": 3.872560024261, + "latitude": 50.770500183105, + "bikes_available": 2, + "capacity": 2, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008895505" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 2.876270055771, + 50.848400115967 + ] + }, + "generatedAtTime": "2021-10-12T06:20:41.845Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Ieper", + "@type": "http://schema.org/ParkingFacility", + "name": "Ieper", + "longitude": 2.876270055771, + "latitude": 50.848400115967, + "bikes_available": 5, + "capacity": 10, + "docks_available": 5, + "nearby": "http://irail.be/stations/NMBS/008896503" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.583220005035, + 51.128299713135 + ] + }, + "generatedAtTime": "2021-10-12T06:20:42.586Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/LierVeemarkt", + "@type": "http://schema.org/ParkingFacility", + "name": "Lier Veemarkt", + "longitude": 4.583220005035, + "latitude": 51.128299713135, + "bikes_available": 7, + "capacity": 7, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821600" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.573339939117, + 51.125301361084 + ] + }, + "generatedAtTime": "2021-10-12T06:20:43.366Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/LierP%2BR%20De%20Mol", + "@type": "http://schema.org/ParkingFacility", + "name": "Lier P+R De Mol", + "longitude": 4.573339939117, + "latitude": 51.125301361084, + "bikes_available": 8, + "capacity": 8, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821600" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.494080066681, + 51.16849899292 + ] + }, + "generatedAtTime": "2021-10-12T06:20:44.070Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/BoechoutP%2BR%20Capenberg", + "@type": "http://schema.org/ParkingFacility", + "name": "Boechout P+R Capenberg", + "longitude": 4.494080066681, + "latitude": 51.16849899292, + "bikes_available": 5, + "capacity": 5, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821634" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.449820041656, + 51.092498779297 + ] + }, + "generatedAtTime": "2021-10-12T06:20:44.789Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Aalter", + "@type": "http://schema.org/ParkingFacility", + "name": "Aalter", + "longitude": 3.449820041656, + "latitude": 51.092498779297, + "bikes_available": 6, + "capacity": 6, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008891140" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.525919914246, + 50.982398986816 + ] + }, + "generatedAtTime": "2021-10-12T06:20:45.522Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/DeinzeLeiespiegel", + "@type": "http://schema.org/ParkingFacility", + "name": "Deinze Leiespiegel", + "longitude": 3.525919914246, + "latitude": 50.982398986816, + "bikes_available": 9, + "capacity": 11, + "docks_available": 2, + "nearby": "http://irail.be/stations/NMBS/008892106" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.92561006546, + 50.808200836182 + ] + }, + "generatedAtTime": "2021-10-12T06:20:46.217Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Tienen", + "@type": "http://schema.org/ParkingFacility", + "name": "Tienen ", + "longitude": 4.92561006546, + "latitude": 50.808200836182, + "bikes_available": 5, + "capacity": 13, + "docks_available": 8, + "nearby": "http://irail.be/stations/NMBS/008833308" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 2.735719919205, + 50.854499816895 + ] + }, + "generatedAtTime": "2021-10-12T06:20:46.915Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Poperinge", + "@type": "http://schema.org/ParkingFacility", + "name": "Poperinge", + "longitude": 2.735719919205, + "latitude": 50.854499816895, + "bikes_available": 7, + "capacity": 7, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008896735" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.632410049438, + 51.35710144043 + ] + }, + "generatedAtTime": "2021-10-12T06:20:47.633Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Noorderkempen", + "@type": "http://schema.org/ParkingFacility", + "name": "Noorderkempen", + "longitude": 4.632410049438, + "latitude": 51.35710144043, + "bikes_available": 10, + "capacity": 10, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821105" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.460559844971, + 51.365100860596 + ] + }, + "generatedAtTime": "2021-10-12T06:20:48.339Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Heide", + "@type": "http://schema.org/ParkingFacility", + "name": "Heide", + "longitude": 4.460559844971, + "latitude": 51.365100860596, + "bikes_available": 11, + "capacity": 12, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008821519" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.484320163727, + 51.029300689697 + ] + }, + "generatedAtTime": "2021-10-12T06:20:49.040Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/MechelenVeemarkt", + "@type": "http://schema.org/ParkingFacility", + "name": "Mechelen Veemarkt", + "longitude": 4.484320163727, + "latitude": 51.029300689697, + "bikes_available": 6, + "capacity": 7, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008822343" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.476119995117, + 51.134300231934 + ] + }, + "generatedAtTime": "2021-10-12T06:20:49.729Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Kontich-Lint", + "@type": "http://schema.org/ParkingFacility", + "name": "Kontich-Lint", + "longitude": 4.476119995117, + "latitude": 51.134300231934, + "bikes_available": 5, + "capacity": 7, + "docks_available": 2, + "nearby": "http://irail.be/stations/NMBS/008821311" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.329819917679, + 50.990600585938 + ] + }, + "generatedAtTime": "2021-10-12T06:20:50.446Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Tielt", + "@type": "http://schema.org/ParkingFacility", + "name": "Tielt", + "longitude": 3.329819917679, + "latitude": 50.990600585938, + "bikes_available": 7, + "capacity": 7, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008892254" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.470870018005, + 50.885200500488 + ] + }, + "generatedAtTime": "2021-10-12T06:20:51.139Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Zaventem", + "@type": "http://schema.org/ParkingFacility", + "name": "Zaventem", + "longitude": 4.470870018005, + "latitude": 50.885200500488, + "bikes_available": 8, + "capacity": 8, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008811221" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.390810012817, + 51.198001861572 + ] + }, + "generatedAtTime": "2021-10-12T06:20:51.829Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Antwerpen-Zuid", + "@type": "http://schema.org/ParkingFacility", + "name": "Antwerpen-Zuid", + "longitude": 4.390810012817, + "latitude": 51.198001861572, + "bikes_available": 8, + "capacity": 8, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821196" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.214819908142, + 50.9208984375 + ] + }, + "generatedAtTime": "2021-10-12T06:20:52.530Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Izegem", + "@type": "http://schema.org/ParkingFacility", + "name": "Izegem", + "longitude": 3.214819908142, + "latitude": 50.9208984375, + "bikes_available": 6, + "capacity": 8, + "docks_available": 2, + "nearby": "http://irail.be/stations/NMBS/008896909" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.711899995804, + 51.035999298096 + ] + }, + "generatedAtTime": "2021-10-12T06:20:53.258Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Gent-Sint-Pieters(M.%20Hendrikaplein)", + "@type": "http://schema.org/ParkingFacility", + "name": "Gent-Sint-Pieters (M. Hendrikaplein)", + "longitude": 3.711899995804, + "latitude": 51.035999298096, + "bikes_available": 16, + "capacity": 26, + "docks_available": 10, + "nearby": "http://irail.be/stations/NMBS/008892007" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.716639995575, + 50.882499694824 + ] + }, + "generatedAtTime": "2021-10-12T06:20:53.955Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Leuven(Kop%20van%20Kessel-Lo)", + "@type": "http://schema.org/ParkingFacility", + "name": "Leuven (Kop van Kessel-Lo)", + "longitude": 4.716639995575, + "latitude": 50.882499694824, + "bikes_available": 16, + "capacity": 24, + "docks_available": 8, + "nearby": "http://irail.be/stations/NMBS/008833001" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.216079950333, + 51.196800231934 + ] + }, + "generatedAtTime": "2021-10-12T06:20:54.646Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Brugge(Kamgebouw)", + "@type": "http://schema.org/ParkingFacility", + "name": "Brugge (Kamgebouw)", + "longitude": 3.216079950333, + "latitude": 51.196800231934, + "bikes_available": 17, + "capacity": 27, + "docks_available": 10, + "nearby": "http://irail.be/stations/NMBS/008891009" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.260260105133, + 51.208499908447 + ] + }, + "generatedAtTime": "2021-10-12T06:20:55.643Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Beveren", + "@type": "http://schema.org/ParkingFacility", + "name": "Beveren", + "longitude": 4.260260105133, + "latitude": 51.208499908447, + "bikes_available": 10, + "capacity": 11, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008894748" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.695439815521, + 50.862598419189 + ] + }, + "generatedAtTime": "2021-10-12T06:20:56.370Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Heverlee", + "@type": "http://schema.org/ParkingFacility", + "name": "Heverlee ", + "longitude": 4.695439815521, + "latitude": 50.862598419189, + "bikes_available": 7, + "capacity": 9, + "docks_available": 2, + "nearby": "http://irail.be/stations/NMBS/008833126" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.31374001503, + 50.855701446533 + ] + }, + "generatedAtTime": "2021-10-12T06:20:57.256Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Harelbeke", + "@type": "http://schema.org/ParkingFacility", + "name": "Harelbeke", + "longitude": 3.31374001503, + "latitude": 50.855701446533, + "bikes_available": 8, + "capacity": 9, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008896115" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.64013004303, + 50.929401397705 + ] + }, + "generatedAtTime": "2021-10-12T06:20:57.946Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Gavere-Asper", + "@type": "http://schema.org/ParkingFacility", + "name": "Gavere-Asper", + "longitude": 3.64013004303, + "latitude": 50.929401397705, + "bikes_available": 7, + "capacity": 7, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008892643" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.654690027237, + 50.930500030518 + ] + }, + "generatedAtTime": "2021-10-12T06:20:58.653Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Gaveremobipunt", + "@type": "http://schema.org/ParkingFacility", + "name": "Gavere mobipunt", + "longitude": 3.654690027237, + "latitude": 50.930500030518, + "bikes_available": 4, + "capacity": 4, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008892643" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.43501996994, + 51.284198760986 + ] + }, + "generatedAtTime": "2021-10-12T06:20:59.366Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Ekeren(perron%201)", + "@type": "http://schema.org/ParkingFacility", + "name": "Ekeren (perron 1)", + "longitude": 4.43501996994, + "latitude": 51.284198760986, + "bikes_available": 3, + "capacity": 4, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008821071" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.434390068054, + 51.281398773193 + ] + }, + "generatedAtTime": "2021-10-12T06:21:00.100Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Ekeren(perron%202)", + "@type": "http://schema.org/ParkingFacility", + "name": "Ekeren (perron 2)", + "longitude": 4.434390068054, + "latitude": 51.281398773193, + "bikes_available": 7, + "capacity": 7, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821071" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.424610137939, + 51.245498657227 + ] + }, + "generatedAtTime": "2021-10-12T06:21:00.798Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Antwerpen-Luchtbal", + "@type": "http://schema.org/ParkingFacility", + "name": "Antwerpen-Luchtbal", + "longitude": 4.424610137939, + "latitude": 51.245498657227, + "bikes_available": 5, + "capacity": 5, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821063" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.42755, + 51.2617 + ] + }, + "generatedAtTime": "2021-10-12T06:21:01.494Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Antwerpen-Noorderdokken", + "@type": "http://schema.org/ParkingFacility", + "name": "Antwerpen-Noorderdokken", + "longitude": 4.42755, + "latitude": 51.2617, + "bikes_available": 7, + "capacity": 7, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821089" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.447710037231, + 50.889099121094 + ] + }, + "generatedAtTime": "2021-10-12T06:21:02.217Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Diegem", + "@type": "http://schema.org/ParkingFacility", + "name": "Diegem", + "longitude": 4.447710037231, + "latitude": 50.889099121094, + "bikes_available": 4, + "capacity": 4, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008811213" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.426719903946, + 50.881999969482 + ] + }, + "generatedAtTime": "2021-10-12T06:21:02.941Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/WaregemExpo%20P%2BR", + "@type": "http://schema.org/ParkingFacility", + "name": "Waregem Expo P+R", + "longitude": 3.426719903946, + "latitude": 50.881999969482, + "bikes_available": 4, + "capacity": 4, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008896149" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.281699, + 51.03833 + ] + }, + "generatedAtTime": "2021-10-12T06:21:03.654Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Heusden", + "@type": "http://schema.org/ParkingFacility", + "name": "Heusden", + "longitude": 5.281699, + "latitude": 51.03833, + "bikes_available": 8, + "capacity": 8, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008832243" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.329922, + 51.033617 + ] + }, + "generatedAtTime": "2021-10-12T06:21:04.377Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Zolder", + "@type": "http://schema.org/ParkingFacility", + "name": "Zolder", + "longitude": 5.329922, + "latitude": 51.033617, + "bikes_available": 8, + "capacity": 8, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008832250" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.329820851913, + 50.932948204381 + ] + }, + "generatedAtTime": "2021-10-12T06:21:05.094Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Meise", + "@type": "http://schema.org/ParkingFacility", + "name": "Meise", + "longitude": 4.329820851913, + "latitude": 50.932948204381, + "bikes_available": 8, + "capacity": 8, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008812047" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.746427, + 51.165508 + ] + }, + "generatedAtTime": "2021-10-12T06:21:05.822Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Bouwel", + "@type": "http://schema.org/ParkingFacility", + "name": "Bouwel", + "longitude": 4.746427, + "latitude": 51.165508, + "bikes_available": 6, + "capacity": 6, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821725" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.613315, + 50.966578 + ] + }, + "generatedAtTime": "2021-10-12T06:21:06.544Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Haacht", + "@type": "http://schema.org/ParkingFacility", + "name": "Haacht", + "longitude": 4.613315, + "latitude": 50.966578, + "bikes_available": 7, + "capacity": 8, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008822517" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.814341, + 50.86948 + ] + }, + "generatedAtTime": "2021-10-12T06:21:07.245Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Zottegem", + "@type": "http://schema.org/ParkingFacility", + "name": "Zottegem ", + "longitude": 3.814341, + "latitude": 50.86948, + "bikes_available": 7, + "capacity": 8, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008895208" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.221333, + 51.125867 + ] + }, + "generatedAtTime": "2021-10-12T06:21:07.957Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Temse", + "@type": "http://schema.org/ParkingFacility", + "name": "Temse", + "longitude": 4.221333, + "latitude": 51.125867, + "bikes_available": 5, + "capacity": 5, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008894672" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.041885, + 51.073108 + ] + }, + "generatedAtTime": "2021-10-12T06:21:08.664Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Zele", + "@type": "http://schema.org/ParkingFacility", + "name": "Zele", + "longitude": 4.041885, + "latitude": 51.073108, + "bikes_available": 6, + "capacity": 7, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008894235" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.182521, + 50.810926 + ] + }, + "generatedAtTime": "2021-10-12T06:21:09.366Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Wevelgem", + "@type": "http://schema.org/ParkingFacility", + "name": "Wevelgem", + "longitude": 3.182521, + "latitude": 50.810926, + "bikes_available": 8, + "capacity": 8, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008896370" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.312579, + 51.21177 + ] + }, + "generatedAtTime": "2021-10-12T06:21:10.091Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Lommel", + "@type": "http://schema.org/ParkingFacility", + "name": "Lommel", + "longitude": 5.312579, + "latitude": 51.21177, + "bikes_available": 6, + "capacity": 6, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008832565" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.464733, + 51.1541195 + ] + }, + "generatedAtTime": "2021-10-12T06:21:10.807Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Hove", + "@type": "http://schema.org/ParkingFacility", + "name": "Hove", + "longitude": 4.464733, + "latitude": 51.1541195, + "bikes_available": 7, + "capacity": 7, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008821337" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.451532, + 51.462176 + ] + }, + "generatedAtTime": "2021-10-12T06:21:11.779Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Essen", + "@type": "http://schema.org/ParkingFacility", + "name": "Essen", + "longitude": 4.451532, + "latitude": 51.462176, + "bikes_available": 7, + "capacity": 8, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008821402" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.079402, + 50.74829 + ] + }, + "generatedAtTime": "2021-10-12T06:21:12.535Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Landen", + "@type": "http://schema.org/ParkingFacility", + "name": "Landen", + "longitude": 5.079402, + "latitude": 50.74829, + "bikes_available": 7, + "capacity": 8, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008833605" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 2.601611, + 51.077376 + ] + }, + "generatedAtTime": "2021-10-12T06:21:13.294Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/DePanne", + "@type": "http://schema.org/ParkingFacility", + "name": "De Panne", + "longitude": 2.601611, + "latitude": 51.077376, + "bikes_available": 8, + "capacity": 8, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008892338" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.1338, + 51.3119 + ] + }, + "generatedAtTime": "2021-10-12T06:21:14.201Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Blankenberge", + "@type": "http://schema.org/ParkingFacility", + "name": "Blankenberge", + "longitude": 3.1338, + "latitude": 51.3119, + "bikes_available": 6, + "capacity": 7, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008891405" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.195476, + 51.32651 + ] + }, + "generatedAtTime": "2021-10-12T06:21:14.998Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Zeebrugge-dorp", + "@type": "http://schema.org/ParkingFacility", + "name": "Zeebrugge-dorp", + "longitude": 3.195476, + "latitude": 51.32651, + "bikes_available": 2, + "capacity": 2, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008891553" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.239668, + 51.334168 + ] + }, + "generatedAtTime": "2021-10-12T06:21:15.934Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Heist", + "@type": "http://schema.org/ParkingFacility", + "name": "Heist", + "longitude": 3.239668, + "latitude": 51.334168, + "bikes_available": 5, + "capacity": 5, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008891645" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.28454, + 51.34018 + ] + }, + "generatedAtTime": "2021-10-12T06:21:16.629Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Knokke", + "@type": "http://schema.org/ParkingFacility", + "name": "Knokke", + "longitude": 3.28454, + "latitude": 51.34018, + "bikes_available": 7, + "capacity": 8, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008891660" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.361626, + 50.747786 + ] + }, + "generatedAtTime": "2021-10-12T06:21:17.346Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Sint-Genesius-Rode", + "@type": "http://schema.org/ParkingFacility", + "name": "Sint-Genesius-Rode", + "longitude": 4.361626, + "latitude": 50.747786, + "bikes_available": 9, + "capacity": 9, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008814167" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.575647, + 51.064208 + ] + }, + "generatedAtTime": "2021-10-12T06:21:18.066Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Landegem", + "@type": "http://schema.org/ParkingFacility", + "name": "Landegem", + "longitude": 3.575647, + "latitude": 51.064208, + "bikes_available": 8, + "capacity": 8, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008892056" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.413591, + 50.923187 + ] + }, + "generatedAtTime": "2021-10-12T06:21:18.772Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/VilvoordeHoppin%20De%20Kassei", + "@type": "http://schema.org/ParkingFacility", + "name": "Vilvoorde Hoppin De Kassei", + "longitude": 4.413591, + "latitude": 50.923187, + "bikes_available": 6, + "capacity": 7, + "docks_available": 1, + "nearby": "http://irail.be/stations/NMBS/008811189" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.333756181866, + 50.835529468767 + ] + }, + "generatedAtTime": "2021-10-12T06:21:19.556Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Brussel-Zuid", + "@type": "http://schema.org/ParkingFacility", + "name": "Brussel-Zuid", + "longitude": 4.333756181866, + "latitude": 50.835529468767, + "bikes_available": 4, + "capacity": 6, + "docks_available": 2, + "nearby": "http://irail.be/stations/NMBS/008814001" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 5.112421, + 51.18119 + ] + }, + "generatedAtTime": "2021-10-12T06:21:20.277Z", + "properties": { + "@id": "http://irail.be/stations/bluebike/Mol-%20De%20Nete", + "@type": "http://schema.org/ParkingFacility", + "name": "Mol - De Nete", + "longitude": 5.112421, + "latitude": 51.18119, + "bikes_available": 7, + "capacity": 7, + "docks_available": 0, + "nearby": "http://irail.be/stations/NMBS/008832409" + } + } + ] +} From 9b7839672a7bf37f421e570f64edbcc95cf8819f Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Sun, 20 Feb 2022 13:59:17 +0100 Subject: [PATCH 090/609] cli: Main: add Target metadata to output --- src/main/java/be/ugent/rml/cli/Main.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/be/ugent/rml/cli/Main.java b/src/main/java/be/ugent/rml/cli/Main.java index c650ff6a..7b542fc4 100644 --- a/src/main/java/be/ugent/rml/cli/Main.java +++ b/src/main/java/be/ugent/rml/cli/Main.java @@ -8,6 +8,7 @@ import be.ugent.rml.functions.lib.IDLabFunctions; import be.ugent.rml.metadata.MetadataGenerator; import be.ugent.rml.records.RecordsFactory; +import be.ugent.rml.store.Quad; import be.ugent.rml.store.QuadStore; import be.ugent.rml.store.RDF4JStore; import be.ugent.rml.store.SimpleQuadStore; @@ -428,9 +429,10 @@ private static void writeOutputTargets(HashMap targets, QuadSto logger.info(store.size() + " quad was generated " + term + " Target"); } - Target target = targetFactory.getTarget(term, rmlStore); + Target target = targetFactory.getTarget(term, rmlStore, store); String serializationFormat = target.getSerializationFormat(); OutputStream output = target.getOutputStream(); + store.addQuads(target.getMetadata()); // Set character encoding Writer out = new BufferedWriter(new OutputStreamWriter(output, Charset.defaultCharset())); From a3c477a4111c3c3fa79acec1863d591096fc23d3 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Sun, 20 Feb 2022 13:59:57 +0100 Subject: [PATCH 091/609] CHANGELOG: LDES Logical Target is now supported --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ada1946f..4c14cead 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +### Added + +- Add support for LDES Logical Target + ### Fixed - Clarified Readme for quick start From c1944ae23559ade3c7b6c6e1660fd0b9ddcda876 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Sun, 20 Feb 2022 14:47:36 +0100 Subject: [PATCH 092/609] test: logical-target: sparql: fix test SPARQL Logical Target contained an error in the mapping file. The LDES Logical Target changes increased the test coverage and catched this bug! --- .../resources/web-of-things/logical-target/sparql/mapping.ttl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/resources/web-of-things/logical-target/sparql/mapping.ttl b/src/test/resources/web-of-things/logical-target/sparql/mapping.ttl index fa26fa4a..9d50b514 100644 --- a/src/test/resources/web-of-things/logical-target/sparql/mapping.ttl +++ b/src/test/resources/web-of-things/logical-target/sparql/mapping.ttl @@ -28,7 +28,7 @@ . <#TargetSPARQL> a rmlt:LogicalTarget; - rmlt:target ; + rmlt:target <#SPARQLEndpoint>; . <#WoTWebResource> a td:PropertyAffordance; From 3b3429b5365db88b998f363a3beb8246065c5732 Mon Sep 17 00:00:00 2001 From: Ben De Meester Date: Thu, 3 Mar 2022 09:22:10 +0000 Subject: [PATCH 093/609] MappingExtractor: No cartesian product for same Logical Sources No more cartesian product when logical source is the same (i.e., has the same URI). If you still want the cartesian product, update your mappings to refer to different logical sources with the same contents. Fixes https://github.com/RMLio/rmlmapper-java/issues/28 --- CHANGELOG.md | 2 + .../java/be/ugent/rml/MappingFactory.java | 41 +++++++++++++ .../be/ugent/rml/extractor/HashExtractor.java | 36 ++++++++++++ .../java/be/ugent/rml/Mapper_JSON_Test.java | 10 ++++ .../test-cases/RMLTC1028-JSON/mapping.ttl | 54 +++++++++++++++++ .../test-cases/RMLTC1028-JSON/output.nq | 12 ++++ .../test-cases/RMLTC1028-JSON/venue.json | 22 +++++++ .../test-cases/RMLTC1028b-JSON/mapping.ttl | 58 +++++++++++++++++++ .../test-cases/RMLTC1028b-JSON/output.nq | 14 +++++ .../test-cases/RMLTC1028b-JSON/venue.json | 22 +++++++ 10 files changed, 271 insertions(+) create mode 100644 src/main/java/be/ugent/rml/extractor/HashExtractor.java create mode 100644 src/test/resources/test-cases/RMLTC1028-JSON/mapping.ttl create mode 100644 src/test/resources/test-cases/RMLTC1028-JSON/output.nq create mode 100644 src/test/resources/test-cases/RMLTC1028-JSON/venue.json create mode 100644 src/test/resources/test-cases/RMLTC1028b-JSON/mapping.ttl create mode 100644 src/test/resources/test-cases/RMLTC1028b-JSON/output.nq create mode 100644 src/test/resources/test-cases/RMLTC1028b-JSON/venue.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c14cead..e3265ae8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed - Clarified Readme for quick start +- No cartesian product when referring to the same logical source (see [issue 28](https://github.com/RMLio/rmlmapper-java/issues/28)]) + - If you still want the cartesian product, update your mappings to refer to different logical sources with the same contents ### Changed diff --git a/src/main/java/be/ugent/rml/MappingFactory.java b/src/main/java/be/ugent/rml/MappingFactory.java index c805e4ca..924a9534 100644 --- a/src/main/java/be/ugent/rml/MappingFactory.java +++ b/src/main/java/be/ugent/rml/MappingFactory.java @@ -1,6 +1,7 @@ package be.ugent.rml; import be.ugent.rml.extractor.ConstantExtractor; +import be.ugent.rml.extractor.HashExtractor; import be.ugent.rml.extractor.ReferenceExtractor; import be.ugent.rml.functions.*; import be.ugent.rml.store.QuadStore; @@ -282,6 +283,26 @@ private void parseObjectMapWithCallback(Term objectmap, BiConsumer logicalSources = Utils.getObjectsFromQuads(store.getQuads(this.triplesMap, new NamedNode(NAMESPACES.RML + "logicalSource"), null)); + Term logicalSource = null; + if (!logicalSources.isEmpty()) { + logicalSource = logicalSources.get(0); + } + + List parentLogicalSources = Utils.getObjectsFromQuads(store.getQuads(parentTriplesMap, new NamedNode(NAMESPACES.RML + "logicalSource"), null)); + Term parentLogicalSource = null; + if (!parentLogicalSources.isEmpty()) { + parentLogicalSource = parentLogicalSources.get(0); + } + // Check if there is at least one Logical Source. + // If logical sources are the same (i.e., have the same IRI): the condition is 'join on same record' + if (logicalSource.equals(parentLogicalSource)) { + // TODO this is a _WILDLY_ inefficient way of handling this: a join is still executed, but then on the hashcode of the record. + // I'm not sure what a more elegant solution would entail in the current architecture: joins are currently hard to optimize + joinConditionFunctionExecutors.add(generateSameLogicalSourceJoinConditionFunctionTermMap()); + } + if (refObjectMapCallback != null) { refObjectMapCallback.accept(parentTriplesMap, joinConditionFunctionExecutors); } @@ -456,6 +477,26 @@ private MultipleRecordsFunctionExecutor parseJoinConditionFunctionTermMap(Term f return new DynamicMultipleRecordsFunctionExecutor(params, functionLoader); } + /** + * Generate a join condition that only returns true if the same record hash is encountered + * @return + * @throws IOException + */ + private MultipleRecordsFunctionExecutor generateSameLogicalSourceJoinConditionFunctionTermMap() throws IOException { + FunctionModel equal = functionLoader.getFunction(new NamedNode("http://example.com/idlab/function/equal")); + Map parameters = new HashMap<>(); + + SingleRecordFunctionExecutor parent = new HashExtractor(); + Object[] detailsParent = {"parent", parent}; + parameters.put("http://users.ugent.be/~bjdmeest/function/grel.ttl#valueParameter", detailsParent); + + SingleRecordFunctionExecutor child = new HashExtractor(); + Object[] detailsChild = {"child", child}; + parameters.put("http://users.ugent.be/~bjdmeest/function/grel.ttl#valueParameter2", detailsChild); + + return new StaticMultipleRecordsFunctionExecutor(equal, parameters); + } + private List parseObjectMapsAndShortcuts(Term pom) throws IOException { List mappingInfos = new ArrayList<>(); diff --git a/src/main/java/be/ugent/rml/extractor/HashExtractor.java b/src/main/java/be/ugent/rml/extractor/HashExtractor.java new file mode 100644 index 00000000..a2d66022 --- /dev/null +++ b/src/main/java/be/ugent/rml/extractor/HashExtractor.java @@ -0,0 +1,36 @@ +package be.ugent.rml.extractor; + +import be.ugent.rml.functions.SingleRecordFunctionExecutor; +import be.ugent.rml.records.Record; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class HashExtractor implements Extractor, SingleRecordFunctionExecutor { + + public HashExtractor() { + } + + @Override + public List extract(Record record) { + ArrayList result = new ArrayList<>(); + result.add("" + record.hashCode()); + return result; + } + + @Override + public Object execute(Record record) throws IOException { + return extract(record); + } + + /** + * to String method + * + * @return string + */ + @Override + public String toString() { + return "HashExtractor"; + } +} diff --git a/src/test/java/be/ugent/rml/Mapper_JSON_Test.java b/src/test/java/be/ugent/rml/Mapper_JSON_Test.java index 73a34814..e4d28f69 100644 --- a/src/test/java/be/ugent/rml/Mapper_JSON_Test.java +++ b/src/test/java/be/ugent/rml/Mapper_JSON_Test.java @@ -274,4 +274,14 @@ public void evaluate_1027_JSON() { public void evaluate_1027_JSONL() { doMapping("./test-cases/RMLTC1027-JSONL/mapping.ttl", "./test-cases/RMLTC1027-JSONL/output.nq"); } + + @Test + public void evaluate_1028_JSON() { + doMapping("./test-cases/RMLTC1028-JSON/mapping.ttl", "./test-cases/RMLTC1028-JSON/output.nq"); + } + + @Test + public void evaluate_1028b_JSON() { + doMapping("./test-cases/RMLTC1028b-JSON/mapping.ttl", "./test-cases/RMLTC1028b-JSON/output.nq"); + } } diff --git a/src/test/resources/test-cases/RMLTC1028-JSON/mapping.ttl b/src/test/resources/test-cases/RMLTC1028-JSON/mapping.ttl new file mode 100644 index 00000000..1816e936 --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1028-JSON/mapping.ttl @@ -0,0 +1,54 @@ +@prefix rml: . +@prefix rr: . +@prefix ql: . +@prefix ex: . +@base . + +<#LogicalSource> rml:source "venue.json"; + rml:referenceFormulation ql:JSONPath ; + rml:iterator "$.venue[*]" . + +<#VenueMapping> a rr:TriplesMap; + rml:logicalSource <#LogicalSource> ; + rr:subjectMap + [ + rr:template "http://loc.example.com/city/{location.city}"; + rr:class ex:City; + ]; + rr:predicateObjectMap [ + rr:predicate ex:latlong; + rr:objectMap [ + rr:parentTriplesMap <#LocationMapping_JSON> + ] + ]; + rr:predicateObjectMap [ + rr:predicate ex:countryCode; + rr:objectMap [ + rml:reference "location.country" + ] + ]; + rr:predicateObjectMap [ + rr:predicate ex:onContinent; + rr:objectMap [ + rml:reference "location.continent" + ] + ]. + +<#LocationMapping_JSON> a rr:TriplesMap; + rml:logicalSource <#LogicalSource> ; + rr:subjectMap [ + rr:template "http://loc.example.com/latlong/{latitude},{longitude}" + ]; + rr:predicateObjectMap [ + rr:predicate ex:lat; + rr:objectMap [ + rml:reference "latitude" + ] + ]; + + rr:predicateObjectMap [ + rr:predicate ex:long; + rr:objectMap [ + rml:reference "longitude" + ] + ]. \ No newline at end of file diff --git a/src/test/resources/test-cases/RMLTC1028-JSON/output.nq b/src/test/resources/test-cases/RMLTC1028-JSON/output.nq new file mode 100644 index 00000000..b6f2b5e8 --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1028-JSON/output.nq @@ -0,0 +1,12 @@ + . + . + "BE" . + "EU" . + "50.901389" . + "4.484444" . + . + . + "GB" . + "EU" . + "51.51334" . + "-0.08901" . diff --git a/src/test/resources/test-cases/RMLTC1028-JSON/venue.json b/src/test/resources/test-cases/RMLTC1028-JSON/venue.json new file mode 100644 index 00000000..defa5e98 --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1028-JSON/venue.json @@ -0,0 +1,22 @@ +{ + "venue": [ + { + "latitude": "50.901389", + "longitude": "4.484444", + "location": { + "continent": "EU", + "country": "BE", + "city": "Brussels" + } + }, + { + "latitude": "51.51334", + "longitude": "-0.08901", + "location": { + "continent": "EU", + "country": "GB", + "city": "London" + } + } + ] +} \ No newline at end of file diff --git a/src/test/resources/test-cases/RMLTC1028b-JSON/mapping.ttl b/src/test/resources/test-cases/RMLTC1028b-JSON/mapping.ttl new file mode 100644 index 00000000..012791c7 --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1028b-JSON/mapping.ttl @@ -0,0 +1,58 @@ +@prefix rml: . +@prefix rr: . +@prefix ql: . +@prefix ex: . +@base . + +<#LogicalSource> rml:source "venue.json"; + rml:referenceFormulation ql:JSONPath ; + rml:iterator "$.venue[*]" . + +<#LogicalSource2> rml:source "venue.json"; + rml:referenceFormulation ql:JSONPath ; + rml:iterator "$.venue[*]" . + +<#VenueMapping> a rr:TriplesMap; + rml:logicalSource <#LogicalSource> ; + rr:subjectMap + [ + rr:template "http://loc.example.com/city/{location.city}"; + rr:class ex:City; + ]; + rr:predicateObjectMap [ + rr:predicate ex:latlong; + rr:objectMap [ + rr:parentTriplesMap <#LocationMapping_JSON> + ] + ]; + rr:predicateObjectMap [ + rr:predicate ex:countryCode; + rr:objectMap [ + rml:reference "location.country" + ] + ]; + rr:predicateObjectMap [ + rr:predicate ex:onContinent; + rr:objectMap [ + rml:reference "location.continent" + ] + ]. + +<#LocationMapping_JSON> a rr:TriplesMap; + rml:logicalSource <#LogicalSource2> ; + rr:subjectMap [ + rr:template "http://loc.example.com/latlong/{latitude},{longitude}" + ]; + rr:predicateObjectMap [ + rr:predicate ex:lat; + rr:objectMap [ + rml:reference "latitude" + ] + ]; + + rr:predicateObjectMap [ + rr:predicate ex:long; + rr:objectMap [ + rml:reference "longitude" + ] + ]. \ No newline at end of file diff --git a/src/test/resources/test-cases/RMLTC1028b-JSON/output.nq b/src/test/resources/test-cases/RMLTC1028b-JSON/output.nq new file mode 100644 index 00000000..09baddf4 --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1028b-JSON/output.nq @@ -0,0 +1,14 @@ + . + . + . + "BE" . + "EU" . + "50.901389" . + "4.484444" . + . + . + . + "GB" . + "EU" . + "51.51334" . + "-0.08901" . diff --git a/src/test/resources/test-cases/RMLTC1028b-JSON/venue.json b/src/test/resources/test-cases/RMLTC1028b-JSON/venue.json new file mode 100644 index 00000000..defa5e98 --- /dev/null +++ b/src/test/resources/test-cases/RMLTC1028b-JSON/venue.json @@ -0,0 +1,22 @@ +{ + "venue": [ + { + "latitude": "50.901389", + "longitude": "4.484444", + "location": { + "continent": "EU", + "country": "BE", + "city": "Brussels" + } + }, + { + "latitude": "51.51334", + "longitude": "-0.08901", + "location": { + "continent": "EU", + "country": "GB", + "city": "London" + } + } + ] +} \ No newline at end of file From 28672c5dba43e71a47356ef322525172ee49b2ab Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 12:22:29 +0100 Subject: [PATCH 094/609] Updated org.eclipse.jetty and com.github.tomakehurst to compatible version. --- CHANGELOG.md | 2 ++ pom.xml | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3265ae8..f3a609dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Clarified Readme for quick start - No cartesian product when referring to the same logical source (see [issue 28](https://github.com/RMLio/rmlmapper-java/issues/28)]) - If you still want the cartesian product, update your mappings to refer to different logical sources with the same contents +- Upgrade jetty-server and jetty-security to 9.4.44.v20210927 +- Upgrade wiremock-jre8 to 2.32.0 ### Changed diff --git a/pom.xml b/pom.xml index 712dcad5..5ad28510 100644 --- a/pom.xml +++ b/pom.xml @@ -211,20 +211,20 @@ org.eclipse.jetty jetty-server - 9.4.17.v20190418 + 9.4.44.v20210927 test org.eclipse.jetty jetty-security - 9.4.17.v20190418 + 9.4.44.v20210927 test com.github.tomakehurst wiremock-jre8 - 2.23.2 + 2.32.0 test From 4eac94cf5329b053269f7e5196f1abbf0fad5ee9 Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 13:41:29 +0100 Subject: [PATCH 095/609] ch.qos.logback upgraded ch.qos.logback from 1.2.3 -> 1.2.10 --- CHANGELOG.md | 3 ++- pom.xml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3a609dd..dd667d39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - No cartesian product when referring to the same logical source (see [issue 28](https://github.com/RMLio/rmlmapper-java/issues/28)]) - If you still want the cartesian product, update your mappings to refer to different logical sources with the same contents - Upgrade jetty-server and jetty-security to 9.4.44.v20210927 -- Upgrade wiremock-jre8 to 2.32.0 +- Upgrade wiremock-jre8 to 2.32.0 +- Upgrade ch.qos.logback to 1.2.10 ### Changed diff --git a/pom.xml b/pom.xml index 5ad28510..d332dea4 100644 --- a/pom.xml +++ b/pom.xml @@ -125,7 +125,7 @@ ch.qos.logback logback-classic - 1.2.3 + 1.2.10 commons-lang From d9c45ab06260f5d5ac11a86f959630d1fa3a652d Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 13:52:53 +0100 Subject: [PATCH 096/609] commons-cli upgraded from 1.4 -> 1.5.0 --- CHANGELOG.md | 1 + pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd667d39..d45f38a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Upgrade jetty-server and jetty-security to 9.4.44.v20210927 - Upgrade wiremock-jre8 to 2.32.0 - Upgrade ch.qos.logback to 1.2.10 +- Upgrade commons-cli to 1.5.0 ### Changed diff --git a/pom.xml b/pom.xml index d332dea4..0b7f95dc 100644 --- a/pom.xml +++ b/pom.xml @@ -135,7 +135,7 @@ commons-cli commons-cli - 1.4 + 1.5.0 org.eclipse.rdf4j From cb3dd95ee0953427f6473333130534d767a11139 Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 14:04:27 +0100 Subject: [PATCH 097/609] com.jayway.jsonpath upgraded from 2.6.0 -> 2.7.0 --- CHANGELOG.md | 1 + pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d45f38a5..d1cef24c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Upgrade wiremock-jre8 to 2.32.0 - Upgrade ch.qos.logback to 1.2.10 - Upgrade commons-cli to 1.5.0 +- Upgrade com.jayway.jsonpath to 2.7.0 ### Changed diff --git a/pom.xml b/pom.xml index 0b7f95dc..cf311828 100644 --- a/pom.xml +++ b/pom.xml @@ -157,7 +157,7 @@ com.jayway.jsonpath json-path - 2.6.0 + 2.7.0 javax.xml.parsers From b02f3e08e5767a296bd0626da7e320d2637ce1a3 Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 14:14:32 +0100 Subject: [PATCH 098/609] ch.vorburger.mariaDB4j upgraded from 2.4.0 -> 2.5.3 --- CHANGELOG.md | 1 + pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1cef24c..6a3c31ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Upgrade ch.qos.logback to 1.2.10 - Upgrade commons-cli to 1.5.0 - Upgrade com.jayway.jsonpath to 2.7.0 +- Upgrade ch.vorburger.mariaDB4j to 2.5.3 ### Changed diff --git a/pom.xml b/pom.xml index cf311828..7157599a 100644 --- a/pom.xml +++ b/pom.xml @@ -172,7 +172,7 @@ ch.vorburger.mariaDB4j mariaDB4j - 2.4.0 + 2.5.3 test From 5f037cf719a7619380cc96029686f44abd5f548e Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 14:47:25 +0100 Subject: [PATCH 099/609] com.microsoft.sqlserver upgraded from 7.2.2.jre8 -> 9.4.0.jre8 --- CHANGELOG.md | 1 + pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a3c31ad..f47ba712 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Upgrade commons-cli to 1.5.0 - Upgrade com.jayway.jsonpath to 2.7.0 - Upgrade ch.vorburger.mariaDB4j to 2.5.3 +- Upgrade com.microsoft.sqlserver to 9.4.0.jre8 ### Changed diff --git a/pom.xml b/pom.xml index 7157599a..852a23d7 100644 --- a/pom.xml +++ b/pom.xml @@ -184,7 +184,7 @@ com.microsoft.sqlserver mssql-jdbc - 7.2.2.jre8 + 9.4.0.jre8 test From bfd0ddaff7a398a61305de34f6e5ce2e9b94cd41 Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 15:21:47 +0100 Subject: [PATCH 100/609] com.fasterxml.jackson.core upgraded to 2.12.1 -> 2.13.1 --- CHANGELOG.md | 1 + pom.xml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f47ba712..352d40b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Upgrade com.jayway.jsonpath to 2.7.0 - Upgrade ch.vorburger.mariaDB4j to 2.5.3 - Upgrade com.microsoft.sqlserver to 9.4.0.jre8 +- Upgrade com.fasterxml.jackson.core to 2.13.1 ### Changed diff --git a/pom.xml b/pom.xml index 852a23d7..eda66975 100644 --- a/pom.xml +++ b/pom.xml @@ -196,17 +196,17 @@ com.fasterxml.jackson.core jackson-core - 2.12.1 + 2.13.1 com.fasterxml.jackson.core jackson-databind - 2.12.1 + 2.13.1 com.fasterxml.jackson.core jackson-annotations - 2.12.1 + 2.13.1 org.eclipse.jetty From 765ceca6d50ac98dc61e4301bc150ce271fbab6f Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 15:30:13 +0100 Subject: [PATCH 101/609] org.jsoup upgrade from 1.14.2 -> 1.14.3 --- CHANGELOG.md | 1 + pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 352d40b6..e2de6509 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Upgrade ch.vorburger.mariaDB4j to 2.5.3 - Upgrade com.microsoft.sqlserver to 9.4.0.jre8 - Upgrade com.fasterxml.jackson.core to 2.13.1 +- Upgrade org.jsoup to 1.14.3 ### Changed diff --git a/pom.xml b/pom.xml index eda66975..6fc753b2 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ org.jsoup jsoup - 1.14.2 + 1.14.3 From 719d8bccc9857f3f4d20894217810cb59381056b Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 15:46:42 +0100 Subject: [PATCH 102/609] org.apache.poi upgraded to 4.1.0 -> 5.0.0 --- CHANGELOG.md | 1 + pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2de6509..78424149 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Upgrade com.microsoft.sqlserver to 9.4.0.jre8 - Upgrade com.fasterxml.jackson.core to 2.13.1 - Upgrade org.jsoup to 1.14.3 +- Upgrade org.apache.poi to 5.0.0 ### Changed diff --git a/pom.xml b/pom.xml index 6fc753b2..19a430ff 100644 --- a/pom.xml +++ b/pom.xml @@ -281,7 +281,7 @@ org.apache.poi poi-ooxml - 4.1.0 + 5.0.0 From 669ffdcf34c69eabf66cb59332b626d744d41f29 Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 15:54:42 +0100 Subject: [PATCH 103/609] org.apache.jena upgraded from 3.8.0 -> 4.3.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 19a430ff..9924fd38 100644 --- a/pom.xml +++ b/pom.xml @@ -231,7 +231,7 @@ org.apache.jena apache-jena-libs pom - 3.8.0 + 4.3.2 test From 594aa4df0ad3c7ba6c77785de37292855597a349 Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 16:03:23 +0100 Subject: [PATCH 104/609] com.microsoft.sqlserver upgrade from 9.4.0.jre8 -> 10.2.0.jre8 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9924fd38..d300aaf5 100644 --- a/pom.xml +++ b/pom.xml @@ -184,7 +184,7 @@ com.microsoft.sqlserver mssql-jdbc - 9.4.0.jre8 + 10.2.0.jre8 test From 1d52aecb255bc6838cda02dc40da6c721fe77967 Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 16:12:31 +0100 Subject: [PATCH 105/609] org.eclipse.rdf4j upgrade from 2.5.5 -> 3.7.4 --- CHANGELOG.md | 22 ++++++++++++---------- pom.xml | 2 +- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78424149..81154620 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,16 +16,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Clarified Readme for quick start - No cartesian product when referring to the same logical source (see [issue 28](https://github.com/RMLio/rmlmapper-java/issues/28)]) - If you still want the cartesian product, update your mappings to refer to different logical sources with the same contents -- Upgrade jetty-server and jetty-security to 9.4.44.v20210927 -- Upgrade wiremock-jre8 to 2.32.0 -- Upgrade ch.qos.logback to 1.2.10 -- Upgrade commons-cli to 1.5.0 -- Upgrade com.jayway.jsonpath to 2.7.0 -- Upgrade ch.vorburger.mariaDB4j to 2.5.3 -- Upgrade com.microsoft.sqlserver to 9.4.0.jre8 -- Upgrade com.fasterxml.jackson.core to 2.13.1 -- Upgrade org.jsoup to 1.14.3 -- Upgrade org.apache.poi to 5.0.0 +- Upgraded jetty-server and jetty-security to 9.4.44.v20210927 +- Upgraded wiremock-jre8 to 2.32.0 +- Upgraded ch.qos.logback to 1.2.10 +- Upgraded commons-cli to 1.5.0 +- Upgraded com.jayway.jsonpath to 2.7.0 +- Upgraded ch.vorburger.mariaDB4j to 2.5.3 +- Upgraded com.microsoft.sqlserver to 10.2.0.jre8 +- Upgraded com.fasterxml.jackson.core to 2.13.1 +- Upgraded org.jsoup to 1.14.3 +- Upgraded org.apache.poi to 5.0.0 +- Upgraded org.apache.jena to 4.3.2 +- Upgraded org.eclipse.rdf4j to 3.7.4 ### Changed diff --git a/pom.xml b/pom.xml index d300aaf5..afb4fa0d 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ org.eclipse.rdf4j rdf4j-runtime - 2.5.5 + 3.7.4 junit From 28e9c76fda409f8bd1ca7d088d50626fa865f265 Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 17:18:46 +0100 Subject: [PATCH 106/609] rollback org.apache.jena to 3.8.0 rollback org.eclipse.rdf4j to 2.5.5 --- CHANGELOG.md | 3 +-- pom.xml | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81154620..f2b602fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,8 +26,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Upgraded com.fasterxml.jackson.core to 2.13.1 - Upgraded org.jsoup to 1.14.3 - Upgraded org.apache.poi to 5.0.0 -- Upgraded org.apache.jena to 4.3.2 -- Upgraded org.eclipse.rdf4j to 3.7.4 + ### Changed diff --git a/pom.xml b/pom.xml index afb4fa0d..19a430ff 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ org.eclipse.rdf4j rdf4j-runtime - 3.7.4 + 2.5.5 junit @@ -184,7 +184,7 @@ com.microsoft.sqlserver mssql-jdbc - 10.2.0.jre8 + 9.4.0.jre8 test @@ -231,7 +231,7 @@ org.apache.jena apache-jena-libs pom - 4.3.2 + 3.8.0 test From 96264b9aafab83ca4b0a5adfa0c2bfa078fd06ec Mon Sep 17 00:00:00 2001 From: tihabils Date: Mon, 21 Feb 2022 17:54:32 +0100 Subject: [PATCH 107/609] TestCore: enable debug logs with VERBOSE env var. Tests will output logs on debug level when the environment variable VERBOSE was set prior their execution. --- CHANGELOG.md | 1 + src/test/java/be/ugent/rml/TestCore.java | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2b602fa..4d451161 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## Unreleased +- TestCore: enable debug logs when VERBOSE env variable is set (see [issue 230](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/230)) ### Added diff --git a/src/test/java/be/ugent/rml/TestCore.java b/src/test/java/be/ugent/rml/TestCore.java index f131ad0d..b227db4e 100644 --- a/src/test/java/be/ugent/rml/TestCore.java +++ b/src/test/java/be/ugent/rml/TestCore.java @@ -11,6 +11,7 @@ import be.ugent.rml.target.TargetFactory; import be.ugent.rml.term.NamedNode; import be.ugent.rml.term.Term; +import ch.qos.logback.classic.Level; import org.eclipse.rdf4j.rio.RDFFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,15 +32,22 @@ public abstract class TestCore { - final Logger logger = LoggerFactory.getLogger(this.getClass()); final String DEFAULT_BASE_IRI = "http://example.com/base/"; - + final ch.qos.logback.classic.Logger logger = (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME); // Mapping options to be applied by the MappingConformer protected static Map mappingOptions = new HashMap<>(); + TestCore(){ + if(System.getenv("VERBOSE") != null){ + logger.setLevel(Level.DEBUG); + }; + } + + /** * Note: the created Executor will run in best effort mode */ + Executor createExecutor(String mapPath) throws Exception { return createExecutor(mapPath, new ArrayList<>(), null, BEST_EFFORT); } From 1aae0a849e460b7f7ac4749fc2bb578bc7c4ec7d Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Thu, 17 Feb 2022 16:56:30 +0100 Subject: [PATCH 108/609] added vocabulary to support ldes generation --- src/main/resources/functions_idlab.ttl | 41 +++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/src/main/resources/functions_idlab.ttl b/src/main/resources/functions_idlab.ttl index ba6ec694..8b334879 100644 --- a/src/main/resources/functions_idlab.ttl +++ b/src/main/resources/functions_idlab.ttl @@ -200,6 +200,17 @@ idlab-fn:normalizeDateTime lib:class "IDLabFunctions" ; lib:method "normalizeDateTimeWithLang" ] . +idlab-fn:generateUniqueIRI + a fno:Function; + fno:name "generateUniqueIRI"; + rdfs:label "generateUniqueIRI"; + dcterms:description "Generates a unique IRI by watching the given property value" + fno:expects ( idlab-fn:iri idlab-fn:watchedProperty idlab-fn:unique idlab-fn:_state); + fno:returns ( idlab-fn:_str ); + lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; + lib:class "IDLabFunctions" ; + lib:method "generateUniqueIRI" ] . + grel:valueParam a fno:Parameter ; @@ -357,4 +368,32 @@ idlab-fn:_lang fno:name "string representing a BCP 47 language tag" ; rdfs:label "string representing a BCP 47 language tag" ; fno:type xsd:string ; - fno:predicate idlab-fn:lang . \ No newline at end of file + fno:predicate idlab-fn:lang . + +idlab-fn:iri + a fno:Parameter; + fno:name "String representation of an IRI"; + rdfs:label "String representation of an IRI"; + fno:type xsd:string; + fno:predicate idlab-fn:iri . + +idlab-fn:watchedProperty + a fno:Parameter; + fno:name "Property field of the data record to be watched"; + rdfs:label "Property field of the data record to be watched"; + fno:type xsd:string ; + fno:predicate idlab-fn:watchedProperty. + +idlab-fn:unique + a fno:Parameter; + fno:name "Indicates if a field contains unique values"; + rdfs:label "Indicates if a field contains unique values"; + fno:type xsd:boolean ; + fno:predicate idlab-fn:unique. + +idlab-fn:_state + a fno:Parameter; + fno:name "string representing the name of the state file"; + rdfs:label "string representing the name of the state file"; + fno:type xsd:string ; + fno:predicate idlab-fn:state . \ No newline at end of file From 9f311ec29eef13580076c91dc6473f292d465608 Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Sat, 19 Feb 2022 13:14:18 +0100 Subject: [PATCH 109/609] implemented unique iri generation for ldes --- src/main/java/be/ugent/rml/NAMESPACES.java | 1 + .../rml/functions/lib/IDLabFunctions.java | 74 ++++++++++++++++++- 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/src/main/java/be/ugent/rml/NAMESPACES.java b/src/main/java/be/ugent/rml/NAMESPACES.java index 13c9a3f2..eccec4c9 100644 --- a/src/main/java/be/ugent/rml/NAMESPACES.java +++ b/src/main/java/be/ugent/rml/NAMESPACES.java @@ -10,6 +10,7 @@ public class NAMESPACES { public static final String D2RQ = "http://www.wiwiss.fu-berlin.de/suhl/bizer/D2RQ/0.1#"; public static final String SD = "http://www.w3.org/ns/sparql-service-description#"; public static final String FNO_S = "https://w3id.org/function/ontology#"; + public static final String LDES = "https://w3id.org/ldes#"; @Deprecated public static final String FNO = "http://w3id.org/function/ontology#"; @Deprecated diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index 227cf83d..94f8fec8 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -13,17 +13,21 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; +import java.io.*; +import java.nio.file.Files; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLEncoder; +import java.nio.file.Path; +import java.nio.file.Paths; import java.text.SimpleDateFormat; import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; public class IDLabFunctions { @@ -288,6 +292,70 @@ public static String normalizeDateTime(String dateTimeStr, String pattern) { + public static String generateUniqueIRI(String template, String watchedValue, boolean isUnique, String stateDirPathStr) { + //TODO: move this to the parameter of the function? (might get too bloated in RML definitions) + int m_buckets = 10; + if (isUnique){ + return template; + } + int templateHash = template.hashCode(); + String hexBucketFileName= Integer.toHexString(templateHash % m_buckets); + String hexKey = Integer.toHexString(templateHash); + String hexPairs = String.format("%s:%s", hexKey, watchedValue); + + + Path stateFilePath = Paths.get(String.format("%s/%s.log", stateDirPathStr, hexBucketFileName)); + + List outputPairs = new ArrayList<>(); + + File stateFile = stateFilePath.toFile(); + AtomicBoolean found = new AtomicBoolean(false); + AtomicBoolean isDifferent = new AtomicBoolean(true); + + try{ + + if (Files.notExists(stateFilePath) ) { + Files.createDirectories(stateFilePath); + }; + + try (BufferedReader reader = new BufferedReader(new FileReader(stateFile))) { + + outputPairs = reader.lines().map(s -> { + String[] strings = s.split(":"); + String key = strings[0]; + String val = strings[1]; + if (hexKey.equals(key)) { + isDifferent.set(!val.equals(watchedValue)); + if (isDifferent.get()){ + val = watchedValue; + } + found.set(true); + } + return String.format("%s:%s", key,val); + }).collect(Collectors.toList()); + + } + + if (!found.get()) { + outputPairs.add(hexPairs); + } + + + } catch (IOException e){ + e.printStackTrace(); + } + + String output = null; + if (isDifferent.get()){ + OffsetDateTime now = OffsetDateTime.now(); + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; + output = String.format("%s#%s", template, formatter.format(now) ); + } + + return output; + } + + // TODO check whether this is the right place for this public static String jsonize(Object s) throws JsonProcessingException { ObjectWriter ow = new ObjectMapper().writer().withDefaultPrettyPrinter(); From 93c68116eb38c8192a35a3424e73b729d94774df Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Sat, 19 Feb 2022 16:44:05 +0100 Subject: [PATCH 110/609] added state updates --- .../rml/functions/lib/IDLabFunctions.java | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index 94f8fec8..c81b8c03 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -306,19 +306,22 @@ public static String generateUniqueIRI(String template, String watchedValue, boo Path stateFilePath = Paths.get(String.format("%s/%s.log", stateDirPathStr, hexBucketFileName)); - List outputPairs = new ArrayList<>(); + System.out.println(String.format("State file path for ldes is: %s", stateFilePath)); + + List outputPairs; - File stateFile = stateFilePath.toFile(); AtomicBoolean found = new AtomicBoolean(false); AtomicBoolean isDifferent = new AtomicBoolean(true); + String output = null; try{ + Files.createDirectories(Paths.get(stateDirPathStr)); if (Files.notExists(stateFilePath) ) { - Files.createDirectories(stateFilePath); + Files.createFile(stateFilePath); }; - try (BufferedReader reader = new BufferedReader(new FileReader(stateFile))) { + try (BufferedReader reader = new BufferedReader(new FileReader(stateFilePath.toFile()))) { outputPairs = reader.lines().map(s -> { String[] strings = s.split(":"); @@ -339,18 +342,23 @@ public static String generateUniqueIRI(String template, String watchedValue, boo if (!found.get()) { outputPairs.add(hexPairs); } - + try (BufferedWriter writer= new BufferedWriter(new FileWriter(stateFilePath.toFile()))){ + for(String line : outputPairs){ + writer.write(line); + writer.newLine(); + } + } + if (isDifferent.get()){ + OffsetDateTime now = OffsetDateTime.now(); + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; + output = String.format("%s#%s", template, formatter.format(now) ); + } } catch (IOException e){ e.printStackTrace(); + output = null; } - String output = null; - if (isDifferent.get()){ - OffsetDateTime now = OffsetDateTime.now(); - DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; - output = String.format("%s#%s", template, formatter.format(now) ); - } return output; } From 9edaf30eba1597efc4e6700e38dce1b01f811a88 Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Sun, 20 Feb 2022 18:33:35 +0100 Subject: [PATCH 111/609] added unit tests --- .../rml/functions/lib/IDLabFunctionsTest.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java b/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java index 1e8c9cb7..79a1b10e 100644 --- a/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java +++ b/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java @@ -1,6 +1,7 @@ package be.ugent.rml.functions.lib; import org.hamcrest.CoreMatchers; +import org.junit.BeforeClass; import org.junit.Test; import java.util.ArrayList; @@ -219,4 +220,43 @@ public void normalizeDateTime() { assertEquals("2020-04-01T09:04:00", IDLabFunctions.normalizeDateTime(input3, format3)); } + public static class LDESGenerationTests{ + + private static final String STATE_DIRECTORY = "/tmp/test-state"; + + + @Test + public void skipGenerateUniqueIRI(){ + String template = "http://example.com/sensor1/"; + String value = "5"; + boolean isUnique = false; + + String generated_iri = IDLabFunctions.generateUniqueIRI(template, value, isUnique, STATE_DIRECTORY); + assertNull(generated_iri); + } + + + @Test + public void generateUniqueIRI(){ + String template = "http://example.com/sensor2/"; + String value = "5"; + boolean isUnique = true; + + String generated_iri = IDLabFunctions.generateUniqueIRI(template, value, isUnique, STATE_DIRECTORY); + assertEquals(generated_iri, template); + + } + + @Test + public void generateUniqueIRIWithDate(){ + + String template = "http://example.com/sensor2/"; + String value = "5"; + boolean isUnique = false; + + String generated_iri = IDLabFunctions.generateUniqueIRI(template, value, isUnique, STATE_DIRECTORY); + assertEquals(generated_iri, template); + } + } + } From acad367cbb6c2dfafd6911e47bac72043a9bc473 Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Mon, 21 Feb 2022 13:56:54 +0100 Subject: [PATCH 112/609] added test case for ldes generation --- src/main/resources/functions_idlab.ttl | 6 +-- .../java/be/ugent/rml/Mapper_LDES_Test.java | 29 +++++++++++ .../web-of-things/ldes/generation/data1.csv | 4 ++ .../web-of-things/ldes/generation/data2.csv | 3 ++ .../web-of-things/ldes/generation/mapping.ttl | 50 +++++++++++++++++++ .../web-of-things/ldes/generation/output.nq | 0 6 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 src/test/java/be/ugent/rml/Mapper_LDES_Test.java create mode 100644 src/test/resources/web-of-things/ldes/generation/data1.csv create mode 100644 src/test/resources/web-of-things/ldes/generation/data2.csv create mode 100644 src/test/resources/web-of-things/ldes/generation/mapping.ttl create mode 100644 src/test/resources/web-of-things/ldes/generation/output.nq diff --git a/src/main/resources/functions_idlab.ttl b/src/main/resources/functions_idlab.ttl index 8b334879..10b011ff 100644 --- a/src/main/resources/functions_idlab.ttl +++ b/src/main/resources/functions_idlab.ttl @@ -204,9 +204,9 @@ idlab-fn:generateUniqueIRI a fno:Function; fno:name "generateUniqueIRI"; rdfs:label "generateUniqueIRI"; - dcterms:description "Generates a unique IRI by watching the given property value" - fno:expects ( idlab-fn:iri idlab-fn:watchedProperty idlab-fn:unique idlab-fn:_state); - fno:returns ( idlab-fn:_str ); + dcterms:description "Generates a unique IRI by watching the given property value"; + fno:expects ( idlab-fn:iri idlab-fn:watchedProperty idlab-fn:unique idlab-fn:_state ); + fno:returns ( idlab-fn:_stringOut ); lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; lib:class "IDLabFunctions" ; lib:method "generateUniqueIRI" ] . diff --git a/src/test/java/be/ugent/rml/Mapper_LDES_Test.java b/src/test/java/be/ugent/rml/Mapper_LDES_Test.java new file mode 100644 index 00000000..52f9a060 --- /dev/null +++ b/src/test/java/be/ugent/rml/Mapper_LDES_Test.java @@ -0,0 +1,29 @@ +package be.ugent.rml; + +import be.ugent.rml.functions.FunctionLoader; +import be.ugent.rml.store.QuadStore; +import be.ugent.rml.store.RDF4JStore; +import org.eclipse.rdf4j.rio.RDFFormat; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; + +public class Mapper_LDES_Test extends TestCore { + + private static FunctionLoader LOADER; + + @BeforeClass + public static void setups() throws Exception { + QuadStore functionDescriptionTriples = new RDF4JStore(); + functionDescriptionTriples.read(Utils.getInputStreamFromFile(new File("./src/main/resources/functions_idlab.ttl")), null, RDFFormat.TURTLE); + LOADER = new FunctionLoader(functionDescriptionTriples); + + } + + @Test + public void evaluate_basic_LDES () throws Exception { + Executor executor = this.createExecutor("./web-of-things/ldes/generation/mapping.ttl", LOADER); + doMapping(executor, "./web-of-things/ldes/generation/output.nq"); + } +} diff --git a/src/test/resources/web-of-things/ldes/generation/data1.csv b/src/test/resources/web-of-things/ldes/generation/data1.csv new file mode 100644 index 00000000..29143cd0 --- /dev/null +++ b/src/test/resources/web-of-things/ldes/generation/data1.csv @@ -0,0 +1,4 @@ +id,value +sensor1,3 +sensor2,3 +sensor3,4 \ No newline at end of file diff --git a/src/test/resources/web-of-things/ldes/generation/data2.csv b/src/test/resources/web-of-things/ldes/generation/data2.csv new file mode 100644 index 00000000..0ef3eb59 --- /dev/null +++ b/src/test/resources/web-of-things/ldes/generation/data2.csv @@ -0,0 +1,3 @@ +id,value +sensor2,10 +sensor3,4 diff --git a/src/test/resources/web-of-things/ldes/generation/mapping.ttl b/src/test/resources/web-of-things/ldes/generation/mapping.ttl new file mode 100644 index 00000000..7e906479 --- /dev/null +++ b/src/test/resources/web-of-things/ldes/generation/mapping.ttl @@ -0,0 +1,50 @@ +@prefix rr: . +@prefix ex: . +@prefix rdfs: . +@prefix xsd: . +@prefix ql: . +@prefix rml: . +@prefix rdf: . +@prefix fnml: . +@prefix fno: . +@prefix dcterms: . +@prefix idlab-fn: . +@base . + + + a rr:TriplesMap; + + rml:logicalSource [ + rml:source "data1.csv"; + rml:referenceFormulation ql:CSV + ]; + + rr:subjectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant idlab-fn:generateUniqueIRI ] ] ; + # IRI template + rr:predicateObjectMap [ + rr:predicate idlab-fn:iri ; + rr:objectMap [ rr:template "http://ex.org/{id}/" ] + ]; + # Properties to watch, can be one or multiple + # Note that the template does not need to match with these properties! + rr:predicateObjectMap [ + rr:predicate idlab-fn:watchedProperty ; + rr:objectMap [ rml:reference "value" ] + ]; + # Flag to indicate if the properties are unique on their own + rr:predicateObjectMap [ + rr:predicate idlab-fn:unique ; + rr:objectMap [ rr:constant "true"; rr:termType xsd:boolean; ] + ]; + # Directory path where the state for this function is/will be stored + rr:predicateObjectMap [ + rr:predicate idlab-fn:state ; + rr:objectMap [ rr:constant "/tmp/test-case"; rr:termType xsd:string; ] + ]; + ]; + ]; + . diff --git a/src/test/resources/web-of-things/ldes/generation/output.nq b/src/test/resources/web-of-things/ldes/generation/output.nq new file mode 100644 index 00000000..e69de29b From d9943059d3d5fd62b802096a17adaa0f1b78e675 Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Mon, 21 Feb 2022 15:08:25 +0100 Subject: [PATCH 113/609] Fixed function not found exception --- .../rml/functions/lib/IDLabFunctions.java | 2 +- src/main/resources/functions_idlab.ttl | 18 +++++++++--------- .../web-of-things/ldes/generation/mapping.ttl | 5 +++++ 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index c81b8c03..19f7b933 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -292,7 +292,7 @@ public static String normalizeDateTime(String dateTimeStr, String pattern) { - public static String generateUniqueIRI(String template, String watchedValue, boolean isUnique, String stateDirPathStr) { + public static String generateUniqueIRI(String template, String watchedValue, Boolean isUnique, String stateDirPathStr) { //TODO: move this to the parameter of the function? (might get too bloated in RML definitions) int m_buckets = 10; if (isUnique){ diff --git a/src/main/resources/functions_idlab.ttl b/src/main/resources/functions_idlab.ttl index 10b011ff..6e161b1b 100644 --- a/src/main/resources/functions_idlab.ttl +++ b/src/main/resources/functions_idlab.ttl @@ -201,12 +201,12 @@ idlab-fn:normalizeDateTime lib:method "normalizeDateTimeWithLang" ] . idlab-fn:generateUniqueIRI - a fno:Function; - fno:name "generateUniqueIRI"; - rdfs:label "generateUniqueIRI"; - dcterms:description "Generates a unique IRI by watching the given property value"; - fno:expects ( idlab-fn:iri idlab-fn:watchedProperty idlab-fn:unique idlab-fn:_state ); - fno:returns ( idlab-fn:_stringOut ); + a fno:Function ; + fno:name "generateUniqueIRI" ; + rdfs:label "generateUniqueIRI" ; + dcterms:description "Generates a unique IRI by watching the given property value" ; + fno:expects ( idlab-fn:_iri idlab-fn:_watchedProperty idlab-fn:_unique idlab-fn:_state ) ; + fno:returns ( idlab-fn:_stringOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; lib:class "IDLabFunctions" ; lib:method "generateUniqueIRI" ] . @@ -370,21 +370,21 @@ idlab-fn:_lang fno:type xsd:string ; fno:predicate idlab-fn:lang . -idlab-fn:iri +idlab-fn:_iri a fno:Parameter; fno:name "String representation of an IRI"; rdfs:label "String representation of an IRI"; fno:type xsd:string; fno:predicate idlab-fn:iri . -idlab-fn:watchedProperty +idlab-fn:_watchedProperty a fno:Parameter; fno:name "Property field of the data record to be watched"; rdfs:label "Property field of the data record to be watched"; fno:type xsd:string ; fno:predicate idlab-fn:watchedProperty. -idlab-fn:unique +idlab-fn:_unique a fno:Parameter; fno:name "Indicates if a field contains unique values"; rdfs:label "Indicates if a field contains unique values"; diff --git a/src/test/resources/web-of-things/ldes/generation/mapping.ttl b/src/test/resources/web-of-things/ldes/generation/mapping.ttl index 7e906479..5a917c7d 100644 --- a/src/test/resources/web-of-things/ldes/generation/mapping.ttl +++ b/src/test/resources/web-of-things/ldes/generation/mapping.ttl @@ -47,4 +47,9 @@ ]; ]; ]; + + rr:predicateObjectMap [ + rr:predicate ex:value; + rr:objectMap [ rml:reference "value" ]; + ]; . From d0685ec1cdc82898f78bb6b9ba4e5038f1b9da9e Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Mon, 21 Feb 2022 22:55:01 +0100 Subject: [PATCH 114/609] added integration tests --- .../rml/functions/lib/IDLabFunctions.java | 1 - .../java/be/ugent/rml/Mapper_LDES_Test.java | 37 ++++++++++++- .../ldes/generation/basic/mapping.ttl | 55 +++++++++++++++++++ .../ldes/generation/basic/output.nq | 3 + .../ldes/generation/partial/mapping.ttl | 55 +++++++++++++++++++ .../ldes/generation/partial/mapping2.ttl | 55 +++++++++++++++++++ .../ldes/generation/repeat/mapping.ttl | 55 +++++++++++++++++++ .../ldes/generation/{ => repeat}/output.nq | 0 8 files changed, 257 insertions(+), 4 deletions(-) create mode 100644 src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl create mode 100644 src/test/resources/web-of-things/ldes/generation/basic/output.nq create mode 100644 src/test/resources/web-of-things/ldes/generation/partial/mapping.ttl create mode 100644 src/test/resources/web-of-things/ldes/generation/partial/mapping2.ttl create mode 100644 src/test/resources/web-of-things/ldes/generation/repeat/mapping.ttl rename src/test/resources/web-of-things/ldes/generation/{ => repeat}/output.nq (100%) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index 19f7b933..48f8f8aa 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -306,7 +306,6 @@ public static String generateUniqueIRI(String template, String watchedValue, Boo Path stateFilePath = Paths.get(String.format("%s/%s.log", stateDirPathStr, hexBucketFileName)); - System.out.println(String.format("State file path for ldes is: %s", stateFilePath)); List outputPairs; diff --git a/src/test/java/be/ugent/rml/Mapper_LDES_Test.java b/src/test/java/be/ugent/rml/Mapper_LDES_Test.java index 52f9a060..5d8bb160 100644 --- a/src/test/java/be/ugent/rml/Mapper_LDES_Test.java +++ b/src/test/java/be/ugent/rml/Mapper_LDES_Test.java @@ -3,15 +3,25 @@ import be.ugent.rml.functions.FunctionLoader; import be.ugent.rml.store.QuadStore; import be.ugent.rml.store.RDF4JStore; +import be.ugent.rml.term.NamedNode; +import org.apache.commons.io.FileUtils; import org.eclipse.rdf4j.rio.RDFFormat; +import org.junit.After; import org.junit.BeforeClass; import org.junit.Test; +import static org.junit.Assert.*; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; public class Mapper_LDES_Test extends TestCore { private static FunctionLoader LOADER; + @After + public void cleanUp() throws IOException { + FileUtils.deleteDirectory(new File("/tmp/ldes-test")); + } @BeforeClass public static void setups() throws Exception { @@ -22,8 +32,29 @@ public static void setups() throws Exception { } @Test - public void evaluate_basic_LDES () throws Exception { - Executor executor = this.createExecutor("./web-of-things/ldes/generation/mapping.ttl", LOADER); - doMapping(executor, "./web-of-things/ldes/generation/output.nq"); + public void evaluate_unique_LDES () throws Exception { + Executor executor = this.createExecutor("./web-of-things/ldes/generation/basic/mapping.ttl", LOADER); + doMapping(executor, "./web-of-things/ldes/generation/basic/output.nq"); } + + @Test + public void evaluate_repeat_LDES() throws Exception { + Executor executor = this.createExecutor("./web-of-things/ldes/generation/repeat/mapping.ttl", LOADER); + executor.executeV5(null).get(new NamedNode("rmlmapper://default.store")); + executor = this.createExecutor("./web-of-things/ldes/generation/repeat/mapping.ttl", LOADER); + doMapping(executor, "./web-of-things/ldes/generation/repeat/output.nq"); + } + + @Test + public void evaluate_partial_repeat_LDES() throws Exception { + Executor executor = this.createExecutor("./web-of-things/ldes/generation/partial/mapping.ttl", LOADER); + QuadStore result = executor.executeV5(null).get(new NamedNode("rmlmapper://default.store")); + executor = this.createExecutor("./web-of-things/ldes/generation/partial/mapping2.ttl", LOADER); + QuadStore result_second = executor.executeV5(null).get(new NamedNode("rmlmapper://default.store")); + assertEquals(3, result.size()); + assertEquals(1, result_second.size()); + + } + + } diff --git a/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl b/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl new file mode 100644 index 00000000..781e48a5 --- /dev/null +++ b/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl @@ -0,0 +1,55 @@ +@prefix rr: . +@prefix ex: . +@prefix rdfs: . +@prefix xsd: . +@prefix ql: . +@prefix rml: . +@prefix rdf: . +@prefix fnml: . +@prefix fno: . +@prefix dcterms: . +@prefix idlab-fn: . +@base . + + + a rr:TriplesMap; + + rml:logicalSource [ + rml:source "../data1.csv"; + rml:referenceFormulation ql:CSV + ]; + + rr:subjectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant idlab-fn:generateUniqueIRI ] ] ; + # IRI template + rr:predicateObjectMap [ + rr:predicate idlab-fn:iri ; + rr:objectMap [ rr:template "http://ex.org/{id}/" ] + ]; + # Properties to watch, can be one or multiple + # Note that the template does not need to match with these properties! + rr:predicateObjectMap [ + rr:predicate idlab-fn:watchedProperty ; + rr:objectMap [ rml:reference "value" ] + ]; + # Flag to indicate if the properties are unique on their own + rr:predicateObjectMap [ + rr:predicate idlab-fn:unique ; + rr:objectMap [ rr:constant "true"; rr:termType xsd:boolean; ] + ]; + # Directory path where the state for this function is/will be stored + rr:predicateObjectMap [ + rr:predicate idlab-fn:state ; + rr:objectMap [ rr:constant "/tmp/ldes-test/test-case-basic"; rr:termType xsd:string; ] + ]; + ]; + ]; + + rr:predicateObjectMap [ + rr:predicate ex:value; + rr:objectMap [ rml:reference "value" ]; + ]; + . diff --git a/src/test/resources/web-of-things/ldes/generation/basic/output.nq b/src/test/resources/web-of-things/ldes/generation/basic/output.nq new file mode 100644 index 00000000..9e6b8ddd --- /dev/null +++ b/src/test/resources/web-of-things/ldes/generation/basic/output.nq @@ -0,0 +1,3 @@ + "3". + "3". + "4". diff --git a/src/test/resources/web-of-things/ldes/generation/partial/mapping.ttl b/src/test/resources/web-of-things/ldes/generation/partial/mapping.ttl new file mode 100644 index 00000000..27a2bc9b --- /dev/null +++ b/src/test/resources/web-of-things/ldes/generation/partial/mapping.ttl @@ -0,0 +1,55 @@ +@prefix rr: . +@prefix ex: . +@prefix rdfs: . +@prefix xsd: . +@prefix ql: . +@prefix rml: . +@prefix rdf: . +@prefix fnml: . +@prefix fno: . +@prefix dcterms: . +@prefix idlab-fn: . +@base . + + + a rr:TriplesMap; + + rml:logicalSource [ + rml:source "../data1.csv"; + rml:referenceFormulation ql:CSV + ]; + + rr:subjectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant idlab-fn:generateUniqueIRI ] ] ; + # IRI template + rr:predicateObjectMap [ + rr:predicate idlab-fn:iri ; + rr:objectMap [ rr:template "http://ex.org/{id}/" ] + ]; + # Properties to watch, can be one or multiple + # Note that the template does not need to match with these properties! + rr:predicateObjectMap [ + rr:predicate idlab-fn:watchedProperty ; + rr:objectMap [ rml:reference "value" ] + ]; + # Flag to indicate if the properties are unique on their own + rr:predicateObjectMap [ + rr:predicate idlab-fn:unique ; + rr:objectMap [ rr:constant "false"; rr:termType xsd:boolean; ] + ]; + # Directory path where the state for this function is/will be stored + rr:predicateObjectMap [ + rr:predicate idlab-fn:state ; + rr:objectMap [ rr:constant "/tmp/ldes-test/partial-test-case"; rr:termType xsd:string; ] + ]; + ]; + ]; + + rr:predicateObjectMap [ + rr:predicate ex:value; + rr:objectMap [ rml:reference "value" ]; + ]; + . diff --git a/src/test/resources/web-of-things/ldes/generation/partial/mapping2.ttl b/src/test/resources/web-of-things/ldes/generation/partial/mapping2.ttl new file mode 100644 index 00000000..0b3d257d --- /dev/null +++ b/src/test/resources/web-of-things/ldes/generation/partial/mapping2.ttl @@ -0,0 +1,55 @@ +@prefix rr: . +@prefix ex: . +@prefix rdfs: . +@prefix xsd: . +@prefix ql: . +@prefix rml: . +@prefix rdf: . +@prefix fnml: . +@prefix fno: . +@prefix dcterms: . +@prefix idlab-fn: . +@base . + + + a rr:TriplesMap; + + rml:logicalSource [ + rml:source "../data2.csv"; + rml:referenceFormulation ql:CSV + ]; + + rr:subjectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant idlab-fn:generateUniqueIRI ] ] ; + # IRI template + rr:predicateObjectMap [ + rr:predicate idlab-fn:iri ; + rr:objectMap [ rr:template "http://ex.org/{id}/" ] + ]; + # Properties to watch, can be one or multiple + # Note that the template does not need to match with these properties! + rr:predicateObjectMap [ + rr:predicate idlab-fn:watchedProperty ; + rr:objectMap [ rml:reference "value" ] + ]; + # Flag to indicate if the properties are unique on their own + rr:predicateObjectMap [ + rr:predicate idlab-fn:unique ; + rr:objectMap [ rr:constant "false"; rr:termType xsd:boolean; ] + ]; + # Directory path where the state for this function is/will be stored + rr:predicateObjectMap [ + rr:predicate idlab-fn:state ; + rr:objectMap [ rr:constant "/tmp/ldes-test/partial-test-case"; rr:termType xsd:string; ] + ]; + ]; + ]; + + rr:predicateObjectMap [ + rr:predicate ex:value; + rr:objectMap [ rml:reference "value" ]; + ]; + . diff --git a/src/test/resources/web-of-things/ldes/generation/repeat/mapping.ttl b/src/test/resources/web-of-things/ldes/generation/repeat/mapping.ttl new file mode 100644 index 00000000..1cca254f --- /dev/null +++ b/src/test/resources/web-of-things/ldes/generation/repeat/mapping.ttl @@ -0,0 +1,55 @@ +@prefix rr: . +@prefix ex: . +@prefix rdfs: . +@prefix xsd: . +@prefix ql: . +@prefix rml: . +@prefix rdf: . +@prefix fnml: . +@prefix fno: . +@prefix dcterms: . +@prefix idlab-fn: . +@base . + + + a rr:TriplesMap; + + rml:logicalSource [ + rml:source "../data1.csv"; + rml:referenceFormulation ql:CSV + ]; + + rr:subjectMap [ + fnml:functionValue [ + rr:predicateObjectMap [ + rr:predicate fno:executes ; + rr:objectMap [ rr:constant idlab-fn:generateUniqueIRI ] ] ; + # IRI template + rr:predicateObjectMap [ + rr:predicate idlab-fn:iri ; + rr:objectMap [ rr:template "http://ex.org/{id}/" ] + ]; + # Properties to watch, can be one or multiple + # Note that the template does not need to match with these properties! + rr:predicateObjectMap [ + rr:predicate idlab-fn:watchedProperty ; + rr:objectMap [ rml:reference "value" ] + ]; + # Flag to indicate if the properties are unique on their own + rr:predicateObjectMap [ + rr:predicate idlab-fn:unique ; + rr:objectMap [ rr:constant "false"; rr:termType xsd:boolean; ] + ]; + # Directory path where the state for this function is/will be stored + rr:predicateObjectMap [ + rr:predicate idlab-fn:state ; + rr:objectMap [ rr:constant "/tmp/ldes-test/repeat-test-case"; rr:termType xsd:string; ] + ]; + ]; + ]; + + rr:predicateObjectMap [ + rr:predicate ex:value; + rr:objectMap [ rml:reference "value" ]; + ]; + . diff --git a/src/test/resources/web-of-things/ldes/generation/output.nq b/src/test/resources/web-of-things/ldes/generation/repeat/output.nq similarity index 100% rename from src/test/resources/web-of-things/ldes/generation/output.nq rename to src/test/resources/web-of-things/ldes/generation/repeat/output.nq From 4a636207809f9c75c83fc665e608b511e78b028f Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Wed, 23 Feb 2022 10:39:39 +0100 Subject: [PATCH 115/609] updated iri generation to watch multiple properties --- .../rml/functions/lib/IDLabFunctions.java | 122 ++++++++++++++---- .../ldes/generation/basic/mapping.ttl | 6 +- .../ldes/generation/basic/output.nq | 6 +- .../web-of-things/ldes/generation/data1.csv | 8 +- .../web-of-things/ldes/generation/data2.csv | 6 +- .../ldes/generation/partial/mapping.ttl | 6 +- .../ldes/generation/partial/mapping2.ttl | 6 +- .../ldes/generation/repeat/mapping.ttl | 6 +- 8 files changed, 116 insertions(+), 50 deletions(-) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index 48f8f8aa..3959f07d 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -291,26 +291,86 @@ public static String normalizeDateTime(String dateTimeStr, String pattern) { } + /** + * Returns a HashMap containing the key,value pairs of the watched properties from the given + * template string of the form "key1=val1&key2=val2&...." + * + * @param watchedValueTemplate Input string template representing the key,value pairs of + * the form "key1=val1&key2=val2&..." + * @return A map containing the parsed pairs of the properties which are being watched. + */ + private static Map parseWatchedPropertyTemplate(String watchedValueTemplate){ + Map result = new HashMap<>(); - public static String generateUniqueIRI(String template, String watchedValue, Boolean isUnique, String stateDirPathStr) { - //TODO: move this to the parameter of the function? (might get too bloated in RML definitions) - int m_buckets = 10; - if (isUnique){ - return template; + Arrays.stream(watchedValueTemplate.split("&")) + .map(s -> s.split("=")) + .filter(sArr -> sArr.length == 2) + .forEach(sArr -> result.put(sArr[0], sArr[1])); + + return result; + } + + private static String updateStatePropertyRecord(Map watchedMap, + String hexKey, AtomicBoolean found, AtomicBoolean isDifferent, + String storedStateRecord) { + int split_idx = storedStateRecord.indexOf(':'); + String storedHexKey = storedStateRecord.substring(0, split_idx); + if (!storedHexKey.equals(hexKey)){ + return storedStateRecord; } + + found.set(true); + + List propertyValPairs = + Arrays.stream(storedStateRecord.substring(split_idx+1) + .split("&")) + .map(str -> { + String[] propVal = str.split("="); + String property = propVal[0]; + String storeVal = propVal[1]; + String watchedVal = watchedMap.getOrDefault(property, null); + + if(!watchedVal.equals(storeVal)){ + isDifferent.set(true); + storeVal = watchedVal; + } + + return String.format("%s=%s", property, storeVal); + }) + .collect(Collectors.toList()); + + return String.format("%s:%s", storedHexKey, String.join("&", propertyValPairs)); + } + + /** + * Handles the case if the isUniqueIRI flag is not set for the method {@link #generateUniqueIRI(String, String, Boolean, String)}. + * The generation of the IRI depends on the value of the watched properties. + * If any of the watched properties changes in value or gets dropped, a unique IRI will be + * generated. Otherwise, null String will be returned. + * In order to check if the watched properties have changed, a file state is written to keep track of + * previously seen property values. + * A unique IRI will be generated from the provided "template" string by appending the current + * date timestamp. + * + * @param template The template string used to generate unique IRI by appending current date timestamp + * @param watchedValueTemplate The template string containing the key-value pairs of properties being watched + * @param stateDirPathStr String representation of the directory path in which the state of the function + * will be stored + * @param m_buckets The number of bucket files to be created + * @return A unique IRI will be generated from the provided "template" string by appending the current + * date timestamp if possible. Otherwise, null is returned + */ + private static String handleNonUniqueIRI(String template, String watchedValueTemplate, String stateDirPathStr, int m_buckets) { + Map watchedMap = parseWatchedPropertyTemplate(watchedValueTemplate); + int templateHash = template.hashCode(); String hexBucketFileName= Integer.toHexString(templateHash % m_buckets); String hexKey = Integer.toHexString(templateHash); - String hexPairs = String.format("%s:%s", hexKey, watchedValue); - Path stateFilePath = Paths.get(String.format("%s/%s.log", stateDirPathStr, hexBucketFileName)); - - List outputPairs; - AtomicBoolean found = new AtomicBoolean(false); - AtomicBoolean isDifferent = new AtomicBoolean(true); + AtomicBoolean isDifferent = new AtomicBoolean(false); String output = null; try{ @@ -321,33 +381,25 @@ public static String generateUniqueIRI(String template, String watchedValue, Boo }; try (BufferedReader reader = new BufferedReader(new FileReader(stateFilePath.toFile()))) { - - outputPairs = reader.lines().map(s -> { - String[] strings = s.split(":"); - String key = strings[0]; - String val = strings[1]; - if (hexKey.equals(key)) { - isDifferent.set(!val.equals(watchedValue)); - if (isDifferent.get()){ - val = watchedValue; - } - found.set(true); - } - return String.format("%s:%s", key,val); - }).collect(Collectors.toList()); - + outputPairs = reader.lines() + .map(s -> updateStatePropertyRecord(watchedMap, hexKey, found, isDifferent, s)) + .collect(Collectors.toList()); } if (!found.get()) { + String hexPairs = String.format("%s:%s", hexKey, watchedValueTemplate); outputPairs.add(hexPairs); } + System.out.println(Arrays.toString(outputPairs.toArray())); + try (BufferedWriter writer= new BufferedWriter(new FileWriter(stateFilePath.toFile()))){ for(String line : outputPairs){ writer.write(line); writer.newLine(); } } - if (isDifferent.get()){ + + if (isDifferent.get() || !found.get()){ OffsetDateTime now = OffsetDateTime.now(); DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; output = String.format("%s#%s", template, formatter.format(now) ); @@ -355,14 +407,28 @@ public static String generateUniqueIRI(String template, String watchedValue, Boo } catch (IOException e){ e.printStackTrace(); - output = null; } + return output; + } + + + public static String generateUniqueIRI(String template, String watchedValueTemplate, Boolean isUnique, String stateDirPathStr) { + //TODO: move this to the parameter of the function? (might get too bloated in RML definitions) + int m_buckets = 10; + + if (isUnique){ + return template; + } + + String output = handleNonUniqueIRI(template, watchedValueTemplate, stateDirPathStr, m_buckets); + return output; } + // TODO check whether this is the right place for this public static String jsonize(Object s) throws JsonProcessingException { ObjectWriter ow = new ObjectMapper().writer().withDefaultPrettyPrinter(); diff --git a/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl b/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl index 781e48a5..add4ad33 100644 --- a/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl +++ b/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl @@ -33,7 +33,7 @@ # Note that the template does not need to match with these properties! rr:predicateObjectMap [ rr:predicate idlab-fn:watchedProperty ; - rr:objectMap [ rml:reference "value" ] + rr:objectMap [ rr:template "pressure={pressure}&temperature={temperature}" ] ]; # Flag to indicate if the properties are unique on their own rr:predicateObjectMap [ @@ -49,7 +49,7 @@ ]; rr:predicateObjectMap [ - rr:predicate ex:value; - rr:objectMap [ rml:reference "value" ]; + rr:predicate ex:temp; + rr:objectMap [ rml:reference "temperature" ]; ]; . diff --git a/src/test/resources/web-of-things/ldes/generation/basic/output.nq b/src/test/resources/web-of-things/ldes/generation/basic/output.nq index 9e6b8ddd..4fd0afca 100644 --- a/src/test/resources/web-of-things/ldes/generation/basic/output.nq +++ b/src/test/resources/web-of-things/ldes/generation/basic/output.nq @@ -1,3 +1,3 @@ - "3". - "3". - "4". + "3". + "3". + "4". diff --git a/src/test/resources/web-of-things/ldes/generation/data1.csv b/src/test/resources/web-of-things/ldes/generation/data1.csv index 29143cd0..5d3c4c89 100644 --- a/src/test/resources/web-of-things/ldes/generation/data1.csv +++ b/src/test/resources/web-of-things/ldes/generation/data1.csv @@ -1,4 +1,4 @@ -id,value -sensor1,3 -sensor2,3 -sensor3,4 \ No newline at end of file +id,temperature,pressure +sensor1,3,10 +sensor2,3,10 +sensor3,4,10 \ No newline at end of file diff --git a/src/test/resources/web-of-things/ldes/generation/data2.csv b/src/test/resources/web-of-things/ldes/generation/data2.csv index 0ef3eb59..a2cfdd29 100644 --- a/src/test/resources/web-of-things/ldes/generation/data2.csv +++ b/src/test/resources/web-of-things/ldes/generation/data2.csv @@ -1,3 +1,3 @@ -id,value -sensor2,10 -sensor3,4 +id,temperature,pressure +sensor2,10,10 +sensor3,4,10 diff --git a/src/test/resources/web-of-things/ldes/generation/partial/mapping.ttl b/src/test/resources/web-of-things/ldes/generation/partial/mapping.ttl index 27a2bc9b..6373a85f 100644 --- a/src/test/resources/web-of-things/ldes/generation/partial/mapping.ttl +++ b/src/test/resources/web-of-things/ldes/generation/partial/mapping.ttl @@ -33,7 +33,7 @@ # Note that the template does not need to match with these properties! rr:predicateObjectMap [ rr:predicate idlab-fn:watchedProperty ; - rr:objectMap [ rml:reference "value" ] + rr:objectMap [ rr:template "temperature={temperature}&pressure={pressure}" ] ]; # Flag to indicate if the properties are unique on their own rr:predicateObjectMap [ @@ -49,7 +49,7 @@ ]; rr:predicateObjectMap [ - rr:predicate ex:value; - rr:objectMap [ rml:reference "value" ]; + rr:predicate ex:temp; + rr:objectMap [ rml:reference "temperature" ]; ]; . diff --git a/src/test/resources/web-of-things/ldes/generation/partial/mapping2.ttl b/src/test/resources/web-of-things/ldes/generation/partial/mapping2.ttl index 0b3d257d..4216bdeb 100644 --- a/src/test/resources/web-of-things/ldes/generation/partial/mapping2.ttl +++ b/src/test/resources/web-of-things/ldes/generation/partial/mapping2.ttl @@ -33,7 +33,7 @@ # Note that the template does not need to match with these properties! rr:predicateObjectMap [ rr:predicate idlab-fn:watchedProperty ; - rr:objectMap [ rml:reference "value" ] + rr:objectMap [ rr:template "pressure={pressure}&temperature={temperature}" ] ]; # Flag to indicate if the properties are unique on their own rr:predicateObjectMap [ @@ -49,7 +49,7 @@ ]; rr:predicateObjectMap [ - rr:predicate ex:value; - rr:objectMap [ rml:reference "value" ]; + rr:predicate ex:temp; + rr:objectMap [ rml:reference "temperature" ]; ]; . diff --git a/src/test/resources/web-of-things/ldes/generation/repeat/mapping.ttl b/src/test/resources/web-of-things/ldes/generation/repeat/mapping.ttl index 1cca254f..f46783aa 100644 --- a/src/test/resources/web-of-things/ldes/generation/repeat/mapping.ttl +++ b/src/test/resources/web-of-things/ldes/generation/repeat/mapping.ttl @@ -33,7 +33,7 @@ # Note that the template does not need to match with these properties! rr:predicateObjectMap [ rr:predicate idlab-fn:watchedProperty ; - rr:objectMap [ rml:reference "value" ] + rr:objectMap [ rml:template "temperature={temperature}&pressure={pressure}" ] ]; # Flag to indicate if the properties are unique on their own rr:predicateObjectMap [ @@ -49,7 +49,7 @@ ]; rr:predicateObjectMap [ - rr:predicate ex:value; - rr:objectMap [ rml:reference "value" ]; + rr:predicate ex:temperature; + rr:objectMap [ rml:reference "temperature" ]; ]; . From 82e4cd0ff07626ddc6a072fc418ee3f0094fef6c Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Wed, 23 Feb 2022 14:00:06 +0100 Subject: [PATCH 116/609] CHANGELOG: LDES unique IRIs --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d451161..70df71f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - Add support for LDES Logical Target +- Add support for generating unique reproducible IRIs for LDES ### Fixed From 853b3cd399610b7fb78e626160c0195d715c895f Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Wed, 23 Feb 2022 14:07:23 +0100 Subject: [PATCH 117/609] NAMESPACES: only define LDES once --- src/main/java/be/ugent/rml/NAMESPACES.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/be/ugent/rml/NAMESPACES.java b/src/main/java/be/ugent/rml/NAMESPACES.java index eccec4c9..13c9a3f2 100644 --- a/src/main/java/be/ugent/rml/NAMESPACES.java +++ b/src/main/java/be/ugent/rml/NAMESPACES.java @@ -10,7 +10,6 @@ public class NAMESPACES { public static final String D2RQ = "http://www.wiwiss.fu-berlin.de/suhl/bizer/D2RQ/0.1#"; public static final String SD = "http://www.w3.org/ns/sparql-service-description#"; public static final String FNO_S = "https://w3id.org/function/ontology#"; - public static final String LDES = "https://w3id.org/ldes#"; @Deprecated public static final String FNO = "http://w3id.org/function/ontology#"; @Deprecated From 7bf75719de321786ad233ed327e322b4a2311e80 Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Wed, 23 Feb 2022 18:44:40 +0100 Subject: [PATCH 118/609] implemented isUnique=true flag path of ldes iri generation --- .../rml/functions/lib/IDLabFunctions.java | 73 +++++++++++-------- .../ldes/generation/basic/mapping.ttl | 6 -- 2 files changed, 43 insertions(+), 36 deletions(-) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index 3959f07d..aeecc766 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -312,7 +312,17 @@ private static Map parseWatchedPropertyTemplate(String watchedVa private static String updateStatePropertyRecord(Map watchedMap, String hexKey, AtomicBoolean found, AtomicBoolean isDifferent, + boolean isUnique, String storedStateRecord) { + //No need to check the properties, we just need to know if the + //given hexKey has already been seen in the state file. + if (isUnique){ + found.set(storedStateRecord.equals(hexKey)); + isDifferent.set(found.get()); + return storedStateRecord; + } + + int split_idx = storedStateRecord.indexOf(':'); String storedHexKey = storedStateRecord.substring(0, split_idx); if (!storedHexKey.equals(hexKey)){ @@ -342,8 +352,14 @@ private static String updateStatePropertyRecord(Map watchedMap, return String.format("%s:%s", storedHexKey, String.join("&", propertyValPairs)); } + + private static Path getStateFilePath(String stateDirPathStr, int m_buckets, int templateHash) { + String hexBucketFileName= Integer.toHexString(templateHash % m_buckets); + return Paths.get(String.format("%s/%s.log", stateDirPathStr, hexBucketFileName)); + } + + /** - * Handles the case if the isUniqueIRI flag is not set for the method {@link #generateUniqueIRI(String, String, Boolean, String)}. * The generation of the IRI depends on the value of the watched properties. * If any of the watched properties changes in value or gets dropped, a unique IRI will be * generated. Otherwise, null String will be returned. @@ -354,21 +370,26 @@ private static String updateStatePropertyRecord(Map watchedMap, * * @param template The template string used to generate unique IRI by appending current date timestamp * @param watchedValueTemplate The template string containing the key-value pairs of properties being watched + * @param isUnique A flag to indicate if the given template already creates unique IRI * @param stateDirPathStr String representation of the directory path in which the state of the function * will be stored - * @param m_buckets The number of bucket files to be created * @return A unique IRI will be generated from the provided "template" string by appending the current * date timestamp if possible. Otherwise, null is returned */ - private static String handleNonUniqueIRI(String template, String watchedValueTemplate, String stateDirPathStr, int m_buckets) { + public static String generateUniqueIRI(String template, String watchedValueTemplate, Boolean isUnique, String stateDirPathStr) { + //TODO: move this to the parameter of the function? (might get too bloated in RML definitions) + int m_buckets = 10; + + // null check just in case idlab-fn:_watchedProperty is not provided in the mapping file + watchedValueTemplate = (watchedValueTemplate == null)? "" : watchedValueTemplate; Map watchedMap = parseWatchedPropertyTemplate(watchedValueTemplate); int templateHash = template.hashCode(); - String hexBucketFileName= Integer.toHexString(templateHash % m_buckets); + Path stateFilePath = getStateFilePath(stateDirPathStr, m_buckets, templateHash); + String hexKey = Integer.toHexString(templateHash); - Path stateFilePath = Paths.get(String.format("%s/%s.log", stateDirPathStr, hexBucketFileName)); - List outputPairs; + List outputRecords; AtomicBoolean found = new AtomicBoolean(false); AtomicBoolean isDifferent = new AtomicBoolean(false); String output = null; @@ -380,29 +401,36 @@ private static String handleNonUniqueIRI(String template, String watchedValueTem Files.createFile(stateFilePath); }; + // Get and update the state records which will be rewritten back to the state file by overwriting try (BufferedReader reader = new BufferedReader(new FileReader(stateFilePath.toFile()))) { - outputPairs = reader.lines() - .map(s -> updateStatePropertyRecord(watchedMap, hexKey, found, isDifferent, s)) + outputRecords = reader.lines() + .map(s -> updateStatePropertyRecord(watchedMap, hexKey, found, isDifferent, isUnique, s)) .collect(Collectors.toList()); } if (!found.get()) { - String hexPairs = String.format("%s:%s", hexKey, watchedValueTemplate); - outputPairs.add(hexPairs); + String hexedStateRecord = (isUnique)? hexKey:String.format("%s:%s", hexKey, watchedValueTemplate); + outputRecords.add(hexedStateRecord); } - System.out.println(Arrays.toString(outputPairs.toArray())); try (BufferedWriter writer= new BufferedWriter(new FileWriter(stateFilePath.toFile()))){ - for(String line : outputPairs){ + for(String line : outputRecords){ writer.write(line); writer.newLine(); } } + + // The unique IRI will be generated if there's any differences found with the corresponding record in + // the state file if (isDifferent.get() || !found.get()){ - OffsetDateTime now = OffsetDateTime.now(); - DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; - output = String.format("%s#%s", template, formatter.format(now) ); + if (isUnique) { + output= template; + }else { + OffsetDateTime now = OffsetDateTime.now(); + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; + output = String.format("%s#%s", template, formatter.format(now)); + } } } catch (IOException e){ @@ -413,21 +441,6 @@ private static String handleNonUniqueIRI(String template, String watchedValueTem } - public static String generateUniqueIRI(String template, String watchedValueTemplate, Boolean isUnique, String stateDirPathStr) { - //TODO: move this to the parameter of the function? (might get too bloated in RML definitions) - int m_buckets = 10; - - if (isUnique){ - return template; - } - - String output = handleNonUniqueIRI(template, watchedValueTemplate, stateDirPathStr, m_buckets); - - - return output; - } - - // TODO check whether this is the right place for this public static String jsonize(Object s) throws JsonProcessingException { diff --git a/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl b/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl index add4ad33..3a207445 100644 --- a/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl +++ b/src/test/resources/web-of-things/ldes/generation/basic/mapping.ttl @@ -29,12 +29,6 @@ rr:predicate idlab-fn:iri ; rr:objectMap [ rr:template "http://ex.org/{id}/" ] ]; - # Properties to watch, can be one or multiple - # Note that the template does not need to match with these properties! - rr:predicateObjectMap [ - rr:predicate idlab-fn:watchedProperty ; - rr:objectMap [ rr:template "pressure={pressure}&temperature={temperature}" ] - ]; # Flag to indicate if the properties are unique on their own rr:predicateObjectMap [ rr:predicate idlab-fn:unique ; From d3fa592a2ab720a61c7124fdfbacaf808cbc23ef Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Thu, 24 Feb 2022 09:34:37 +0100 Subject: [PATCH 119/609] update unit test cases --- .../rml/functions/lib/IDLabFunctionsTest.java | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java b/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java index 79a1b10e..d7ab69fb 100644 --- a/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java +++ b/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java @@ -1,9 +1,13 @@ package be.ugent.rml.functions.lib; +import org.apache.commons.io.FileUtils; import org.hamcrest.CoreMatchers; +import org.junit.After; import org.junit.BeforeClass; import org.junit.Test; +import java.io.IOException; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -225,12 +229,18 @@ public static class LDESGenerationTests{ private static final String STATE_DIRECTORY = "/tmp/test-state"; + @After + public void cleanUp() throws IOException { + FileUtils.deleteDirectory(Paths.get(STATE_DIRECTORY).toFile()); + } + @Test public void skipGenerateUniqueIRI(){ String template = "http://example.com/sensor1/"; - String value = "5"; + String value = "pressure=5"; boolean isUnique = false; + IDLabFunctions.generateUniqueIRI(template, value, isUnique, STATE_DIRECTORY); String generated_iri = IDLabFunctions.generateUniqueIRI(template, value, isUnique, STATE_DIRECTORY); assertNull(generated_iri); } @@ -239,7 +249,7 @@ public void skipGenerateUniqueIRI(){ @Test public void generateUniqueIRI(){ String template = "http://example.com/sensor2/"; - String value = "5"; + String value = "pressure=5"; boolean isUnique = true; String generated_iri = IDLabFunctions.generateUniqueIRI(template, value, isUnique, STATE_DIRECTORY); @@ -251,11 +261,12 @@ public void generateUniqueIRI(){ public void generateUniqueIRIWithDate(){ String template = "http://example.com/sensor2/"; - String value = "5"; + String value = "pressure=5"; boolean isUnique = false; String generated_iri = IDLabFunctions.generateUniqueIRI(template, value, isUnique, STATE_DIRECTORY); - assertEquals(generated_iri, template); + assertNotNull(generated_iri); + assertTrue(generated_iri.contains(template)); } } From 9925b5d27442e1bd197f9a88200d3edf3408de03 Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Thu, 24 Feb 2022 11:30:40 +0100 Subject: [PATCH 120/609] fixed isUnique=true generation of iri --- src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index aeecc766..65bf8af5 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -318,7 +318,6 @@ private static String updateStatePropertyRecord(Map watchedMap, //given hexKey has already been seen in the state file. if (isUnique){ found.set(storedStateRecord.equals(hexKey)); - isDifferent.set(found.get()); return storedStateRecord; } From d0b151ade6da4364691f394f8448431fae1b3841 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Thu, 24 Feb 2022 12:10:49 +0100 Subject: [PATCH 121/609] IDLabFunctions: fix IRI generation for unique IRIs --- src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index 65bf8af5..e34bd9ad 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -317,7 +317,9 @@ private static String updateStatePropertyRecord(Map watchedMap, //No need to check the properties, we just need to know if the //given hexKey has already been seen in the state file. if (isUnique){ - found.set(storedStateRecord.equals(hexKey)); + if (storedStateRecord.equals(hexKey)) { + found.set(true); + } return storedStateRecord; } @@ -423,6 +425,7 @@ public static String generateUniqueIRI(String template, String watchedValueTempl // The unique IRI will be generated if there's any differences found with the corresponding record in // the state file if (isDifferent.get() || !found.get()){ + logger.debug("Update found! Generating IRI..."); if (isUnique) { output= template; }else { From 08b2a586b6ca0be28c1bf22c257ad6fb05eb2a19 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Tue, 1 Mar 2022 11:05:31 +0100 Subject: [PATCH 122/609] IDLabFunctions: handle missing data for LDES diff check --- .../ugent/rml/functions/lib/IDLabFunctions.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index e34bd9ad..486875b3 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -323,7 +323,6 @@ private static String updateStatePropertyRecord(Map watchedMap, return storedStateRecord; } - int split_idx = storedStateRecord.indexOf(':'); String storedHexKey = storedStateRecord.substring(0, split_idx); if (!storedHexKey.equals(hexKey)){ @@ -336,12 +335,20 @@ private static String updateStatePropertyRecord(Map watchedMap, Arrays.stream(storedStateRecord.substring(split_idx+1) .split("&")) .map(str -> { - String[] propVal = str.split("="); + String[] propVal = str.split("="); String property = propVal[0]; - String storeVal = propVal[1]; + String storeVal = "NULL"; + // Data may not provide a given property value so try and fail gracefully + try { + storeVal = propVal[1]; + } + catch (IndexOutOfBoundsException e) { + + } + String watchedVal = watchedMap.getOrDefault(property, null); - if(!watchedVal.equals(storeVal)){ + if(watchedVal != null && !watchedVal.equals(storeVal)){ isDifferent.set(true); storeVal = watchedVal; } From 71b112cf1459b41b48354308ad2c241cb8999dd2 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Thu, 3 Mar 2022 15:51:32 +0100 Subject: [PATCH 123/609] IDLabFunctions: handle edge case --- src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index 486875b3..8092de77 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -324,6 +324,10 @@ private static String updateStatePropertyRecord(Map watchedMap, } int split_idx = storedStateRecord.indexOf(':'); + // Ignore if no split is possible + if (split_idx == -1) { + return storedStateRecord; + } String storedHexKey = storedStateRecord.substring(0, split_idx); if (!storedHexKey.equals(hexKey)){ return storedStateRecord; @@ -432,7 +436,6 @@ public static String generateUniqueIRI(String template, String watchedValueTempl // The unique IRI will be generated if there's any differences found with the corresponding record in // the state file if (isDifferent.get() || !found.get()){ - logger.debug("Update found! Generating IRI..."); if (isUnique) { output= template; }else { From 71a747bd6e2bc1623620fff9e866afd3b667f260 Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Thu, 3 Mar 2022 11:54:37 +0100 Subject: [PATCH 124/609] refactored to use in-memory states --- CHANGELOG.md | 4 + src/main/java/be/ugent/rml/cli/Main.java | 2 + .../rml/functions/lib/IDLabFunctions.java | 158 ++++++++++++++++-- .../java/be/ugent/rml/Mapper_LDES_Test.java | 4 + .../rml/functions/lib/IDLabFunctionsTest.java | 2 +- 5 files changed, 153 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70df71f6..9f05d125 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add support for LDES Logical Target - Add support for generating unique reproducible IRIs for LDES +### Changed + +- Write LDES state to disk when mapping execution is complete. + ### Fixed - Clarified Readme for quick start diff --git a/src/main/java/be/ugent/rml/cli/Main.java b/src/main/java/be/ugent/rml/cli/Main.java index 7b542fc4..fd4e228e 100644 --- a/src/main/java/be/ugent/rml/cli/Main.java +++ b/src/main/java/be/ugent/rml/cli/Main.java @@ -386,6 +386,8 @@ public static void main(String[] args, String basePath) { String outputFile = getPriorityOptionValue(outputfileOption, lineArgs, configFile); result.copyNameSpaces(rmlStore); + IDLabFunctions.saveState(); + writeOutputTargets(targets, rmlStore, basePath, outputFile, outputFormat); } catch (Exception e) { logger.error(e.getMessage()); diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index 8092de77..acd4c7eb 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -14,6 +14,7 @@ import org.slf4j.LoggerFactory; import java.io.*; +import java.nio.Buffer; import java.nio.file.Files; import java.net.HttpURLConnection; import java.net.URL; @@ -28,10 +29,13 @@ import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +import java.util.stream.Stream; public class IDLabFunctions { private static final Logger logger = LoggerFactory.getLogger(IDLabFunctions.class); + private static Map> LDES_FILE_STATE_MAP = new HashMap<>(); + private static Path STATE_DIR = null; public static boolean stringContainsOtherString(String str, String otherStr, String delimiter) { String[] split = str.split(delimiter); @@ -299,7 +303,7 @@ public static String normalizeDateTime(String dateTimeStr, String pattern) { * the form "key1=val1&key2=val2&..." * @return A map containing the parsed pairs of the properties which are being watched. */ - private static Map parseWatchedPropertyTemplate(String watchedValueTemplate){ + private static Map parsePropertyValueTemplate(String watchedValueTemplate){ Map result = new HashMap<>(); Arrays.stream(watchedValueTemplate.split("&")) @@ -388,13 +392,13 @@ private static Path getStateFilePath(String stateDirPathStr, int m_buckets, int * @return A unique IRI will be generated from the provided "template" string by appending the current * date timestamp if possible. Otherwise, null is returned */ - public static String generateUniqueIRI(String template, String watchedValueTemplate, Boolean isUnique, String stateDirPathStr) { + public static String generateUniqueIRIOLD(String template, String watchedValueTemplate, Boolean isUnique, String stateDirPathStr) { //TODO: move this to the parameter of the function? (might get too bloated in RML definitions) int m_buckets = 10; // null check just in case idlab-fn:_watchedProperty is not provided in the mapping file watchedValueTemplate = (watchedValueTemplate == null)? "" : watchedValueTemplate; - Map watchedMap = parseWatchedPropertyTemplate(watchedValueTemplate); + Map watchedMap = parsePropertyValueTemplate(watchedValueTemplate); int templateHash = template.hashCode(); Path stateFilePath = getStateFilePath(stateDirPathStr, m_buckets, templateHash); @@ -411,7 +415,7 @@ public static String generateUniqueIRI(String template, String watchedValueTempl Files.createDirectories(Paths.get(stateDirPathStr)); if (Files.notExists(stateFilePath) ) { Files.createFile(stateFilePath); - }; + } // Get and update the state records which will be rewritten back to the state file by overwriting try (BufferedReader reader = new BufferedReader(new FileReader(stateFilePath.toFile()))) { @@ -432,18 +436,7 @@ public static String generateUniqueIRI(String template, String watchedValueTempl } } - - // The unique IRI will be generated if there's any differences found with the corresponding record in - // the state file - if (isDifferent.get() || !found.get()){ - if (isUnique) { - output= template; - }else { - OffsetDateTime now = OffsetDateTime.now(); - DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; - output = String.format("%s#%s", template, formatter.format(now)); - } - } + output = getOutput(template, isUnique, found, isDifferent); } catch (IOException e){ e.printStackTrace(); @@ -452,6 +445,139 @@ public static String generateUniqueIRI(String template, String watchedValueTempl return output; } + private static String getOutput(String template, Boolean isUnique, AtomicBoolean found, AtomicBoolean isDifferent) { + // The unique IRI will be generated if there's any differences found with the corresponding record in + // the state file + String output = null; + if (isDifferent.get() || !found.get()){ + logger.debug("Update found! Generating IRI..."); + if (isUnique) { + output = template; + }else { + OffsetDateTime now = OffsetDateTime.now(); + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; + output = String.format("%s#%s", template, formatter.format(now)); + } + } + return output; + } + + private static void initStateFile(Path stateDirPath){ + try (Stream statePaths = Files.find(stateDirPath, 1, (filePath, fileAttr) -> fileAttr.isRegularFile())){ + + Stream readers = statePaths.map(path -> { + try { + return new AbstractMap.SimpleImmutableEntry(path, new BufferedReader(new FileReader(path.toFile()))); + } catch (FileNotFoundException e) { + return null; + } + }).filter(Objects::nonNull); + + + readers.forEach( entry -> { + try (BufferedReader bReader = (BufferedReader) entry.getValue()){ + String fileName = ((Path) entry.getKey()).toString(); + bReader.lines().forEach( + line -> { + String[] pairs = line.split(":"); + String key = pairs[0]; + String val = null; + try { + val = pairs[1]; + }catch (IndexOutOfBoundsException e) { + e.printStackTrace(); + } + LDES_FILE_STATE_MAP.computeIfAbsent(fileName, k -> new HashMap<>()).put(key, val); + } + ); + } catch (IOException e) { + e.printStackTrace(); + } + + }); + + }catch (IOException e){ + e.printStackTrace(); + } + } + + public static void saveState() throws IOException { + if (Objects.nonNull(STATE_DIR)){ + + Files.createDirectories(STATE_DIR); + for (Map.Entry> entry: LDES_FILE_STATE_MAP.entrySet()){ + + Path file = Paths.get(entry.getKey()); + Map storedMap = entry.getValue(); + + if (Files.notExists(file)){ + Files.createFile(file); + } + + + try( BufferedWriter writer = new BufferedWriter(new FileWriter(file.toFile()))){ + for( Map.Entry record: storedMap.entrySet()){ + String line = String.format("%s:%s", record.getKey(), record.getValue()); + writer.write(line); + writer.newLine(); + } + } + + } + } + + } + + public static void resetState(){ + LDES_FILE_STATE_MAP = new HashMap<>(); + STATE_DIR = null; + } + public static String generateUniqueIRI(String template, String watchedValueTemplate, Boolean isUnique, String stateDirPathStr) { + if (STATE_DIR == null){ + STATE_DIR = Paths.get(stateDirPathStr); + initStateFile(STATE_DIR); + } + int m_buckets = 10; + + // null check just in case idlab-fn:_watchedProperty is not provided in the mapping file + watchedValueTemplate = (watchedValueTemplate == null)? "" : watchedValueTemplate; + Map watchedMap = parsePropertyValueTemplate(watchedValueTemplate); + + int templateHash = template.hashCode(); + String stateFilePathStr = getStateFilePath(stateDirPathStr, m_buckets, templateHash).toString(); + + String hexKey = Integer.toHexString(templateHash); + + AtomicBoolean found = new AtomicBoolean(false); + AtomicBoolean isDifferent = new AtomicBoolean(false); + + Map keyValMap = LDES_FILE_STATE_MAP.computeIfAbsent(stateFilePathStr, f -> new HashMap<>()); + if (keyValMap.containsKey(hexKey)){ + found.set(true); + String storedProp = keyValMap.get(hexKey); + Map storedPropMap = parsePropertyValueTemplate(storedProp); + for (Map.Entry kv : watchedMap.entrySet()){ + String prop = kv.getKey(); + String val = kv.getValue(); + + String storedVal = storedPropMap.getOrDefault(prop, null); + if (!val.equals(storedVal)){ + isDifferent.set(true); + } + } + + } + keyValMap.put(hexKey, watchedValueTemplate); + + + + + + + return getOutput(template,isUnique, found, isDifferent); + + } + // TODO check whether this is the right place for this diff --git a/src/test/java/be/ugent/rml/Mapper_LDES_Test.java b/src/test/java/be/ugent/rml/Mapper_LDES_Test.java index 5d8bb160..92d2ca10 100644 --- a/src/test/java/be/ugent/rml/Mapper_LDES_Test.java +++ b/src/test/java/be/ugent/rml/Mapper_LDES_Test.java @@ -1,6 +1,7 @@ package be.ugent.rml; import be.ugent.rml.functions.FunctionLoader; +import be.ugent.rml.functions.lib.IDLabFunctions; import be.ugent.rml.store.QuadStore; import be.ugent.rml.store.RDF4JStore; import be.ugent.rml.term.NamedNode; @@ -20,6 +21,7 @@ public class Mapper_LDES_Test extends TestCore { private static FunctionLoader LOADER; @After public void cleanUp() throws IOException { + IDLabFunctions.resetState(); FileUtils.deleteDirectory(new File("/tmp/ldes-test")); } @@ -41,6 +43,7 @@ public void evaluate_unique_LDES () throws Exception { public void evaluate_repeat_LDES() throws Exception { Executor executor = this.createExecutor("./web-of-things/ldes/generation/repeat/mapping.ttl", LOADER); executor.executeV5(null).get(new NamedNode("rmlmapper://default.store")); + IDLabFunctions.saveState(); executor = this.createExecutor("./web-of-things/ldes/generation/repeat/mapping.ttl", LOADER); doMapping(executor, "./web-of-things/ldes/generation/repeat/output.nq"); } @@ -49,6 +52,7 @@ public void evaluate_repeat_LDES() throws Exception { public void evaluate_partial_repeat_LDES() throws Exception { Executor executor = this.createExecutor("./web-of-things/ldes/generation/partial/mapping.ttl", LOADER); QuadStore result = executor.executeV5(null).get(new NamedNode("rmlmapper://default.store")); + IDLabFunctions.saveState(); executor = this.createExecutor("./web-of-things/ldes/generation/partial/mapping2.ttl", LOADER); QuadStore result_second = executor.executeV5(null).get(new NamedNode("rmlmapper://default.store")); assertEquals(3, result.size()); diff --git a/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java b/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java index d7ab69fb..8d0c6e03 100644 --- a/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java +++ b/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java @@ -253,7 +253,7 @@ public void generateUniqueIRI(){ boolean isUnique = true; String generated_iri = IDLabFunctions.generateUniqueIRI(template, value, isUnique, STATE_DIRECTORY); - assertEquals(generated_iri, template); + assertEquals(template, generated_iri); } From 62bc22c1015f2614eabee1c12c1c8dad20a9cb71 Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Mon, 7 Mar 2022 10:20:03 +0100 Subject: [PATCH 125/609] handle null string case for property parsing --- .../be/ugent/rml/functions/lib/IDLabFunctions.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index acd4c7eb..8e91f2a9 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -306,10 +306,12 @@ public static String normalizeDateTime(String dateTimeStr, String pattern) { private static Map parsePropertyValueTemplate(String watchedValueTemplate){ Map result = new HashMap<>(); - Arrays.stream(watchedValueTemplate.split("&")) - .map(s -> s.split("=")) - .filter(sArr -> sArr.length == 2) - .forEach(sArr -> result.put(sArr[0], sArr[1])); + if (watchedValueTemplate != null || watchedValueTemplate.length() > 0) { + Arrays.stream(watchedValueTemplate.split("&")) + .map(s -> s.split("=")) + .filter(sArr -> sArr.length == 2) + .forEach(sArr -> result.put(sArr[0], sArr[1])); + } return result; } From eee6cc29e4fb713d1f8aca8ace6c8517d8bca220 Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Mon, 7 Mar 2022 11:12:14 +0100 Subject: [PATCH 126/609] fix isUnique=True case of iri generation --- .../rml/functions/lib/IDLabFunctions.java | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java index 8e91f2a9..d56889f9 100644 --- a/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java +++ b/src/main/java/be/ugent/rml/functions/lib/IDLabFunctions.java @@ -303,11 +303,12 @@ public static String normalizeDateTime(String dateTimeStr, String pattern) { * the form "key1=val1&key2=val2&..." * @return A map containing the parsed pairs of the properties which are being watched. */ - private static Map parsePropertyValueTemplate(String watchedValueTemplate){ + private static Map parsePropertyValueTemplate(Optional watchedValueTemplate){ Map result = new HashMap<>(); + String watchedVal = watchedValueTemplate.orElse(""); - if (watchedValueTemplate != null || watchedValueTemplate.length() > 0) { - Arrays.stream(watchedValueTemplate.split("&")) + if (watchedVal.length() > 0) { + Arrays.stream(watchedVal.split("&")) .map(s -> s.split("=")) .filter(sArr -> sArr.length == 2) .forEach(sArr -> result.put(sArr[0], sArr[1])); @@ -398,9 +399,7 @@ public static String generateUniqueIRIOLD(String template, String watchedValueTe //TODO: move this to the parameter of the function? (might get too bloated in RML definitions) int m_buckets = 10; - // null check just in case idlab-fn:_watchedProperty is not provided in the mapping file - watchedValueTemplate = (watchedValueTemplate == null)? "" : watchedValueTemplate; - Map watchedMap = parsePropertyValueTemplate(watchedValueTemplate); + Map watchedMap = parsePropertyValueTemplate(Optional.ofNullable(watchedValueTemplate)); int templateHash = template.hashCode(); Path stateFilePath = getStateFilePath(stateDirPathStr, m_buckets, templateHash); @@ -519,7 +518,11 @@ public static void saveState() throws IOException { try( BufferedWriter writer = new BufferedWriter(new FileWriter(file.toFile()))){ for( Map.Entry record: storedMap.entrySet()){ - String line = String.format("%s:%s", record.getKey(), record.getValue()); + String line = record.getKey(); + if (record.getValue() != null) { + line = String.format("%s:%s", record.getKey(), record.getValue()); + } + writer.write(line); writer.newLine(); } @@ -542,8 +545,9 @@ public static String generateUniqueIRI(String template, String watchedValueTempl int m_buckets = 10; // null check just in case idlab-fn:_watchedProperty is not provided in the mapping file - watchedValueTemplate = (watchedValueTemplate == null)? "" : watchedValueTemplate; - Map watchedMap = parsePropertyValueTemplate(watchedValueTemplate); + + Optional watchedValueOption = Optional.ofNullable(watchedValueTemplate); + Map watchedMap = parsePropertyValueTemplate(watchedValueOption); int templateHash = template.hashCode(); String stateFilePathStr = getStateFilePath(stateDirPathStr, m_buckets, templateHash).toString(); @@ -557,7 +561,7 @@ public static String generateUniqueIRI(String template, String watchedValueTempl if (keyValMap.containsKey(hexKey)){ found.set(true); String storedProp = keyValMap.get(hexKey); - Map storedPropMap = parsePropertyValueTemplate(storedProp); + Map storedPropMap = parsePropertyValueTemplate(Optional.ofNullable(storedProp)); for (Map.Entry kv : watchedMap.entrySet()){ String prop = kv.getKey(); String val = kv.getValue(); From 1aad23be6a284bc2c91b524095938c447806bbeb Mon Sep 17 00:00:00 2001 From: Sitt Min Oo Date: Mon, 7 Mar 2022 14:05:40 +0100 Subject: [PATCH 127/609] fixed tests --- src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java b/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java index 8d0c6e03..82dd0d3b 100644 --- a/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java +++ b/src/test/java/be/ugent/rml/functions/lib/IDLabFunctionsTest.java @@ -231,6 +231,7 @@ public static class LDESGenerationTests{ @After public void cleanUp() throws IOException { + IDLabFunctions.resetState(); FileUtils.deleteDirectory(Paths.get(STATE_DIRECTORY).toFile()); } From 1435c2fa6c953c63097d3b58e01fc9c4059dedd3 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Tue, 8 Mar 2022 08:23:45 +0100 Subject: [PATCH 128/609] resources: functions_grel: use xsd:integer xsd:number does not exist, use xsd:integer instead. Fixes https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/234 --- CHANGELOG.md | 2 ++ src/main/resources/functions_grel.ttl | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f05d125..b44391fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Upgraded com.fasterxml.jackson.core to 2.13.1 - Upgraded org.jsoup to 1.14.3 - Upgraded org.apache.poi to 5.0.0 +- Resources: functions_grel: use xsd:integer (see [issue 234](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/234)) + ### Changed diff --git a/src/main/resources/functions_grel.ttl b/src/main/resources/functions_grel.ttl index 9a4bb572..7ce38d0f 100644 --- a/src/main/resources/functions_grel.ttl +++ b/src/main/resources/functions_grel.ttl @@ -412,7 +412,7 @@ grel:output_number fno:name "number" ; rdfs:label "number" ; fno:predicate grel:o_number ; - fno:type xsd:number . + fno:type xsd:integer . #TOSTRING From a3edeb3e76c695141e48005933b8d9024a158750 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Wed, 9 Mar 2022 14:45:03 +0100 Subject: [PATCH 129/609] RecordsFactory: print error if reference formulation is unknown --- CHANGELOG.md | 3 +-- src/main/java/be/ugent/rml/records/RecordsFactory.java | 7 +++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b44391fa..74cba1b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,8 +33,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Upgraded org.jsoup to 1.14.3 - Upgraded org.apache.poi to 5.0.0 - Resources: functions_grel: use xsd:integer (see [issue 234](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/234)) - - +- Print error if referenceformulation is unsupported ### Changed diff --git a/src/main/java/be/ugent/rml/records/RecordsFactory.java b/src/main/java/be/ugent/rml/records/RecordsFactory.java index ec01b39d..83fc9a9f 100644 --- a/src/main/java/be/ugent/rml/records/RecordsFactory.java +++ b/src/main/java/be/ugent/rml/records/RecordsFactory.java @@ -6,8 +6,11 @@ import be.ugent.rml.access.AccessFactory; import be.ugent.rml.store.Quad; import be.ugent.rml.store.QuadStore; +import be.ugent.rml.target.TargetFactory; import be.ugent.rml.term.NamedNode; import be.ugent.rml.term.Term; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; @@ -23,6 +26,7 @@ public class RecordsFactory { private Map>>> recordCache; private AccessFactory accessFactory; private Map referenceFormulationRecordFactoryMap; + private static final Logger logger = LoggerFactory.getLogger(RecordsFactory.class); public RecordsFactory(String basePath) { accessFactory = new AccessFactory(basePath); @@ -133,6 +137,9 @@ private List getRecords(Access access, Term logicalSource, String refere if (records == null) { try { // Select the Record Factory based on the reference formulation. + if (!referenceFormulationRecordFactoryMap.containsKey(referenceFormulation)) { + logger.error("Referenceformulation " + referenceFormulation + " is unsupported!"); + } ReferenceFormulationRecordFactory factory = referenceFormulationRecordFactoryMap.get(referenceFormulation); records = factory.getRecords(access, logicalSource, rmlStore); From ddcb370066231e846804392c9d14c19a7a9f34a7 Mon Sep 17 00:00:00 2001 From: tihabils Date: Thu, 10 Mar 2022 16:05:08 +0100 Subject: [PATCH 130/609] Changed to (deprecated) simple-odf fork. --- pom.xml | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index 19a430ff..64c9b124 100644 --- a/pom.xml +++ b/pom.xml @@ -283,18 +283,11 @@ poi-ooxml 5.0.0 - + - - org.apache.odftoolkit + org.odftoolkit simple-odf - 0.8.2-incubating + 0.9.0 tools @@ -302,6 +295,26 @@ + + + + + + + + + + + + + + From 2f62a589ad32e5140bd0d9c7631ccd614d8d8bb7 Mon Sep 17 00:00:00 2001 From: tihabils Date: Thu, 10 Mar 2022 16:24:04 +0100 Subject: [PATCH 131/609] clean up of pom file. --- pom.xml | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/pom.xml b/pom.xml index 64c9b124..f76f2f0e 100644 --- a/pom.xml +++ b/pom.xml @@ -288,33 +288,7 @@ org.odftoolkit simple-odf 0.9.0 - - - tools - com.sun - - - - - - - - - - - - - - - - From 81893dba668b075ea7209274b11619d88f3a9b23 Mon Sep 17 00:00:00 2001 From: tihabils Date: Thu, 10 Mar 2022 17:18:52 +0100 Subject: [PATCH 132/609] added CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74cba1b4..78f1cc12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased - TestCore: enable debug logs when VERBOSE env variable is set (see [issue 230](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/230)) +- Switched to fork of ODFtoolkit (see [issue 237](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/237)) ### Added From b918714dd4b5367360eb8b99d4941efb9454cec4 Mon Sep 17 00:00:00 2001 From: Dylan Van Assche Date: Fri, 11 Mar 2022 10:45:00 +0000 Subject: [PATCH 133/609] CHANGELOG: release v5.0.0 --- CHANGELOG.md | 3 +++ buildNumber.properties | 4 ++-- pom.xml | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78f1cc12..43eaf512 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## Unreleased + +## [5.0.0] - 2022-03-11 - TestCore: enable debug logs when VERBOSE env variable is set (see [issue 230](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/230)) - Switched to fork of ODFtoolkit (see [issue 237](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/237)) @@ -505,6 +507,7 @@ and [169](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/169)) - support for accessing remote files (via HTTP GET) - basic support for functions +[5.0.0]: https://github.com/RMLio/rmlmapper-java/compare/v4.15.0...v5.0.0 [4.15.0]: https://github.com/RMLio/rmlmapper-java/compare/v4.14.3...v4.15.0 [4.14.3]: https://github.com/RMLio/rmlmapper-java/compare/v4.14.2...v4.14.3 [4.14.2]: https://github.com/RMLio/rmlmapper-java/compare/v4.14.1...v4.14.2 diff --git a/buildNumber.properties b/buildNumber.properties index ad7e6c73..c5b85fe0 100644 --- a/buildNumber.properties +++ b/buildNumber.properties @@ -1,3 +1,3 @@ #maven.buildNumber.plugin properties file -#Tue Feb 01 10:31:54 GMT 2022 -buildNumber0=360 +#Fri Mar 11 10:44:24 GMT 2022 +buildNumber0=361 diff --git a/pom.xml b/pom.xml index f76f2f0e..4b20349a 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ be.ugent.rml rmlmapper RMLMapper - 4.15.0 + 5.0.0 The RMLMapper executes RML rules to generate high quality Linked Data from multiple originally (semi-)structured data sources. From ae2a9a657178d20a32c40b5a2cddb65a5dfee63d Mon Sep 17 00:00:00 2001 From: Gerald Haesendonck Date: Thu, 10 Mar 2022 08:09:37 +0100 Subject: [PATCH 134/609] Fix pipeline by changing chagelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43eaf512..41b6d079 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Add support for LDES Logical Target - Add support for generating unique reproducible IRIs for LDES +- Integration of independent function handler ### Changed From 86dbeb388c87a808416bc1f3213d7fd6390fa5c0 Mon Sep 17 00:00:00 2001 From: Gerald Haesendonck Date: Tue, 15 Mar 2022 15:41:15 +0100 Subject: [PATCH 135/609] WIP: integrating Function Agent --- pom.xml | 12 +++ src/main/java/be/ugent/rml/Executor.java | 25 +++--- src/main/java/be/ugent/rml/Initializer.java | 5 +- .../java/be/ugent/rml/MappingFactory.java | 15 ++-- src/main/java/be/ugent/rml/cli/Main.java | 24 ++++-- ...ynamicMultipleRecordsFunctionExecutor.java | 85 +++++++------------ .../DynamicSingleRecordFunctionExecutor.java | 6 +- ...StaticMultipleRecordsFunctionExecutor.java | 26 ++++-- .../StaticSingleRecordFunctionExecutor.java | 5 +- .../rml/functions/lib/IDLabFunctions.java | 5 +- src/main/resources/functions_idlab.ttl | 38 ++++----- .../ugent/rml/Custom_RML_FnO_Mapper_Test.java | 37 +++++--- .../java/be/ugent/rml/Mapper_LDES_Test.java | 18 ++-- src/test/java/be/ugent/rml/TestCore.java | 8 +- .../java/be/ugent/rml/TestFunctionCore.java | 12 ++- .../ugent/rml/readme/ReadmeFunctionTest.java | 5 +- .../java/be/ugent/rml/readme/ReadmeTest.java | 7 +- .../{logback.xml => logback-test.xml} | 2 +- 18 files changed, 191 insertions(+), 144 deletions(-) rename src/test/resources/{logback.xml => logback-test.xml} (93%) diff --git a/pom.xml b/pom.xml index 4b20349a..2b534287 100644 --- a/pom.xml +++ b/pom.xml @@ -290,6 +290,18 @@ 0.9.0 + + + be.ugent.idlab.knows + function-agent-java + 1.0-SNAPSHOT + + + org.apache.jena + * + + + diff --git a/src/main/java/be/ugent/rml/Executor.java b/src/main/java/be/ugent/rml/Executor.java index 2ebbabab..e2f06a14 100644 --- a/src/main/java/be/ugent/rml/Executor.java +++ b/src/main/java/be/ugent/rml/Executor.java @@ -1,15 +1,16 @@ package be.ugent.rml; +import be.ugent.idlab.knows.functions.agent.Agent; import be.ugent.rml.functions.FunctionLoader; import be.ugent.rml.functions.MultipleRecordsFunctionExecutor; import be.ugent.rml.metadata.Metadata; import be.ugent.rml.metadata.MetadataGenerator; import be.ugent.rml.records.Record; import be.ugent.rml.records.RecordsFactory; -import be.ugent.rml.store.RDF4JStore; -import be.ugent.rml.term.ProvenancedQuad; import be.ugent.rml.store.QuadStore; +import be.ugent.rml.store.RDF4JStore; import be.ugent.rml.term.NamedNode; +import be.ugent.rml.term.ProvenancedQuad; import be.ugent.rml.term.ProvenancedTerm; import be.ugent.rml.term.Term; import org.slf4j.Logger; @@ -39,23 +40,23 @@ public class Executor { private final StrictMode strictMode; public Executor(QuadStore rmlStore, RecordsFactory recordsFactory, String baseIRI, StrictMode strictMode) throws Exception { - this(rmlStore, recordsFactory, null, null, baseIRI, strictMode); + this(rmlStore, recordsFactory, null, null, baseIRI, strictMode, null); } - public Executor(QuadStore rmlStore, RecordsFactory recordsFactory, FunctionLoader functionLoader, String baseIRI, StrictMode strictMode) throws Exception { - this(rmlStore, recordsFactory, functionLoader, null, baseIRI, strictMode); + public Executor(QuadStore rmlStore, RecordsFactory recordsFactory, FunctionLoader functionLoader, String baseIRI, StrictMode strictMode, final Agent functionAgent) throws Exception { + this(rmlStore, recordsFactory, functionLoader, null, baseIRI, strictMode, functionAgent); } /** * Defaults to best effort operation. For strict mode, - * use {@link Executor#Executor(QuadStore, RecordsFactory, FunctionLoader, QuadStore, String, StrictMode)} + * use {@link Executor#Executor(QuadStore, RecordsFactory, FunctionLoader, QuadStore, String, StrictMode, Agent)} */ - public Executor(QuadStore rmlStore, RecordsFactory recordsFactory, FunctionLoader functionLoader, QuadStore resultingQuads, String baseIRI) throws Exception { - this(rmlStore, recordsFactory, functionLoader, resultingQuads, baseIRI, StrictMode.BEST_EFFORT); + public Executor(QuadStore rmlStore, RecordsFactory recordsFactory, FunctionLoader functionLoader, QuadStore resultingQuads, String baseIRI, final Agent functionAgent) throws Exception { + this(rmlStore, recordsFactory, functionLoader, resultingQuads, baseIRI, StrictMode.BEST_EFFORT, functionAgent); } - public Executor(QuadStore rmlStore, RecordsFactory recordsFactory, FunctionLoader functionLoader, QuadStore resultingQuads, String baseIRI, StrictMode strictMode) throws Exception { - this.initializer = new Initializer(rmlStore, functionLoader); + public Executor(QuadStore rmlStore, RecordsFactory recordsFactory, FunctionLoader functionLoader, QuadStore resultingQuads, String baseIRI, StrictMode strictMode, final Agent functionAgent) throws Exception { + this.initializer = new Initializer(rmlStore, functionLoader, functionAgent); this.mappings = this.initializer.getMappings(); this.rmlStore = rmlStore; this.recordsFactory = recordsFactory; @@ -105,8 +106,8 @@ public Executor(QuadStore rmlStore, RecordsFactory recordsFactory, FunctionLoade } } - public Executor(RDF4JStore rmlStore, RecordsFactory factory, FunctionLoader functionLoader, QuadStore outputStore) throws Exception { - this(rmlStore, factory, functionLoader, outputStore, rmlStore.getBase()); + public Executor(RDF4JStore rmlStore, RecordsFactory factory, FunctionLoader functionLoader, QuadStore outputStore, final Agent functionAgent) throws Exception { + this(rmlStore, factory, functionLoader, outputStore, rmlStore.getBase(), functionAgent); } /* diff --git a/src/main/java/be/ugent/rml/Initializer.java b/src/main/java/be/ugent/rml/Initializer.java index 6ee162b4..3cdee167 100644 --- a/src/main/java/be/ugent/rml/Initializer.java +++ b/src/main/java/be/ugent/rml/Initializer.java @@ -1,5 +1,6 @@ package be.ugent.rml; +import be.ugent.idlab.knows.functions.agent.Agent; import be.ugent.rml.functions.FunctionLoader; import be.ugent.rml.store.QuadStore; import be.ugent.rml.term.NamedNode; @@ -17,7 +18,7 @@ public class Initializer { private List triplesMaps; private HashMap mappings; - public Initializer(QuadStore rmlStore, FunctionLoader functionLoader) throws Exception { + public Initializer(QuadStore rmlStore, FunctionLoader functionLoader, final Agent functionAgent) throws Exception { this.rmlStore = rmlStore; //we get all the TriplesMaps from the mapping this.triplesMaps = this.getAllTriplesMaps(); @@ -29,7 +30,7 @@ public Initializer(QuadStore rmlStore, FunctionLoader functionLoader) throws Exc this.functionLoader = functionLoader; } - this.factory = new MappingFactory(this.functionLoader); + this.factory = new MappingFactory(this.functionLoader, functionAgent); extractMappings(); } diff --git a/src/main/java/be/ugent/rml/MappingFactory.java b/src/main/java/be/ugent/rml/MappingFactory.java index 924a9534..b3948f10 100644 --- a/src/main/java/be/ugent/rml/MappingFactory.java +++ b/src/main/java/be/ugent/rml/MappingFactory.java @@ -1,5 +1,6 @@ package be.ugent.rml; +import be.ugent.idlab.knows.functions.agent.Agent; import be.ugent.rml.extractor.ConstantExtractor; import be.ugent.rml.extractor.HashExtractor; import be.ugent.rml.extractor.ReferenceExtractor; @@ -25,7 +26,8 @@ import static be.ugent.rml.Utils.isValidrrLanguage; public class MappingFactory { - private final FunctionLoader functionLoader; + private final FunctionLoader functionLoader; // TODO: remove + private final Agent functionAgent; private MappingInfo subjectMappingInfo; private List graphMappingInfos; private Term triplesMap; @@ -36,8 +38,9 @@ public class MappingFactory { private boolean ignoreDoubleQuotes; protected Logger logger = LoggerFactory.getLogger(this.getClass()); - public MappingFactory(FunctionLoader functionLoader) { + public MappingFactory(FunctionLoader functionLoader, Agent functionAgent) { this.functionLoader = functionLoader; + this.functionAgent = functionAgent; } public Mapping createMapping(Term triplesMap, QuadStore store) throws Exception { @@ -274,7 +277,7 @@ private void parseObjectMapWithCallback(Term objectmap, BiConsumer parseObjectMapsAndShortcuts(Term pom) throws IOException { diff --git a/src/main/java/be/ugent/rml/cli/Main.java b/src/main/java/be/ugent/rml/cli/Main.java index fd4e228e..a0195637 100644 --- a/src/main/java/be/ugent/rml/cli/Main.java +++ b/src/main/java/be/ugent/rml/cli/Main.java @@ -1,5 +1,7 @@ package be.ugent.rml.cli; +import be.ugent.idlab.knows.functions.agent.Agent; +import be.ugent.idlab.knows.functions.agent.AgentFactory; import be.ugent.rml.Executor; import be.ugent.rml.StrictMode; import be.ugent.rml.Utils; @@ -8,7 +10,6 @@ import be.ugent.rml.functions.lib.IDLabFunctions; import be.ugent.rml.metadata.MetadataGenerator; import be.ugent.rml.records.RecordsFactory; -import be.ugent.rml.store.Quad; import be.ugent.rml.store.QuadStore; import be.ugent.rml.store.RDF4JStore; import be.ugent.rml.store.SimpleQuadStore; @@ -18,7 +19,6 @@ import be.ugent.rml.term.Term; import ch.qos.logback.classic.Level; import org.apache.commons.cli.*; -import org.apache.commons.io.IOUtils; import org.eclipse.rdf4j.rio.RDFFormat; import org.eclipse.rdf4j.rio.RDFParseException; import org.slf4j.Logger; @@ -34,7 +34,8 @@ import java.util.*; import java.util.stream.Collectors; -import static be.ugent.rml.StrictMode.*; +import static be.ugent.rml.StrictMode.BEST_EFFORT; +import static be.ugent.rml.StrictMode.STRICT; public class Main { @@ -301,13 +302,24 @@ public static void main(String[] args, String basePath) { } String[] fOptionValue = getOptionValues(functionfileOption, lineArgs, configFile); - FunctionLoader functionLoader; + final FunctionLoader functionLoader; + final Agent functionAgent; // Read function description files. if (fOptionValue == null) { functionLoader = new FunctionLoader(); + // default initialisation with IDLab functions and GREL functions... + functionAgent = AgentFactory.createFromFnO( + "functions_idlab.ttl", + "https://users.ugent.be/~bjdmeest/function/grel.ttl", + "grel_java_mapping.ttl"); } else { logger.debug("Using custom path to functions.ttl file: " + Arrays.toString(fOptionValue)); + String[] optionWithIDLabFunctionArgs = new String[fOptionValue.length + 1]; + optionWithIDLabFunctionArgs[0] = "functions_idlab.ttl" ; + System.arraycopy(fOptionValue, 0, optionWithIDLabFunctionArgs, 1, optionWithIDLabFunctionArgs.length); + functionAgent = AgentFactory.createFromFnO(optionWithIDLabFunctionArgs); + RDF4JStore functionDescriptionTriples = new RDF4JStore(); functionDescriptionTriples.read(Utils.getInputStreamFromFile(Utils.getFile("functions_idlab.ttl")), null, RDFFormat.TURTLE); Map libraryMap = new HashMap<>(); @@ -319,6 +331,8 @@ public static void main(String[] args, String basePath) { functionDescriptionTriples.read(lisF.get(i), null, RDFFormat.TURTLE); } functionLoader = new FunctionLoader(functionDescriptionTriples, libraryMap); + + } if (mOptionValue != null) { @@ -347,7 +361,7 @@ public static void main(String[] args, String basePath) { } } - executor = new Executor(rmlStore, factory, functionLoader, outputStore, baseIRI, strictMode); + executor = new Executor(rmlStore, factory, functionLoader, outputStore, baseIRI, strictMode, functionAgent); List triplesMaps = new ArrayList<>(); diff --git a/src/main/java/be/ugent/rml/functions/DynamicMultipleRecordsFunctionExecutor.java b/src/main/java/be/ugent/rml/functions/DynamicMultipleRecordsFunctionExecutor.java index 2ffb5c06..a04c1a35 100644 --- a/src/main/java/be/ugent/rml/functions/DynamicMultipleRecordsFunctionExecutor.java +++ b/src/main/java/be/ugent/rml/functions/DynamicMultipleRecordsFunctionExecutor.java @@ -1,5 +1,7 @@ package be.ugent.rml.functions; +import be.ugent.idlab.knows.functions.agent.Agent; +import be.ugent.idlab.knows.functions.agent.Arguments; import be.ugent.rml.NAMESPACES; import be.ugent.rml.records.Record; import be.ugent.rml.term.NamedNode; @@ -7,24 +9,35 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; -import java.util.stream.Collectors; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; public class DynamicMultipleRecordsFunctionExecutor implements MultipleRecordsFunctionExecutor { private static final Logger logger = LoggerFactory.getLogger(DynamicMultipleRecordsFunctionExecutor.class); - private List parameterValuePairs; - private FunctionLoader functionLoader; + private final List parameterValuePairs; - public DynamicMultipleRecordsFunctionExecutor(List parameterValuePairs, FunctionLoader functionLoader) { + private final Agent functionAgent; + + public DynamicMultipleRecordsFunctionExecutor(final List parameterValuePairs, final Agent functionAgent) { this.parameterValuePairs = parameterValuePairs; - this.functionLoader = functionLoader; + this.functionAgent = functionAgent; + +// // TODO: this will be passed as constructor argument +// try { +// functionAgent = AgentFactory.createFromFnO("functions_idlab.ttl", +// "https://users.ugent.be/~bjdmeest/function/grel.ttl", "grel_java_mapping.ttl"); +// } catch (FnOException e) { +// functionAgent = null; +// e.printStackTrace(); +// } } @Override public Object execute(Map records) throws Exception { final ArrayList fnTerms = new ArrayList<>(); - final ArrayList args = new ArrayList<>(); + final Arguments arguments = new Arguments(); parameterValuePairs.forEach(pv -> { ArrayList parameters = new ArrayList<>(); @@ -56,59 +69,19 @@ public Object execute(Map records) throws Exception { fnTerms.add(values.get(0)); } else { - parameters.forEach(parameter -> { - ArrayList temp = new ArrayList<>(); - - values.forEach(value -> { - temp.add(value.getValue()); - }); - - args.add(new Argument(parameter.getValue(), temp)); - }); + for (Term parameter : parameters) { + for (Term value : values) { + arguments.add(parameter.getValue(), value.getValue()); + } + } } }); - final Map> mergedArgs = new HashMap<>(); - //TODO check if function is list? - args.forEach(arg -> { - if (!mergedArgs.containsKey(arg.getParameter())) { - mergedArgs.put(arg.getParameter(), arg.getArguments()); - } else { - mergedArgs.get(arg.getParameter()).addAll(arg.getArguments()); - } - }); if (fnTerms.isEmpty()) { - throw new Exception("No function was defined for parameters: " + mergedArgs.keySet()); + throw new Exception("No function was defined for parameters: " + arguments.getArgumentNames()); } else { - FunctionModel function = functionLoader.getFunction(fnTerms.get(0)); - return function.execute((Map) mergedArgs); + final String functionId = fnTerms.get(0).getValue(); + return functionAgent.execute(functionId, arguments); } } -} - -/** - * Helper class to combine a parameter and his arguments in one object - */ -class Argument { - /** - * Function Parameter URI - */ - private String parameter; - /** - * All the actual generated values for this parameter - */ - private List arguments; - - Argument(String parameter, List arguments) { - this.parameter = parameter; - this.arguments = arguments; - } - - public String getParameter() { - return parameter; - } - - public List getArguments() { - return arguments; - } -} +} \ No newline at end of file diff --git a/src/main/java/be/ugent/rml/functions/DynamicSingleRecordFunctionExecutor.java b/src/main/java/be/ugent/rml/functions/DynamicSingleRecordFunctionExecutor.java index 0d8f6393..509e0731 100644 --- a/src/main/java/be/ugent/rml/functions/DynamicSingleRecordFunctionExecutor.java +++ b/src/main/java/be/ugent/rml/functions/DynamicSingleRecordFunctionExecutor.java @@ -1,11 +1,13 @@ package be.ugent.rml.functions; +import be.ugent.idlab.knows.functions.agent.Agent; + import java.util.ArrayList; import java.util.List; public class DynamicSingleRecordFunctionExecutor extends AbstractSingleRecordFunctionExecutor { - public DynamicSingleRecordFunctionExecutor(List parameterValuePairs, FunctionLoader functionLoader) { + public DynamicSingleRecordFunctionExecutor(List parameterValuePairs, final Agent functionAgent) { ArrayList pairs = new ArrayList<>(); parameterValuePairs.forEach(pair -> { @@ -18,6 +20,6 @@ public DynamicSingleRecordFunctionExecutor(List parameterVal pairs.add(new ParameterValueOriginPair(pair.getParameterGenerators(), objectGeneratorOriginPairs)); }); - functionExecutor = new DynamicMultipleRecordsFunctionExecutor(pairs, functionLoader); + functionExecutor = new DynamicMultipleRecordsFunctionExecutor(pairs, functionAgent); } } \ No newline at end of file diff --git a/src/main/java/be/ugent/rml/functions/StaticMultipleRecordsFunctionExecutor.java b/src/main/java/be/ugent/rml/functions/StaticMultipleRecordsFunctionExecutor.java index c38fb573..099b1b5a 100644 --- a/src/main/java/be/ugent/rml/functions/StaticMultipleRecordsFunctionExecutor.java +++ b/src/main/java/be/ugent/rml/functions/StaticMultipleRecordsFunctionExecutor.java @@ -1,5 +1,7 @@ package be.ugent.rml.functions; +import be.ugent.idlab.knows.functions.agent.Agent; +import be.ugent.idlab.knows.functions.agent.Arguments; import be.ugent.rml.records.Record; import java.util.HashMap; @@ -7,17 +9,23 @@ public class StaticMultipleRecordsFunctionExecutor implements MultipleRecordsFunctionExecutor { - private final FunctionModel functionModel; + private final FunctionModel functionModel; // TODO: remove private final Map parameters; - public StaticMultipleRecordsFunctionExecutor(FunctionModel model, Map parameters) { + private final Agent functionAgent; + private final String functionId; + + public StaticMultipleRecordsFunctionExecutor(FunctionModel model, Map parameters, Agent functionAgent, String functionId) { this.functionModel = model; this.parameters = parameters; + this.functionAgent = functionAgent; + this.functionId = functionId; } @Override public Object execute(Map records) throws Exception { Map filledInParameters = new HashMap<>(); + final Arguments functionArguments = new Arguments(); for (Map.Entry entry : this.parameters.entrySet()) { SingleRecordFunctionExecutor executor = (SingleRecordFunctionExecutor) entry.getValue()[1]; @@ -25,14 +33,14 @@ public Object execute(Map records) throws Exception { Object o = executor.execute(records.get(recordType)); - if (o != null) { - filledInParameters.put(entry.getKey(), o); - } else { - // TODO check whether key is actually optional! - filledInParameters.put(entry.getKey(), null); - } + // TODO check whether key is actually optional! + filledInParameters.put(entry.getKey(), o); + functionArguments.add(entry.getKey(), o); } - return this.functionModel.execute(filledInParameters); + Object result1 = functionModel.execute(filledInParameters); + Object result2 = functionAgent.execute(functionId, functionArguments); + assert(result1.equals(result2)); + return result1; } } diff --git a/src/main/java/be/ugent/rml/functions/StaticSingleRecordFunctionExecutor.java b/src/main/java/be/ugent/rml/functions/StaticSingleRecordFunctionExecutor.java index 18ebec0e..94fed0ca 100644 --- a/src/main/java/be/ugent/rml/functions/StaticSingleRecordFunctionExecutor.java +++ b/src/main/java/be/ugent/rml/functions/StaticSingleRecordFunctionExecutor.java @@ -1,5 +1,6 @@ package be.ugent.rml.functions; +import be.ugent.idlab.knows.functions.agent.Agent; import be.ugent.rml.Template; import java.util.HashMap; @@ -8,7 +9,7 @@ public class StaticSingleRecordFunctionExecutor extends AbstractSingleRecordFunctionExecutor { - public StaticSingleRecordFunctionExecutor(FunctionModel model, Map> parameters) { + public StaticSingleRecordFunctionExecutor(FunctionModel model, final Agent functionAgent, final String functionId, Map> parameters) { HashMap parametersForOtherExecutor = new HashMap<>(); parameters.keySet().forEach(parameter -> { @@ -20,6 +21,6 @@ public StaticSingleRecordFunctionExecutor(FunctionModel model, Map dbpediaSpotlight(String text, String endpoint) { return new ArrayList<>(); } - public static Object trueCondition(String bool, String value) { + public static String trueCondition(String bool, String value) { if (bool == null || !bool.equals("true")) { return null; } else { diff --git a/src/main/resources/functions_idlab.ttl b/src/main/resources/functions_idlab.ttl index 6e161b1b..6de68e56 100644 --- a/src/main/resources/functions_idlab.ttl +++ b/src/main/resources/functions_idlab.ttl @@ -21,7 +21,7 @@ idlab-fn:equal fno:expects ( grel:valueParam grel:valueParam2 ) ; fno:returns ( grel:stringOut ) ; lib:providedBy [ lib:localLibrary "UtilFunctions.jar" ; - lib:class "UtilFunctions" ; + lib:class "be.ugent.rml.functions.lib.UtilFunctions" ; lib:method "equal" ] . idlab-fn:notEqual @@ -32,7 +32,7 @@ idlab-fn:notEqual fno:expects ( grel:valueParam grel:valueParam2 ) ; fno:returns ( grel:stringOut ) ; lib:providedBy [ lib:localLibrary "UtilFunctions.jar" ; - lib:class "UtilFunctions" ; + lib:class "be.ugent.rml.functions.lib.UtilFunctions" ; lib:method "notEqual" ] . idlab-fn:stringContainsOtherString @@ -43,7 +43,7 @@ idlab-fn:stringContainsOtherString fno:expects ( idlab-fn:_str idlab-fn:_otherStr idlab-fn:_delimiter ) ; fno:returns ( idlab-fn:_stringOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; - lib:class "IDLabFunctions" ; + lib:class "be.ugent.rml.functions.lib.IDLabFunctions" ; lib:method "stringContainsOtherString" ] . idlab-fn:listContainsElement @@ -54,7 +54,7 @@ idlab-fn:listContainsElement fno:expects ( idlab-fn:_list idlab-fn:_str ) ; fno:returns ( idlab-fn:_stringOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; - lib:class "IDLabFunctions" ; + lib:class "be.ugent.rml.functions.lib.IDLabFunctions" ; lib:method "listContainsElement" ] . idlab-fn:getMIMEType @@ -65,7 +65,7 @@ idlab-fn:getMIMEType fno:expects ( idlab-fn:_str ) ; fno:returns ( idlab-fn:_stringOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; - lib:class "IDLabFunctions" ; + lib:class "be.ugent.rml.functions.lib.IDLabFunctions" ; lib:method "getMIMEType" ] . idlab-fn:decide @@ -76,7 +76,7 @@ idlab-fn:decide fno:expects ( idlab-fn:_str idlab-fn:_expectedStr idlab-fn:_result ) ; fno:returns ( idlab-fn:_stringOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; - lib:class "IDLabFunctions" ; + lib:class "be.ugent.rml.functions.lib.IDLabFunctions" ; lib:method "decide" ] . idlab-fn:isNull @@ -87,7 +87,7 @@ idlab-fn:isNull fno:expects ( idlab-fn:_str ) ; fno:returns ( idlab-fn:_boolOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; - lib:class "IDLabFunctions" ; + lib:class "be.ugent.rml.functions.lib.IDLabFunctions" ; lib:method "isNull" ] . idlab-fn:dbpediaSpotlight @@ -98,7 +98,7 @@ idlab-fn:dbpediaSpotlight fno:expects ( idlab-fn:_str idlab-fn:_endpoint ) ; fno:returns ( idlab-fn:_stringOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; - lib:class "IDLabFunctions" ; + lib:class "be.ugent.rml.functions.lib.IDLabFunctions" ; lib:method "dbpediaSpotlight" ] . idlab-fn:trueCondition @@ -109,7 +109,7 @@ idlab-fn:trueCondition fno:expects ( idlab-fn:_strBoolean idlab-fn:_str ) ; fno:returns ( idlab-fn:_stringOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; - lib:class "IDLabFunctions" ; + lib:class "be.ugent.rml.functions.lib.IDLabFunctions" ; lib:method "trueCondition" ] . idlab-fn:readFile @@ -120,7 +120,7 @@ idlab-fn:readFile fno:expects ( idlab-fn:_path ) ; fno:returns ( idlab-fn:_stringOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; - lib:class "IDLabFunctions" ; + lib:class "be.ugent.rml.functions.lib.IDLabFunctions" ; lib:method "readFile" ] . idlab-fn:random @@ -131,7 +131,7 @@ idlab-fn:random fno:expects ( ) ; fno:returns ( idlab-fn:_stringOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; - lib:class "IDLabFunctions" ; + lib:class "be.ugent.rml.functions.lib.IDLabFunctions" ; lib:method "random" ] . idlab-fn:inRange @@ -142,7 +142,7 @@ idlab-fn:inRange fno:expects ( idlab-fn:_test idlab-fn:_from idlab-fn:_to ) ; fno:returns ( idlab-fn:_boolOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; - lib:class "IDLabFunctions" ; + lib:class "be.ugent.rml.functions.lib.IDLabFunctions" ; lib:method "inRange" ] . idlab-fn:slugify @@ -153,7 +153,7 @@ idlab-fn:slugify fno:expects ( idlab-fn:_str ) ; fno:returns ( idlab-fn:_stringOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; - lib:class "IDLabFunctions" ; + lib:class "be.ugent.rml.functions.lib.IDLabFunctions" ; lib:method "slugify" ] . idlab-fn:normalizeDate @@ -164,7 +164,7 @@ idlab-fn:normalizeDate fno:expects ( idlab-fn:_strDate idlab-fn:_pattern ) ; fno:returns ( idlab-fn:_stringOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; - lib:class "IDLabFunctions" ; + lib:class "be.ugent.rml.functions.lib.IDLabFunctions" ; lib:method "normalizeDate" ] . idlab-fn:normalizeDateWithLang @@ -175,7 +175,7 @@ idlab-fn:normalizeDate fno:expects ( idlab-fn:_strDate idlab-fn:_pattern idlab-fn:_lang ) ; fno:returns ( idlab-fn:_stringOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; - lib:class "IDLabFunctions" ; + lib:class "be.ugent.rml.functions.lib.IDLabFunctions" ; lib:method "normalizeDateWithLang" ] . idlab-fn:normalizeDateTime @@ -186,7 +186,7 @@ idlab-fn:normalizeDateTime fno:expects ( idlab-fn:_strDate idlab-fn:_pattern ) ; fno:returns ( idlab-fn:_stringOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; - lib:class "IDLabFunctions" ; + lib:class "be.ugent.rml.functions.lib.IDLabFunctions" ; lib:method "normalizeDateTime" ] . idlab-fn:normalizeDateTimeWithLang @@ -197,7 +197,7 @@ idlab-fn:normalizeDateTime fno:expects ( idlab-fn:_strDate idlab-fn:_pattern idlab-fn:_lang ) ; fno:returns ( idlab-fn:_stringOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; - lib:class "IDLabFunctions" ; + lib:class "be.ugent.rml.functions.lib.IDLabFunctions" ; lib:method "normalizeDateTimeWithLang" ] . idlab-fn:generateUniqueIRI @@ -208,7 +208,7 @@ idlab-fn:generateUniqueIRI fno:expects ( idlab-fn:_iri idlab-fn:_watchedProperty idlab-fn:_unique idlab-fn:_state ) ; fno:returns ( idlab-fn:_stringOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; - lib:class "IDLabFunctions" ; + lib:class "be.ugent.rml.functions.lib.IDLabFunctions" ; lib:method "generateUniqueIRI" ] . @@ -241,7 +241,7 @@ idlab-fn:toUpperCaseURL fno:expects ( idlab-fn:_str ) ; fno:returns ( idlab-fn:_stringOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; - lib:class "IDLabFunctions" ; + lib:class "be.ugent.rml.functions.lib.IDLabFunctions" ; lib:method "toUpperCaseURL" ] . idlab-fn:_path diff --git a/src/test/java/be/ugent/rml/Custom_RML_FnO_Mapper_Test.java b/src/test/java/be/ugent/rml/Custom_RML_FnO_Mapper_Test.java index 7e5d6877..d448fbda 100644 --- a/src/test/java/be/ugent/rml/Custom_RML_FnO_Mapper_Test.java +++ b/src/test/java/be/ugent/rml/Custom_RML_FnO_Mapper_Test.java @@ -1,7 +1,8 @@ package be.ugent.rml; +import be.ugent.idlab.knows.functions.agent.Agent; +import be.ugent.idlab.knows.functions.agent.AgentFactory; import be.ugent.rml.functions.FunctionLoader; -import be.ugent.rml.functions.FunctionUtils; import be.ugent.rml.store.QuadStore; import be.ugent.rml.store.RDF4JStore; import org.eclipse.rdf4j.rio.RDFFormat; @@ -10,7 +11,6 @@ import java.io.File; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; public class Custom_RML_FnO_Mapper_Test extends TestFunctionCore { @@ -19,9 +19,13 @@ public void evaluate_A001() throws Exception { QuadStore functionDescriptionTriples = new RDF4JStore(); functionDescriptionTriples.read(Utils.getInputStreamFromFile(new File("./src/test/resources/rml-fno-test-cases/functions_dynamic.ttl")), null, RDFFormat.TURTLE); FunctionLoader functionLoader = new FunctionLoader(functionDescriptionTriples); - Executor executor = this.createExecutor("./rml-fno-test-cases/RMLFNOTC0001-CSV/mapping.ttl", functionLoader); + Agent functionAgent = AgentFactory.createFromFnO( + "./src/test/resources/rml-fno-test-cases/functions_dynamic.ttl", + "https://users.ugent.be/~bjdmeest/function/grel.ttl", + "grel_java_mapping.ttl"); + Executor executor = this.createExecutor("./rml-fno-test-cases/RMLFNOTC0001-CSV/mapping.ttl", functionLoader, functionAgent); doMapping(executor, "./rml-fno-test-cases/RMLFNOTC0001-CSV/output.ttl"); - assertTrue(functionLoader.getLibraryPath("GrelFunctions").endsWith("GrelFunctions_dynamic.jar")); + //assertTrue(functionLoader.getLibraryPath("GrelFunctions").endsWith("GrelFunctions_dynamic.jar")); } /** @@ -33,9 +37,11 @@ public void evaluate_A001_missing_params() throws Exception { QuadStore functionDescriptionTriples = new RDF4JStore(); functionDescriptionTriples.read(Utils.getInputStreamFromFile(new File("./src/test/resources/rml-fno-test-cases/functions_dynamic_missing_params.ttl")), null, RDFFormat.TURTLE); FunctionLoader functionLoader = new FunctionLoader(functionDescriptionTriples); - Executor executor = this.createExecutor("./rml-fno-test-cases/RMLFNOTC0001-CSV/mapping.ttl", functionLoader); + Agent functionAgent = AgentFactory.createFromFnO( + "./src/test/resources/rml-fno-test-cases/functions_dynamic_missing_params.ttl"); + Executor executor = this.createExecutor("./rml-fno-test-cases/RMLFNOTC0001-CSV/mapping.ttl", functionLoader, functionAgent); doMapping(executor, "./rml-fno-test-cases/RMLFNOTC0001-CSV/output.ttl"); - assertTrue(functionLoader.getLibraryPath("GrelFunctions").endsWith("GrelFunctions_dynamic.jar")); + //assertTrue(functionLoader.getLibraryPath("GrelFunctions").endsWith("GrelFunctions_dynamic.jar")); } @Test @@ -49,12 +55,17 @@ public void evaluate_A003() throws Exception { QuadStore functionDescriptionTriples = new RDF4JStore(); functionDescriptionTriples.read(Utils.getInputStreamFromFile(new File("./src/test/resources/rml-fno-test-cases/functions_dynamic.ttl")), null, RDFFormat.TURTLE); FunctionLoader functionLoader = new FunctionLoader(functionDescriptionTriples); + Agent functionAgent = AgentFactory.createFromFnO( + "./src/test/resources/rml-fno-test-cases/functions_dynamic.ttl", + "https://users.ugent.be/~bjdmeest/function/grel.ttl", + "grel_java_mapping.ttl"); // You first need to execute the mapping, bc the libraryMap of loaded Jars is dynamically built - Executor executor = this.createExecutor("./rml-fno-test-cases/RMLFNOTC0001-CSV/mapping.ttl", functionLoader); + // TODO: what kind of test is this? + Executor executor = this.createExecutor("./rml-fno-test-cases/RMLFNOTC0001-CSV/mapping.ttl", functionLoader, functionAgent); doMapping(executor, "./rml-fno-test-cases/RMLFNOTC0001-CSV/output.ttl"); - String libPath = functionLoader.getLibraryPath("GrelFunctions"); - Class cls = FunctionUtils.functionRequire(new File(libPath), "GrelFunctions"); - assertEquals("GrelFunctions", cls.getName()); + //String libPath = functionLoader.getLibraryPath("GrelFunctions"); + //Class cls = FunctionUtils.functionRequire(new File(libPath), "GrelFunctions"); + //assertEquals("GrelFunctions", cls.getName()); } /** @@ -192,9 +203,10 @@ public void evaluate_AB0001() throws Exception { QuadStore functionDescriptionTriples = new RDF4JStore(); functionDescriptionTriples.read(Utils.getInputStreamFromFile(new File("./src/test/resources/aaabimfunctions/aaabim_java_mapping.ttl")), null, RDFFormat.TURTLE); FunctionLoader functionLoader = new FunctionLoader(functionDescriptionTriples); + Agent functionAgent = AgentFactory.createFromFnO("./src/test/resources/aaabimfunctions/aaabim_java_mapping.ttl"); // You first need to execute the mapping, bc the libraryMap of loaded Jars is dynamically built - Executor executor = this.createExecutor("rml-fno-test-cases/RMLFNOTCAB0001-JSON/mapping.ttl", functionLoader); + Executor executor = this.createExecutor("rml-fno-test-cases/RMLFNOTCAB0001-JSON/mapping.ttl", functionLoader, functionAgent); // execute mapping doMapping(executor, "rml-fno-test-cases/RMLFNOTCAB0001-JSON/output.ttl"); @@ -209,9 +221,10 @@ public void evaluate_AB0002() throws Exception { QuadStore functionDescriptionTriples = new RDF4JStore(); functionDescriptionTriples.read(Utils.getInputStreamFromFile(new File("./src/test/resources/aaabimfunctions/aaabim_java_mapping.ttl")), null, RDFFormat.TURTLE); FunctionLoader functionLoader = new FunctionLoader(functionDescriptionTriples); + Agent functionAgent = AgentFactory.createFromFnO("./src/test/resources/aaabimfunctions/aaabim_java_mapping.ttl"); // You first need to execute the mapping, bc the libraryMap of loaded Jars is dynamically built - Executor executor = this.createExecutor("rml-fno-test-cases/RMLFNOTCAB0002-JSON/mapping.ttl", functionLoader); + Executor executor = this.createExecutor("rml-fno-test-cases/RMLFNOTCAB0002-JSON/mapping.ttl", functionLoader, functionAgent); // execute mapping doMapping(executor, "rml-fno-test-cases/RMLFNOTCAB0002-JSON/output.ttl"); diff --git a/src/test/java/be/ugent/rml/Mapper_LDES_Test.java b/src/test/java/be/ugent/rml/Mapper_LDES_Test.java index 92d2ca10..cee56cc4 100644 --- a/src/test/java/be/ugent/rml/Mapper_LDES_Test.java +++ b/src/test/java/be/ugent/rml/Mapper_LDES_Test.java @@ -1,5 +1,7 @@ package be.ugent.rml; +import be.ugent.idlab.knows.functions.agent.Agent; +import be.ugent.idlab.knows.functions.agent.AgentFactory; import be.ugent.rml.functions.FunctionLoader; import be.ugent.rml.functions.lib.IDLabFunctions; import be.ugent.rml.store.QuadStore; @@ -11,14 +13,15 @@ import org.junit.BeforeClass; import org.junit.Test; -import static org.junit.Assert.*; import java.io.File; import java.io.IOException; -import java.nio.file.Files; + +import static org.junit.Assert.assertEquals; public class Mapper_LDES_Test extends TestCore { private static FunctionLoader LOADER; + private static Agent AGENT; @After public void cleanUp() throws IOException { IDLabFunctions.resetState(); @@ -30,30 +33,31 @@ public static void setups() throws Exception { QuadStore functionDescriptionTriples = new RDF4JStore(); functionDescriptionTriples.read(Utils.getInputStreamFromFile(new File("./src/main/resources/functions_idlab.ttl")), null, RDFFormat.TURTLE); LOADER = new FunctionLoader(functionDescriptionTriples); + AGENT = AgentFactory.createFromFnO("functions_idlab.ttl"); } @Test public void evaluate_unique_LDES () throws Exception { - Executor executor = this.createExecutor("./web-of-things/ldes/generation/basic/mapping.ttl", LOADER); + Executor executor = this.createExecutor("./web-of-things/ldes/generation/basic/mapping.ttl", LOADER, AGENT); doMapping(executor, "./web-of-things/ldes/generation/basic/output.nq"); } @Test public void evaluate_repeat_LDES() throws Exception { - Executor executor = this.createExecutor("./web-of-things/ldes/generation/repeat/mapping.ttl", LOADER); + Executor executor = this.createExecutor("./web-of-things/ldes/generation/repeat/mapping.ttl", LOADER, AGENT); executor.executeV5(null).get(new NamedNode("rmlmapper://default.store")); IDLabFunctions.saveState(); - executor = this.createExecutor("./web-of-things/ldes/generation/repeat/mapping.ttl", LOADER); + executor = this.createExecutor("./web-of-things/ldes/generation/repeat/mapping.ttl", LOADER, AGENT); doMapping(executor, "./web-of-things/ldes/generation/repeat/output.nq"); } @Test public void evaluate_partial_repeat_LDES() throws Exception { - Executor executor = this.createExecutor("./web-of-things/ldes/generation/partial/mapping.ttl", LOADER); + Executor executor = this.createExecutor("./web-of-things/ldes/generation/partial/mapping.ttl", LOADER, AGENT); QuadStore result = executor.executeV5(null).get(new NamedNode("rmlmapper://default.store")); IDLabFunctions.saveState(); - executor = this.createExecutor("./web-of-things/ldes/generation/partial/mapping2.ttl", LOADER); + executor = this.createExecutor("./web-of-things/ldes/generation/partial/mapping2.ttl", LOADER, AGENT); QuadStore result_second = executor.executeV5(null).get(new NamedNode("rmlmapper://default.store")); assertEquals(3, result.size()); assertEquals(1, result_second.size()); diff --git a/src/test/java/be/ugent/rml/TestCore.java b/src/test/java/be/ugent/rml/TestCore.java index b227db4e..e6f1c177 100644 --- a/src/test/java/be/ugent/rml/TestCore.java +++ b/src/test/java/be/ugent/rml/TestCore.java @@ -1,5 +1,6 @@ package be.ugent.rml; +import be.ugent.idlab.knows.functions.agent.Agent; import be.ugent.rml.cli.Main; import be.ugent.rml.conformer.MappingConformer; import be.ugent.rml.functions.FunctionLoader; @@ -26,8 +27,7 @@ import java.util.List; import java.util.Map; -import static be.ugent.rml.StrictMode.*; - +import static be.ugent.rml.StrictMode.BEST_EFFORT; import static org.junit.Assert.*; public abstract class TestCore { @@ -129,14 +129,14 @@ Executor createExecutor(String mapPath, String parentPath, StrictMode strictMode /** * Note: the created Executor will run in best effort mode */ - Executor createExecutor(String mapPath, FunctionLoader functionLoader) throws Exception { + Executor createExecutor(String mapPath, FunctionLoader functionLoader, final Agent functionAgent) throws Exception { ClassLoader classLoader = getClass().getClassLoader(); // execute mapping file File mappingFile = new File(classLoader.getResource(mapPath).getFile()); QuadStore rmlStore = QuadStoreFactory.read(mappingFile); return new Executor(rmlStore, new RecordsFactory(mappingFile.getParent()), - functionLoader, DEFAULT_BASE_IRI, BEST_EFFORT); + functionLoader, DEFAULT_BASE_IRI, BEST_EFFORT, functionAgent); } /** diff --git a/src/test/java/be/ugent/rml/TestFunctionCore.java b/src/test/java/be/ugent/rml/TestFunctionCore.java index 76841547..4b882baa 100644 --- a/src/test/java/be/ugent/rml/TestFunctionCore.java +++ b/src/test/java/be/ugent/rml/TestFunctionCore.java @@ -1,5 +1,7 @@ package be.ugent.rml; +import be.ugent.idlab.knows.functions.agent.Agent; +import be.ugent.idlab.knows.functions.agent.AgentFactory; import be.ugent.rml.functions.FunctionLoader; import be.ugent.rml.functions.lib.IDLabTestFunctions; import be.ugent.rml.store.RDF4JStore; @@ -26,7 +28,15 @@ Executor doPreloadMapping(String mapPath, String outPath) throws Exception { functionDescriptionTriples.read(Utils.getInputStreamFromFile(Utils.getFile("grel_java_mapping.ttl")), null, RDFFormat.TURTLE); // File myFile = Utils.getFile("rml-fno-test-cases/functions_test.ttl"); FunctionLoader functionLoader = new FunctionLoader(functionDescriptionTriples, libraryMap); - Executor executor = this.createExecutor(mapPath, functionLoader); + + Agent functionAgent = AgentFactory.createFromFnO( + "functions_idlab.ttl", + "rml-fno-test-cases/functions_test.ttl", + "grel_java_mapping.ttl", + "https://users.ugent.be/~bjdmeest/function/grel.ttl" + ); + + Executor executor = this.createExecutor(mapPath, functionLoader, functionAgent); doMapping(executor, outPath); return executor; } diff --git a/src/test/java/be/ugent/rml/readme/ReadmeFunctionTest.java b/src/test/java/be/ugent/rml/readme/ReadmeFunctionTest.java index b10f7bac..e7addc39 100644 --- a/src/test/java/be/ugent/rml/readme/ReadmeFunctionTest.java +++ b/src/test/java/be/ugent/rml/readme/ReadmeFunctionTest.java @@ -1,5 +1,7 @@ package be.ugent.rml.readme; +import be.ugent.idlab.knows.functions.agent.Agent; +import be.ugent.idlab.knows.functions.agent.AgentFactory; import be.ugent.rml.Executor; import be.ugent.rml.Utils; import be.ugent.rml.functions.FunctionLoader; @@ -43,12 +45,13 @@ public void function() { File functionsFile = Utils.getFile(functionPath); FunctionLoader functionLoader = new FunctionLoader(QuadStoreFactory.read(functionsFile), libraryMap); + Agent functionAgent = AgentFactory.createFromFnO(functionPath); // Set up the outputstore (needed when you want to output something else than nquads QuadStore outputStore = new RDF4JStore(); // Create the Executor - Executor executor = new Executor(rmlStore, factory, functionLoader, outputStore, Utils.getBaseDirectiveTurtle(mappingStream)); + Executor executor = new Executor(rmlStore, factory, functionLoader, outputStore, Utils.getBaseDirectiveTurtle(mappingStream), functionAgent); // Execute the mapping QuadStore result = executor.executeV5(null).get(new NamedNode("rmlmapper://default.store")); diff --git a/src/test/java/be/ugent/rml/readme/ReadmeTest.java b/src/test/java/be/ugent/rml/readme/ReadmeTest.java index 80a26dc7..acdd5a58 100644 --- a/src/test/java/be/ugent/rml/readme/ReadmeTest.java +++ b/src/test/java/be/ugent/rml/readme/ReadmeTest.java @@ -1,5 +1,7 @@ package be.ugent.rml.readme; +import be.ugent.idlab.knows.functions.agent.Agent; +import be.ugent.idlab.knows.functions.agent.AgentFactory; import be.ugent.rml.Executor; import be.ugent.rml.Utils; import be.ugent.rml.functions.FunctionLoader; @@ -15,7 +17,7 @@ import java.util.HashMap; import java.util.Map; -import static org.junit.Assert.*; +import static org.junit.Assert.fail; public class ReadmeTest { @@ -39,12 +41,13 @@ public void standard() { libraryMap.put("IDLabFunctions", IDLabFunctions.class); FunctionLoader functionLoader = new FunctionLoader(null, libraryMap); + Agent functionAgent = AgentFactory.createFromFnO("functions_idlab.ttl"); // Set up the outputstore (needed when you want to output something else than nquads QuadStore outputStore = new RDF4JStore(); // Create the Executor - Executor executor = new Executor(rmlStore, factory, functionLoader, outputStore, Utils.getBaseDirectiveTurtle(mappingStream)); + Executor executor = new Executor(rmlStore, factory, functionLoader, outputStore, Utils.getBaseDirectiveTurtle(mappingStream), functionAgent); // Execute the mapping QuadStore result = executor.executeV5(null).get(new NamedNode("rmlmapper://default.store")); diff --git a/src/test/resources/logback.xml b/src/test/resources/logback-test.xml similarity index 93% rename from src/test/resources/logback.xml rename to src/test/resources/logback-test.xml index 240a7728..6c518707 100644 --- a/src/test/resources/logback.xml +++ b/src/test/resources/logback-test.xml @@ -7,7 +7,7 @@ - + From b80251f5d9094f20d39c8dcba11972b1b492d21a Mon Sep 17 00:00:00 2001 From: Gerald Haesendonck Date: Tue, 15 Mar 2022 16:56:42 +0100 Subject: [PATCH 136/609] Initialise function agent correctly when passed as null value; correct some errors in FnO documents. --- src/main/java/be/ugent/rml/Initializer.java | 15 ++++++++++----- src/main/resources/functions_idlab.ttl | 6 +++--- .../aaabimfunctions/aaabim_java_mapping.ttl | 8 ++++---- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/main/java/be/ugent/rml/Initializer.java b/src/main/java/be/ugent/rml/Initializer.java index 3cdee167..ff5e80f8 100644 --- a/src/main/java/be/ugent/rml/Initializer.java +++ b/src/main/java/be/ugent/rml/Initializer.java @@ -1,6 +1,7 @@ package be.ugent.rml; import be.ugent.idlab.knows.functions.agent.Agent; +import be.ugent.idlab.knows.functions.agent.AgentFactory; import be.ugent.rml.functions.FunctionLoader; import be.ugent.rml.store.QuadStore; import be.ugent.rml.term.NamedNode; @@ -13,10 +14,10 @@ public class Initializer { private final MappingFactory factory; - private QuadStore rmlStore; - private FunctionLoader functionLoader; - private List triplesMaps; - private HashMap mappings; + private final QuadStore rmlStore; + private final FunctionLoader functionLoader; + private final List triplesMaps; + private final HashMap mappings; public Initializer(QuadStore rmlStore, FunctionLoader functionLoader, final Agent functionAgent) throws Exception { this.rmlStore = rmlStore; @@ -30,7 +31,11 @@ public Initializer(QuadStore rmlStore, FunctionLoader functionLoader, final Agen this.functionLoader = functionLoader; } - this.factory = new MappingFactory(this.functionLoader, functionAgent); + final Agent initialisedFunctionAgent = functionAgent == null ? + AgentFactory.createFromFnO("functions_idlab.ttl") + : functionAgent; + + this.factory = new MappingFactory(this.functionLoader, initialisedFunctionAgent); extractMappings(); } diff --git a/src/main/resources/functions_idlab.ttl b/src/main/resources/functions_idlab.ttl index 6de68e56..3c694a2e 100644 --- a/src/main/resources/functions_idlab.ttl +++ b/src/main/resources/functions_idlab.ttl @@ -41,7 +41,7 @@ idlab-fn:stringContainsOtherString rdfs:label "stringContainsOtherString" ; dcterms:description "Returns true if a string is found in a delimited string." ; fno:expects ( idlab-fn:_str idlab-fn:_otherStr idlab-fn:_delimiter ) ; - fno:returns ( idlab-fn:_stringOut ) ; + fno:returns ( idlab-fn:_boolOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; lib:class "be.ugent.rml.functions.lib.IDLabFunctions" ; lib:method "stringContainsOtherString" ] . @@ -52,7 +52,7 @@ idlab-fn:listContainsElement rdfs:label "listContainsElement" ; dcterms:description "Returns true if the string is found in the list" ; fno:expects ( idlab-fn:_list idlab-fn:_str ) ; - fno:returns ( idlab-fn:_stringOut ) ; + fno:returns ( idlab-fn:_boolOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; lib:class "be.ugent.rml.functions.lib.IDLabFunctions" ; lib:method "listContainsElement" ] . @@ -96,7 +96,7 @@ idlab-fn:dbpediaSpotlight rdfs:label "DBpedia Spotlight annotate" ; dcterms:description "Returns the DBpedia URLs of the detected entities in the input text" ; fno:expects ( idlab-fn:_str idlab-fn:_endpoint ) ; - fno:returns ( idlab-fn:_stringOut ) ; + fno:returns ( idlab-fn:_listOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; lib:class "be.ugent.rml.functions.lib.IDLabFunctions" ; lib:method "dbpediaSpotlight" ] . diff --git a/src/test/resources/aaabimfunctions/aaabim_java_mapping.ttl b/src/test/resources/aaabimfunctions/aaabim_java_mapping.ttl index f57ae1c9..7d466f97 100644 --- a/src/test/resources/aaabimfunctions/aaabim_java_mapping.ttl +++ b/src/test/resources/aaabimfunctions/aaabim_java_mapping.ttl @@ -246,18 +246,18 @@ aaabim:stringOut a fno:Output ; # CLASSES # ########### -aaabimm:geoFunctions a fnoi:javaClass ; +aaabimm:geoFunctions a fnoi:JavaClass ; doap:download-page "./src/test/resources/aaabimfunctions/AaabimFunctions.jar" ; fnoi:class-name "io.fno.aaabim.GeoFunctions" . -aaabimm:timeFunctions a fnoi:javaClass ; +aaabimm:timeFunctions a fnoi:JavaClass ; doap:download-page "./src/test/resources/aaabimfunctions/AaabimFunctions.jar" ; fnoi:class-name "io.fno.aaabim.TimeFunctions" . -aaabimm:obeliskFunctions a fnoi:javaClass ; +aaabimm:obeliskFunctions a fnoi:JavaClass ; doap:download-page "./src/test/resources/aaabimfunctions/AaabimFunctions.jar" ; fnoi:class-name "io.fno.aaabim.ObeliskFunctions" . -aaabimm:stringFunctions a fnoi:javaClass ; +aaabimm:stringFunctions a fnoi:JavaClass ; doap:download-page "./src/test/resources/aaabimfunctions/AaabimFunctions.jar" ; fnoi:class-name "io.fno.aaabim.StringFunctions" . \ No newline at end of file From c1cf71bc77a26e9d222a2dc6b3e3a73998e4d468 Mon Sep 17 00:00:00 2001 From: Gerald Haesendonck Date: Tue, 15 Mar 2022 22:30:25 +0100 Subject: [PATCH 137/609] Load GREL functions by default as well... --- src/main/java/be/ugent/rml/Initializer.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/be/ugent/rml/Initializer.java b/src/main/java/be/ugent/rml/Initializer.java index ff5e80f8..15535b91 100644 --- a/src/main/java/be/ugent/rml/Initializer.java +++ b/src/main/java/be/ugent/rml/Initializer.java @@ -32,7 +32,9 @@ public Initializer(QuadStore rmlStore, FunctionLoader functionLoader, final Agen } final Agent initialisedFunctionAgent = functionAgent == null ? - AgentFactory.createFromFnO("functions_idlab.ttl") + AgentFactory.createFromFnO("functions_idlab.ttl", + "https://users.ugent.be/~bjdmeest/function/grel.ttl", + "grel_java_mapping.ttl") : functionAgent; this.factory = new MappingFactory(this.functionLoader, initialisedFunctionAgent); From 26cacdb4ab1519a1b698cfcd7b962504bab017fe Mon Sep 17 00:00:00 2001 From: Gerald Haesendonck Date: Thu, 17 Mar 2022 11:55:27 +0100 Subject: [PATCH 138/609] Some corrections in FnO docs --- src/main/resources/functions_idlab.ttl | 4 ++-- src/test/resources/rml-fno-test-cases/functions_test.ttl | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/resources/functions_idlab.ttl b/src/main/resources/functions_idlab.ttl index 3c694a2e..6ca6047d 100644 --- a/src/main/resources/functions_idlab.ttl +++ b/src/main/resources/functions_idlab.ttl @@ -19,7 +19,7 @@ idlab-fn:equal rdfs:label "equal" ; dcterms:description "Returns true if strings are equal." ; fno:expects ( grel:valueParam grel:valueParam2 ) ; - fno:returns ( grel:stringOut ) ; + fno:returns ( idlab-fn:_boolOut ) ; lib:providedBy [ lib:localLibrary "UtilFunctions.jar" ; lib:class "be.ugent.rml.functions.lib.UtilFunctions" ; lib:method "equal" ] . @@ -30,7 +30,7 @@ idlab-fn:notEqual rdfs:label "not equal" ; dcterms:description "Returns true if strings are not equal." ; fno:expects ( grel:valueParam grel:valueParam2 ) ; - fno:returns ( grel:stringOut ) ; + fno:returns ( idlab-fn:_boolOut ) ; lib:providedBy [ lib:localLibrary "UtilFunctions.jar" ; lib:class "be.ugent.rml.functions.lib.UtilFunctions" ; lib:method "notEqual" ] . diff --git a/src/test/resources/rml-fno-test-cases/functions_test.ttl b/src/test/resources/rml-fno-test-cases/functions_test.ttl index 334ae9c7..9011e629 100644 --- a/src/test/resources/rml-fno-test-cases/functions_test.ttl +++ b/src/test/resources/rml-fno-test-cases/functions_test.ttl @@ -21,7 +21,7 @@ idlab-fn:generateNull fno:expects ( ) ; fno:returns ( idlab-fn:_stringOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; - lib:class "IDLabFunctions" ; + lib:class "be.ugent.rml.functions.lib.IDLabTestFunctions" ; lib:method "getNull" ] . idlab-fn:generateA @@ -32,7 +32,7 @@ idlab-fn:generateA fno:expects ( ) ; fno:returns ( idlab-fn:_stringOut ) ; lib:providedBy [ lib:localLibrary "IDLabFunctions.jar" ; - lib:class "IDLabFunctions" ; + lib:class "be.ugent.rml.functions.lib.IDLabTestFunctions" ; lib:method "generateA" ] . idlab-fn:_stringOut From 97bdc59d3184182be202b0dc1ef88daedf4168e3 Mon Sep 17 00:00:00 2001 From: Gerald Haesendonck Date: Fri, 18 Mar 2022 15:32:57 +0100 Subject: [PATCH 139/609] * Removed FunctionLoader from code; * Converted functions_idlab.ttl the new way as to be able to provide other implementations for testing; * Some more tests succeeding --- src/main/java/be/ugent/rml/Executor.java | 31 +-- src/main/java/be/ugent/rml/Initializer.java | 15 +- .../java/be/ugent/rml/MappingFactory.java | 10 +- src/main/java/be/ugent/rml/MyFileUtils.java | 3 + src/main/java/be/ugent/rml/cli/Main.java | 26 +- .../rml/extractor/ConstantExtractor.java | 2 +- .../ugent/rml/functions/ConcatFunction.java | 7 +- ...ynamicMultipleRecordsFunctionExecutor.java | 9 - .../ugent/rml/functions/FunctionLoader.java | 232 ----------------- .../be/ugent/rml/functions/FunctionModel.java | 137 ---------- .../be/ugent/rml/functions/FunctionUtils.java | 230 +---------------- ...StaticMultipleRecordsFunctionExecutor.java | 12 +- .../StaticSingleRecordFunctionExecutor.java | 26 -- .../rml/termgenerator/BlankNodeGenerator.java | 3 +- .../rml/termgenerator/LiteralGenerator.java | 19 +- .../rml/termgenerator/NamedNodeGenerator.java | 3 +- src/main/resources/functions_idlab.ttl | 235 +++++++++++------- .../functions_idlab_classes_java_mapping.ttl | 18 ++ .../java/be/ugent/rml/Arguments_Test.java | 9 +- ..._Test.java => CustomRMLFnOMapperTest.java} | 42 +--- .../rml/Custom_RML_FnO_Mapper_CSV_Test.java | 6 +- ...per_LDES_Test.java => MapperLDESTest.java} | 21 +- src/test/java/be/ugent/rml/TestCore.java | 6 +- .../java/be/ugent/rml/TestFunctionCore.java | 26 +- .../ugent/rml/readme/ReadmeFunctionTest.java | 5 +- .../java/be/ugent/rml/readme/ReadmeTest.java | 12 +- ...tions_idlab_classes_java_mapping_tests.ttl | 18 ++ 27 files changed, 259 insertions(+), 904 deletions(-) delete mode 100644 src/main/java/be/ugent/rml/functions/FunctionLoader.java delete mode 100644 src/main/java/be/ugent/rml/functions/FunctionModel.java delete mode 100644 src/main/java/be/ugent/rml/functions/StaticSingleRecordFunctionExecutor.java create mode 100644 src/main/resources/functions_idlab_classes_java_mapping.ttl rename src/test/java/be/ugent/rml/{Custom_RML_FnO_Mapper_Test.java => CustomRMLFnOMapperTest.java} (76%) rename src/test/java/be/ugent/rml/{Mapper_LDES_Test.java => MapperLDESTest.java} (72%) create mode 100644 src/test/resources/functions_idlab_classes_java_mapping_tests.ttl diff --git a/src/main/java/be/ugent/rml/Executor.java b/src/main/java/be/ugent/rml/Executor.java index e2f06a14..579b2eea 100644 --- a/src/main/java/be/ugent/rml/Executor.java +++ b/src/main/java/be/ugent/rml/Executor.java @@ -1,7 +1,6 @@ package be.ugent.rml; import be.ugent.idlab.knows.functions.agent.Agent; -import be.ugent.rml.functions.FunctionLoader; import be.ugent.rml.functions.MultipleRecordsFunctionExecutor; import be.ugent.rml.metadata.Metadata; import be.ugent.rml.metadata.MetadataGenerator; @@ -40,31 +39,31 @@ public class Executor { private final StrictMode strictMode; public Executor(QuadStore rmlStore, RecordsFactory recordsFactory, String baseIRI, StrictMode strictMode) throws Exception { - this(rmlStore, recordsFactory, null, null, baseIRI, strictMode, null); + this(rmlStore, recordsFactory, null, baseIRI, strictMode, null); } - public Executor(QuadStore rmlStore, RecordsFactory recordsFactory, FunctionLoader functionLoader, String baseIRI, StrictMode strictMode, final Agent functionAgent) throws Exception { - this(rmlStore, recordsFactory, functionLoader, null, baseIRI, strictMode, functionAgent); + public Executor(QuadStore rmlStore, RecordsFactory recordsFactory, String baseIRI, StrictMode strictMode, final Agent functionAgent) throws Exception { + this(rmlStore, recordsFactory, null, baseIRI, strictMode, functionAgent); } /** * Defaults to best effort operation. For strict mode, - * use {@link Executor#Executor(QuadStore, RecordsFactory, FunctionLoader, QuadStore, String, StrictMode, Agent)} + * use {@link Executor#Executor(QuadStore, RecordsFactory, QuadStore, String, StrictMode, Agent)} */ - public Executor(QuadStore rmlStore, RecordsFactory recordsFactory, FunctionLoader functionLoader, QuadStore resultingQuads, String baseIRI, final Agent functionAgent) throws Exception { - this(rmlStore, recordsFactory, functionLoader, resultingQuads, baseIRI, StrictMode.BEST_EFFORT, functionAgent); + public Executor(QuadStore rmlStore, RecordsFactory recordsFactory, QuadStore resultingQuads, String baseIRI, final Agent functionAgent) throws Exception { + this(rmlStore, recordsFactory, resultingQuads, baseIRI, StrictMode.BEST_EFFORT, functionAgent); } - public Executor(QuadStore rmlStore, RecordsFactory recordsFactory, FunctionLoader functionLoader, QuadStore resultingQuads, String baseIRI, StrictMode strictMode, final Agent functionAgent) throws Exception { - this.initializer = new Initializer(rmlStore, functionLoader, functionAgent); + public Executor(QuadStore rmlStore, RecordsFactory recordsFactory, QuadStore resultingQuads, String baseIRI, StrictMode strictMode, final Agent functionAgent) throws Exception { + this.initializer = new Initializer(rmlStore, functionAgent); this.mappings = this.initializer.getMappings(); this.rmlStore = rmlStore; this.recordsFactory = recordsFactory; this.baseIRI = baseIRI; this.strictMode = strictMode; - this.recordsHolders = new HashMap>(); - this.subjectCache = new HashMap>(); - this.targetStores = new HashMap(); + this.recordsHolders = new HashMap<>(); + this.subjectCache = new HashMap<>(); + this.targetStores = new HashMap<>(); Executor.blankNodeCounter = 0; // Default store if no Targets are available for a triple @@ -106,8 +105,8 @@ public Executor(QuadStore rmlStore, RecordsFactory recordsFactory, FunctionLoade } } - public Executor(RDF4JStore rmlStore, RecordsFactory factory, FunctionLoader functionLoader, QuadStore outputStore, final Agent functionAgent) throws Exception { - this(rmlStore, factory, functionLoader, outputStore, rmlStore.getBase(), functionAgent); + public Executor(RDF4JStore rmlStore, RecordsFactory factory, QuadStore outputStore, final Agent functionAgent) throws Exception { + this(rmlStore, factory, outputStore, rmlStore.getBase(), functionAgent); } /* @@ -433,10 +432,6 @@ private List getRecords(Term triplesMap) throws Exception { return this.recordsHolders.get(triplesMap); } - public FunctionLoader getFunctionLoader() { - return this.initializer.getFunctionLoader(); - } - private List combineMultiplePOGs(List predicates, List objects, List graphs) { ArrayList results = new ArrayList<>(); diff --git a/src/main/java/be/ugent/rml/Initializer.java b/src/main/java/be/ugent/rml/Initializer.java index 15535b91..d3888363 100644 --- a/src/main/java/be/ugent/rml/Initializer.java +++ b/src/main/java/be/ugent/rml/Initializer.java @@ -2,7 +2,6 @@ import be.ugent.idlab.knows.functions.agent.Agent; import be.ugent.idlab.knows.functions.agent.AgentFactory; -import be.ugent.rml.functions.FunctionLoader; import be.ugent.rml.store.QuadStore; import be.ugent.rml.term.NamedNode; import be.ugent.rml.term.Term; @@ -15,29 +14,24 @@ public class Initializer { private final MappingFactory factory; private final QuadStore rmlStore; - private final FunctionLoader functionLoader; private final List triplesMaps; private final HashMap mappings; - public Initializer(QuadStore rmlStore, FunctionLoader functionLoader, final Agent functionAgent) throws Exception { + public Initializer(QuadStore rmlStore, final Agent functionAgent) throws Exception { this.rmlStore = rmlStore; //we get all the TriplesMaps from the mapping this.triplesMaps = this.getAllTriplesMaps(); this.mappings = new HashMap(); - if (functionLoader == null) { - this.functionLoader = new FunctionLoader(); - } else { - this.functionLoader = functionLoader; - } final Agent initialisedFunctionAgent = functionAgent == null ? AgentFactory.createFromFnO("functions_idlab.ttl", + "functions_idlab_classes_java_mapping.ttl", "https://users.ugent.be/~bjdmeest/function/grel.ttl", "grel_java_mapping.ttl") : functionAgent; - this.factory = new MappingFactory(this.functionLoader, initialisedFunctionAgent); + this.factory = new MappingFactory(initialisedFunctionAgent); extractMappings(); } @@ -76,7 +70,4 @@ public List getTriplesMaps() { return this.triplesMaps; } - public FunctionLoader getFunctionLoader() { - return this.functionLoader; - } } diff --git a/src/main/java/be/ugent/rml/MappingFactory.java b/src/main/java/be/ugent/rml/MappingFactory.java index b3948f10..109e354f 100644 --- a/src/main/java/be/ugent/rml/MappingFactory.java +++ b/src/main/java/be/ugent/rml/MappingFactory.java @@ -26,7 +26,6 @@ import static be.ugent.rml.Utils.isValidrrLanguage; public class MappingFactory { - private final FunctionLoader functionLoader; // TODO: remove private final Agent functionAgent; private MappingInfo subjectMappingInfo; private List graphMappingInfos; @@ -38,8 +37,7 @@ public class MappingFactory { private boolean ignoreDoubleQuotes; protected Logger logger = LoggerFactory.getLogger(this.getClass()); - public MappingFactory(FunctionLoader functionLoader, Agent functionAgent) { - this.functionLoader = functionLoader; + public MappingFactory(Agent functionAgent) { this.functionAgent = functionAgent; } @@ -265,7 +263,6 @@ private void parseObjectMapWithCallback(Term objectmap, BiConsumer parameters = new HashMap<>(); boolean ignoreDoubleQuotesInParent = this.areDoubleQuotesIgnored(store, parentTriplesMap); @@ -277,7 +274,7 @@ private void parseObjectMapWithCallback(Term objectmap, BiConsumer parameters = new HashMap<>(); SingleRecordFunctionExecutor parent = new HashExtractor(); @@ -497,7 +493,7 @@ private MultipleRecordsFunctionExecutor generateSameLogicalSourceJoinConditionFu Object[] detailsChild = {"child", child}; parameters.put("http://users.ugent.be/~bjdmeest/function/grel.ttl#valueParameter2", detailsChild); - return new StaticMultipleRecordsFunctionExecutor(equal, parameters, functionAgent, "http://example.com/idlab/function/equal"); + return new StaticMultipleRecordsFunctionExecutor(parameters, functionAgent, "http://example.com/idlab/function/equal"); } private List parseObjectMapsAndShortcuts(Term pom) throws IOException { diff --git a/src/main/java/be/ugent/rml/MyFileUtils.java b/src/main/java/be/ugent/rml/MyFileUtils.java index 2ac42926..9ee3b77b 100644 --- a/src/main/java/be/ugent/rml/MyFileUtils.java +++ b/src/main/java/be/ugent/rml/MyFileUtils.java @@ -2,6 +2,8 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; @@ -75,6 +77,7 @@ public File getFile() throws IOException { } public static class URLClassLoaderFileResource implements FileResource { + private static Logger logger = LoggerFactory.getLogger(URLClassLoaderFileResource.class); private ClassLoader cl; private String resource; diff --git a/src/main/java/be/ugent/rml/cli/Main.java b/src/main/java/be/ugent/rml/cli/Main.java index a0195637..e37aa27d 100644 --- a/src/main/java/be/ugent/rml/cli/Main.java +++ b/src/main/java/be/ugent/rml/cli/Main.java @@ -6,7 +6,6 @@ import be.ugent.rml.StrictMode; import be.ugent.rml.Utils; import be.ugent.rml.conformer.MappingConformer; -import be.ugent.rml.functions.FunctionLoader; import be.ugent.rml.functions.lib.IDLabFunctions; import be.ugent.rml.metadata.MetadataGenerator; import be.ugent.rml.records.RecordsFactory; @@ -302,37 +301,22 @@ public static void main(String[] args, String basePath) { } String[] fOptionValue = getOptionValues(functionfileOption, lineArgs, configFile); - final FunctionLoader functionLoader; final Agent functionAgent; // Read function description files. if (fOptionValue == null) { - functionLoader = new FunctionLoader(); // default initialisation with IDLab functions and GREL functions... functionAgent = AgentFactory.createFromFnO( - "functions_idlab.ttl", + "functions_idlab.ttl", "functions_idlab_classes_java_mapping.ttl", "https://users.ugent.be/~bjdmeest/function/grel.ttl", "grel_java_mapping.ttl"); } else { logger.debug("Using custom path to functions.ttl file: " + Arrays.toString(fOptionValue)); - String[] optionWithIDLabFunctionArgs = new String[fOptionValue.length + 1]; + String[] optionWithIDLabFunctionArgs = new String[fOptionValue.length + 2]; optionWithIDLabFunctionArgs[0] = "functions_idlab.ttl" ; - System.arraycopy(fOptionValue, 0, optionWithIDLabFunctionArgs, 1, optionWithIDLabFunctionArgs.length); + optionWithIDLabFunctionArgs[1] = "functions_idlab_classes_java_mapping.ttl" ; + System.arraycopy(fOptionValue, 0, optionWithIDLabFunctionArgs, 2, fOptionValue.length); functionAgent = AgentFactory.createFromFnO(optionWithIDLabFunctionArgs); - - RDF4JStore functionDescriptionTriples = new RDF4JStore(); - functionDescriptionTriples.read(Utils.getInputStreamFromFile(Utils.getFile("functions_idlab.ttl")), null, RDFFormat.TURTLE); - Map libraryMap = new HashMap<>(); - libraryMap.put("IDLabFunctions", IDLabFunctions.class); - List lisF = Arrays.stream(fOptionValue) - .map(Utils::getInputStreamFromFileOrContentString) - .collect(Collectors.toList()); - for (int i = 0; i < lisF.size(); i++) { - functionDescriptionTriples.read(lisF.get(i), null, RDFFormat.TURTLE); - } - functionLoader = new FunctionLoader(functionDescriptionTriples, libraryMap); - - } if (mOptionValue != null) { @@ -361,7 +345,7 @@ public static void main(String[] args, String basePath) { } } - executor = new Executor(rmlStore, factory, functionLoader, outputStore, baseIRI, strictMode, functionAgent); + executor = new Executor(rmlStore, factory, outputStore, baseIRI, strictMode, functionAgent); List triplesMaps = new ArrayList<>(); diff --git a/src/main/java/be/ugent/rml/extractor/ConstantExtractor.java b/src/main/java/be/ugent/rml/extractor/ConstantExtractor.java index 3e9ab608..259a51f7 100644 --- a/src/main/java/be/ugent/rml/extractor/ConstantExtractor.java +++ b/src/main/java/be/ugent/rml/extractor/ConstantExtractor.java @@ -9,7 +9,7 @@ public class ConstantExtractor implements Extractor, SingleRecordFunctionExecutor { - private String constant; + private final String constant; public ConstantExtractor(String constant) { this.constant = constant; diff --git a/src/main/java/be/ugent/rml/functions/ConcatFunction.java b/src/main/java/be/ugent/rml/functions/ConcatFunction.java index ed63aeb4..7fb19a3c 100644 --- a/src/main/java/be/ugent/rml/functions/ConcatFunction.java +++ b/src/main/java/be/ugent/rml/functions/ConcatFunction.java @@ -22,10 +22,6 @@ public ConcatFunction(List extractors, boolean encodeURI) { this.encodeURI = encodeURI; } - public ConcatFunction(List extractors) { - this(extractors, false); - } - @Override public List execute(Record record) { return concat(record); @@ -44,8 +40,7 @@ private List concat(Record record) { for (int i = 0; allValuesFound && i < extractors.size(); i++) { Extractor extractor = extractors.get(i); - List extractedValues = new ArrayList<>(); - FunctionUtils.functionObjectToList(extractor.extract(record), extractedValues); + List extractedValues = FunctionUtils.functionObjectToList(extractor.extract(record)); if (!extractedValues.isEmpty()) { ArrayList temp = new ArrayList<>(); diff --git a/src/main/java/be/ugent/rml/functions/DynamicMultipleRecordsFunctionExecutor.java b/src/main/java/be/ugent/rml/functions/DynamicMultipleRecordsFunctionExecutor.java index a04c1a35..e8c7f904 100644 --- a/src/main/java/be/ugent/rml/functions/DynamicMultipleRecordsFunctionExecutor.java +++ b/src/main/java/be/ugent/rml/functions/DynamicMultipleRecordsFunctionExecutor.java @@ -23,15 +23,6 @@ public class DynamicMultipleRecordsFunctionExecutor implements MultipleRecordsFu public DynamicMultipleRecordsFunctionExecutor(final List parameterValuePairs, final Agent functionAgent) { this.parameterValuePairs = parameterValuePairs; this.functionAgent = functionAgent; - -// // TODO: this will be passed as constructor argument -// try { -// functionAgent = AgentFactory.createFromFnO("functions_idlab.ttl", -// "https://users.ugent.be/~bjdmeest/function/grel.ttl", "grel_java_mapping.ttl"); -// } catch (FnOException e) { -// functionAgent = null; -// e.printStackTrace(); -// } } @Override diff --git a/src/main/java/be/ugent/rml/functions/FunctionLoader.java b/src/main/java/be/ugent/rml/functions/FunctionLoader.java deleted file mode 100644 index 1b04ed5a..00000000 --- a/src/main/java/be/ugent/rml/functions/FunctionLoader.java +++ /dev/null @@ -1,232 +0,0 @@ -package be.ugent.rml.functions; - -import be.ugent.rml.NAMESPACES; -import be.ugent.rml.functions.lib.IDLabFunctions; -import be.ugent.rml.store.RDF4JStore; -import be.ugent.rml.term.NamedNode; -import be.ugent.rml.term.Term; -import be.ugent.rml.Utils; -import be.ugent.rml.functions.lib.UtilFunctions; -import be.ugent.rml.store.QuadStore; -import org.eclipse.rdf4j.rio.RDFFormat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class FunctionLoader { - - private static final Logger logger = LoggerFactory.getLogger(FunctionLoader.class); - - private final QuadStore functionDescriptionTriples; - - // updated dynamically - /** - * Cache for loaded classes - */ - private Map classMap; - /** - * Cache for library paths - */ - private Map libraryMap; - /** - * Cache for loaded functions - */ - private Map loadedMethods; - - public FunctionLoader() throws Exception { - this(null, null); - } - - public FunctionLoader(QuadStore functionDescriptionTriples) throws Exception { - this(functionDescriptionTriples, null); - } - - public FunctionLoader(QuadStore functionDescriptionTriples, Map libraryMap) throws Exception { - if (functionDescriptionTriples == null) { - functionDescriptionTriples = new RDF4JStore(); - functionDescriptionTriples.read(Utils.getInputStreamFromFile(Utils.getFile("functions_idlab.ttl")), null, RDFFormat.TURTLE); - functionDescriptionTriples.read(Utils.getInputStreamFromFile(Utils.getFile("functions_grel.ttl")), null, RDFFormat.TURTLE); - functionDescriptionTriples.read(Utils.getInputStreamFromFile(Utils.getFile("grel_java_mapping.ttl")), null, RDFFormat.TURTLE); - } - - this.functionDescriptionTriples = functionDescriptionTriples; - - this.libraryMap = new HashMap<>(); - - if (libraryMap == null) { - this.classMap = new HashMap<>(); - this.classMap.put("IDLabFunctions", IDLabFunctions.class); - this.classMap.put("io.fno.grel.ArrayFunctions", io.fno.grel.ArrayFunctions.class); - this.classMap.put("io.fno.grel.BooleanFunctions", io.fno.grel.BooleanFunctions.class); - this.classMap.put("io.fno.grel.ControlsFunctions", io.fno.grel.ControlsFunctions.class); - this.classMap.put("io.fno.grel.StringFunctions", io.fno.grel.StringFunctions.class); - this.libraryMap.put("IDLabFunctions", "__local"); - this.libraryMap.put("io.fno.grel.ArrayFunctions", "__local"); - this.libraryMap.put("io.fno.grel.BooleanFunctions", "__local"); - this.libraryMap.put("io.fno.grel.ControlsFunctions", "__local"); - this.libraryMap.put("io.fno.grel.StringFunctions", "__local"); - } else { - this.classMap = libraryMap; - for (String key : libraryMap.keySet()) { - this.libraryMap.put(key, "__local"); - } - } - - this.classMap.put("UtilFunctions", UtilFunctions.class); - this.libraryMap.put("UtilFunctions", "__local"); - - this.loadedMethods = new HashMap<>(); - } - - public FunctionModel getFunction(Term iri) throws IOException { - if (!this.loadedMethods.containsKey(iri)) { - logger.debug("Loading function: " + iri); - try { - findMethodOldWay(iri); - logger.warn("Found a function using the old `lib:` way, this is deprecated"); - } catch (IOException e) { - findMethodNewWay(iri); - } - } - - return this.loadedMethods.get(iri); - } - - public String getLibraryPath(String className) { - return this.libraryMap.get(className); - } - - private void findMethodOldWay(Term iri) throws IOException { - List libraries = Utils.getObjectsFromQuads(this.functionDescriptionTriples.getQuads(iri, new NamedNode(NAMESPACES.LIB + "providedBy"), null)); - - if (libraries.size() > 0) { - List pathNames = Utils.getObjectsFromQuads(this.functionDescriptionTriples.getQuads(libraries.get(0), new NamedNode(NAMESPACES.LIB + "localLibrary"), null)); - List classes = Utils.getObjectsFromQuads(this.functionDescriptionTriples.getQuads(libraries.get(0), new NamedNode(NAMESPACES.LIB + "class"), null)); - - if (pathNames.size() > 0 && classes.size() > 0) { - String pathName = pathNames.get(0).getValue(); - String className = classes.get(0).getValue(); - Class cls; - if (this.classMap.containsKey(className)) { - cls = this.classMap.get(className); - } else { - File functionFile = Utils.getFile(pathName); - cls = FunctionUtils.functionRequire(functionFile, className); - this.classMap.put(className, cls); - this.libraryMap.put(className, functionFile.getCanonicalPath()); - } - - List parameters = new ArrayList<>(); - List expectList = Utils.getObjectsFromQuads(FunctionUtils.getQuadsByFunctionPrefix(this.functionDescriptionTriples, iri, "expects", null)); - - if (expectList.size() > 0) { - parameters = Utils.getList(this.functionDescriptionTriples, expectList.get(0)); - } - - List methods = Utils.getObjectsFromQuads(this.functionDescriptionTriples.getQuads(libraries.get(0), new NamedNode(NAMESPACES.LIB + "method"), null)); - - List outputs = Utils.getList(this.functionDescriptionTriples, Utils.getObjectsFromQuads(FunctionUtils.getQuadsByFunctionPrefix(this.functionDescriptionTriples, iri, "returns", null)).get(0)); - // Validation of parameters within FunctionUtils.getFunctionParameterUris - List fnParameterUris = FunctionUtils.getFunctionParameterUris(this.functionDescriptionTriples, parameters); - List fnOutputUris = FunctionUtils.getFunctionParameterUris(this.functionDescriptionTriples, outputs); - - Class[] orderedParameters = FunctionUtils.parseFunctionParameters(this.functionDescriptionTriples, parameters); - if (methods.size() > 0) { - Method fn = null; - try { - fn = cls.getMethod(methods.get(0).getValue(), orderedParameters); - } catch (NoSuchMethodException e) { - throw new IOException("Declared method " + methods.get(0) + " does not exist for class " + classes.get(0) + "."); - } - - FunctionModel fnm = new FunctionModel(iri, fn, fnParameterUris, fnOutputUris); - - this.loadedMethods.put(iri, fnm); - } - } else { - throw new IOException("No library or class was found for the function with IRI " + iri + " in the function descriptions."); - } - } else { - throw new IOException("No library or class was found for the function with IRI " + iri + " in the function descriptions."); - } - } - - private void findMethodNewWay(Term iri) throws IOException { - List mappings = Utils.getSubjectsFromQuads(FunctionUtils.getQuadsByFunctionPrefix(this.functionDescriptionTriples, null, "function", iri)); - - if (mappings.size() == 0) { - throw new IOException("No mapping was found for the function with IRI " + iri + " in the function descriptions."); - } - - List libraries = Utils.getObjectsFromQuads(FunctionUtils.getQuadsByFunctionPrefix(this.functionDescriptionTriples, mappings.get(0), "implementation", null)); - - if (libraries.size() == 0) { - throw new IOException("No library was found for the mapping with IRI " + mappings.get(0) + " in the function descriptions."); - } - List pathNames = Utils.getObjectsFromQuads(this.functionDescriptionTriples.getQuads(libraries.get(0), new NamedNode(NAMESPACES.DOAP + "download-page"), null)); - List classes = Utils.getObjectsFromQuads(this.functionDescriptionTriples.getQuads(libraries.get(0), new NamedNode(NAMESPACES.FNOI + "class-name"), null)); - - if (pathNames.size() == 0 || classes.size() == 0) { - throw new IOException("No path or class found for the library with IRI " + libraries.get(0) + " in the function descriptions."); - } - - String pathName = pathNames.get(0).getValue(); - String className = classes.get(0).getValue(); - Class cls; - - if (this.classMap.containsKey(className)) { - cls = this.classMap.get(className); - } else { - File functionFile = Utils.getFile(pathName); - cls = FunctionUtils.functionRequire(functionFile, className); - this.classMap.put(className, cls); - this.libraryMap.put(className, functionFile.getCanonicalPath()); - } - - List parameters = new ArrayList<>(); - List expectList = Utils.getObjectsFromQuads(FunctionUtils.getQuadsByFunctionPrefix(this.functionDescriptionTriples, iri, "expects", null)); - - if (expectList.size() > 0) { - parameters = Utils.getList(this.functionDescriptionTriples, expectList.get(0)); - } - - - List returns = Utils.getObjectsFromQuads(FunctionUtils.getQuadsByFunctionPrefix(this.functionDescriptionTriples, iri, "returns", null)); - if (returns.isEmpty()) { - throw new IOException("Missing " + NAMESPACES.FNO_S + "returns for " + iri + " in the function descriptions."); - } - List outputs = Utils.getList(this.functionDescriptionTriples, returns.get(0)); - - List methodMappings = Utils.getObjectsFromQuads(FunctionUtils.getQuadsByFunctionPrefix(this.functionDescriptionTriples, mappings.get(0), "methodMapping", null)); - if (methodMappings.size() == 0) { - throw new IOException("No methodmapping found for the mapping with IRI " + mappings.get(0) + " in the function descriptions."); - } - - List methods = Utils.getObjectsFromQuads(this.functionDescriptionTriples.getQuads(methodMappings.get(0), new NamedNode(NAMESPACES.FNOM + "method-name"), null)); - if (methods.size() == 0) { - throw new IOException("No method found for the mapping with IRI " + mappings.get(0) + " in the function descriptions."); - } - - Class[] orderedParameters = FunctionUtils.parseFunctionParameters(this.functionDescriptionTriples, parameters); - Method fn = null; - try { - fn = cls.getDeclaredMethod(methods.get(0).getValue(), orderedParameters); - } catch (NoSuchMethodException e) { - throw new IOException("Declared method " + methods.get(0) + " does not exist for class " + classes.get(0) + "."); - } - - List fnParameterUris = FunctionUtils.getFunctionParameterUris(this.functionDescriptionTriples, parameters); - List fnOutputUris = FunctionUtils.getFunctionParameterUris(this.functionDescriptionTriples, outputs); - - FunctionModel fnm = new FunctionModel(iri, fn, fnParameterUris, fnOutputUris); - - this.loadedMethods.put(iri, fnm); - } -} diff --git a/src/main/java/be/ugent/rml/functions/FunctionModel.java b/src/main/java/be/ugent/rml/functions/FunctionModel.java deleted file mode 100644 index 2acb503e..00000000 --- a/src/main/java/be/ugent/rml/functions/FunctionModel.java +++ /dev/null @@ -1,137 +0,0 @@ -package be.ugent.rml.functions; - -import be.ugent.rml.term.Term; -import net.minidev.json.parser.JSONParser; -import net.minidev.json.parser.ParseException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.lang.reflect.Type; -import java.time.*; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Function Model - * - * @author bjdmeest - */ -public class FunctionModel { - - private final List parameters; // parameters urls - private final List outputs; // output types - private Term URI; - private Method method; - - protected Logger logger = LoggerFactory.getLogger(this.getClass()); - - public FunctionModel(Term URI, Method m, List parameters, List outputs) { - this.URI = URI; - this.method = m; - this.parameters = parameters; - this.outputs = outputs; - } - - public Object execute(Map args) { - Object[] parameters = this.getParameters(args); - try { - return this.method.invoke(null, parameters); -// ArrayList result = this.toValue(object, this.getDataType(args)); - } catch (IllegalAccessException | InvocationTargetException e) { - // Nothing to do? - e.printStackTrace(); // maybe this? :p - } - - return null; - } - - public Term getURI() { - return URI; - } - - private Object[] getParameters(Map parameters) { - Object[] args = new Object[this.parameters.size()]; - Type[] paramTypes = this.method.getGenericParameterTypes(); - - for (int i = 0; i < this.parameters.size(); i++) { - if (parameters.get(this.parameters.get(i).getValue()) != null) { - args[i] = parseParameter(parameters.get(this.parameters.get(i).getValue()), paramTypes[i].getTypeName()); - } else { - logger.debug("No argument was found for following parameter: " + this.parameters.get(i).getValue()); - args[i] = null; - } - } - - return args; - } - - private Object parseParameter(Object parameter, String typeName) { - String javaList = "java.util.List"; - if (typeName.contains(javaList)) { - if (parameter instanceof String) { - JSONParser parser = new JSONParser(JSONParser.MODE_PERMISSIVE); - try { - //this should return a JSONArray, which implements java.util.List - return parser.parse((String) parameter); - } catch (ParseException e) { - e.printStackTrace(); - throw new Error("Could not get a List from " + parameter); - } - } else if(parameter instanceof List && typeName.contains("<") && typeName.contains(">")) { - // Must have contents to be able to recursively parse - String listElementType = typeName.substring(javaList.length() + 1, typeName.length() - 1); - return ((List) parameter).stream() - .map(o -> parseParameter(o, listElementType)) // recursively convert List elements - .collect(Collectors.toList()); - } else { - return parameter; - } - } - if (parameter instanceof List) { - List l = (List) parameter; - - if (l.isEmpty()) { - return null; - } else { - parameter = l.get(0); - } - } - switch (typeName) { - case "java.lang.Object": - case "java.lang.String": - return parameter.toString(); - case "int": - case "java.lang.Integer": - return Integer.parseInt(parameter.toString()); - case "double": - case "java.lang.Double": - return Double.parseDouble(parameter.toString()); - case "long": - case "java.lang.Long": - return Long.parseLong(parameter.toString()); - case "java.lang.Boolean": - return Boolean.parseBoolean(parameter.toString()); - case "java.time.LocalDate": - return LocalDate.parse(parameter.toString()); - case "java.time.LocalDateTime": - return LocalDateTime.parse(parameter.toString()); - case "java.time.ZonedDateTime": - return ZonedDateTime.parse(parameter.toString()); - case "java.time.Duration": - return Duration.parse(parameter.toString()); - case "java.time.Month": - return Month.valueOf(parameter.toString()); - case "java.time.MonthDay": - return MonthDay.parse(parameter.toString()); - case "java.time.Year": - return Year.parse(parameter.toString()); - case "java.time.YearMonth": - return YearMonth.parse(parameter.toString()); - default: - throw new Error("Couldn't derive " + typeName + " from " + parameter); - } - } -} diff --git a/src/main/java/be/ugent/rml/functions/FunctionUtils.java b/src/main/java/be/ugent/rml/functions/FunctionUtils.java index 2a1c0ef4..21b6d72d 100644 --- a/src/main/java/be/ugent/rml/functions/FunctionUtils.java +++ b/src/main/java/be/ugent/rml/functions/FunctionUtils.java @@ -1,82 +1,10 @@ package be.ugent.rml.functions; -import be.ugent.rml.NAMESPACES; -import be.ugent.rml.Utils; -import be.ugent.rml.store.Quad; -import be.ugent.rml.store.QuadStore; -import be.ugent.rml.term.NamedNode; -import be.ugent.rml.term.Term; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.tools.JavaCompiler; -import javax.tools.ToolProvider; -import java.io.File; -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLClassLoader; -import java.time.*; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; public class FunctionUtils { - private static final Logger logger = LoggerFactory.getLogger(FunctionUtils.class); - - public static Class functionRequire(File file, String className) throws IOException { - String path = file.getPath(); - if (path.endsWith(".jar")) { - return FunctionUtils.getClass(file, className, "application/java-archive"); - } else if (path.endsWith(".java")) { - return FunctionUtils.getClass(file, className, "text/x-java-source"); - } - - throw new IOException("Not a valid path for a JAVA implementation: " + path); - } - - /** - * Returns and validates parameters - * @param store - * @param parameterResources - * @return - */ - public static List getFunctionParameterUris(QuadStore store, List parameterResources) { - List parameterPredicates = new ArrayList<>(); - - try { - for (Term subject : parameterResources) { - parameterPredicates.add(Utils.getObjectsFromQuads(getQuadsByFunctionPrefix(store, subject, "predicate", null)).get(0)); - } - } catch (Exception e) { - logger.error("Missing function parameters in {}", parameterResources); - } - - return parameterPredicates; - } - - public static Class[] parseFunctionParameters(QuadStore store, List parameterResources) - throws IOException { - Class[] args = new Class[parameterResources.size()]; - - for (int i = 0; i < parameterResources.size(); i++) { - Term subject = parameterResources.get(i); - List types = Utils.getObjectsFromQuads(getQuadsByFunctionPrefix(store, subject, "type", null)); - if (types.isEmpty()) { - throw new IOException("Missing " + NAMESPACES.FNO_S + "type for " + subject + " in function descriptions."); - } - Term type = types.get(0); - - try { - args[i] = FunctionUtils.getParamType(type); - } catch (Exception e) { - args[i] = String.class; - } - } - return args; - } - /** * Generates strings from a function object. Possible lists/sets/bags/... in the object are unrolled recursively * and a string value is generated from each "simple" (i.e., not a list/set/bag/...) child object. @@ -84,12 +12,13 @@ public static Class[] parseFunctionParameters(QuadStore store, List par * @param o Function object, can be iterable. * @param result A string list to which string values of objects are added */ - public static void functionObjectToList(Object o, List result) { + public static List functionObjectToList(Object o) { + final List result = new ArrayList<>(); if (o != null) { // if o has child objects, recursively call this function on each child if (o instanceof Iterable) { ((Iterable) o).forEach(item -> { - functionObjectToList(item, result); + result.addAll(functionObjectToList(item)); }); } // if o has no children, call toString() to serialize it into a string @@ -100,157 +29,6 @@ public static void functionObjectToList(Object o, List result) { result.add(o.toString()); } } - } - - private static Class getParamType(Term type) { - String typeStr = type.getValue(); - - switch (typeStr) { - // This is quite crude, based on https://www.w3.org/TR/xmlschema11-2/#built-in-datatypes - case "http://www.w3.org/2001/XMLSchema#any": - return Object.class; - case "http://www.w3.org/2001/XMLSchema#string": - return String.class; - case "http://www.w3.org/2001/XMLSchema#unsignedLong": - case "http://www.w3.org/2001/XMLSchema#long": - return Long.class; - case "http://www.w3.org/2001/XMLSchema#integer": - case "http://www.w3.org/2001/XMLSchema#int": - case "http://www.w3.org/2001/XMLSchema#short": - case "http://www.w3.org/2001/XMLSchema#byte": - case "http://www.w3.org/2001/XMLSchema#nonNegativeInteger": - case "http://www.w3.org/2001/XMLSchema#positiveInteger": - case "http://www.w3.org/2001/XMLSchema#unsignedInt": - case "http://www.w3.org/2001/XMLSchema#unsignedShort": - case "http://www.w3.org/2001/XMLSchema#unsignedByte": - case "http://www.w3.org/2001/XMLSchema#nonPositiveInteger": - case "http://www.w3.org/2001/XMLSchema#negativeInteger": - return Integer.class; - case "http://www.w3.org/2001/XMLSchema#boolean": - return Boolean.class; - case "http://www.w3.org/2001/XMLSchema#date": - // "Local" just means "without a time zone" - return LocalDate.class; - case "http://www.w3.org/2001/XMLSchema#dateTime": - // again "Local" means "without a time zone" - // (An xsd:dateTime actually has an OPTIONAL time zone, so there is a small semantic difference - // with java.time.LocalDateTime, this is a best effort.) - return LocalDateTime.class; - case "http://www.w3.org/2001/XMLSchema#dateTimeStamp": - return ZonedDateTime.class; - case "http://www.w3.org/2001/XMLSchema#dayTimeDuration": - case "http://www.w3.org/2001/XMLSchema#yearMonthDuration": - return Duration.class; - case "http://www.w3.org/2001/XMLSchema#gDay": - // TODO there is no java.time equivalent of xsd:day - // (There is java.time.DayOfWeek, but xsd:day would corresponds to java.time.DayOfMonth .) - throw new DateTimeException("There is no java.time equivalent of xsd:day. Crashing."); - case "http://www.w3.org/2001/XMLSchema#gMonth": - return Month.class; - case "http://www.w3.org/2001/XMLSchema#gMonthDay": - return MonthDay.class; - case "http://www.w3.org/2001/XMLSchema#gYear": - return Year.class; - case "http://www.w3.org/2001/XMLSchema#gYearMonth": - return YearMonth.class; - case "http://www.w3.org/2001/XMLSchema#decimal": - case "http://www.w3.org/2001/XMLSchema#double": - case "http://www.w3.org/2001/XMLSchema#float": - return Double.class; - case "http://www.w3.org/1999/02/22-rdf-syntax-ns#List": - return List.class; - default: - throw new Error("Couldn't derive type from " + type); - } - } - - private static Class getClass(File sourceFile, String className, String mime) throws IOException { - logger.info("Found class on path " + sourceFile.getCanonicalPath()); - - switch (mime) { - case "text/x-java-source": - return FunctionUtils.getClassFromJAVA(sourceFile, className); - case "application/java-archive": - return FunctionUtils.getClassFromJAR(sourceFile, className); - } - - return null; - } - - private static Class getClassFromJAVA(File sourceFile, String className) { - Class cls = null; - - // TODO let's not recompile every time - // Compile source file. - JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); - int res = compiler.run(null, null, null, sourceFile.getPath()); - - if (res != 0) { - return null; - } - - // Load and instantiate compiled class. - URLClassLoader classLoader = null; - try { - classLoader = URLClassLoader.newInstance(new URL[]{(new File(sourceFile.getParent())).toURI().toURL()}); - cls = Class.forName(className, true, classLoader); - } catch (MalformedURLException | ClassNotFoundException e) { - e.printStackTrace(); - } - - return cls; - } - - private static Class getClassFromJAR(File sourceFile, String className) { - Class cls = null; - - URLClassLoader child = null; - try { - child = URLClassLoader.newInstance(new URL[]{sourceFile.toURI().toURL()}); - cls = Class.forName(className, true, child); - } catch (MalformedURLException | ClassNotFoundException e) { - e.printStackTrace(); - } - - return cls; - } - - /** - * Retrieve triples of a store based on a predicate, taking into account deprecated FnO prefixed predicates - * @param store The triple store to retrieve the triples from - * @param s the subject - * @param functionTerm the unprefixed function term - * @param o the object - * @return the quads that are conform to the triple pattern fragment - */ - static List getQuadsByFunctionPrefix(QuadStore store, Term s, String functionTerm, Term o) { - List prefices = Arrays.asList(NAMESPACES.FNO_S, NAMESPACES.FNO, NAMESPACES.FNO_OLD); - return getQuadsByPrefix(store, s, functionTerm, o, prefices); - } - - /** - * Retrieve triples of a store based on a predicate, taking into account multiple prefixes - * @param store The triple store to retrieve the triples from - * @param s the subject - * @param pString the unprefixed predicate term - * @param o the object - * @param prefices the list of prefices on which to look for, in order of 'correctness' (all prefixes except for the first one are assumed deprecated) - * @return the quads that are conform to the triple pattern fragment - */ - private static List getQuadsByPrefix(QuadStore store, Term s, String pString, Term o, List prefices) { - String preferredPrefix = prefices.get(0); - Term realTerm; - List quads = new ArrayList<>(); - for (int i = 0; i < prefices.size(); i++) { - realTerm = new NamedNode(prefices.get(i) + pString); - quads = store.getQuads(s, realTerm, o); - if (quads.size() > 0) { - if (i != 0) { - logger.warn(prefices.get(i) + "is a deprecated prefix, please use " + preferredPrefix); - } - return quads; - } - } - return quads; + return result; } } diff --git a/src/main/java/be/ugent/rml/functions/StaticMultipleRecordsFunctionExecutor.java b/src/main/java/be/ugent/rml/functions/StaticMultipleRecordsFunctionExecutor.java index 099b1b5a..3ed1916e 100644 --- a/src/main/java/be/ugent/rml/functions/StaticMultipleRecordsFunctionExecutor.java +++ b/src/main/java/be/ugent/rml/functions/StaticMultipleRecordsFunctionExecutor.java @@ -4,19 +4,16 @@ import be.ugent.idlab.knows.functions.agent.Arguments; import be.ugent.rml.records.Record; -import java.util.HashMap; import java.util.Map; public class StaticMultipleRecordsFunctionExecutor implements MultipleRecordsFunctionExecutor { - private final FunctionModel functionModel; // TODO: remove private final Map parameters; private final Agent functionAgent; private final String functionId; - public StaticMultipleRecordsFunctionExecutor(FunctionModel model, Map parameters, Agent functionAgent, String functionId) { - this.functionModel = model; + public StaticMultipleRecordsFunctionExecutor(final Map parameters, Agent functionAgent, String functionId) { this.parameters = parameters; this.functionAgent = functionAgent; this.functionId = functionId; @@ -24,7 +21,6 @@ public StaticMultipleRecordsFunctionExecutor(FunctionModel model, Map records) throws Exception { - Map filledInParameters = new HashMap<>(); final Arguments functionArguments = new Arguments(); for (Map.Entry entry : this.parameters.entrySet()) { @@ -34,13 +30,9 @@ public Object execute(Map records) throws Exception { Object o = executor.execute(records.get(recordType)); // TODO check whether key is actually optional! - filledInParameters.put(entry.getKey(), o); functionArguments.add(entry.getKey(), o); } - Object result1 = functionModel.execute(filledInParameters); - Object result2 = functionAgent.execute(functionId, functionArguments); - assert(result1.equals(result2)); - return result1; + return functionAgent.execute(functionId, functionArguments); } } diff --git a/src/main/java/be/ugent/rml/functions/StaticSingleRecordFunctionExecutor.java b/src/main/java/be/ugent/rml/functions/StaticSingleRecordFunctionExecutor.java deleted file mode 100644 index 94fed0ca..00000000 --- a/src/main/java/be/ugent/rml/functions/StaticSingleRecordFunctionExecutor.java +++ /dev/null @@ -1,26 +0,0 @@ -package be.ugent.rml.functions; - -import be.ugent.idlab.knows.functions.agent.Agent; -import be.ugent.rml.Template; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class StaticSingleRecordFunctionExecutor extends AbstractSingleRecordFunctionExecutor { - - public StaticSingleRecordFunctionExecutor(FunctionModel model, final Agent functionAgent, final String functionId, Map> parameters) { - HashMap parametersForOtherExecutor = new HashMap<>(); - - parameters.keySet().forEach(parameter -> { - List