From 3eea75bcfaeffa85d9e0c91e05f241e3760448db Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 18 Nov 2019 12:47:07 -0600 Subject: [PATCH 001/309] pio module for constructing and executing pdal pipelines in a pythonic way and returning results --- pdal/pio.py | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 pdal/pio.py diff --git a/pdal/pio.py b/pdal/pio.py new file mode 100644 index 00000000..fd1334f7 --- /dev/null +++ b/pdal/pio.py @@ -0,0 +1,111 @@ +import json +from functools import partial +from collections import defaultdict +from itertools import chain +import copy +import warnings + +import pdal + +DEFAULT_STAGE_PARAMS = defaultdict(dict) +DEFAULT_STAGE_PARAMS.update({ +# TODO: add stage specific default configurations +}) + +class StageSpec(object): + def __init__(self, prefix, **kwargs): + self.prefix = prefix + key = ".".join([self.prefix, kwargs.get("type", "")]) + self.spec = DEFAULT_STAGE_PARAMS[key].copy() + self.spec.update(kwargs) + self.spec["type"] = key + # NOTE: special case to support reading files without passing an explicit reader + if (self.prefix == "readers") and kwargs.get("type") == "auto": + del self.spec["type"] + + @property + def pipeline(self): + output = PipelineSpec() + output.add_stage(self) + return output + + def __getattr__(self, name): + return partial(self.__class__, self.prefix, type=name) + + def __str__(self): + return json.dumps(self.spec, indent=4) + + def __add__(self, other): + return self.pipeline + other + + def execute(self): + return self.pipeline.execute() + + +readers = StageSpec("readers") +filters = StageSpec("filters") +writers = StageSpec("writers") + + +class PipelineSpec(object): + readers = [] + filters = [] + writer = None + + def __init__(self, other=None): + if other is not None: + self.readers = copy.copy(other.readers) + self.filters = copy.copy(other.filters) + self.writer = copy.copy(other.writer) + + @property + def spec(self): + return { + "pipeline": [stage.spec for stage in self.stages] + } + + @property + def stages(self): + if self.writer is not None: + return chain(self.readers, self.filters, [self.writer]) + else: + return chain(self.readers, self.filters) + + def add_stage(self, stage): + assert isinstance(stage, StageSpec), "Expected StageSpec" + if stage.prefix == "writers": + if self.writer is not None: + warnings.warn("Currently assigned output stage will be replaced.") + self.writer = stage + elif stage.prefix == "readers": + self.readers.append(stage) + elif stage.prefix == "filters": + self.filters.append(stage) + else: + warnings.warn("Unknown stage type. Skipping.") + + return self + + def __str__(self): + return json.dumps(self.spec, indent=4) + + def __add__(self, stage_or_pipeline): + assert isinstance(stage_or_pipeline, (StageSpec, PipelineSpec)), "Expected StageSpec or PipelineSpec" + + output = self.__class__(self) + if isinstance(stage_or_pipeline, StageSpec): + output.add_stage(stage_or_pipeline) + elif isinstance(stage_or_pipeline, PipelineSpec): + for stage in stage_or_pipeline.stages: + output.add_stage(stage) + return output + + def execute(self): + # TODO: do some validation before calling execute + + # TODO: some exception/error handling around pdal + pipeline = pdal.Pipeline(json.dumps(self.spec)) + # pipeline.validate() # NOTE: disabling this because it causes segfaults in certain cases + pipeline.execute() + + return pipeline.arrays[0] # NOTE: are there situation where arrays has multiple elements? From 7b9d056ff7f1f537b3431471a82610edbeae8de7 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 18 Nov 2019 13:00:05 -0600 Subject: [PATCH 002/309] one basic testcase --- test/test_pio.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 test/test_pio.py diff --git a/test/test_pio.py b/test/test_pio.py new file mode 100644 index 00000000..a06b945e --- /dev/null +++ b/test/test_pio.py @@ -0,0 +1,47 @@ +import unittest +import json + +from pdal import pio + +dummy_pipeline = """{ + "pipeline": [ + { + "type": "readers.ply", + "filename": "dummyinput.ply" + }, + { + "type": "filters.outlier", + "method": "statistical", + "mean_k": 16, + "multiplier": 1.0 + }, + { + "type": "filters.range", + "limits": "Classification![7:7]" + }, + { + "type": "filters.normal" + }, + { + "type": "writers.ply", + "storage_mode": "ascii", + "precision": 4, + "filename": "dummyoutput.ply", + "dims": "X,Y,Z,Red,Green,Blue,NormalX,NormalY,NormalZ" + } + ] +}""" + + + +class TestPIOBasics(unittest.TestCase): + def test_pipeline_construction(self): + pipeline = (pio.readers.ply(filename="dummyfile.ply") + + pio.filters.outlier(method="statistical", mean_k=16, multiplier=1.0) + + pio.filters.range(limits="Classification![7:7]") + + pio.filters.normal() + pio.writers.ply(storage_mode="ascii", precision=4, filename=output, + dims="X,Y,Z,Red,Green,Blue,NormalX,NormalY,NormalZ")) + + self.assertTrue(isinstance(pipeline, pio.PipelineSpec)) + self.assertEqual(len(list(pipeline.stages)), 5) + self.assertEqual(json.dumps(pipeline.spec, indent=2, dummy_pipeline) From cc49f84c934090cf799eddd98bdcb612a384b085 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 18 Nov 2019 13:54:19 -0600 Subject: [PATCH 003/309] writer doesn't need to be copied since it isn't a list --- pdal/pio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdal/pio.py b/pdal/pio.py index fd1334f7..b7d6ef3b 100644 --- a/pdal/pio.py +++ b/pdal/pio.py @@ -56,7 +56,7 @@ def __init__(self, other=None): if other is not None: self.readers = copy.copy(other.readers) self.filters = copy.copy(other.filters) - self.writer = copy.copy(other.writer) + self.writer = other.writer @property def spec(self): From ae11bb128dfd473a2b7f8094d06c0a20a3bb0105 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 18 Nov 2019 13:54:46 -0600 Subject: [PATCH 004/309] update and expand testcase --- test/test_pio.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/test/test_pio.py b/test/test_pio.py index a06b945e..8051e94e 100644 --- a/test/test_pio.py +++ b/test/test_pio.py @@ -36,12 +36,21 @@ class TestPIOBasics(unittest.TestCase): def test_pipeline_construction(self): - pipeline = (pio.readers.ply(filename="dummyfile.ply") + + pipeline = (pio.readers.ply(filename="dummyinput.ply") + pio.filters.outlier(method="statistical", mean_k=16, multiplier=1.0) + pio.filters.range(limits="Classification![7:7]") + - pio.filters.normal() + pio.writers.ply(storage_mode="ascii", precision=4, filename=output, + pio.filters.normal() + pio.writers.ply(storage_mode="ascii", precision=4, filename="dummyoutput.ply", dims="X,Y,Z,Red,Green,Blue,NormalX,NormalY,NormalZ")) - self.assertTrue(isinstance(pipeline, pio.PipelineSpec)) + self.assertIsInstance(pipeline, pio.PipelineSpec) self.assertEqual(len(list(pipeline.stages)), 5) - self.assertEqual(json.dumps(pipeline.spec, indent=2, dummy_pipeline) + self.assertEqual(json.dumps(pipeline.spec, indent=2), dummy_pipeline) + + pipeline = pipeline + pio.readers.auto(filename="anotherdummmyfile.laz") + + self.assertEqual(len(pipeline.readers), 2) # we can have multiple readers + + null_writer = pio.writers.null() + self.assertIsNot(pipeline.writer, null_writer) + pipeline = pipeline + null_writer + self.assertIs(pipeline.writer, null_writer) From f161adc2a7622b62dc42c21ff707e8de3567cb09 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 18 Nov 2019 16:43:34 -0600 Subject: [PATCH 005/309] validation of available pdal drivers using introspected data --- pdal/pio.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/pdal/pio.py b/pdal/pio.py index b7d6ef3b..3db6c1eb 100644 --- a/pdal/pio.py +++ b/pdal/pio.py @@ -1,4 +1,6 @@ +import types import json +import subprocess from functools import partial from collections import defaultdict from itertools import chain @@ -7,18 +9,24 @@ import pdal +PDAL_DRIVERS_JSON = subprocess.run(["pdal", "--drivers", "--showjson"], capture_output=True).stdout +PDAL_DRIVERS = json.loads(PDAL_DRIVERS_JSON) +modules = set([e["name"].split(".")[0] for e in PDAL_DRIVERS]) + + DEFAULT_STAGE_PARAMS = defaultdict(dict) DEFAULT_STAGE_PARAMS.update({ # TODO: add stage specific default configurations }) + class StageSpec(object): def __init__(self, prefix, **kwargs): self.prefix = prefix - key = ".".join([self.prefix, kwargs.get("type", "")]) - self.spec = DEFAULT_STAGE_PARAMS[key].copy() + self.key = ".".join([self.prefix, kwargs.get("type", "")]) + self.spec = DEFAULT_STAGE_PARAMS[self.key].copy() self.spec.update(kwargs) - self.spec["type"] = key + self.spec["type"] = self.key # NOTE: special case to support reading files without passing an explicit reader if (self.prefix == "readers") and kwargs.get("type") == "auto": del self.spec["type"] @@ -30,6 +38,7 @@ def pipeline(self): return output def __getattr__(self, name): + assert name in dir(self), "Invalid or unsupported stage" return partial(self.__class__, self.prefix, type=name) def __str__(self): @@ -38,6 +47,10 @@ def __str__(self): def __add__(self, other): return self.pipeline + other + def __dir__(self): + extra_keys = [e["name"][len(self.key):] for e in PDAL_DRIVERS if e["name"].startswith(self.key)] + return super().__dir__() + [e for e in extra_keys if len(e) > 0] + def execute(self): return self.pipeline.execute() From 5ee8f728edffae3da5ef0b768b0a5a01057116ed Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 18 Nov 2019 16:47:17 -0600 Subject: [PATCH 006/309] raise in AttributeError instead of an AssertionError when accessing an invalid stage --- pdal/pio.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pdal/pio.py b/pdal/pio.py index 3db6c1eb..06aadaa8 100644 --- a/pdal/pio.py +++ b/pdal/pio.py @@ -38,7 +38,8 @@ def pipeline(self): return output def __getattr__(self, name): - assert name in dir(self), "Invalid or unsupported stage" + if name not in dir(self): + raise AttributeError(f"'{self.prefix}.{name}' is an invalid or unsupported PDAL stage") return partial(self.__class__, self.prefix, type=name) def __str__(self): From 10e30accd8b8e972ded7a88274714bc7755a18e1 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 18 Nov 2019 16:50:11 -0600 Subject: [PATCH 007/309] remove unused var --- pdal/pio.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pdal/pio.py b/pdal/pio.py index 06aadaa8..7d3b07f1 100644 --- a/pdal/pio.py +++ b/pdal/pio.py @@ -11,8 +11,6 @@ PDAL_DRIVERS_JSON = subprocess.run(["pdal", "--drivers", "--showjson"], capture_output=True).stdout PDAL_DRIVERS = json.loads(PDAL_DRIVERS_JSON) -modules = set([e["name"].split(".")[0] for e in PDAL_DRIVERS]) - DEFAULT_STAGE_PARAMS = defaultdict(dict) DEFAULT_STAGE_PARAMS.update({ From 811067ec933b8890ba4e81723f6dd085d3ecaaeb Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 18 Nov 2019 17:00:33 -0600 Subject: [PATCH 008/309] make validation conditional in case pdal isn't on the path --- pdal/pio.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pdal/pio.py b/pdal/pio.py index 7d3b07f1..8150fa07 100644 --- a/pdal/pio.py +++ b/pdal/pio.py @@ -9,8 +9,13 @@ import pdal -PDAL_DRIVERS_JSON = subprocess.run(["pdal", "--drivers", "--showjson"], capture_output=True).stdout -PDAL_DRIVERS = json.loads(PDAL_DRIVERS_JSON) +try: + PDAL_DRIVERS_JSON = subprocess.run(["pdal", "--drivers", "--showjson"], capture_output=True).stdout + PDAL_DRIVERS = json.loads(PDAL_DRIVERS_JSON) + _PDAL_VALIDATE = True +except: + PDAL_DRIVERS = [] + _PDAL_VALIDATE = False DEFAULT_STAGE_PARAMS = defaultdict(dict) DEFAULT_STAGE_PARAMS.update({ @@ -36,7 +41,7 @@ def pipeline(self): return output def __getattr__(self, name): - if name not in dir(self): + if _PDAL_VALIDATE and (name not in dir(self)): raise AttributeError(f"'{self.prefix}.{name}' is an invalid or unsupported PDAL stage") return partial(self.__class__, self.prefix, type=name) From fd8dd0b235c2cbb76b5050350faa382f7e511e80 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Tue, 19 Nov 2019 11:27:01 -0600 Subject: [PATCH 009/309] need to include auto in extra_keys --- pdal/pio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdal/pio.py b/pdal/pio.py index 8150fa07..26a4001d 100644 --- a/pdal/pio.py +++ b/pdal/pio.py @@ -52,7 +52,7 @@ def __add__(self, other): return self.pipeline + other def __dir__(self): - extra_keys = [e["name"][len(self.key):] for e in PDAL_DRIVERS if e["name"].startswith(self.key)] + extra_keys = [e["name"][len(self.key):] for e in PDAL_DRIVERS if e["name"].startswith(self.key)] + ["auto"] return super().__dir__() + [e for e in extra_keys if len(e) > 0] def execute(self): From 8a2f74f0a7662dc32687dec2900965c45adab1d4 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 25 Nov 2019 15:50:41 -0600 Subject: [PATCH 010/309] removing tracking of stage types --- pdal/pio.py | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/pdal/pio.py b/pdal/pio.py index 26a4001d..bf8cf60c 100644 --- a/pdal/pio.py +++ b/pdal/pio.py @@ -65,15 +65,11 @@ def execute(self): class PipelineSpec(object): - readers = [] - filters = [] - writer = None + stages = [] def __init__(self, other=None): if other is not None: - self.readers = copy.copy(other.readers) - self.filters = copy.copy(other.filters) - self.writer = other.writer + self.stages = copy.copy(other.stages) @property def spec(self): @@ -81,26 +77,10 @@ def spec(self): "pipeline": [stage.spec for stage in self.stages] } - @property - def stages(self): - if self.writer is not None: - return chain(self.readers, self.filters, [self.writer]) - else: - return chain(self.readers, self.filters) - def add_stage(self, stage): assert isinstance(stage, StageSpec), "Expected StageSpec" - if stage.prefix == "writers": - if self.writer is not None: - warnings.warn("Currently assigned output stage will be replaced.") - self.writer = stage - elif stage.prefix == "readers": - self.readers.append(stage) - elif stage.prefix == "filters": - self.filters.append(stage) - else: - warnings.warn("Unknown stage type. Skipping.") + self.stages.append(stage) return self def __str__(self): From bab8f1df61302bdf5c38362e92300452c8b6d074 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 25 Nov 2019 16:00:11 -0600 Subject: [PATCH 011/309] update test to not check for the readers/writer/filters behavior --- test/test_pio.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/test/test_pio.py b/test/test_pio.py index 8051e94e..de0b2471 100644 --- a/test/test_pio.py +++ b/test/test_pio.py @@ -45,12 +45,3 @@ def test_pipeline_construction(self): self.assertIsInstance(pipeline, pio.PipelineSpec) self.assertEqual(len(list(pipeline.stages)), 5) self.assertEqual(json.dumps(pipeline.spec, indent=2), dummy_pipeline) - - pipeline = pipeline + pio.readers.auto(filename="anotherdummmyfile.laz") - - self.assertEqual(len(pipeline.readers), 2) # we can have multiple readers - - null_writer = pio.writers.null() - self.assertIsNot(pipeline.writer, null_writer) - pipeline = pipeline + null_writer - self.assertIs(pipeline.writer, null_writer) From 5cf38859e73be500426978d9b8f4cff06a2e6803 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Tue, 3 Dec 2019 10:11:11 -0500 Subject: [PATCH 012/309] adding writers.auto case --- pdal/pio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdal/pio.py b/pdal/pio.py index bf8cf60c..013003fe 100644 --- a/pdal/pio.py +++ b/pdal/pio.py @@ -31,7 +31,7 @@ def __init__(self, prefix, **kwargs): self.spec.update(kwargs) self.spec["type"] = self.key # NOTE: special case to support reading files without passing an explicit reader - if (self.prefix == "readers") and kwargs.get("type") == "auto": + if (self.prefix in ["readers", "writers"]) and kwargs.get("type") == "auto": del self.spec["type"] @property From 6c82060adeeb1d7b83c473c5022710551de455b4 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Tue, 3 Dec 2019 10:11:19 -0500 Subject: [PATCH 013/309] adding tests for auto readers and writers --- test/test_pio.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/test_pio.py b/test/test_pio.py index de0b2471..bcde6bcb 100644 --- a/test/test_pio.py +++ b/test/test_pio.py @@ -45,3 +45,13 @@ def test_pipeline_construction(self): self.assertIsInstance(pipeline, pio.PipelineSpec) self.assertEqual(len(list(pipeline.stages)), 5) self.assertEqual(json.dumps(pipeline.spec, indent=2), dummy_pipeline) + + auto_reader = pio.readers.auto(filename="dummyinput.las") + auto_writer = pio.writers.auto(filename="dummyoutput.las") + + self.assertIn("filename", auto_reader.spec) + self.assertNotIn("type", auto_reader.spec) + self.assertIn("filename", auto_reader.spec) + self.assertNotIn("type", auto_writer.spec) + self.assertEqual(auto_reader.prefix, "readers") + self.assertEqual(auto_writer.prefix, "writers") From b613647d5f12099455a74ac851b720849b534823 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 9 Dec 2019 17:38:15 -0600 Subject: [PATCH 014/309] adding module docstrings --- pdal/pio.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/pdal/pio.py b/pdal/pio.py index 013003fe..fbbc15e0 100644 --- a/pdal/pio.py +++ b/pdal/pio.py @@ -1,3 +1,27 @@ +""" +This module provides a python-syntax interface for constructing and executing pdal-python json +pipelines. The API is not explicitly defined but stage names are validated against the pdal executable's drivers when possible. + +To construct pipeline stages, access the driver name from this module. This will create +a callable function where driver parameters can be specified as keyword arguments. For example: + +>>> from pdal import pio +>>> las_reader = pio.readers.las(filename="test.las") + +To construct a pipeline, sum stages together. + +>>> pipeline = pio.readers.las(filename="test.las") + pio.writers.ply(filename="test.ply") + +To execute a pipeline and return results, call `execute`. + +>>> arr = pipeline.execute() # returns a numpy structured array + +To access the pipelines as a dict (which may be dumped to json), call `spec`. + +>>> json.dumps(pipeline.spec) + +""" + import types import json import subprocess @@ -36,6 +60,10 @@ def __init__(self, prefix, **kwargs): @property def pipeline(self): + """ + Promote this stage to a `pdal.pio.PipelineSpec` with one `pdal.pio.StageSpec` + and return it. + """ output = PipelineSpec() output.add_stage(self) return output @@ -73,11 +101,17 @@ def __init__(self, other=None): @property def spec(self): + """ + Return a `dict` containing the pdal pipeline suitable for dumping to json + """ return { "pipeline": [stage.spec for stage in self.stages] } def add_stage(self, stage): + """ + Add a StageSpec to the end of this pipeline, and return the updated result. + """ assert isinstance(stage, StageSpec), "Expected StageSpec" self.stages.append(stage) @@ -98,6 +132,9 @@ def __add__(self, stage_or_pipeline): return output def execute(self): + """ + Shortcut to execute and return the results of the pipeline. + """ # TODO: do some validation before calling execute # TODO: some exception/error handling around pdal From 3d94698d789f6dcb498a2cb127c06e49d1cd89e1 Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Wed, 18 Dec 2019 13:26:15 -0500 Subject: [PATCH 015/309] Make dimension number case more flexible. --- test/test_pipeline.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 823ae1ff..e16124d9 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -196,10 +196,8 @@ class TestDimensions(PDALTest): def test_fetch_dimensions(self): """Ask PDAL for its valid dimensions list""" dims = pdal.dimensions - if Version(pdal.info.version) < Version('1.8'): - self.assertEqual(len(dims), 71) - else: - self.assertEqual(len(dims), 72) + self.assertLess(len(dims), 100) + self.assertGreater(len(dims), 71) def test_suite(): return unittest.TestSuite( From 80eea96fada65f734ed4dd26a08681d1c2d5c415 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Thu, 19 Dec 2019 10:22:33 -0600 Subject: [PATCH 016/309] initial builds of readers.numpy and filters.python outside of PDAL main tree using distutils --- pdal/__init__.py | 2 +- pdal/filters/PythonFilter.cpp | 159 + pdal/filters/PythonFilter.hpp | 72 + pdal/io/NumpyReader.cpp | 520 + pdal/io/NumpyReader.hpp | 124 + pdal/nlohmann/json.hpp | 20894 ++++++++++++++++++++++++++++++++ pdal/plang/Environment.cpp | 381 + pdal/plang/Environment.hpp | 85 + pdal/plang/Invocation.cpp | 485 + pdal/plang/Invocation.hpp | 88 + pdal/plang/Redirector.cpp | 208 + pdal/plang/Redirector.hpp | 53 + pdal/plang/Script.cpp | 66 + pdal/plang/Script.hpp | 78 + setup.py | 152 +- 15 files changed, 23314 insertions(+), 53 deletions(-) create mode 100644 pdal/filters/PythonFilter.cpp create mode 100644 pdal/filters/PythonFilter.hpp create mode 100644 pdal/io/NumpyReader.cpp create mode 100644 pdal/io/NumpyReader.hpp create mode 100644 pdal/nlohmann/json.hpp create mode 100644 pdal/plang/Environment.cpp create mode 100644 pdal/plang/Environment.hpp create mode 100644 pdal/plang/Invocation.cpp create mode 100644 pdal/plang/Invocation.hpp create mode 100644 pdal/plang/Redirector.cpp create mode 100644 pdal/plang/Redirector.hpp create mode 100644 pdal/plang/Script.cpp create mode 100644 pdal/plang/Script.hpp diff --git a/pdal/__init__.py b/pdal/__init__.py index 4d775cb6..c1b548d7 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__='2.2.2' +__version__='2.3.0' from .pipeline import Pipeline from .array import Array diff --git a/pdal/filters/PythonFilter.cpp b/pdal/filters/PythonFilter.cpp new file mode 100644 index 00000000..6b8f5399 --- /dev/null +++ b/pdal/filters/PythonFilter.cpp @@ -0,0 +1,159 @@ +/****************************************************************************** +* Copyright (c) 2011, Michael P. Gerlek (mpg@flaxen.com) +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following +* conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided +* with the distribution. +* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the +* names of its contributors may be used to endorse or promote +* products derived from this software without specific prior +* written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +* OF SUCH DAMAGE. +****************************************************************************/ + +#include "PythonFilter.hpp" + +#include "../nlohmann/json.hpp" + +#include +#include +#include +#include + +namespace pdal +{ + +static PluginInfo const s_info +{ + "filters.python", + "Manipulate data using inline Python", + "http://pdal.io/stages/filters.python.html" +}; + +CREATE_SHARED_STAGE(PythonFilter, s_info) + +struct PythonFilter::Args +{ + std::string m_module; + std::string m_function; + std::string m_source; + std::string m_scriptFile; + StringList m_addDimensions; + NL::json m_pdalargs; +}; + +PythonFilter::PythonFilter() : + m_script(nullptr), m_pythonMethod(nullptr), m_args(new Args) +{} + + +PythonFilter::~PythonFilter() +{} + + +std::string PythonFilter::getName() const +{ + return s_info.name; +} + + +void PythonFilter::addArgs(ProgramArgs& args) +{ + args.add("module", "Python module containing the function to run", + m_args->m_module).setPositional(); + args.add("function", "Function to call", + m_args->m_function).setPositional(); + args.add("source", "Python script to run", m_args->m_source); + args.add("script", "File containing script to run", m_args->m_scriptFile); + args.add("add_dimension", "Dimensions to add", m_args->m_addDimensions); + args.add("pdalargs", "Dictionary to add to module globals when " + "calling function", m_args->m_pdalargs); +} + + +void PythonFilter::addDimensions(PointLayoutPtr layout) +{ + for (const std::string& s : m_args->m_addDimensions) + { + StringList spec = Utils::split(s, '='); + Utils::trim(spec[0]); + if (spec.size() == 2) + { + Utils::trim(spec[1]); + auto type = pdal::Dimension::type(spec[1]); + if (type == pdal::Dimension::Type::None) + throwError("Invalid dimension type specified '" + spec[1] + + "'. See documentation for valid dimension types"); + layout->registerOrAssignDim(spec[0], type); + } + else if (spec.size() == 1) + layout->registerOrAssignDim(s, pdal::Dimension::Type::Double); + else + throwError("Invalid dimension specified '" + s + + "'. Need or =."); + } +} + + +void PythonFilter::prepared(PointTableRef table) +{ + if (m_args->m_source.size() && m_args->m_scriptFile.size()) + throwError("Can't set both 'source' and 'script' options."); + if (!m_args->m_source.size() && !m_args->m_scriptFile.size()) + throwError("Must set one of 'source' and 'script' options."); +} + + +void PythonFilter::ready(PointTableRef table) +{ + if (m_args->m_source.empty()) + m_args->m_source = FileUtils::readFileIntoString(m_args->m_scriptFile); + std::ostream *out = log()->getLogStream(); + plang::EnvironmentPtr env = plang::Environment::get(); + env->set_stdout(out); + m_script.reset(new plang::Script(m_args->m_source, m_args->m_module, + m_args->m_function)); + m_pythonMethod.reset(new plang::Invocation(*m_script, table.metadata(), + m_args->m_pdalargs.dump(1))); +} + + +PointViewSet PythonFilter::run(PointViewPtr view) +{ + log()->get(LogLevel::Debug5) << "filters.python " << *m_script << + " processing " << view->size() << " points." << std::endl; + m_pythonMethod->execute(view, getMetadata()); + + PointViewSet viewSet; + viewSet.insert(view); + return viewSet; +} + + +void PythonFilter::done(PointTableRef table) +{ + static_cast(plang::Environment::get())->reset_stdout(); +} + +} // namespace pdal diff --git a/pdal/filters/PythonFilter.hpp b/pdal/filters/PythonFilter.hpp new file mode 100644 index 00000000..19f3d036 --- /dev/null +++ b/pdal/filters/PythonFilter.hpp @@ -0,0 +1,72 @@ +/****************************************************************************** +* Copyright (c) 2011, Michael P. Gerlek (mpg@flaxen.com) +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following +* conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided +* with the distribution. +* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the +* names of its contributors may be used to endorse or promote +* products derived from this software without specific prior +* written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +* OF SUCH DAMAGE. +****************************************************************************/ + +#pragma once + +#include +#include + +#include "../plang/Invocation.hpp" + + +namespace pdal +{ + +class PDAL_DLL PythonFilter : public Filter +{ +public: + PythonFilter(); + virtual ~PythonFilter(); + + std::string getName() const; + +private: + PythonFilter& operator=(const PythonFilter&) = delete; + PythonFilter(const PythonFilter&) = delete; + + virtual void addArgs(ProgramArgs& args); + virtual void addDimensions(PointLayoutPtr layout); + virtual void prepared(PointTableRef table); + virtual void ready(PointTableRef table); + virtual PointViewSet run(PointViewPtr view); + virtual void done(PointTableRef table); + + std::unique_ptr m_script; + std::unique_ptr m_pythonMethod; + + struct Args; + std::unique_ptr m_args; +}; + +} // namespace pdal diff --git a/pdal/io/NumpyReader.cpp b/pdal/io/NumpyReader.cpp new file mode 100644 index 00000000..d67e1fc9 --- /dev/null +++ b/pdal/io/NumpyReader.cpp @@ -0,0 +1,520 @@ +/****************************************************************************** +* Copyright (c) 2018, Howard Butler, howard@hobu.co +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following +* conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided +* with the distribution. +* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the +* names of its contributors may be used to endorse or promote +* products derived from this software without specific prior +* written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +* OF SUCH DAMAGE. +****************************************************************************/ + +#include "NumpyReader.hpp" + +#include +#include +#include +#include +#include +#include + +#include "../plang/Environment.hpp" + + +std::string toString(PyObject *pname) +{ + std::stringstream mssg; + PyObject* r = PyObject_Str(pname); + if (!r) + throw pdal::pdal_error("couldn't make string representation value"); +#if PY_MAJOR_VERSION >= 3 + Py_ssize_t size; + const char *d = PyUnicode_AsUTF8AndSize(r, &size); +#else + const char *d = PyString_AsString(r); +#endif + mssg << d; + return mssg.str(); +} + + +namespace pdal +{ + +static PluginInfo const s_info +{ + "readers.numpy", + "Read data from .npy files.", + "" +}; + +CREATE_SHARED_STAGE(NumpyReader, s_info) + +std::string NumpyReader::getName() const { return s_info.name; } + +std::ostream& operator << (std::ostream& out, + const NumpyReader::Order& order) +{ + switch (order) + { + case NumpyReader::Order::Row: + out << "row"; + break; + case NumpyReader::Order::Column: + out << "column"; + break; + } + return out; +} + + +std::istream& operator >> (std::istream& in, NumpyReader::Order& order) +{ + std::string s; + in >> s; + + s = Utils::tolower(s); + if (s == "row") + order = NumpyReader::Order::Row; + else if (s == "column") + order = NumpyReader::Order::Column; + else + in.setstate(std::ios_base::failbit); + return in; +} + + +void NumpyReader::setArray(PyObject* array) +{ + plang::Environment::get(); + if (!PyArray_Check(array)) + throw pdal::pdal_error("object provided to setArray is not a python numpy array!"); + + m_array = (PyArrayObject*)array; + Py_XINCREF(m_array); +} + +PyArrayObject* load_npy(std::string const& filename) +{ + + PyObject *py_filename = PyUnicode_FromString(filename.c_str()); + PyObject *numpy_module = PyImport_ImportModule("numpy"); + if (!numpy_module) + throw pdal::pdal_error(plang::getTraceback()); + + PyObject *numpy_mod_dict = PyModule_GetDict(numpy_module); + if (!numpy_mod_dict) + throw pdal::pdal_error(plang::getTraceback()); + + PyObject *loads_func = PyDict_GetItemString(numpy_mod_dict, "load"); + if (!loads_func) + throw pdal::pdal_error(plang::getTraceback()); + + PyObject *numpy_args = PyTuple_New(1); + if (!numpy_args) + throw pdal::pdal_error(plang::getTraceback()); + + if (PyTuple_SetItem(numpy_args, 0, py_filename)) + throw pdal::pdal_error(plang::getTraceback()); + + PyObject* array = PyObject_CallObject(loads_func, numpy_args); + if (!array) + throw pdal::pdal_error(plang::getTraceback()); + + return reinterpret_cast(array); +} + + +void NumpyReader::initialize() +{ + plang::Environment::get(); + m_numPoints = 0; + m_chunkCount = 0; + m_ndims = 0; + + m_iter = NULL; + + p_data = NULL; + m_dataptr = NULL; + m_strideptr = NULL; + m_innersizeptr = NULL; + m_dtype = NULL; + + if (m_filename.size()) + { + m_array = load_npy(m_filename); + } + if (m_array) + if (!PyArray_Check(m_array)) + throw pdal::pdal_error("Object in file '" + m_filename + + "' is not a numpy array"); +} + + +void NumpyReader::wakeUpNumpyArray() +{ + // TODO pivot whether we are a 1d, 2d, or named arrays + + if (PyArray_SIZE(m_array) == 0) + throw pdal::pdal_error("Array cannot be empty!"); + + m_iter = NpyIter_New(m_array, + NPY_ITER_EXTERNAL_LOOP | NPY_ITER_READONLY| NPY_ITER_REFS_OK, + NPY_KEEPORDER, NPY_NO_CASTING, + NULL); + if (!m_iter) + { + std::ostringstream oss; + + oss << "Unable to create iterator from array in '" + << m_filename + "' with traceback: '" + << plang::getTraceback() <<"'"; + throw pdal::pdal_error(oss.str()); + } + + char* itererr; + m_iternext = NpyIter_GetIterNext(m_iter, &itererr); + if (!m_iternext) + { + NpyIter_Deallocate(m_iter); + throw pdal::pdal_error(itererr); + } + + m_dtype = PyArray_DTYPE(m_array); + if (!m_dtype) + throw pdal_error(plang::getTraceback()); + + m_ndims = PyArray_NDIM(m_array); + m_shape = PyArray_SHAPE(m_array); + if (!m_shape) + throw pdal_error(plang::getTraceback()); + + npy_intp* shape = PyArray_SHAPE(m_array); + if (!shape) + throw pdal_error(plang::getTraceback()); + m_numPoints = 1; + for (int i = 0; i < m_ndims; ++i) + m_numPoints *= m_shape[i]; + + // If the order arg wasn't set, set order based on the internal order of + // the array. + if (!m_orderArg->set()) + { + int flags = PyArray_FLAGS(m_array); + if (flags & NPY_ARRAY_F_CONTIGUOUS) + m_order = Order::Column; + else + m_order = Order::Row; + } +} + + +void NumpyReader::addArgs(ProgramArgs& args) +{ + args.add("dimension", "In an unstructured array, the dimension name to " + "map to values.", m_defaultDimension, "Intensity"); + m_orderArg = &args.add("order", "Order of dimension interpretation " + "of the array. Either 'row'-major (C) or 'column'-major (Fortran)", + m_order); +} + + +// Try to map dimension names to existing PDAL dimension names by +// checking them with certain characters removed. +Dimension::Id NumpyReader::registerDim(PointLayoutPtr layout, + const std::string& name, Dimension::Type pdalType) +{ + Dimension::Id id; + + auto registerName = [&layout, &pdalType, &id](std::string name, char elim) + { + if (elim != '\0') + Utils::remove(name, elim); + Dimension::Id tempId = Dimension::id(name); + if (tempId != Dimension::Id::Unknown) + { + id = tempId; + layout->registerDim(id, pdalType); + return true; + } + return false; + }; + + // Try registering the name in various ways. If that doesn't work, + // just punt and use the name as is. + if (!registerName(name, '\0') && !registerName(name, '-') && + !registerName(name, ' ') && !registerName(name, '_')) + id = layout->registerOrAssignDim(name, pdalType); + return id; +} + +namespace +{ + + +Dimension::Type getType(PyArray_Descr *dtype, const std::string& name) +{ + if (!dtype) + throw pdal_error("Can't fetch data type for numpy field."); + + Dimension::Type pdalType = + plang::Environment::getPDALDataType(dtype->type_num); + if (pdalType == Dimension::Type::None) + { + std::ostringstream oss; + oss << "Unable to map dimension '" << name << "' because its " + "type '" << dtype->type_num <<"' is not mappable to PDAL"; + throw pdal_error(oss.str()); + } + return pdalType; +} + +} // unnamed namespace + + +void NumpyReader::createFields(PointLayoutPtr layout) +{ + Dimension::Id id; + Dimension::Type type; + int offset; + + m_numFields = 0; + if (m_dtype->fields != Py_None) + m_numFields = static_cast(PyDict_Size(m_dtype->fields)); + + // Array isn't structured - just a bunch of data. + if (m_numFields <= 0) + { + type = getType(m_dtype, m_defaultDimension); + id = registerDim(layout, m_defaultDimension, type); + m_fields.push_back({id, type, 0}); + } + else + { + PyObject* names_dict = m_dtype->fields; + PyObject* names = PyDict_Keys(names_dict); + PyObject* values = PyDict_Values(names_dict); + if (!names || !values) + throw pdal_error("Bad field specification for numpy array layout."); + + for (int i = 0; i < m_numFields; ++i) + { + std::string name = toString(PyList_GetItem(names, i)); + PyObject *tup = PyList_GetItem(values, i); + if (!tup) + throw pdal_error(plang::getTraceback()); + + // Get offset. + PyObject* offset_o = PySequence_Fast_GET_ITEM(tup, 1); + if (!offset_o) + throw pdal_error(plang::getTraceback()); + offset = PyLong_AsLong(offset_o); + + // Get type. + type = getType((PyArray_Descr *)PySequence_Fast_GET_ITEM(tup, 0), + name); + id = registerDim(layout, name, type); + m_fields.push_back({id, type, offset}); + } + } +} + + +void NumpyReader::addDimensions(PointLayoutPtr layout) +{ + using namespace Dimension; + + wakeUpNumpyArray(); + createFields(layout); + + m_storeXYZ = true; + // If we already have an X dimension, we're done. + for (const Field& field : m_fields) + if (field.m_id == Id::X || field.m_id == Id::Y || field.m_id == Id::Z) + { + m_storeXYZ = false; + return; + } + + // We're storing a calculated XYZ, so register the dims. + layout->registerDim(Id::X, Type::Signed32); + if (m_ndims > 1) + { + layout->registerDim(Id::Y, Type::Signed32); + if (m_ndims > 2) + layout->registerDim(Id::Z, Type::Signed32); + } + if (m_order == Order::Row) + { + m_xIter = m_shape[m_ndims - 1]; + m_xDiv = 1; + if (m_ndims > 1) + { + m_yDiv = 1; + m_xDiv = m_xIter; + m_xIter *= m_shape[m_ndims - 2]; + m_yIter = m_shape[m_ndims - 1]; + if (m_ndims > 2) + { + m_xDiv = m_xIter; + m_yDiv = m_yIter; + m_zDiv = 1; + m_xIter *= m_shape[m_ndims - 3]; + m_yIter *= m_shape[m_ndims - 2]; + m_zIter = m_shape[m_ndims - 1]; + } + } + } + else + { + m_xIter = m_shape[0]; + m_xDiv = 1; + if (m_ndims > 1) + { + m_yIter = m_shape[0] * m_shape[1]; + m_yDiv = m_xIter; + if (m_ndims > 2) + { + m_zIter = m_shape[0] * m_shape[1] * m_shape[2]; + m_zDiv = m_yIter; + } + } + } +} + + +void NumpyReader::ready(PointTableRef table) +{ + plang::Environment::get()->set_stdout(log()->getLogStream()); + + // Set our iterators + // The location of the data pointer which the iterator may update + m_dataptr = NpyIter_GetDataPtrArray(m_iter); + + // The location of the stride which the iterator may update + m_strideptr = NpyIter_GetInnerStrideArray(m_iter); + + // The location of the inner loop size which the iterator may update + m_innersizeptr = NpyIter_GetInnerLoopSizePtr(m_iter); + + p_data = *m_dataptr; + m_chunkCount = *m_innersizeptr; + m_index = 0; + + log()->get(LogLevel::Debug) << "Initializing Numpy array for file '" << + m_filename << "'" << std::endl; + log()->get(LogLevel::Debug) << "numpy inner stride '" << + *m_strideptr << "'" << std::endl; + log()->get(LogLevel::Debug) << "numpy inner stride size '" << + *m_innersizeptr << "'" << std::endl; + log()->get(LogLevel::Debug) << "numpy number of points '" << + m_numPoints << "'" << std::endl; + log()->get(LogLevel::Debug) << "numpy number of dimensions '" << + m_ndims << "'" << std::endl; + for (npy_intp i = 0; i < m_ndims; ++i) + log()->get(LogLevel::Debug) << "numpy shape dimension number '" << + i << "' is '" << m_shape[i] <<"'" << std::endl; +} + +bool NumpyReader::nextPoint() +{ + // If we are at the end of this chunk, try to fetch another. Otherwise, + // just advance by the stride. + if (--m_chunkCount == 0) + { + // If we can't fetch the next ite + if (!m_iternext(m_iter)) + return false; + m_chunkCount = *m_innersizeptr; + p_data = *m_dataptr; + } + else + p_data += *m_strideptr; + return true; +} + + +bool NumpyReader::loadPoint(PointRef& point, point_count_t position) +{ + using namespace Dimension; + + for (const Field& f : m_fields) + point.setField(f.m_id, f.m_type, (void*)(p_data + f.m_offset)); + + if (m_storeXYZ) + { + point.setField(Dimension::Id::X, (position % m_xIter) / m_xDiv); + if (m_ndims > 1) + { + point.setField(Dimension::Id::Y, (position % m_yIter) / m_yDiv); + if (m_ndims > 2) + point.setField(Dimension::Id::Z, (position % m_zIter) / m_zDiv); + } + } + + return (nextPoint()); +} + + +bool NumpyReader::processOne(PointRef& point) +{ + if (m_index >= m_numPoints) + return false; + return loadPoint(point, m_index++); +} + + +point_count_t NumpyReader::read(PointViewPtr view, point_count_t numToRead) +{ + PointId idx = view->size(); + point_count_t numRead(0); + + while (numRead < numToRead) + { + PointRef point(*view, idx); + if (!processOne(point)) + break; + numRead++; + idx++; + } + return numRead; +} + + +void NumpyReader::done(PointTableRef) +{ + // Dereference everything we're using + + if (m_iter) + NpyIter_Deallocate(m_iter); + + Py_XDECREF(m_array); +} + + +} // namespace pdal + + diff --git a/pdal/io/NumpyReader.hpp b/pdal/io/NumpyReader.hpp new file mode 100644 index 00000000..31a18102 --- /dev/null +++ b/pdal/io/NumpyReader.hpp @@ -0,0 +1,124 @@ +/****************************************************************************** +* Copyright (c) 2018, Howard Butler, howard@hobu.co +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following +* conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided +* with the distribution. +* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the +* names of its contributors may be used to endorse or promote +* products derived from this software without specific prior +* written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +* OF SUCH DAMAGE. +****************************************************************************/ + +#pragma once + +#include +#include + +#include +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API +#define NO_IMPORT_ARRAY +#include + +namespace pdal +{ + +class PDAL_DLL NumpyReader : public Reader, public Streamable +{ + enum class Order + { + Row, + Column + }; + friend std::ostream& operator << (std::ostream& out, + const NumpyReader::Order& order); + friend std::istream& operator >> (std::istream& in, + NumpyReader::Order& order); +public: + NumpyReader& operator=(const NumpyReader&) = delete; + NumpyReader(const NumpyReader&) = delete; + NumpyReader() : m_array(NULL) + {} + + void setArray(PyObject* array); + std::string getName() const; + +private: + virtual void initialize(); + virtual void addArgs(ProgramArgs& args); + virtual void addDimensions(PointLayoutPtr layout); + virtual void ready(PointTableRef table); + virtual point_count_t read(PointViewPtr view, point_count_t count); + virtual bool processOne(PointRef& point); + virtual void done(PointTableRef table); + + void createFields(PointLayoutPtr layout); + bool nextPoint(); + bool loadPoint(PointRef& point, point_count_t position); + void wakeUpNumpyArray(); + Dimension::Id registerDim(PointLayoutPtr layout, const std::string& name, + Dimension::Type pdalType); + void prepareFieldsArray(PointLayoutPtr layout); + void prepareRasterArray(PointLayoutPtr layout); + + // Py_XDECREF these on the way out + PyArrayObject* m_array; + NpyIter* m_iter; + NpyIter_IterNextFunc* m_iternext; + PyArray_Descr* m_dtype; + + char** m_dataptr; + char* p_data; + npy_intp m_nonzero_count; + npy_intp* m_strideptr, *m_innersizeptr; + npy_intp* m_shape; + npy_intp m_chunkCount; + point_count_t m_numPoints; + int m_numFields; + + Arg *m_orderArg; + int m_ndims; + std::string m_defaultDimension; + Order m_order; + bool m_storeXYZ; + size_t m_xIter; + size_t m_yIter; + size_t m_zIter; + size_t m_xDiv; + size_t m_yDiv; + size_t m_zDiv; + + struct Field + { + Dimension::Id m_id; + Dimension::Type m_type; + int m_offset; + }; + std::vector m_fields; + point_count_t m_index; +}; + +} // namespace pdal diff --git a/pdal/nlohmann/json.hpp b/pdal/nlohmann/json.hpp new file mode 100644 index 00000000..3c28a9d6 --- /dev/null +++ b/pdal/nlohmann/json.hpp @@ -0,0 +1,20894 @@ +/* + __ _____ _____ _____ + __| | __| | | | JSON for Modern C++ +| | |__ | | | | | | version 3.6.1 +|_____|_____|_____|_|___| https://github.com/nlohmann/json + +Licensed under the MIT License . +SPDX-License-Identifier: MIT +Copyright (c) 2013-2019 Niels Lohmann . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +#ifndef INCLUDE_NLOHMANN_JSON_HPP_ +#define INCLUDE_NLOHMANN_JSON_HPP_ + +#define NLOHMANN_JSON_VERSION_MAJOR 3 +#define NLOHMANN_JSON_VERSION_MINOR 6 +#define NLOHMANN_JSON_VERSION_PATCH 1 + +#include // all_of, find, for_each +#include // assert +#include // and, not, or +#include // nullptr_t, ptrdiff_t, size_t +#include // hash, less +#include // initializer_list +#include // istream, ostream +#include // random_access_iterator_tag +#include // unique_ptr +#include // accumulate +#include // string, stoi, to_string +#include // declval, forward, move, pair, swap +#include // vector + +// #include + + +#include + +// #include + + +#include // transform +#include // array +#include // and, not +#include // forward_list +#include // inserter, front_inserter, end +#include // map +#include // string +#include // tuple, make_tuple +#include // is_arithmetic, is_same, is_enum, underlying_type, is_convertible +#include // unordered_map +#include // pair, declval +#include // valarray + +// #include + + +#include // exception +#include // runtime_error +#include // to_string + +// #include + + +#include // size_t + +namespace nlohmann +{ +namespace detail +{ +/// struct to capture the start position of the current token +struct position_t +{ + /// the total number of characters read + std::size_t chars_read_total = 0; + /// the number of characters read in the current line + std::size_t chars_read_current_line = 0; + /// the number of lines read + std::size_t lines_read = 0; + + /// conversion to size_t to preserve SAX interface + constexpr operator size_t() const + { + return chars_read_total; + } +}; + +} // namespace detail +} // namespace nlohmann + + +namespace nlohmann +{ +namespace detail +{ +//////////////// +// exceptions // +//////////////// + +/*! +@brief general exception of the @ref basic_json class + +This class is an extension of `std::exception` objects with a member @a id for +exception ids. It is used as the base class for all exceptions thrown by the +@ref basic_json class. This class can hence be used as "wildcard" to catch +exceptions. + +Subclasses: +- @ref parse_error for exceptions indicating a parse error +- @ref invalid_iterator for exceptions indicating errors with iterators +- @ref type_error for exceptions indicating executing a member function with + a wrong type +- @ref out_of_range for exceptions indicating access out of the defined range +- @ref other_error for exceptions indicating other library errors + +@internal +@note To have nothrow-copy-constructible exceptions, we internally use + `std::runtime_error` which can cope with arbitrary-length error messages. + Intermediate strings are built with static functions and then passed to + the actual constructor. +@endinternal + +@liveexample{The following code shows how arbitrary library exceptions can be +caught.,exception} + +@since version 3.0.0 +*/ +class exception : public std::exception +{ + public: + /// returns the explanatory string + const char* what() const noexcept override + { + return m.what(); + } + + /// the id of the exception + const int id; + + protected: + exception(int id_, const char* what_arg) : id(id_), m(what_arg) {} + + static std::string name(const std::string& ename, int id_) + { + return "[json.exception." + ename + "." + std::to_string(id_) + "] "; + } + + private: + /// an exception object as storage for error messages + std::runtime_error m; +}; + +/*! +@brief exception indicating a parse error + +This exception is thrown by the library when a parse error occurs. Parse errors +can occur during the deserialization of JSON text, CBOR, MessagePack, as well +as when using JSON Patch. + +Member @a byte holds the byte index of the last read character in the input +file. + +Exceptions have ids 1xx. + +name / id | example message | description +------------------------------ | --------------- | ------------------------- +json.exception.parse_error.101 | parse error at 2: unexpected end of input; expected string literal | This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member @a byte indicates the error position. +json.exception.parse_error.102 | parse error at 14: missing or wrong low surrogate | JSON uses the `\uxxxx` format to describe Unicode characters. Code points above above 0xFFFF are split into two `\uxxxx` entries ("surrogate pairs"). This error indicates that the surrogate pair is incomplete or contains an invalid code point. +json.exception.parse_error.103 | parse error: code points above 0x10FFFF are invalid | Unicode supports code points up to 0x10FFFF. Code points above 0x10FFFF are invalid. +json.exception.parse_error.104 | parse error: JSON patch must be an array of objects | [RFC 6902](https://tools.ietf.org/html/rfc6902) requires a JSON Patch document to be a JSON document that represents an array of objects. +json.exception.parse_error.105 | parse error: operation must have string member 'op' | An operation of a JSON Patch document must contain exactly one "op" member, whose value indicates the operation to perform. Its value must be one of "add", "remove", "replace", "move", "copy", or "test"; other values are errors. +json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number without a leading `0`. +json.exception.parse_error.107 | parse error: JSON pointer must be empty or begin with '/' - was: 'foo' | A JSON Pointer must be a Unicode string containing a sequence of zero or more reference tokens, each prefixed by a `/` character. +json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences. +json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number. +json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read. +json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xF8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read. +json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read. +json.exception.parse_error.114 | parse error: Unsupported BSON record type 0x0F | The parsing of the corresponding BSON record type is not implemented (yet). + +@note For an input with n bytes, 1 is the index of the first character and n+1 + is the index of the terminating null byte or the end of file. This also + holds true when reading a byte vector (CBOR or MessagePack). + +@liveexample{The following code shows how a `parse_error` exception can be +caught.,parse_error} + +@sa - @ref exception for the base class of the library exceptions +@sa - @ref invalid_iterator for exceptions indicating errors with iterators +@sa - @ref type_error for exceptions indicating executing a member function with + a wrong type +@sa - @ref out_of_range for exceptions indicating access out of the defined range +@sa - @ref other_error for exceptions indicating other library errors + +@since version 3.0.0 +*/ +class parse_error : public exception +{ + public: + /*! + @brief create a parse error exception + @param[in] id_ the id of the exception + @param[in] pos the position where the error occurred (or with + chars_read_total=0 if the position cannot be + determined) + @param[in] what_arg the explanatory string + @return parse_error object + */ + static parse_error create(int id_, const position_t& pos, const std::string& what_arg) + { + std::string w = exception::name("parse_error", id_) + "parse error" + + position_string(pos) + ": " + what_arg; + return parse_error(id_, pos.chars_read_total, w.c_str()); + } + + static parse_error create(int id_, std::size_t byte_, const std::string& what_arg) + { + std::string w = exception::name("parse_error", id_) + "parse error" + + (byte_ != 0 ? (" at byte " + std::to_string(byte_)) : "") + + ": " + what_arg; + return parse_error(id_, byte_, w.c_str()); + } + + /*! + @brief byte index of the parse error + + The byte index of the last read character in the input file. + + @note For an input with n bytes, 1 is the index of the first character and + n+1 is the index of the terminating null byte or the end of file. + This also holds true when reading a byte vector (CBOR or MessagePack). + */ + const std::size_t byte; + + private: + parse_error(int id_, std::size_t byte_, const char* what_arg) + : exception(id_, what_arg), byte(byte_) {} + + static std::string position_string(const position_t& pos) + { + return " at line " + std::to_string(pos.lines_read + 1) + + ", column " + std::to_string(pos.chars_read_current_line); + } +}; + +/*! +@brief exception indicating errors with iterators + +This exception is thrown if iterators passed to a library function do not match +the expected semantics. + +Exceptions have ids 2xx. + +name / id | example message | description +----------------------------------- | --------------- | ------------------------- +json.exception.invalid_iterator.201 | iterators are not compatible | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. +json.exception.invalid_iterator.202 | iterator does not fit current value | In an erase or insert function, the passed iterator @a pos does not belong to the JSON value for which the function was called. It hence does not define a valid position for the deletion/insertion. +json.exception.invalid_iterator.203 | iterators do not fit current value | Either iterator passed to function @ref erase(IteratorType first, IteratorType last) does not belong to the JSON value from which values shall be erased. It hence does not define a valid range to delete values from. +json.exception.invalid_iterator.204 | iterators out of range | When an iterator range for a primitive type (number, boolean, or string) is passed to a constructor or an erase function, this range has to be exactly (@ref begin(), @ref end()), because this is the only way the single stored value is expressed. All other ranges are invalid. +json.exception.invalid_iterator.205 | iterator out of range | When an iterator for a primitive type (number, boolean, or string) is passed to an erase function, the iterator has to be the @ref begin() iterator, because it is the only way to address the stored value. All other iterators are invalid. +json.exception.invalid_iterator.206 | cannot construct with iterators from null | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) belong to a JSON null value and hence to not define a valid range. +json.exception.invalid_iterator.207 | cannot use key() for non-object iterators | The key() member function can only be used on iterators belonging to a JSON object, because other types do not have a concept of a key. +json.exception.invalid_iterator.208 | cannot use operator[] for object iterators | The operator[] to specify a concrete offset cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. +json.exception.invalid_iterator.209 | cannot use offsets with object iterators | The offset operators (+, -, +=, -=) cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. +json.exception.invalid_iterator.210 | iterators do not fit | The iterator range passed to the insert function are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. +json.exception.invalid_iterator.211 | passed iterators may not belong to container | The iterator range passed to the insert function must not be a subrange of the container to insert to. +json.exception.invalid_iterator.212 | cannot compare iterators of different containers | When two iterators are compared, they must belong to the same container. +json.exception.invalid_iterator.213 | cannot compare order of object iterators | The order of object iterators cannot be compared, because JSON objects are unordered. +json.exception.invalid_iterator.214 | cannot get value | Cannot get value for iterator: Either the iterator belongs to a null value or it is an iterator to a primitive type (number, boolean, or string), but the iterator is different to @ref begin(). + +@liveexample{The following code shows how an `invalid_iterator` exception can be +caught.,invalid_iterator} + +@sa - @ref exception for the base class of the library exceptions +@sa - @ref parse_error for exceptions indicating a parse error +@sa - @ref type_error for exceptions indicating executing a member function with + a wrong type +@sa - @ref out_of_range for exceptions indicating access out of the defined range +@sa - @ref other_error for exceptions indicating other library errors + +@since version 3.0.0 +*/ +class invalid_iterator : public exception +{ + public: + static invalid_iterator create(int id_, const std::string& what_arg) + { + std::string w = exception::name("invalid_iterator", id_) + what_arg; + return invalid_iterator(id_, w.c_str()); + } + + private: + invalid_iterator(int id_, const char* what_arg) + : exception(id_, what_arg) {} +}; + +/*! +@brief exception indicating executing a member function with a wrong type + +This exception is thrown in case of a type error; that is, a library function is +executed on a JSON value whose type does not match the expected semantics. + +Exceptions have ids 3xx. + +name / id | example message | description +----------------------------- | --------------- | ------------------------- +json.exception.type_error.301 | cannot create object from initializer list | To create an object from an initializer list, the initializer list must consist only of a list of pairs whose first element is a string. When this constraint is violated, an array is created instead. +json.exception.type_error.302 | type must be object, but is array | During implicit or explicit value conversion, the JSON type must be compatible to the target type. For instance, a JSON string can only be converted into string types, but not into numbers or boolean types. +json.exception.type_error.303 | incompatible ReferenceType for get_ref, actual type is object | To retrieve a reference to a value stored in a @ref basic_json object with @ref get_ref, the type of the reference must match the value type. For instance, for a JSON array, the @a ReferenceType must be @ref array_t &. +json.exception.type_error.304 | cannot use at() with string | The @ref at() member functions can only be executed for certain JSON types. +json.exception.type_error.305 | cannot use operator[] with string | The @ref operator[] member functions can only be executed for certain JSON types. +json.exception.type_error.306 | cannot use value() with string | The @ref value() member functions can only be executed for certain JSON types. +json.exception.type_error.307 | cannot use erase() with string | The @ref erase() member functions can only be executed for certain JSON types. +json.exception.type_error.308 | cannot use push_back() with string | The @ref push_back() and @ref operator+= member functions can only be executed for certain JSON types. +json.exception.type_error.309 | cannot use insert() with | The @ref insert() member functions can only be executed for certain JSON types. +json.exception.type_error.310 | cannot use swap() with number | The @ref swap() member functions can only be executed for certain JSON types. +json.exception.type_error.311 | cannot use emplace_back() with string | The @ref emplace_back() member function can only be executed for certain JSON types. +json.exception.type_error.312 | cannot use update() with string | The @ref update() member functions can only be executed for certain JSON types. +json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined. +json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers. +json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive. +json.exception.type_error.316 | invalid UTF-8 byte at index 10: 0x7E | The @ref dump function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. | +json.exception.type_error.317 | JSON value cannot be serialized to requested format | The dynamic type of the object cannot be represented in the requested serialization format (e.g. a raw `true` or `null` JSON object cannot be serialized to BSON) | + +@liveexample{The following code shows how a `type_error` exception can be +caught.,type_error} + +@sa - @ref exception for the base class of the library exceptions +@sa - @ref parse_error for exceptions indicating a parse error +@sa - @ref invalid_iterator for exceptions indicating errors with iterators +@sa - @ref out_of_range for exceptions indicating access out of the defined range +@sa - @ref other_error for exceptions indicating other library errors + +@since version 3.0.0 +*/ +class type_error : public exception +{ + public: + static type_error create(int id_, const std::string& what_arg) + { + std::string w = exception::name("type_error", id_) + what_arg; + return type_error(id_, w.c_str()); + } + + private: + type_error(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; + +/*! +@brief exception indicating access out of the defined range + +This exception is thrown in case a library function is called on an input +parameter that exceeds the expected range, for instance in case of array +indices or nonexisting object keys. + +Exceptions have ids 4xx. + +name / id | example message | description +------------------------------- | --------------- | ------------------------- +json.exception.out_of_range.401 | array index 3 is out of range | The provided array index @a i is larger than @a size-1. +json.exception.out_of_range.402 | array index '-' (3) is out of range | The special array index `-` in a JSON Pointer never describes a valid element of the array, but the index past the end. That is, it can only be used to add elements at this position, but not to read it. +json.exception.out_of_range.403 | key 'foo' not found | The provided key was not found in the JSON object. +json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved. +json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value. +json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF. +json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON and BSON only support integer numbers up to 9223372036854775807. | +json.exception.out_of_range.408 | excessive array size: 8658170730974374167 | The size (following `#`) of an UBJSON array or object exceeds the maximal capacity. | +json.exception.out_of_range.409 | BSON key cannot contain code point U+0000 (at byte 2) | Key identifiers to be serialized to BSON cannot contain code point U+0000, since the key is stored as zero-terminated c-string | + +@liveexample{The following code shows how an `out_of_range` exception can be +caught.,out_of_range} + +@sa - @ref exception for the base class of the library exceptions +@sa - @ref parse_error for exceptions indicating a parse error +@sa - @ref invalid_iterator for exceptions indicating errors with iterators +@sa - @ref type_error for exceptions indicating executing a member function with + a wrong type +@sa - @ref other_error for exceptions indicating other library errors + +@since version 3.0.0 +*/ +class out_of_range : public exception +{ + public: + static out_of_range create(int id_, const std::string& what_arg) + { + std::string w = exception::name("out_of_range", id_) + what_arg; + return out_of_range(id_, w.c_str()); + } + + private: + out_of_range(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; + +/*! +@brief exception indicating other library errors + +This exception is thrown in case of errors that cannot be classified with the +other exception types. + +Exceptions have ids 5xx. + +name / id | example message | description +------------------------------ | --------------- | ------------------------- +json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed. + +@sa - @ref exception for the base class of the library exceptions +@sa - @ref parse_error for exceptions indicating a parse error +@sa - @ref invalid_iterator for exceptions indicating errors with iterators +@sa - @ref type_error for exceptions indicating executing a member function with + a wrong type +@sa - @ref out_of_range for exceptions indicating access out of the defined range + +@liveexample{The following code shows how an `other_error` exception can be +caught.,other_error} + +@since version 3.0.0 +*/ +class other_error : public exception +{ + public: + static other_error create(int id_, const std::string& what_arg) + { + std::string w = exception::name("other_error", id_) + what_arg; + return other_error(id_, w.c_str()); + } + + private: + other_error(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; +} // namespace detail +} // namespace nlohmann + +// #include + + +#include // pair + +// This file contains all internal macro definitions +// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them + +// exclude unsupported compilers +#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) + #if defined(__clang__) + #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 + #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 + #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #endif +#endif + +// C++ language standard detection +#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 +#elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) + #define JSON_HAS_CPP_14 +#endif + +// disable float-equal warnings on GCC/clang +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + +// disable documentation warnings on clang +#if defined(__clang__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdocumentation" +#endif + +// allow for portable deprecation warnings +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #define JSON_DEPRECATED __attribute__((deprecated)) +#elif defined(_MSC_VER) + #define JSON_DEPRECATED __declspec(deprecated) +#else + #define JSON_DEPRECATED +#endif + +// allow for portable nodiscard warnings +#if defined(__has_cpp_attribute) + #if __has_cpp_attribute(nodiscard) + #if defined(__clang__) && !defined(JSON_HAS_CPP_17) // issue #1535 + #define JSON_NODISCARD + #else + #define JSON_NODISCARD [[nodiscard]] + #endif + #elif __has_cpp_attribute(gnu::warn_unused_result) + #define JSON_NODISCARD [[gnu::warn_unused_result]] + #else + #define JSON_NODISCARD + #endif +#else + #define JSON_NODISCARD +#endif + +// allow to disable exceptions +#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) + #define JSON_THROW(exception) throw exception + #define JSON_TRY try + #define JSON_CATCH(exception) catch(exception) + #define JSON_INTERNAL_CATCH(exception) catch(exception) +#else + #include + #define JSON_THROW(exception) std::abort() + #define JSON_TRY if(true) + #define JSON_CATCH(exception) if(false) + #define JSON_INTERNAL_CATCH(exception) if(false) +#endif + +// override exception macros +#if defined(JSON_THROW_USER) + #undef JSON_THROW + #define JSON_THROW JSON_THROW_USER +#endif +#if defined(JSON_TRY_USER) + #undef JSON_TRY + #define JSON_TRY JSON_TRY_USER +#endif +#if defined(JSON_CATCH_USER) + #undef JSON_CATCH + #define JSON_CATCH JSON_CATCH_USER + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_CATCH_USER +#endif +#if defined(JSON_INTERNAL_CATCH_USER) + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER +#endif + +// manual branch prediction +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #define JSON_LIKELY(x) __builtin_expect(x, 1) + #define JSON_UNLIKELY(x) __builtin_expect(x, 0) +#else + #define JSON_LIKELY(x) x + #define JSON_UNLIKELY(x) x +#endif + +/*! +@brief macro to briefly define a mapping between an enum and JSON +@def NLOHMANN_JSON_SERIALIZE_ENUM +@since version 3.4.0 +*/ +#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ + template \ + inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [e](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.first == e; \ + }); \ + j = ((it != std::end(m)) ? it : std::begin(m))->second; \ + } \ + template \ + inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [j](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.second == j; \ + }); \ + e = ((it != std::end(m)) ? it : std::begin(m))->first; \ + } + +// Ugly macros to avoid uglier copy-paste when specializing basic_json. They +// may be removed in the future once the class is split. + +#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ + template class ObjectType, \ + template class ArrayType, \ + class StringType, class BooleanType, class NumberIntegerType, \ + class NumberUnsignedType, class NumberFloatType, \ + template class AllocatorType, \ + template class JSONSerializer> + +#define NLOHMANN_BASIC_JSON_TPL \ + basic_json + +// #include + + +#include // not +#include // size_t +#include // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type + +namespace nlohmann +{ +namespace detail +{ +// alias templates to reduce boilerplate +template +using enable_if_t = typename std::enable_if::type; + +template +using uncvref_t = typename std::remove_cv::type>::type; + +// implementation of C++14 index_sequence and affiliates +// source: https://stackoverflow.com/a/32223343 +template +struct index_sequence +{ + using type = index_sequence; + using value_type = std::size_t; + static constexpr std::size_t size() noexcept + { + return sizeof...(Ints); + } +}; + +template +struct merge_and_renumber; + +template +struct merge_and_renumber, index_sequence> + : index_sequence < I1..., (sizeof...(I1) + I2)... > {}; + +template +struct make_index_sequence + : merge_and_renumber < typename make_index_sequence < N / 2 >::type, + typename make_index_sequence < N - N / 2 >::type > {}; + +template<> struct make_index_sequence<0> : index_sequence<> {}; +template<> struct make_index_sequence<1> : index_sequence<0> {}; + +template +using index_sequence_for = make_index_sequence; + +// dispatch utility (taken from ranges-v3) +template struct priority_tag : priority_tag < N - 1 > {}; +template<> struct priority_tag<0> {}; + +// taken from ranges-v3 +template +struct static_const +{ + static constexpr T value{}; +}; + +template +constexpr T static_const::value; +} // namespace detail +} // namespace nlohmann + +// #include + + +#include // not +#include // numeric_limits +#include // false_type, is_constructible, is_integral, is_same, true_type +#include // declval + +// #include + + +#include // random_access_iterator_tag + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template struct make_void +{ + using type = void; +}; +template using void_t = typename make_void::type; +} // namespace detail +} // namespace nlohmann + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template +struct iterator_types {}; + +template +struct iterator_types < + It, + void_t> +{ + using difference_type = typename It::difference_type; + using value_type = typename It::value_type; + using pointer = typename It::pointer; + using reference = typename It::reference; + using iterator_category = typename It::iterator_category; +}; + +// This is required as some compilers implement std::iterator_traits in a way that +// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. +template +struct iterator_traits +{ +}; + +template +struct iterator_traits < T, enable_if_t < !std::is_pointer::value >> + : iterator_types +{ +}; + +template +struct iterator_traits::value>> +{ + using iterator_category = std::random_access_iterator_tag; + using value_type = T; + using difference_type = ptrdiff_t; + using pointer = T*; + using reference = T&; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + +// #include + + +#include + +// #include + + +// http://en.cppreference.com/w/cpp/experimental/is_detected +namespace nlohmann +{ +namespace detail +{ +struct nonesuch +{ + nonesuch() = delete; + ~nonesuch() = delete; + nonesuch(nonesuch const&) = delete; + nonesuch(nonesuch const&&) = delete; + void operator=(nonesuch const&) = delete; + void operator=(nonesuch&&) = delete; +}; + +template class Op, + class... Args> +struct detector +{ + using value_t = std::false_type; + using type = Default; +}; + +template class Op, class... Args> +struct detector>, Op, Args...> +{ + using value_t = std::true_type; + using type = Op; +}; + +template