From 489f8d3e6a60eb55ba01244600662dcb769fc0aa Mon Sep 17 00:00:00 2001 From: Franz Haas Date: Wed, 2 Oct 2024 21:46:14 +0200 Subject: [PATCH 1/4] - compiling GreedyRange --- construct/core.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/construct/core.py b/construct/core.py index 8f5919f0..b4fcf05c 100644 --- a/construct/core.py +++ b/construct/core.py @@ -2633,6 +2633,36 @@ def _sizeof(self, context, path): def _emitfulltype(self, ksy, bitwise): return dict(type=self.subcon._compileprimitivetype(ksy, bitwise), repeat="eos") + def _emitparse(self, code): + fname = f"parse_greadyrange_{code.allocateId()}" + block = f""" + def {fname}(io, this): + list_ = ListContainer() + while True: + try: + obj_ = {self.subcon._compileparse(code)} + if not ({self.discard}): + list_.append(obj_) + except (struct.error, StreamError) as e: + return list_ + """ + code.append(block) + return f"{fname}(io, this)" + + def _emitbuild(self, code): + fname = f"build_greadyrange_{code.allocateId()}" + block = f""" + def {fname}(obj, io, this): + objiter = iter(obj) + list_ = ListContainer() + for item in objiter : + obj_ = reuse(item, lambda obj: {self.subcon._compilebuild(code)}) + list_.append(obj_) + return list_ + """ + code.append(block) + return f"{fname}(obj, io, this)" + class RepeatUntil(Subconstruct): r""" From b5080829489bacdfef537b12aa201ad457338074 Mon Sep 17 00:00:00 2001 From: Franz Haas Date: Sat, 5 Oct 2024 01:00:48 +0200 Subject: [PATCH 2/4] - fixed handling of StopFieldError in GreedyRanges --- construct/core.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/construct/core.py b/construct/core.py index b4fcf05c..62da02ad 100644 --- a/construct/core.py +++ b/construct/core.py @@ -2638,17 +2638,23 @@ def _emitparse(self, code): block = f""" def {fname}(io, this): list_ = ListContainer() - while True: - try: + try: + while True: + fallback = io.tell() obj_ = {self.subcon._compileparse(code)} if not ({self.discard}): list_.append(obj_) - except (struct.error, StreamError) as e: - return list_ + except StopFieldError: + pass + except ExplicitError: + raise + except Exception: + io.seek(fallback) + return list_ """ code.append(block) return f"{fname}(io, this)" - + def _emitbuild(self, code): fname = f"build_greadyrange_{code.allocateId()}" block = f""" From 12f0f340b1608734ff1ef75958c5936180a7564e Mon Sep 17 00:00:00 2001 From: Franz Haas Date: Mon, 7 Oct 2024 20:38:18 +0200 Subject: [PATCH 3/4] - optimised greedyrange --- construct/core.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/construct/core.py b/construct/core.py index 62da02ad..31f00d44 100644 --- a/construct/core.py +++ b/construct/core.py @@ -2634,23 +2634,23 @@ def _emitfulltype(self, ksy, bitwise): return dict(type=self.subcon._compileprimitivetype(ksy, bitwise), repeat="eos") def _emitparse(self, code): - fname = f"parse_greadyrange_{code.allocateId()}" + fname = f"parse_greedyrange_{code.allocateId()}" + work_item = {False: f"list_.append({self.subcon._compileparse(code)})", + True: f"{self.subcon._compileparse(code)}"}[self.discard] block = f""" def {fname}(io, this): - list_ = ListContainer() + list_ = [] try: while True: fallback = io.tell() - obj_ = {self.subcon._compileparse(code)} - if not ({self.discard}): - list_.append(obj_) + {work_item} except StopFieldError: pass except ExplicitError: raise except Exception: io.seek(fallback) - return list_ + return ListContainer(list_) """ code.append(block) return f"{fname}(io, this)" From c00555f31c361648e2a88d0f6e33ffcb5f9b0ed8 Mon Sep 17 00:00:00 2001 From: Franz Haas Date: Mon, 7 Oct 2024 21:12:55 +0200 Subject: [PATCH 4/4] - further compacted compiling greedyrange --- construct/core.py | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/construct/core.py b/construct/core.py index 31f00d44..d335efdd 100644 --- a/construct/core.py +++ b/construct/core.py @@ -2635,11 +2635,10 @@ def _emitfulltype(self, ksy, bitwise): def _emitparse(self, code): fname = f"parse_greedyrange_{code.allocateId()}" - work_item = {False: f"list_.append({self.subcon._compileparse(code)})", + work_item = {False: f"yield {self.subcon._compileparse(code)}", True: f"{self.subcon._compileparse(code)}"}[self.discard] block = f""" def {fname}(io, this): - list_ = [] try: while True: fallback = io.tell() @@ -2650,24 +2649,15 @@ def {fname}(io, this): raise except Exception: io.seek(fallback) - return ListContainer(list_) """ code.append(block) - return f"{fname}(io, this)" + if False == self.discard: + return f"ListContainer({fname}(io, this))" + else: + return f"({fname}(io, this), ListContainer())[1]" def _emitbuild(self, code): - fname = f"build_greadyrange_{code.allocateId()}" - block = f""" - def {fname}(obj, io, this): - objiter = iter(obj) - list_ = ListContainer() - for item in objiter : - obj_ = reuse(item, lambda obj: {self.subcon._compilebuild(code)}) - list_.append(obj_) - return list_ - """ - code.append(block) - return f"{fname}(obj, io, this)" + return f"ListContainer([{self.subcon._compilebuild(code)} for obj in iter(obj)])" class RepeatUntil(Subconstruct):