@@ -1024,31 +1024,27 @@ def parse_all(self, states: list[State], post_parse: bool = True) -> None:
10241024 self .post_parse_all (states )
10251025 return
10261026
1027- sequential_states = []
1028- parallel_states = []
1027+ parallel_states = [state for state in states if state .tree is None ]
10291028 for state in states :
10301029 if state .tree is not None :
10311030 # The file was already parsed.
1031+ state .needs_parse = False
10321032 continue
10331033 parallel_states .append (state )
10341034 if len (parallel_states ) > 1 :
1035- self .parse_parallel (sequential_states , parallel_states )
1036- else :
1037- # Avoid using executor when there is no parallelism.
1038- for state in states :
1039- state .parse_file ()
1035+ self .parse_parallel (parallel_states )
10401036 if post_parse :
10411037 self .post_parse_all (states )
10421038
1043- def parse_parallel (self , sequential_states : list [ State ], parallel_states : list [State ]) -> None :
1039+ def parse_parallel (self , parallel_states : list [State ]) -> None :
10441040 """Perform parallel parsing of states.
10451041
10461042 Note: this duplicates a bit of logic from State.parse_file(). This is done
10471043 as an optimization to parallelize only those parts of the code that can be
10481044 parallelized efficiently.
10491045 """
10501046 parallel_parsed_states , parallel_parsed_states_set = self .parse_files_threaded_raw (
1051- sequential_states , parallel_states
1047+ parallel_states
10521048 )
10531049
10541050 for state in parallel_parsed_states :
@@ -1093,12 +1089,9 @@ def parse_parallel(self, sequential_states: list[State], parallel_states: list[S
10931089 state .check_blockers ()
10941090 state .setup_errors ()
10951091
1096- def parse_files_threaded_raw (
1097- self , sequential_states : list [State ], parallel_states : list [State ]
1098- ) -> tuple [list [State ], set [State ]]:
1099- """Parse files using a thread pool.
1092+ def parse_files_threaded_raw (self , states : list [State ]) -> tuple [list [State ], set [State ]]:
1093+ """Parse files in parallel using a thread pool.
11001094
1101- Also parse sequential states while waiting for the parallel results.
11021095 Trees from the new parser are left in raw (serialized) form.
11031096
11041097 Return (list, set) of states that were actually parsed (not cached).
@@ -1114,7 +1107,7 @@ def parse_files_threaded_raw(
11141107 # parse_file_inner() results in no visible improvement with more than 8 threads.
11151108 # TODO: reuse thread pool and/or batch small files in single submit() call.
11161109 with ThreadPoolExecutor (max_workers = min (available_threads , 8 )) as executor :
1117- for state in parallel_states :
1110+ for state in states :
11181111 state .needs_parse = False
11191112 if state .id not in self .ast_cache :
11201113 self .log (f"Parsing { state .xpath } ({ state .id } )" )
@@ -1129,10 +1122,6 @@ def parse_files_threaded_raw(
11291122 state .tree , state .early_errors , source_hash = self .ast_cache [state .id ]
11301123 state .source_hash = source_hash
11311124
1132- # Parse sequential before waiting on parallel.
1133- for state in sequential_states :
1134- state .parse_file ()
1135-
11361125 for fut in wait (futures ).done :
11371126 fut .result ()
11381127
0 commit comments