File tree 1 file changed +7
-10
lines changed
Filter options
1 file changed +7
-10
lines changed
Original file line number Diff line number Diff line change @@ -62,21 +62,18 @@ def getTreeWalker(treeType, implementation=None, **kwargs):
62
62
63
63
64
64
def concatenateCharacterTokens (tokens ):
65
- charactersToken = None
65
+ pendingCharacters = []
66
66
for token in tokens :
67
67
type = token ["type" ]
68
68
if type in ("Characters" , "SpaceCharacters" ):
69
- if charactersToken is None :
70
- charactersToken = {"type" : "Characters" , "data" : token ["data" ]}
71
- else :
72
- charactersToken ["data" ] += token ["data" ]
69
+ pendingCharacters .append (token ["data" ])
73
70
else :
74
- if charactersToken is not None :
75
- yield charactersToken
76
- charactersToken = None
71
+ if pendingCharacters :
72
+ yield { "type" : "Characters" , "data" : "" . join ( pendingCharacters )}
73
+ pendingCharacters = []
77
74
yield token
78
- if charactersToken is not None :
79
- yield charactersToken
75
+ if pendingCharacters :
76
+ yield { "type" : "Characters" , "data" : "" . join ( pendingCharacters )}
80
77
81
78
82
79
def pprint (tokens ):
You can’t perform that action at this time.
0 commit comments