From 3b670c8e2940c29df6d5b4e8f7495e43ac056bec Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 30 Apr 2022 22:10:25 +0900 Subject: [PATCH 001/138] lexer: define token type --- lexer/token.go | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 lexer/token.go diff --git a/lexer/token.go b/lexer/token.go new file mode 100644 index 0000000..931de38 --- /dev/null +++ b/lexer/token.go @@ -0,0 +1,94 @@ +package lexer + +type TokenType int + +const ( + // Misc characters + Invalid TokenType = iota + Hat + Tilde + TGREATER + TLESS + Equal + Colon + ParenL + ParenR + BraceL + BraceR + SEMICOLON + + // Keyword + Pragma + Constract + Function + Pubilc + Pure + Returns + Return +) + +func asMiscToken(r rune) TokenType { + switch r { + case '^': + return Hat + case '~': + return Tilde + case '<': + return TLESS + case '>': + return TGREATER + case '=': + return Equal + case ':': + return Colon + case ';': + return SEMICOLON + case '(': + return ParenL + case ')': + return ParenR + case '{': + return BraceL + case '}': + return BraceR + } + + return Invalid +} + +func asKeyword(str string) TokenType { + switch str { + case ">": + return TGREATER + case "=": + return Equal + case "pragma": + return Pragma + case "constract": + return Constract + case "function": + return Function + case "pubilc": + return Pubilc + case "pure": + return Pure + case "returns": + return Returns + case "return": + return Return + } + + return Invalid +} + +type Token struct { + TokenType TokenType + Text string +} + +func NewToken(ch string) Token { + return Token{ + TokenType: asKeyword(ch), + Text: ch, + } +} From b487b41e296d6c12770636a2885f0a2064466231 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 30 Apr 2022 23:21:18 +0900 Subject: [PATCH 002/138] go.mod: import go-cmp --- go.mod | 2 ++ go.sum | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 go.sum diff --git a/go.mod b/go.mod index 0189534..7884694 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module github.com/uji/solparser go 1.16 + +require github.com/google/go-cmp v0.5.8 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..e9b099c --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= From ab0169c933f3e44f5ca513d61cbbc1654c0b49d4 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 30 Apr 2022 23:22:19 +0900 Subject: [PATCH 003/138] lexer: implement lexer --- lexer/lexer.go | 102 ++++++++++++++++++++++++++++++++++++++++++++ lexer/lexer_test.go | 64 +++++++++++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 lexer/lexer.go create mode 100644 lexer/lexer_test.go diff --git a/lexer/lexer.go b/lexer/lexer.go new file mode 100644 index 0000000..074c3c9 --- /dev/null +++ b/lexer/lexer.go @@ -0,0 +1,102 @@ +package lexer + +import ( + "bufio" + "io" + "unicode/utf8" +) + +type Lexer struct { + scanner *bufio.Scanner + token Token +} + +// isSpace reports whether the character is a Unicode white space character. +// We avoid dependency on the unicode package, but check validity of the implementation +// in the tests. +func isSpace(r rune) bool { + if r <= '\u00FF' { + // Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs. + switch r { + case ' ', '\t', '\n', '\v', '\f', '\r': + return true + case '\u0085', '\u00A0': + return true + } + return false + } + // High-valued ones. + if '\u2000' <= r && r <= '\u200a' { + return true + } + switch r { + case '\u1680', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000': + return true + } + return false +} + +func isMiscToken(r rune) bool { + return asMiscToken(r) != Invalid +} + +func ScanTokens(data []byte, atEOF bool) (advance int, token []byte, err error) { + // Skip leading spaces. + start := 0 + for width := 0; start < len(data); start += width { + var r rune + r, width = utf8.DecodeRune(data[start:]) + if !isSpace(r) { + break + } + } + // Return misk token. + r, width := utf8.DecodeRune(data[start:]) + if isMiscToken(r) { + return start + width, data[start : start+width], nil + } + // Scan until space, marking end of word. + for width, i := 0, start; i < len(data); i += width { + var r rune + r, width = utf8.DecodeRune(data[i:]) + if isMiscToken(r) { + return i, data[start:i], nil + } + if isSpace(r) { + return i + width, data[start:i], nil + } + } + // If we're at EOF, we have a final, non-empty, non-terminated word. Return it. + if atEOF && len(data) > start { + return len(data), data[start:], nil + } + // Request more data. + return start, nil, nil +} + +func New(input io.Reader) *Lexer { + s := bufio.NewScanner(input) + s.Split(ScanTokens) + + return &Lexer{ + scanner: s, + } +} + +func (l *Lexer) Scan() bool { + if !l.scanner.Scan() { + return false + } + + txt := l.scanner.Text() + l.token = Token{ + TokenType: asKeyword(txt), + Text: txt, + } + + return true +} + +func (l Lexer) Token() Token { + return l.token +} diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go new file mode 100644 index 0000000..06540d9 --- /dev/null +++ b/lexer/lexer_test.go @@ -0,0 +1,64 @@ +package lexer + +import ( + "bufio" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +// Test that the token splitter. +func TestScanTokens(t *testing.T) { + cases := []struct { + input string + want []string + }{ + {"", []string{}}, + {" ", []string{}}, + {"\n", []string{}}, + {"a", []string{"a"}}, + {" a ", []string{"a"}}, + {"abc def", []string{"abc", "def"}}, + {" abc def ", []string{"abc", "def"}}, + { + " abc\tdef\nghi\rjkl\fmno\vpqr\u0085stu\u00a0\n", + []string{ + "abc", + "def", + "ghi", + "jkl", + "mno", + "pqr", + "stu", + }, + }, + {"^0.8.13", []string{"^", "0.8.13"}}, + {"0.8.13;", []string{"0.8.13", ";"}}, + {"pragma solidity ^0.8.13;", []string{"pragma", "solidity", "^", "0.8.13", ";"}}, + } + + for n, c := range cases { + buf := strings.NewReader(c.input) + s := bufio.NewScanner(buf) + s.Split(ScanTokens) + + got := make([]string, 0, len(c.want)) + for i := 0; i < len(c.want); i++ { + if !s.Scan() { + break + } + got = append(got, s.Text()) + } + if s.Scan() { + t.Errorf("#%d: scan ran too long, got %q", n, s.Text()) + } + if diff := cmp.Diff(c.want, got); diff != "" { + t.Errorf("#%d: %s", n, diff) + } + err := s.Err() + if err != nil { + t.Errorf("#%d: %v", n, err) + } + } +} From 7eb4b9bd04b14ae530285ce295ad52df6826894d Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 1 May 2022 01:07:39 +0900 Subject: [PATCH 004/138] lexer: refactoring --- lexer/lexer.go | 12 ++++++++-- lexer/lexer_test.go | 56 +++++++++++++++++++++++++++++++++------------ lexer/token.go | 29 +++++++++++++---------- 3 files changed, 69 insertions(+), 28 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 074c3c9..22d63ad 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -9,6 +9,7 @@ import ( type Lexer struct { scanner *bufio.Scanner token Token + err error } // isSpace reports whether the character is a Unicode white space character. @@ -37,7 +38,7 @@ func isSpace(r rune) bool { } func isMiscToken(r rune) bool { - return asMiscToken(r) != Invalid + return asMiscToken(r) != Unknown } func ScanTokens(data []byte, atEOF bool) (advance int, token []byte, err error) { @@ -87,10 +88,13 @@ func (l *Lexer) Scan() bool { if !l.scanner.Scan() { return false } + if err := l.scanner.Err(); err != nil { + l.err = err + } txt := l.scanner.Text() l.token = Token{ - TokenType: asKeyword(txt), + TokenType: asToken(txt), Text: txt, } @@ -100,3 +104,7 @@ func (l *Lexer) Scan() bool { func (l Lexer) Token() Token { return l.token } + +func (l Lexer) Error() error { + return l.err +} diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 06540d9..106366a 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -1,4 +1,4 @@ -package lexer +package lexer_test import ( "bufio" @@ -6,6 +6,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/uji/solparser/lexer" ) // Test that the token splitter. @@ -21,27 +22,17 @@ func TestScanTokens(t *testing.T) { {" a ", []string{"a"}}, {"abc def", []string{"abc", "def"}}, {" abc def ", []string{"abc", "def"}}, - { - " abc\tdef\nghi\rjkl\fmno\vpqr\u0085stu\u00a0\n", - []string{ - "abc", - "def", - "ghi", - "jkl", - "mno", - "pqr", - "stu", - }, - }, + {" a\tb\nc\rd\fe\vf\u0085g\u00a0\n", []string{"a", "b", "c", "d", "e", "f", "g"}}, {"^0.8.13", []string{"^", "0.8.13"}}, {"0.8.13;", []string{"0.8.13", ";"}}, {"pragma solidity ^0.8.13;", []string{"pragma", "solidity", "^", "0.8.13", ";"}}, + {"contract HelloWorld {", []string{"contract", "HelloWorld", "{"}}, } for n, c := range cases { buf := strings.NewReader(c.input) s := bufio.NewScanner(buf) - s.Split(ScanTokens) + s.Split(lexer.ScanTokens) got := make([]string, 0, len(c.want)) for i := 0; i < len(c.want); i++ { @@ -62,3 +53,40 @@ func TestScanTokens(t *testing.T) { } } } + +func TestLexer_Scan(t *testing.T) { + cases := []struct { + input string + want []lexer.Token + }{ + { + input: "pragma solidity ^0.8.13;", + want: []lexer.Token{ + {lexer.Pragma, "pragma"}, + {lexer.Unknown, "solidity"}, + {lexer.Hat, "^"}, + {lexer.Unknown, "0.8.13"}, + {lexer.Semicolon, ";"}, + }, + }, + } + + for n, c := range cases { + buf := strings.NewReader(c.input) + l := lexer.New(buf) + got := make([]lexer.Token, 0, len(c.want)) + for i := 0; i < len(c.want); i++ { + l.Scan() + got = append(got, l.Token()) + } + if l.Scan() { + t.Errorf("#%d: scan ran too long, got %q", n, got) + } + if diff := cmp.Diff(c.want, got); diff != "" { + t.Errorf("#%d: %s", n, diff) + } + if err := l.Error(); err != nil { + t.Errorf("#%d: %v", n, err) + } + } +} diff --git a/lexer/token.go b/lexer/token.go index 931de38..6a334a6 100644 --- a/lexer/token.go +++ b/lexer/token.go @@ -5,17 +5,18 @@ type TokenType int const ( // Misc characters Invalid TokenType = iota + Unknown Hat Tilde - TGREATER - TLESS + Greater + Less Equal Colon ParenL ParenR BraceL BraceR - SEMICOLON + Semicolon // Keyword Pragma @@ -34,15 +35,15 @@ func asMiscToken(r rune) TokenType { case '~': return Tilde case '<': - return TLESS + return Less case '>': - return TGREATER + return Greater case '=': return Equal case ':': return Colon case ';': - return SEMICOLON + return Semicolon case '(': return ParenL case ')': @@ -53,15 +54,11 @@ func asMiscToken(r rune) TokenType { return BraceR } - return Invalid + return Unknown } func asKeyword(str string) TokenType { switch str { - case ">": - return TGREATER - case "=": - return Equal case "pragma": return Pragma case "constract": @@ -78,7 +75,15 @@ func asKeyword(str string) TokenType { return Return } - return Invalid + return Unknown +} + +func asToken(str string) TokenType { + r := []rune(str) + if len(r) != 1 { + return asKeyword(str) + } + return asMiscToken(r[0]) } type Token struct { From 3329f10513008b640b0b5bb2d0864a77038a61a0 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 1 May 2022 01:29:10 +0900 Subject: [PATCH 005/138] ast: publish ast structs --- ast/ast.go | 49 +++++++++++++++++++++++-------------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 9fecdec..f29a579 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -1,62 +1,59 @@ package ast -type version string -type expression string - type pragmaValue struct { version string - expression expression + expression string } -type pragmaDirective struct { +type PragmaDirective struct { pragmaName string pragmaValue pragmaValue } -type functionDescriptor struct { +type FunctionDescriptor struct { name string } -type stateMutability struct { +type StateMutability struct { pure bool } -type modifierList struct { - stateMutability stateMutability +type ModifierList struct { + stateMutability *StateMutability } -type typeName struct { +type TypeName struct { elementalyTypeName string } -type eventParameter struct { - typeName typeName +type EventParameter struct { + typeName *TypeName } -type parameterList struct { - eventParameter eventParameter +type ParameterList struct { + eventParameter *EventParameter } -type returnParameters struct { - parameterList parameterList +type ReturnParameters struct { + parameterList *ParameterList } -type functionDefinition struct { - functionDescriptor functionDescriptor - modifierList modifierList - returnParameters returnParameters +type FunctionDefinition struct { + functionDescriptor *FunctionDescriptor + modifierList *ModifierList + returnParameters *ReturnParameters } -type contractPart struct { - functionDefinition functionDefinition +type ContractPart struct { + functionDefinition *FunctionDefinition } -type contractDefinition struct { - contractPart contractPart +type ContractDefinition struct { + contractPart *ContractPart } // A File node represents a Solidity source file. type SourceUnit struct { - pragmaDirective pragmaDirective - contractDefinition contractDefinition + PragmaDirective *PragmaDirective + ContractDefinition *ContractDefinition } From b0e68ce9540a46e462c3d9165540210035451e07 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 1 May 2022 01:30:10 +0900 Subject: [PATCH 006/138] main: define parser struct --- solparser.go | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 solparser.go diff --git a/solparser.go b/solparser.go new file mode 100644 index 0000000..5b72dc4 --- /dev/null +++ b/solparser.go @@ -0,0 +1,46 @@ +package solparser + +import ( + "io" + + "github.com/uji/solparser/ast" + "github.com/uji/solparser/lexer" +) + +type Parser struct { + input io.Reader + lexer *lexer.Lexer +} + +func New(input io.Reader) *Parser { + return &Parser{ + input: input, + } +} + +func (p *Parser) Parse(input io.Reader) (*ast.SourceUnit, error) { + pragmaDirective, err := p.parsePragmaDirective() + if err != nil { + return nil, err + } + + contractDefinition, err := p.parseContractDefinition() + if err != nil { + return nil, err + } + + return &ast.SourceUnit{ + PragmaDirective: pragmaDirective, + ContractDefinition: contractDefinition, + }, nil +} + +func (p *Parser) parsePragmaDirective() (*ast.PragmaDirective, error) { + // pragma ~ のパース + return nil, nil +} + +func (p *Parser) parseContractDefinition() (*ast.ContractDefinition, error) { + // contract ~ のパース + return nil, nil +} From 3312a1eb5216fadb60d2225c8225e37177a37c3a Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 1 May 2022 15:16:11 +0900 Subject: [PATCH 007/138] ast: publish PragmaDirectiveField --- ast/ast.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index f29a579..56064ab 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -1,13 +1,13 @@ package ast -type pragmaValue struct { - version string - expression string +type PragmaValue struct { + Version string + Expression string } type PragmaDirective struct { - pragmaName string - pragmaValue pragmaValue + PragmaName string + PragmaValue PragmaValue } type FunctionDescriptor struct { From 9aa7eeabb6ec2e1ed566bd2511e0f515f40b9068 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 1 May 2022 15:16:38 +0900 Subject: [PATCH 008/138] main: implement ParsePragmaDirective --- solparser.go | 42 +++++++++++++++++++++++++++++++++++++++--- solparser_test.go | 45 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 solparser_test.go diff --git a/solparser.go b/solparser.go index 5b72dc4..09a619b 100644 --- a/solparser.go +++ b/solparser.go @@ -1,6 +1,7 @@ package solparser import ( + "errors" "io" "github.com/uji/solparser/ast" @@ -15,11 +16,12 @@ type Parser struct { func New(input io.Reader) *Parser { return &Parser{ input: input, + lexer: lexer.New(input), } } func (p *Parser) Parse(input io.Reader) (*ast.SourceUnit, error) { - pragmaDirective, err := p.parsePragmaDirective() + pragmaDirective, err := p.ParsePragmaDirective() if err != nil { return nil, err } @@ -35,9 +37,43 @@ func (p *Parser) Parse(input io.Reader) (*ast.SourceUnit, error) { }, nil } -func (p *Parser) parsePragmaDirective() (*ast.PragmaDirective, error) { +func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { + p.lexer.Scan() + if p.lexer.Token().TokenType != lexer.Pragma { + return nil, errors.New("not found pragma") + } + + p.lexer.Scan() + pragmaName := p.lexer.Token() + if pragmaName.TokenType != lexer.Unknown { + return nil, errors.New("not found unkknown") + } + + p.lexer.Scan() + expression := p.lexer.Token() + if expression.TokenType != lexer.Hat { + return nil, errors.New("not found expression") + } + + p.lexer.Scan() + version := p.lexer.Token() + if version.TokenType != lexer.Unknown { + return nil, errors.New("not found version") + } + + p.lexer.Scan() + if p.lexer.Token().TokenType != lexer.Semicolon { + return nil, errors.New("not found semicolon") + } + // pragma ~ のパース - return nil, nil + return &ast.PragmaDirective{ + PragmaName: pragmaName.Text, + PragmaValue: ast.PragmaValue{ + Version: version.Text, + Expression: expression.Text, + }, + }, nil } func (p *Parser) parseContractDefinition() (*ast.ContractDefinition, error) { diff --git a/solparser_test.go b/solparser_test.go new file mode 100644 index 0000000..8486656 --- /dev/null +++ b/solparser_test.go @@ -0,0 +1,45 @@ +package solparser_test + +import ( + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" + "github.com/uji/solparser/ast" +) + +func TestParserParsePragmaDirective(t *testing.T) { + cases := []struct { + input string + want *ast.PragmaDirective + err error + }{ + { + input: "pragma solidity ^0.8.13;", + want: &ast.PragmaDirective{ + PragmaName: "solidity", + PragmaValue: ast.PragmaValue{ + Version: "0.8.13", + Expression: "^", + }, + }, + err: nil, + }, + } + + for n, c := range cases { + r := strings.NewReader(c.input) + p := solparser.New(r) + got, err := p.ParsePragmaDirective() + if err != c.err { + t.Errorf("#%d unexpected err want: %s, got: %s", n, c.err, err) + } + if err != nil { + break + } + if diff := cmp.Diff(c.want, got); diff != "" { + t.Errorf("#%d want: %s, got: %s", n, c.want, got) + } + } +} From c505c4d0be2c4265a202a2f7869823f139679b1a Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 1 May 2022 15:46:53 +0900 Subject: [PATCH 009/138] ast: publish all struct field --- ast/ast.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 56064ab..ab197e9 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -11,45 +11,45 @@ type PragmaDirective struct { } type FunctionDescriptor struct { - name string + Name string } type StateMutability struct { - pure bool + Pure bool } type ModifierList struct { - stateMutability *StateMutability + StateMutability *StateMutability } type TypeName struct { - elementalyTypeName string + ElementalyTypeName string } type EventParameter struct { - typeName *TypeName + TypeName *TypeName } type ParameterList struct { - eventParameter *EventParameter + EventParameter *EventParameter } type ReturnParameters struct { - parameterList *ParameterList + ParameterList *ParameterList } type FunctionDefinition struct { - functionDescriptor *FunctionDescriptor - modifierList *ModifierList - returnParameters *ReturnParameters + FunctionDescriptor *FunctionDescriptor + ModifierList *ModifierList + ReturnParameters *ReturnParameters } type ContractPart struct { - functionDefinition *FunctionDefinition + FunctionDefinition *FunctionDefinition } type ContractDefinition struct { - contractPart *ContractPart + ContractPart *ContractPart } // A File node represents a Solidity source file. From ff69e77942cc35b367b4b02f4b43daf252bb9d0c Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 1 May 2022 22:48:06 +0900 Subject: [PATCH 010/138] main: implement ParseContractDefinition --- solparser.go | 52 ++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/solparser.go b/solparser.go index 09a619b..9c7c5ce 100644 --- a/solparser.go +++ b/solparser.go @@ -26,7 +26,7 @@ func (p *Parser) Parse(input io.Reader) (*ast.SourceUnit, error) { return nil, err } - contractDefinition, err := p.parseContractDefinition() + contractDefinition, err := p.ParseContractDefinition() if err != nil { return nil, err } @@ -66,7 +66,6 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { return nil, errors.New("not found semicolon") } - // pragma ~ のパース return &ast.PragmaDirective{ PragmaName: pragmaName.Text, PragmaValue: ast.PragmaValue{ @@ -76,7 +75,52 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { }, nil } -func (p *Parser) parseContractDefinition() (*ast.ContractDefinition, error) { - // contract ~ のパース +func (p *Parser) ParseModirierList() (*ast.ModifierList, error) { return nil, nil } + +func (p *Parser) ParseReturnParameters() (*ast.ReturnParameters, error) { + return nil, nil +} + +func (p *Parser) ParseFunctionDefinition() (*ast.FunctionDefinition, error) { + return nil, nil +} + +func (p *Parser) ParseContractPart() (*ast.ContractPart, error) { + funcDef, err := p.ParseFunctionDefinition() + if err != nil { + return nil, err + } + + return &ast.ContractPart{ + FunctionDefinition: funcDef, + }, nil +} + +func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { + p.lexer.Scan() + keyward := p.lexer.Token() + if keyward.TokenType != lexer.Constract { + return nil, errors.New("not found contract definition") + } + + p.lexer.Scan() + if p.lexer.Token().TokenType != lexer.BraceL { + return nil, errors.New("not found left brace") + } + + part, err := p.ParseContractPart() + if err != nil { + return nil, err + } + + p.lexer.Scan() + if p.lexer.Token().TokenType != lexer.BraceR { + return nil, errors.New("not found right brace") + } + + return &ast.ContractDefinition{ + ContractPart: part, + }, nil +} From d1324c91fd8561eb33fbe23d272476312efa8bb9 Mon Sep 17 00:00:00 2001 From: uji Date: Thu, 5 May 2022 23:54:26 +0900 Subject: [PATCH 011/138] lexer: define Position struct --- lexer/token.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lexer/token.go b/lexer/token.go index 6a334a6..2bb825e 100644 --- a/lexer/token.go +++ b/lexer/token.go @@ -97,3 +97,8 @@ func NewToken(ch string) Token { Text: ch, } } + +type Position struct { + Start int + Size int +} From 2034f59c4c0af629735721aeefd6e74e0bf574d9 Mon Sep 17 00:00:00 2001 From: uji Date: Fri, 13 May 2022 14:17:58 +0900 Subject: [PATCH 012/138] lexer: add line to Position --- lexer/token.go | 1 + 1 file changed, 1 insertion(+) diff --git a/lexer/token.go b/lexer/token.go index 2bb825e..81c2209 100644 --- a/lexer/token.go +++ b/lexer/token.go @@ -101,4 +101,5 @@ func NewToken(ch string) Token { type Position struct { Start int Size int + Line int } From f1472f1a6e79b66fa5f3c32f3a1c9ac39ab53740 Mon Sep 17 00:00:00 2001 From: uji Date: Fri, 13 May 2022 14:20:38 +0900 Subject: [PATCH 013/138] lexer: add Position to Token struct --- lexer/token.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lexer/token.go b/lexer/token.go index 81c2209..86f0497 100644 --- a/lexer/token.go +++ b/lexer/token.go @@ -89,12 +89,14 @@ func asToken(str string) TokenType { type Token struct { TokenType TokenType Text string + Pos Position } -func NewToken(ch string) Token { +func NewToken(ch string, pos Position) Token { return Token{ TokenType: asKeyword(ch), Text: ch, + Pos: pos, } } From 51f95843a7ffcbbd2797557ea0d2b1f97741c1b0 Mon Sep 17 00:00:00 2001 From: uji Date: Fri, 13 May 2022 15:07:27 +0900 Subject: [PATCH 014/138] lexer: manage scanning offset state --- lexer/lexer.go | 15 ++++++++++++--- lexer/lexer_test.go | 10 +++++----- lexer/token.go | 2 +- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 22d63ad..8864dd5 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -10,6 +10,10 @@ type Lexer struct { scanner *bufio.Scanner token Token err error + + // position state + offset int + lineOffset int } // isSpace reports whether the character is a Unicode white space character. @@ -93,10 +97,15 @@ func (l *Lexer) Scan() bool { } txt := l.scanner.Text() - l.token = Token{ - TokenType: asToken(txt), - Text: txt, + size := len([]rune(txt)) + pos := Position{ + Start: l.offset, + Size: size, + Line: l.lineOffset, } + l.token = NewToken(txt, pos) + + l.offset += size return true } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 106366a..10fc206 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -62,11 +62,11 @@ func TestLexer_Scan(t *testing.T) { { input: "pragma solidity ^0.8.13;", want: []lexer.Token{ - {lexer.Pragma, "pragma"}, - {lexer.Unknown, "solidity"}, - {lexer.Hat, "^"}, - {lexer.Unknown, "0.8.13"}, - {lexer.Semicolon, ";"}, + {lexer.Pragma, "pragma", lexer.Position{Start: 0, Size: 6, Line: 0}}, + {lexer.Unknown, "solidity", lexer.Position{Start: 6, Size: 8, Line: 0}}, + {lexer.Hat, "^", lexer.Position{Start: 14, Size: 1, Line: 0}}, + {lexer.Unknown, "0.8.13", lexer.Position{Start: 15, Size: 6, Line: 0}}, + {lexer.Semicolon, ";", lexer.Position{Start: 21, Size: 1, Line: 0}}, }, }, } diff --git a/lexer/token.go b/lexer/token.go index 86f0497..687280c 100644 --- a/lexer/token.go +++ b/lexer/token.go @@ -94,7 +94,7 @@ type Token struct { func NewToken(ch string, pos Position) Token { return Token{ - TokenType: asKeyword(ch), + TokenType: asToken(ch), Text: ch, Pos: pos, } From c3faef7f26b7b52beeb5f123fef722b66fdc0dd1 Mon Sep 17 00:00:00 2001 From: uji Date: Fri, 13 May 2022 23:53:26 +0900 Subject: [PATCH 015/138] lexer: create scanner.go and move code for scanning --- lexer/lexer.go | 64 ----------------------------------------- lexer/lexer_test.go | 46 ------------------------------ lexer/scanner.go | 66 +++++++++++++++++++++++++++++++++++++++++++ lexer/scanner_test.go | 55 ++++++++++++++++++++++++++++++++++++ 4 files changed, 121 insertions(+), 110 deletions(-) create mode 100644 lexer/scanner.go create mode 100644 lexer/scanner_test.go diff --git a/lexer/lexer.go b/lexer/lexer.go index 8864dd5..3ca3f68 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -3,7 +3,6 @@ package lexer import ( "bufio" "io" - "unicode/utf8" ) type Lexer struct { @@ -16,69 +15,6 @@ type Lexer struct { lineOffset int } -// isSpace reports whether the character is a Unicode white space character. -// We avoid dependency on the unicode package, but check validity of the implementation -// in the tests. -func isSpace(r rune) bool { - if r <= '\u00FF' { - // Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs. - switch r { - case ' ', '\t', '\n', '\v', '\f', '\r': - return true - case '\u0085', '\u00A0': - return true - } - return false - } - // High-valued ones. - if '\u2000' <= r && r <= '\u200a' { - return true - } - switch r { - case '\u1680', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000': - return true - } - return false -} - -func isMiscToken(r rune) bool { - return asMiscToken(r) != Unknown -} - -func ScanTokens(data []byte, atEOF bool) (advance int, token []byte, err error) { - // Skip leading spaces. - start := 0 - for width := 0; start < len(data); start += width { - var r rune - r, width = utf8.DecodeRune(data[start:]) - if !isSpace(r) { - break - } - } - // Return misk token. - r, width := utf8.DecodeRune(data[start:]) - if isMiscToken(r) { - return start + width, data[start : start+width], nil - } - // Scan until space, marking end of word. - for width, i := 0, start; i < len(data); i += width { - var r rune - r, width = utf8.DecodeRune(data[i:]) - if isMiscToken(r) { - return i, data[start:i], nil - } - if isSpace(r) { - return i + width, data[start:i], nil - } - } - // If we're at EOF, we have a final, non-empty, non-terminated word. Return it. - if atEOF && len(data) > start { - return len(data), data[start:], nil - } - // Request more data. - return start, nil, nil -} - func New(input io.Reader) *Lexer { s := bufio.NewScanner(input) s.Split(ScanTokens) diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 10fc206..9e761ef 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -1,7 +1,6 @@ package lexer_test import ( - "bufio" "strings" "testing" @@ -9,51 +8,6 @@ import ( "github.com/uji/solparser/lexer" ) -// Test that the token splitter. -func TestScanTokens(t *testing.T) { - cases := []struct { - input string - want []string - }{ - {"", []string{}}, - {" ", []string{}}, - {"\n", []string{}}, - {"a", []string{"a"}}, - {" a ", []string{"a"}}, - {"abc def", []string{"abc", "def"}}, - {" abc def ", []string{"abc", "def"}}, - {" a\tb\nc\rd\fe\vf\u0085g\u00a0\n", []string{"a", "b", "c", "d", "e", "f", "g"}}, - {"^0.8.13", []string{"^", "0.8.13"}}, - {"0.8.13;", []string{"0.8.13", ";"}}, - {"pragma solidity ^0.8.13;", []string{"pragma", "solidity", "^", "0.8.13", ";"}}, - {"contract HelloWorld {", []string{"contract", "HelloWorld", "{"}}, - } - - for n, c := range cases { - buf := strings.NewReader(c.input) - s := bufio.NewScanner(buf) - s.Split(lexer.ScanTokens) - - got := make([]string, 0, len(c.want)) - for i := 0; i < len(c.want); i++ { - if !s.Scan() { - break - } - got = append(got, s.Text()) - } - if s.Scan() { - t.Errorf("#%d: scan ran too long, got %q", n, s.Text()) - } - if diff := cmp.Diff(c.want, got); diff != "" { - t.Errorf("#%d: %s", n, diff) - } - err := s.Err() - if err != nil { - t.Errorf("#%d: %v", n, err) - } - } -} - func TestLexer_Scan(t *testing.T) { cases := []struct { input string diff --git a/lexer/scanner.go b/lexer/scanner.go new file mode 100644 index 0000000..87fc0ec --- /dev/null +++ b/lexer/scanner.go @@ -0,0 +1,66 @@ +package lexer + +import "unicode/utf8" + +// isSpace reports whether the character is a Unicode white space character. +// We avoid dependency on the unicode package, but check validity of the implementation +// in the tests. +func isSpace(r rune) bool { + if r <= '\u00FF' { + // Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs. + switch r { + case ' ', '\t', '\n', '\v', '\f', '\r': + return true + case '\u0085', '\u00A0': + return true + } + return false + } + // High-valued ones. + if '\u2000' <= r && r <= '\u200a' { + return true + } + switch r { + case '\u1680', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000': + return true + } + return false +} + +func isMiscToken(r rune) bool { + return asMiscToken(r) != Unknown +} + +func ScanTokens(data []byte, atEOF bool) (advance int, token []byte, err error) { + // Skip leading spaces. + start := 0 + for width := 0; start < len(data); start += width { + var r rune + r, width = utf8.DecodeRune(data[start:]) + if !isSpace(r) { + break + } + } + // Return misk token. + r, width := utf8.DecodeRune(data[start:]) + if isMiscToken(r) { + return start + width, data[start : start+width], nil + } + // Scan until space, marking end of word. + for width, i := 0, start; i < len(data); i += width { + var r rune + r, width = utf8.DecodeRune(data[i:]) + if isMiscToken(r) { + return i, data[start:i], nil + } + if isSpace(r) { + return i + width, data[start:i], nil + } + } + // If we're at EOF, we have a final, non-empty, non-terminated word. Return it. + if atEOF && len(data) > start { + return len(data), data[start:], nil + } + // Request more data. + return start, nil, nil +} diff --git a/lexer/scanner_test.go b/lexer/scanner_test.go new file mode 100644 index 0000000..b0c8b9d --- /dev/null +++ b/lexer/scanner_test.go @@ -0,0 +1,55 @@ +package lexer_test + +import ( + "bufio" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser/lexer" +) + +// Test that the token splitter. +func TestScanTokens(t *testing.T) { + cases := []struct { + input string + want []string + }{ + {"", []string{}}, + {" ", []string{" "}}, + {"\n", []string{"\n"}}, + {"a", []string{"a"}}, + {" a ", []string{" ", "a", " "}}, + {"abc def ", []string{"abc", " ", "def", " "}}, + {"a\tb\nc\r\td\f", []string{"a", "\t", "b", "\n", "c", "\r\t", "d", "\f"}}, + {"e\vf\u0085g\u00a0\n", []string{"e", "\v", "f", "\u0085", "g", "\u00a0", "\n"}}, + {"^0.8.13", []string{"^", "0.8.13"}}, + {"0.8.13;", []string{"0.8.13", ";"}}, + {"pragma solidity ^0.8.13;", []string{"pragma", "solidity", "^", "0.8.13", ";"}}, + {"contract HelloWorld {", []string{"contract", "HelloWorld", "{"}}, + } + + for n, c := range cases { + buf := strings.NewReader(c.input) + s := bufio.NewScanner(buf) + s.Split(lexer.ScanTokens) + + got := make([]string, 0, len(c.want)) + for i := 0; i < len(c.want); i++ { + if !s.Scan() { + break + } + got = append(got, s.Text()) + } + if s.Scan() { + t.Errorf("#%d: scan ran too long, got %q", n, s.Text()) + } + if diff := cmp.Diff(c.want, got); diff != "" { + t.Errorf("#%d: %s", n, diff) + } + err := s.Err() + if err != nil { + t.Errorf("#%d: %v", n, err) + } + } +} From 40d71689adb7fcc690f6d59dac551599bf233a8b Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 14 May 2022 00:45:45 +0900 Subject: [PATCH 016/138] lexer: scan space for memorizing scanning position --- lexer/lexer_test.go | 8 ++++---- lexer/scanner.go | 24 ++++++++---------------- lexer/scanner_test.go | 4 ++-- 3 files changed, 14 insertions(+), 22 deletions(-) diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 9e761ef..29d45ac 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -17,10 +17,10 @@ func TestLexer_Scan(t *testing.T) { input: "pragma solidity ^0.8.13;", want: []lexer.Token{ {lexer.Pragma, "pragma", lexer.Position{Start: 0, Size: 6, Line: 0}}, - {lexer.Unknown, "solidity", lexer.Position{Start: 6, Size: 8, Line: 0}}, - {lexer.Hat, "^", lexer.Position{Start: 14, Size: 1, Line: 0}}, - {lexer.Unknown, "0.8.13", lexer.Position{Start: 15, Size: 6, Line: 0}}, - {lexer.Semicolon, ";", lexer.Position{Start: 21, Size: 1, Line: 0}}, + {lexer.Unknown, "solidity", lexer.Position{Start: 7, Size: 8, Line: 0}}, + {lexer.Hat, "^", lexer.Position{Start: 16, Size: 1, Line: 0}}, + {lexer.Unknown, "0.8.13", lexer.Position{Start: 17, Size: 6, Line: 0}}, + {lexer.Semicolon, ";", lexer.Position{Start: 23, Size: 1, Line: 0}}, }, }, } diff --git a/lexer/scanner.go b/lexer/scanner.go index 87fc0ec..8e286b0 100644 --- a/lexer/scanner.go +++ b/lexer/scanner.go @@ -9,7 +9,7 @@ func isSpace(r rune) bool { if r <= '\u00FF' { // Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs. switch r { - case ' ', '\t', '\n', '\v', '\f', '\r': + case ' ', '\t', '\v', '\f', '\r': return true case '\u0085', '\u00A0': return true @@ -32,30 +32,22 @@ func isMiscToken(r rune) bool { } func ScanTokens(data []byte, atEOF bool) (advance int, token []byte, err error) { - // Skip leading spaces. start := 0 - for width := 0; start < len(data); start += width { - var r rune - r, width = utf8.DecodeRune(data[start:]) - if !isSpace(r) { - break - } - } - // Return misk token. + + // Return newline code or misk token. r, width := utf8.DecodeRune(data[start:]) - if isMiscToken(r) { + if r == '\n' || isMiscToken(r) { return start + width, data[start : start+width], nil } - // Scan until space, marking end of word. + + tokenIsSpace := isSpace(r) + // Scan until isSpace result changed, marking end of word. for width, i := 0, start; i < len(data); i += width { var r rune r, width = utf8.DecodeRune(data[i:]) - if isMiscToken(r) { + if r == '\n' || isMiscToken(r) || isSpace(r) != tokenIsSpace { return i, data[start:i], nil } - if isSpace(r) { - return i + width, data[start:i], nil - } } // If we're at EOF, we have a final, non-empty, non-terminated word. Return it. if atEOF && len(data) > start { diff --git a/lexer/scanner_test.go b/lexer/scanner_test.go index b0c8b9d..8d277fd 100644 --- a/lexer/scanner_test.go +++ b/lexer/scanner_test.go @@ -25,8 +25,8 @@ func TestScanTokens(t *testing.T) { {"e\vf\u0085g\u00a0\n", []string{"e", "\v", "f", "\u0085", "g", "\u00a0", "\n"}}, {"^0.8.13", []string{"^", "0.8.13"}}, {"0.8.13;", []string{"0.8.13", ";"}}, - {"pragma solidity ^0.8.13;", []string{"pragma", "solidity", "^", "0.8.13", ";"}}, - {"contract HelloWorld {", []string{"contract", "HelloWorld", "{"}}, + {"pragma solidity ^0.8.13;", []string{"pragma", " ", "solidity", " ", "^", "0.8.13", ";"}}, + {"contract HelloWorld {", []string{"contract", " ", "HelloWorld", " ", "{"}}, } for n, c := range cases { From f8ad79cc1fea4bc4decb9e2919e225e9112e64c7 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 14 May 2022 05:06:43 +0900 Subject: [PATCH 017/138] lexer: fix lexer logic --- lexer/lexer.go | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 3ca3f68..4456875 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -3,6 +3,7 @@ package lexer import ( "bufio" "io" + "unicode/utf8" ) type Lexer struct { @@ -24,12 +25,29 @@ func New(input io.Reader) *Lexer { } } -func (l *Lexer) Scan() bool { - if !l.scanner.Scan() { - return false - } - if err := l.scanner.Err(); err != nil { - l.err = err +func (l *Lexer) Scan() (result bool) { + // Scan until next token. + for { + if !l.scanner.Scan() { + return false + } + if err := l.scanner.Err(); err != nil { + l.err = err + } + result = true + + txt := l.scanner.Text() + r, _ := utf8.DecodeRune([]byte(txt)) + if r == '\n' { + l.lineOffset++ + l.offset = 0 + continue + } + if isSpace(r) { + l.offset += len([]rune(txt)) + continue + } + break } txt := l.scanner.Text() From be93f7045920605c20e5e24c2698b2bbfc22ebe6 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 15 May 2022 14:03:21 +0900 Subject: [PATCH 018/138] lexer: handle scanner error --- lexer/lexer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/lexer/lexer.go b/lexer/lexer.go index 4456875..38b1e91 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -33,6 +33,7 @@ func (l *Lexer) Scan() (result bool) { } if err := l.scanner.Err(); err != nil { l.err = err + return false } result = true From c6f59343a209d5aca530ab9d7bd496a27e60fe43 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 15 May 2022 14:32:12 +0900 Subject: [PATCH 019/138] lexer: fix contract keyward type --- lexer/lexer_test.go | 5 ++++- lexer/token.go | 6 +++--- solparser.go | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 29d45ac..b6ee6ff 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -14,13 +14,16 @@ func TestLexer_Scan(t *testing.T) { want []lexer.Token }{ { - input: "pragma solidity ^0.8.13;", + input: "pragma solidity ^0.8.13;\n\ncontract HelloWorld { ", want: []lexer.Token{ {lexer.Pragma, "pragma", lexer.Position{Start: 0, Size: 6, Line: 0}}, {lexer.Unknown, "solidity", lexer.Position{Start: 7, Size: 8, Line: 0}}, {lexer.Hat, "^", lexer.Position{Start: 16, Size: 1, Line: 0}}, {lexer.Unknown, "0.8.13", lexer.Position{Start: 17, Size: 6, Line: 0}}, {lexer.Semicolon, ";", lexer.Position{Start: 23, Size: 1, Line: 0}}, + {lexer.Contract, "contract", lexer.Position{Start: 0, Size: 8, Line: 2}}, + {lexer.Unknown, "HelloWorld", lexer.Position{Start: 9, Size: 10, Line: 2}}, + {lexer.BraceL, "{", lexer.Position{Start: 20, Size: 1, Line: 2}}, }, }, } diff --git a/lexer/token.go b/lexer/token.go index 687280c..e77834f 100644 --- a/lexer/token.go +++ b/lexer/token.go @@ -20,7 +20,7 @@ const ( // Keyword Pragma - Constract + Contract Function Pubilc Pure @@ -61,8 +61,8 @@ func asKeyword(str string) TokenType { switch str { case "pragma": return Pragma - case "constract": - return Constract + case "contract": + return Contract case "function": return Function case "pubilc": diff --git a/solparser.go b/solparser.go index 9c7c5ce..80b0922 100644 --- a/solparser.go +++ b/solparser.go @@ -101,7 +101,7 @@ func (p *Parser) ParseContractPart() (*ast.ContractPart, error) { func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { p.lexer.Scan() keyward := p.lexer.Token() - if keyward.TokenType != lexer.Constract { + if keyward.TokenType != lexer.Contract { return nil, errors.New("not found contract definition") } From e2528b15ef012b271eefae09d8089caed5277c5a Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 15 May 2022 15:44:40 +0900 Subject: [PATCH 020/138] lexer: changed token field from offset to offset + 1 --- lexer/lexer.go | 6 +++--- lexer/lexer_test.go | 16 ++++++++-------- lexer/token.go | 6 +++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 38b1e91..6faa111 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -54,9 +54,9 @@ func (l *Lexer) Scan() (result bool) { txt := l.scanner.Text() size := len([]rune(txt)) pos := Position{ - Start: l.offset, - Size: size, - Line: l.lineOffset, + Column: l.offset + 1, + Size: size, + Line: l.lineOffset + 1, } l.token = NewToken(txt, pos) diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index b6ee6ff..d60b431 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -16,14 +16,14 @@ func TestLexer_Scan(t *testing.T) { { input: "pragma solidity ^0.8.13;\n\ncontract HelloWorld { ", want: []lexer.Token{ - {lexer.Pragma, "pragma", lexer.Position{Start: 0, Size: 6, Line: 0}}, - {lexer.Unknown, "solidity", lexer.Position{Start: 7, Size: 8, Line: 0}}, - {lexer.Hat, "^", lexer.Position{Start: 16, Size: 1, Line: 0}}, - {lexer.Unknown, "0.8.13", lexer.Position{Start: 17, Size: 6, Line: 0}}, - {lexer.Semicolon, ";", lexer.Position{Start: 23, Size: 1, Line: 0}}, - {lexer.Contract, "contract", lexer.Position{Start: 0, Size: 8, Line: 2}}, - {lexer.Unknown, "HelloWorld", lexer.Position{Start: 9, Size: 10, Line: 2}}, - {lexer.BraceL, "{", lexer.Position{Start: 20, Size: 1, Line: 2}}, + {lexer.Pragma, "pragma", lexer.Position{Column: 1, Size: 6, Line: 1}}, + {lexer.Unknown, "solidity", lexer.Position{Column: 8, Size: 8, Line: 1}}, + {lexer.Hat, "^", lexer.Position{Column: 17, Size: 1, Line: 1}}, + {lexer.Unknown, "0.8.13", lexer.Position{Column: 18, Size: 6, Line: 1}}, + {lexer.Semicolon, ";", lexer.Position{Column: 24, Size: 1, Line: 1}}, + {lexer.Contract, "contract", lexer.Position{Column: 1, Size: 8, Line: 3}}, + {lexer.Unknown, "HelloWorld", lexer.Position{Column: 10, Size: 10, Line: 3}}, + {lexer.BraceL, "{", lexer.Position{Column: 21, Size: 1, Line: 3}}, }, }, } diff --git a/lexer/token.go b/lexer/token.go index e77834f..5e40c15 100644 --- a/lexer/token.go +++ b/lexer/token.go @@ -101,7 +101,7 @@ func NewToken(ch string, pos Position) Token { } type Position struct { - Start int - Size int - Line int + Column int + Size int + Line int } From f89f0dcba828f2e9009f29716b2a15304918a712 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 15 May 2022 16:00:52 +0900 Subject: [PATCH 021/138] lexer: imeplement String() in Position --- lexer/token.go | 12 ++++++++++++ lexer/token_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 lexer/token_test.go diff --git a/lexer/token.go b/lexer/token.go index 5e40c15..0149d5b 100644 --- a/lexer/token.go +++ b/lexer/token.go @@ -1,5 +1,7 @@ package lexer +import "fmt" + type TokenType int const ( @@ -105,3 +107,13 @@ type Position struct { Size int Line int } + +func (pos Position) IsValid() bool { return pos.Line > 0 && pos.Column > 0 } + +func (pos Position) String() string { + if !pos.IsValid() { + return "-" + } + + return fmt.Sprintf("%d:%d", pos.Line, pos.Column) +} diff --git a/lexer/token_test.go b/lexer/token_test.go new file mode 100644 index 0000000..9736811 --- /dev/null +++ b/lexer/token_test.go @@ -0,0 +1,24 @@ +package lexer_test + +import ( + "testing" + + "github.com/uji/solparser/lexer" +) + +func TestPosition_String(t *testing.T) { + cases := []struct { + pos lexer.Position + want string + }{ + {lexer.Position{Column: 3, Line: 10}, "10:3"}, + {lexer.Position{Column: 0, Line: 10}, "-"}, + {lexer.Position{Column: 1, Line: 0}, "-"}, + } + + for n, c := range cases { + if c.pos.String() != c.want { + t.Errorf("#%d: got: %s, want: %s", n, c.pos.String(), c.want) + } + } +} From 672645fb949e698c68e72d855b2e3289b836813c Mon Sep 17 00:00:00 2001 From: uji Date: Tue, 17 May 2022 23:23:07 +0900 Subject: [PATCH 022/138] main: implement error struct --- error.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 error.go diff --git a/error.go b/error.go new file mode 100644 index 0000000..fe4dd5f --- /dev/null +++ b/error.go @@ -0,0 +1,24 @@ +package solparser + +import "github.com/uji/solparser/lexer" + +type Error struct { + Token lexer.Token + Msg string +} + +var _ error = &Error{} + +func newError(token lexer.Token, msg string) *Error { + return &Error{ + Token: token, + Msg: msg, + } +} + +func (e *Error) Error() string { + if e.Token.Pos.IsValid() { + return e.Token.Pos.String() + ": " + e.Msg + } + return e.Msg +} From 183c68f0e74786eeca49bfe43adfbd38b030c0b8 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 29 May 2022 16:53:53 +0900 Subject: [PATCH 023/138] lexer: remove Size field from Position --- lexer/lexer.go | 4 +--- lexer/lexer_test.go | 16 ++++++++-------- lexer/token.go | 1 - 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 6faa111..ac145f7 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -52,15 +52,13 @@ func (l *Lexer) Scan() (result bool) { } txt := l.scanner.Text() - size := len([]rune(txt)) pos := Position{ Column: l.offset + 1, - Size: size, Line: l.lineOffset + 1, } l.token = NewToken(txt, pos) - l.offset += size + l.offset += len([]rune(txt)) return true } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index d60b431..092e8de 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -16,14 +16,14 @@ func TestLexer_Scan(t *testing.T) { { input: "pragma solidity ^0.8.13;\n\ncontract HelloWorld { ", want: []lexer.Token{ - {lexer.Pragma, "pragma", lexer.Position{Column: 1, Size: 6, Line: 1}}, - {lexer.Unknown, "solidity", lexer.Position{Column: 8, Size: 8, Line: 1}}, - {lexer.Hat, "^", lexer.Position{Column: 17, Size: 1, Line: 1}}, - {lexer.Unknown, "0.8.13", lexer.Position{Column: 18, Size: 6, Line: 1}}, - {lexer.Semicolon, ";", lexer.Position{Column: 24, Size: 1, Line: 1}}, - {lexer.Contract, "contract", lexer.Position{Column: 1, Size: 8, Line: 3}}, - {lexer.Unknown, "HelloWorld", lexer.Position{Column: 10, Size: 10, Line: 3}}, - {lexer.BraceL, "{", lexer.Position{Column: 21, Size: 1, Line: 3}}, + {lexer.Pragma, "pragma", lexer.Position{Column: 1, Line: 1}}, + {lexer.Unknown, "solidity", lexer.Position{Column: 8, Line: 1}}, + {lexer.Hat, "^", lexer.Position{Column: 17, Line: 1}}, + {lexer.Unknown, "0.8.13", lexer.Position{Column: 18, Line: 1}}, + {lexer.Semicolon, ";", lexer.Position{Column: 24, Line: 1}}, + {lexer.Contract, "contract", lexer.Position{Column: 1, Line: 3}}, + {lexer.Unknown, "HelloWorld", lexer.Position{Column: 10, Line: 3}}, + {lexer.BraceL, "{", lexer.Position{Column: 21, Line: 3}}, }, }, } diff --git a/lexer/token.go b/lexer/token.go index 0149d5b..deb4514 100644 --- a/lexer/token.go +++ b/lexer/token.go @@ -104,7 +104,6 @@ func NewToken(ch string, pos Position) Token { type Position struct { Column int - Size int Line int } From 40d89f60acd746feb8cf7917541148b3f033b57a Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 29 May 2022 22:15:28 +0900 Subject: [PATCH 024/138] main: use solparser.Error in parse function --- error.go | 4 +++- solparser.go | 14 +++++++------- solparser_test.go | 34 +++++++++++++++++++++++++++------- 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/error.go b/error.go index fe4dd5f..b5a2cb8 100644 --- a/error.go +++ b/error.go @@ -1,6 +1,8 @@ package solparser -import "github.com/uji/solparser/lexer" +import ( + "github.com/uji/solparser/lexer" +) type Error struct { Token lexer.Token diff --git a/solparser.go b/solparser.go index 80b0922..86a1c8f 100644 --- a/solparser.go +++ b/solparser.go @@ -39,31 +39,31 @@ func (p *Parser) Parse(input io.Reader) (*ast.SourceUnit, error) { func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { p.lexer.Scan() - if p.lexer.Token().TokenType != lexer.Pragma { - return nil, errors.New("not found pragma") + if tkn := p.lexer.Token(); tkn.TokenType != lexer.Pragma { + return nil, newError(tkn, "not found pragma") } p.lexer.Scan() pragmaName := p.lexer.Token() if pragmaName.TokenType != lexer.Unknown { - return nil, errors.New("not found unkknown") + return nil, newError(pragmaName, "not found unkknown") } p.lexer.Scan() expression := p.lexer.Token() if expression.TokenType != lexer.Hat { - return nil, errors.New("not found expression") + return nil, newError(expression, "not found expression") } p.lexer.Scan() version := p.lexer.Token() if version.TokenType != lexer.Unknown { - return nil, errors.New("not found version") + return nil, newError(version, "not found version") } p.lexer.Scan() - if p.lexer.Token().TokenType != lexer.Semicolon { - return nil, errors.New("not found semicolon") + if tkn := p.lexer.Token(); tkn.TokenType != lexer.Semicolon { + return nil, newError(tkn, "not found semicolon") } return &ast.PragmaDirective{ diff --git a/solparser_test.go b/solparser_test.go index 8486656..ef8c0f3 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -1,19 +1,21 @@ package solparser_test import ( + "errors" "strings" "testing" "github.com/google/go-cmp/cmp" "github.com/uji/solparser" "github.com/uji/solparser/ast" + "github.com/uji/solparser/lexer" ) func TestParserParsePragmaDirective(t *testing.T) { cases := []struct { input string want *ast.PragmaDirective - err error + err *solparser.Error }{ { input: "pragma solidity ^0.8.13;", @@ -26,20 +28,38 @@ func TestParserParsePragmaDirective(t *testing.T) { }, err: nil, }, + { + input: "solidity ^0.8.13;", + want: nil, + err: &solparser.Error{ + Token: lexer.Token{ + TokenType: lexer.Unknown, + Text: "solidity", + Pos: lexer.Position{ + Column: 1, + Line: 1, + }, + }, + Msg: "not found pragma", + }, + }, } for n, c := range cases { r := strings.NewReader(c.input) p := solparser.New(r) + got, err := p.ParsePragmaDirective() - if err != c.err { - t.Errorf("#%d unexpected err want: %s, got: %s", n, c.err, err) - } - if err != nil { - break + + var sErr *solparser.Error + if errors.As(err, &sErr) { + if diff := cmp.Diff(c.err, sErr); diff != "" { + t.Errorf("#%d %s", n, diff) + } } + if diff := cmp.Diff(c.want, got); diff != "" { - t.Errorf("#%d want: %s, got: %s", n, c.want, got) + t.Errorf("#%d %s", n, diff) } } } From 249f101d0fd39970f33e902ae7f1c92a7e6067be Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 29 May 2022 22:21:03 +0900 Subject: [PATCH 025/138] main: replace field of Error from Token to Pos --- error.go | 14 +++++++------- solparser.go | 10 +++++----- solparser_test.go | 10 +++------- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/error.go b/error.go index b5a2cb8..81a9084 100644 --- a/error.go +++ b/error.go @@ -5,22 +5,22 @@ import ( ) type Error struct { - Token lexer.Token - Msg string + Pos lexer.Position + Msg string } var _ error = &Error{} -func newError(token lexer.Token, msg string) *Error { +func newError(pos lexer.Position, msg string) *Error { return &Error{ - Token: token, - Msg: msg, + Pos: pos, + Msg: msg, } } func (e *Error) Error() string { - if e.Token.Pos.IsValid() { - return e.Token.Pos.String() + ": " + e.Msg + if e.Pos.IsValid() { + return e.Pos.String() + ": " + e.Msg } return e.Msg } diff --git a/solparser.go b/solparser.go index 86a1c8f..2205fd2 100644 --- a/solparser.go +++ b/solparser.go @@ -40,30 +40,30 @@ func (p *Parser) Parse(input io.Reader) (*ast.SourceUnit, error) { func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { p.lexer.Scan() if tkn := p.lexer.Token(); tkn.TokenType != lexer.Pragma { - return nil, newError(tkn, "not found pragma") + return nil, newError(tkn.Pos, "not found pragma") } p.lexer.Scan() pragmaName := p.lexer.Token() if pragmaName.TokenType != lexer.Unknown { - return nil, newError(pragmaName, "not found unkknown") + return nil, newError(pragmaName.Pos, "not found unkknown") } p.lexer.Scan() expression := p.lexer.Token() if expression.TokenType != lexer.Hat { - return nil, newError(expression, "not found expression") + return nil, newError(expression.Pos, "not found expression") } p.lexer.Scan() version := p.lexer.Token() if version.TokenType != lexer.Unknown { - return nil, newError(version, "not found version") + return nil, newError(version.Pos, "not found version") } p.lexer.Scan() if tkn := p.lexer.Token(); tkn.TokenType != lexer.Semicolon { - return nil, newError(tkn, "not found semicolon") + return nil, newError(tkn.Pos, "not found semicolon") } return &ast.PragmaDirective{ diff --git a/solparser_test.go b/solparser_test.go index ef8c0f3..c36622c 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -32,13 +32,9 @@ func TestParserParsePragmaDirective(t *testing.T) { input: "solidity ^0.8.13;", want: nil, err: &solparser.Error{ - Token: lexer.Token{ - TokenType: lexer.Unknown, - Text: "solidity", - Pos: lexer.Position{ - Column: 1, - Line: 1, - }, + Pos: lexer.Position{ + Column: 1, + Line: 1, }, Msg: "not found pragma", }, From 856144939d144d778cb2147a721c4e8fa6ad7bb7 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 29 May 2022 22:28:34 +0900 Subject: [PATCH 026/138] main: add error test --- error_test.go | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 error_test.go diff --git a/error_test.go b/error_test.go new file mode 100644 index 0000000..dedefa9 --- /dev/null +++ b/error_test.go @@ -0,0 +1,47 @@ +package solparser_test + +import ( + "testing" + + "github.com/uji/solparser" + "github.com/uji/solparser/lexer" +) + +func TestErrorError(t *testing.T) { + tests := []struct { + name string + err solparser.Error + want string + }{ + { + name: "normal case", + err: solparser.Error{ + Pos: lexer.Position{ + Column: 1, + Line: 1, + }, + Msg: "not found pragma", + }, + want: "1:1: not found pragma", + }, + { + name: "invalid position case", + err: solparser.Error{ + Pos: lexer.Position{ + Column: 0, + Line: 1, + }, + Msg: "not found pragma", + }, + want: "not found pragma", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.err.Error() + if got != tt.want { + t.Fatalf("want %v, but %v:", tt.want, got) + } + }) + } +} From 0e565f5a3c77e6f9be7a0e1fb3ff85062ade3d0a Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 29 May 2022 22:36:08 +0900 Subject: [PATCH 027/138] main, lexer: fix subtest format --- lexer/lexer_test.go | 40 ++++++++++++++++++++++------------------ lexer/scanner_test.go | 4 ++-- solparser_test.go | 31 ++++++++++++++++++------------- 3 files changed, 42 insertions(+), 33 deletions(-) diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 092e8de..8b62a9d 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -9,11 +9,13 @@ import ( ) func TestLexer_Scan(t *testing.T) { - cases := []struct { + tests := []struct { + name string input string want []lexer.Token }{ { + name: "normal case", input: "pragma solidity ^0.8.13;\n\ncontract HelloWorld { ", want: []lexer.Token{ {lexer.Pragma, "pragma", lexer.Position{Column: 1, Line: 1}}, @@ -28,22 +30,24 @@ func TestLexer_Scan(t *testing.T) { }, } - for n, c := range cases { - buf := strings.NewReader(c.input) - l := lexer.New(buf) - got := make([]lexer.Token, 0, len(c.want)) - for i := 0; i < len(c.want); i++ { - l.Scan() - got = append(got, l.Token()) - } - if l.Scan() { - t.Errorf("#%d: scan ran too long, got %q", n, got) - } - if diff := cmp.Diff(c.want, got); diff != "" { - t.Errorf("#%d: %s", n, diff) - } - if err := l.Error(); err != nil { - t.Errorf("#%d: %v", n, err) - } + for _, c := range tests { + t.Run(c.name, func(t *testing.T) { + buf := strings.NewReader(c.input) + l := lexer.New(buf) + got := make([]lexer.Token, 0, len(c.want)) + for i := 0; i < len(c.want); i++ { + l.Scan() + got = append(got, l.Token()) + } + if l.Scan() { + t.Errorf("scan ran too long, got %q", got) + } + if diff := cmp.Diff(c.want, got); diff != "" { + t.Errorf("%s", diff) + } + if err := l.Error(); err != nil { + t.Errorf("%v", err) + } + }) } } diff --git a/lexer/scanner_test.go b/lexer/scanner_test.go index 8d277fd..5625fd1 100644 --- a/lexer/scanner_test.go +++ b/lexer/scanner_test.go @@ -11,7 +11,7 @@ import ( // Test that the token splitter. func TestScanTokens(t *testing.T) { - cases := []struct { + tests := []struct { input string want []string }{ @@ -29,7 +29,7 @@ func TestScanTokens(t *testing.T) { {"contract HelloWorld {", []string{"contract", " ", "HelloWorld", " ", "{"}}, } - for n, c := range cases { + for n, c := range tests { buf := strings.NewReader(c.input) s := bufio.NewScanner(buf) s.Split(lexer.ScanTokens) diff --git a/solparser_test.go b/solparser_test.go index c36622c..44f2282 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -12,12 +12,14 @@ import ( ) func TestParserParsePragmaDirective(t *testing.T) { - cases := []struct { + tests := []struct { + name string input string want *ast.PragmaDirective err *solparser.Error }{ { + name: "normal case", input: "pragma solidity ^0.8.13;", want: &ast.PragmaDirective{ PragmaName: "solidity", @@ -29,6 +31,7 @@ func TestParserParsePragmaDirective(t *testing.T) { err: nil, }, { + name: "there is no pragma keyword", input: "solidity ^0.8.13;", want: nil, err: &solparser.Error{ @@ -41,21 +44,23 @@ func TestParserParsePragmaDirective(t *testing.T) { }, } - for n, c := range cases { - r := strings.NewReader(c.input) - p := solparser.New(r) + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) - got, err := p.ParsePragmaDirective() + got, err := p.ParsePragmaDirective() - var sErr *solparser.Error - if errors.As(err, &sErr) { - if diff := cmp.Diff(c.err, sErr); diff != "" { - t.Errorf("#%d %s", n, diff) + var sErr *solparser.Error + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } } - } - if diff := cmp.Diff(c.want, got); diff != "" { - t.Errorf("#%d %s", n, diff) - } + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) } } From caed658fd9f551cf9bef95e67d1c0ac27a96606e Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 29 May 2022 23:05:20 +0900 Subject: [PATCH 028/138] main: add test --- solparser.go | 6 +++--- solparser_test.go | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/solparser.go b/solparser.go index 2205fd2..07d6050 100644 --- a/solparser.go +++ b/solparser.go @@ -46,13 +46,13 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { p.lexer.Scan() pragmaName := p.lexer.Token() if pragmaName.TokenType != lexer.Unknown { - return nil, newError(pragmaName.Pos, "not found unkknown") + return nil, newError(pragmaName.Pos, "not found pragma name") } p.lexer.Scan() expression := p.lexer.Token() if expression.TokenType != lexer.Hat { - return nil, newError(expression.Pos, "not found expression") + return nil, newError(expression.Pos, "not found Hat expression") } p.lexer.Scan() @@ -63,7 +63,7 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { p.lexer.Scan() if tkn := p.lexer.Token(); tkn.TokenType != lexer.Semicolon { - return nil, newError(tkn.Pos, "not found semicolon") + return nil, newError(tkn.Pos, "not found Semicolon") } return &ast.PragmaDirective{ diff --git a/solparser_test.go b/solparser_test.go index 44f2282..ea21ddf 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -42,6 +42,42 @@ func TestParserParsePragmaDirective(t *testing.T) { Msg: "not found pragma", }, }, + { + name: "there is no pragma name", + input: "pragma ^0.8.13;", + want: nil, + err: &solparser.Error{ + Pos: lexer.Position{ + Column: 8, + Line: 1, + }, + Msg: "not found pragma name", + }, + }, + { + name: "there is no Hat expression", + input: "pragma solidity 0.8.13;", + want: nil, + err: &solparser.Error{ + Pos: lexer.Position{ + Column: 17, + Line: 1, + }, + Msg: "not found Hat expression", + }, + }, + { + name: "there is no Semicolon", + input: "pragma solidity ^0.8.13", + want: nil, + err: &solparser.Error{ + Pos: lexer.Position{ + Column: 18, + Line: 1, + }, + Msg: "not found Semicolon", + }, + }, } for _, tt := range tests { From bae4c3e2f89e1af6e851538d78f95fa7d297c463 Mon Sep 17 00:00:00 2001 From: uji Date: Mon, 30 May 2022 00:36:58 +0900 Subject: [PATCH 029/138] lexer: add all keywords --- lexer/token.go | 278 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 269 insertions(+), 9 deletions(-) diff --git a/lexer/token.go b/lexer/token.go index deb4514..8733dee 100644 --- a/lexer/token.go +++ b/lexer/token.go @@ -20,14 +20,102 @@ const ( BraceR Semicolon + // Reserved Keyword + After + Alias + Apply + Auto + Byte + Case + Copyof + Default + Define + Final + Implements + In + Inline + Let + Macro + Match + Mutable + Null + Of + Partial + Promise + Reference + Relocatable + Sealed + Sizeof + Static + Supports + Switch + Typedef + Typeof + Var + // Keyword - Pragma + Abstract + Address + Anonymous + As + Assembly + Bool + Break + Bytes + Calldata + Catch + Constant + Constructor + Continue Contract + Delete + Do + Else + Emit + Enum + Error + Event + External + Fallback + False + Fixed + For + From Function - Pubilc + Global + Hex + If + Immutable + Import + Indexed + Interface + Internal + Is + Library + Mapping + Memory + Modifier + NewKeyword + Override + Payable + Pragma + Private + Public Pure - Returns + Receive Return + Returns + Revert + Storage + String + Struct + True + Try + Type + Using + View + Virtual + While ) func asMiscToken(r rune) TokenType { @@ -61,20 +149,192 @@ func asMiscToken(r rune) TokenType { func asKeyword(str string) TokenType { switch str { - case "pragma": - return Pragma + case "after": + return After + case "alias": + return Alias + case "apply": + return Apply + case "auto": + return Auto + case "byte": + return Byte + case "case": + return Case + case "copyof": + return Copyof + case "default": + return Default + case "define": + return Define + case "final": + return Final + case "implements": + return Implements + case "in": + return In + case "inline": + return Inline + case "let": + return Let + case "macro": + return Macro + case "match": + return Match + case "mutable": + return Mutable + case "null": + return Null + case "of": + return Of + case "partial": + return Partial + case "promise": + return Promise + case "reference": + return Reference + case "relocatable": + return Relocatable + case "sealed": + return Sealed + case "sizeof": + return Sizeof + case "static": + return Static + case "supports": + return Supports + case "switch": + return Switch + case "typedef": + return Typedef + case "typeof": + return Typeof + case "var": + return Var + case "abstract": + return Abstract + case "address": + return Address + case "anonymous": + return Anonymous + case "as": + return As + case "assembly": + return Assembly + case "bool": + return Bool + case "break": + return Break + case "bytes": + return Bytes + case "calldata": + return Calldata + case "catch": + return Catch + case "constant": + return Constant + case "constructor": + return Constructor + case "continue": + return Continue case "contract": return Contract + case "delete": + return Delete + case "do": + return Do + case "else": + return Else + case "emit": + return Emit + case "enum": + return Enum + case "error": + return Error + case "event": + return Event + case "external": + return External + case "fallback": + return Fallback + case "false": + return False + case "fixed": + return Fixed + case "for": + return For + case "from": + return From case "function": return Function - case "pubilc": - return Pubilc + case "global": + return Global + case "hex": + return Hex + case "if": + return If + case "immutable": + return Immutable + case "import": + return Import + case "indexed": + return Indexed + case "interface": + return Interface + case "internal": + return Internal + case "is": + return Is + case "library": + return Library + case "mapping": + return Mapping + case "memory": + return Memory + case "modifier": + return Modifier + case "newKeyword": + return NewKeyword + case "override": + return Override + case "payable": + return Payable + case "pragma": + return Pragma + case "private": + return Private + case "public": + return Public case "pure": return Pure - case "returns": - return Returns + case "receive": + return Receive case "return": return Return + case "returns": + return Returns + case "revert": + return Revert + case "storage": + return Storage + case "string": + return String + case "struct": + return Struct + case "true": + return True + case "try": + return Try + case "type": + return Type + case "using": + return Using + case "view": + return View + case "virtual": + return Virtual + case "while": + return While } return Unknown From 11f58b31c133c7896540aec87bf6199163418ee9 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 4 Jun 2022 23:31:44 +0900 Subject: [PATCH 030/138] solparser: move ParsePragmaDirective to other file --- pragmaDirective.go | 44 ++++++++++++++++++++ solparser_test.go => pragmaDirective_test.go | 0 solparser.go | 38 ----------------- 3 files changed, 44 insertions(+), 38 deletions(-) create mode 100644 pragmaDirective.go rename solparser_test.go => pragmaDirective_test.go (100%) diff --git a/pragmaDirective.go b/pragmaDirective.go new file mode 100644 index 0000000..da602d1 --- /dev/null +++ b/pragmaDirective.go @@ -0,0 +1,44 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/lexer" +) + +func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { + p.lexer.Scan() + if tkn := p.lexer.Token(); tkn.TokenType != lexer.Pragma { + return nil, newError(tkn.Pos, "not found pragma") + } + + p.lexer.Scan() + pragmaName := p.lexer.Token() + if pragmaName.TokenType != lexer.Unknown { + return nil, newError(pragmaName.Pos, "not found pragma name") + } + + p.lexer.Scan() + expression := p.lexer.Token() + if expression.TokenType != lexer.Hat { + return nil, newError(expression.Pos, "not found Hat expression") + } + + p.lexer.Scan() + version := p.lexer.Token() + if version.TokenType != lexer.Unknown { + return nil, newError(version.Pos, "not found version") + } + + p.lexer.Scan() + if tkn := p.lexer.Token(); tkn.TokenType != lexer.Semicolon { + return nil, newError(tkn.Pos, "not found Semicolon") + } + + return &ast.PragmaDirective{ + PragmaName: pragmaName.Text, + PragmaValue: ast.PragmaValue{ + Version: version.Text, + Expression: expression.Text, + }, + }, nil +} diff --git a/solparser_test.go b/pragmaDirective_test.go similarity index 100% rename from solparser_test.go rename to pragmaDirective_test.go diff --git a/solparser.go b/solparser.go index 07d6050..991776b 100644 --- a/solparser.go +++ b/solparser.go @@ -37,44 +37,6 @@ func (p *Parser) Parse(input io.Reader) (*ast.SourceUnit, error) { }, nil } -func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { - p.lexer.Scan() - if tkn := p.lexer.Token(); tkn.TokenType != lexer.Pragma { - return nil, newError(tkn.Pos, "not found pragma") - } - - p.lexer.Scan() - pragmaName := p.lexer.Token() - if pragmaName.TokenType != lexer.Unknown { - return nil, newError(pragmaName.Pos, "not found pragma name") - } - - p.lexer.Scan() - expression := p.lexer.Token() - if expression.TokenType != lexer.Hat { - return nil, newError(expression.Pos, "not found Hat expression") - } - - p.lexer.Scan() - version := p.lexer.Token() - if version.TokenType != lexer.Unknown { - return nil, newError(version.Pos, "not found version") - } - - p.lexer.Scan() - if tkn := p.lexer.Token(); tkn.TokenType != lexer.Semicolon { - return nil, newError(tkn.Pos, "not found Semicolon") - } - - return &ast.PragmaDirective{ - PragmaName: pragmaName.Text, - PragmaValue: ast.PragmaValue{ - Version: version.Text, - Expression: expression.Text, - }, - }, nil -} - func (p *Parser) ParseModirierList() (*ast.ModifierList, error) { return nil, nil } From f8f7e602cf2445b08b9b6c3c563d3d89b7e47da9 Mon Sep 17 00:00:00 2001 From: uji Date: Thu, 16 Jun 2022 22:47:27 +0900 Subject: [PATCH 031/138] scanner: implement look ahead scanner --- scanner/scanner.go | 75 +++++++++++++++++++++++++++++++++++++++++ scanner/scanner_test.go | 70 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 scanner/scanner.go create mode 100644 scanner/scanner_test.go diff --git a/scanner/scanner.go b/scanner/scanner.go new file mode 100644 index 0000000..f1bfb94 --- /dev/null +++ b/scanner/scanner.go @@ -0,0 +1,75 @@ +package scanner + +import ( + "bufio" + "io" +) + +type Scanner struct { + scanner *bufio.Scanner + result bool + text string + err error + peekResult bool + peekText string + peekErr error + firstPeekEnded bool +} + +func New(r io.Reader) *Scanner { + return &Scanner{scanner: bufio.NewScanner(r)} +} + +func (s *Scanner) Scan() bool { + if s.firstPeekEnded { + s.result = s.peekResult + s.text = s.peekText + s.err = s.peekErr + } else { + s.result = s.scanner.Scan() + s.text = s.scanner.Text() + s.err = s.scanner.Err() + s.firstPeekEnded = true + } + + if !s.result { + return false + } + + if s.err != nil { + return false + } + + s.peekResult = s.scanner.Scan() + s.peekText = s.scanner.Text() + s.peekErr = s.scanner.Err() + return s.result +} + +func (s *Scanner) Buffer(buf []byte, max int) { + s.scanner.Buffer(buf, max) +} + +func (s *Scanner) Bytes() []byte { + return s.scanner.Bytes() +} + +func (s *Scanner) Split(split bufio.SplitFunc) { + s.scanner.Split(split) +} + +func (s *Scanner) Text() string { + return s.text +} + +func (s *Scanner) Err() error { + return s.scanner.Err() +} + +func (s *Scanner) PeekText() string { + return s.peekText +} + +func (s *Scanner) PeekErr() error { + return s.peekErr +} diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go new file mode 100644 index 0000000..12c5185 --- /dev/null +++ b/scanner/scanner_test.go @@ -0,0 +1,70 @@ +package scanner_test + +import ( + "bufio" + "strings" + "testing" + + "github.com/uji/solparser/scanner" +) + +func TestScan(t *testing.T) { + cases := []struct { + name string + input string + splitFunc bufio.SplitFunc + exptText string + exptErr error + exptPeekText string + exptPeekErr error + }{ + { + name: "normal", + input: "abc def", + splitFunc: bufio.ScanWords, + exptText: "abc", + exptErr: nil, + exptPeekText: "def", + exptPeekErr: nil, + }, + { + name: "scan one token", + input: "abc", + splitFunc: bufio.ScanLines, + exptText: "abc", + exptErr: nil, + exptPeekText: "", + exptPeekErr: nil, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + s := scanner.New(strings.NewReader(c.input)) + s.Split(c.splitFunc) + if s.Scan() != true { + t.Errorf("scan returns false") + } + + gotText := s.Text() + if gotText != c.exptText { + t.Errorf("Text() wont %s, get %s", c.exptText, gotText) + } + + gotErr := s.Err() + if gotErr != c.exptErr { + t.Errorf("Err() wont %s, get %s", c.exptErr, gotErr) + } + + gotPeekText := s.PeekText() + if gotPeekText != c.exptPeekText { + t.Errorf("PeekText() wont %s, get %s", c.exptPeekText, gotPeekText) + } + + gotPeekErr := s.PeekErr() + if gotPeekErr != c.exptPeekErr { + t.Errorf("PeekErr() wont %s, get %s", c.exptPeekErr, gotPeekErr) + } + }) + } +} From 221fee793e65e4f31b8a24df8fade26f2b2a61e6 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 19 Jun 2022 04:21:33 +0900 Subject: [PATCH 032/138] Revert "scanner: implement look ahead scanner" This reverts commit f8f7e602cf2445b08b9b6c3c563d3d89b7e47da9. --- scanner/scanner.go | 75 ----------------------------------------- scanner/scanner_test.go | 70 -------------------------------------- 2 files changed, 145 deletions(-) delete mode 100644 scanner/scanner.go delete mode 100644 scanner/scanner_test.go diff --git a/scanner/scanner.go b/scanner/scanner.go deleted file mode 100644 index f1bfb94..0000000 --- a/scanner/scanner.go +++ /dev/null @@ -1,75 +0,0 @@ -package scanner - -import ( - "bufio" - "io" -) - -type Scanner struct { - scanner *bufio.Scanner - result bool - text string - err error - peekResult bool - peekText string - peekErr error - firstPeekEnded bool -} - -func New(r io.Reader) *Scanner { - return &Scanner{scanner: bufio.NewScanner(r)} -} - -func (s *Scanner) Scan() bool { - if s.firstPeekEnded { - s.result = s.peekResult - s.text = s.peekText - s.err = s.peekErr - } else { - s.result = s.scanner.Scan() - s.text = s.scanner.Text() - s.err = s.scanner.Err() - s.firstPeekEnded = true - } - - if !s.result { - return false - } - - if s.err != nil { - return false - } - - s.peekResult = s.scanner.Scan() - s.peekText = s.scanner.Text() - s.peekErr = s.scanner.Err() - return s.result -} - -func (s *Scanner) Buffer(buf []byte, max int) { - s.scanner.Buffer(buf, max) -} - -func (s *Scanner) Bytes() []byte { - return s.scanner.Bytes() -} - -func (s *Scanner) Split(split bufio.SplitFunc) { - s.scanner.Split(split) -} - -func (s *Scanner) Text() string { - return s.text -} - -func (s *Scanner) Err() error { - return s.scanner.Err() -} - -func (s *Scanner) PeekText() string { - return s.peekText -} - -func (s *Scanner) PeekErr() error { - return s.peekErr -} diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go deleted file mode 100644 index 12c5185..0000000 --- a/scanner/scanner_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package scanner_test - -import ( - "bufio" - "strings" - "testing" - - "github.com/uji/solparser/scanner" -) - -func TestScan(t *testing.T) { - cases := []struct { - name string - input string - splitFunc bufio.SplitFunc - exptText string - exptErr error - exptPeekText string - exptPeekErr error - }{ - { - name: "normal", - input: "abc def", - splitFunc: bufio.ScanWords, - exptText: "abc", - exptErr: nil, - exptPeekText: "def", - exptPeekErr: nil, - }, - { - name: "scan one token", - input: "abc", - splitFunc: bufio.ScanLines, - exptText: "abc", - exptErr: nil, - exptPeekText: "", - exptPeekErr: nil, - }, - } - - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - s := scanner.New(strings.NewReader(c.input)) - s.Split(c.splitFunc) - if s.Scan() != true { - t.Errorf("scan returns false") - } - - gotText := s.Text() - if gotText != c.exptText { - t.Errorf("Text() wont %s, get %s", c.exptText, gotText) - } - - gotErr := s.Err() - if gotErr != c.exptErr { - t.Errorf("Err() wont %s, get %s", c.exptErr, gotErr) - } - - gotPeekText := s.PeekText() - if gotPeekText != c.exptPeekText { - t.Errorf("PeekText() wont %s, get %s", c.exptPeekText, gotPeekText) - } - - gotPeekErr := s.PeekErr() - if gotPeekErr != c.exptPeekErr { - t.Errorf("PeekErr() wont %s, get %s", c.exptPeekErr, gotPeekErr) - } - }) - } -} From f8d68b753273ffd69d6a7c105d9d33c2b3376307 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 19 Jun 2022 05:04:29 +0900 Subject: [PATCH 033/138] lexer: replace Lexer.scanner to interface type --- lexer/lexer.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index ac145f7..828ac3c 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -6,10 +6,18 @@ import ( "unicode/utf8" ) +type scanner interface { + Scan() bool + Text() string + Err() error +} + type Lexer struct { - scanner *bufio.Scanner - token Token - err error + scanner scanner + + // scan result + token Token + err error // position state offset int From f61fa40e8fa0be3c607b49146f4261276f6728b2 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 19 Jun 2022 09:40:58 +0900 Subject: [PATCH 034/138] lexer: implemente Lexer.Peek --- lexer/lexer.go | 76 ++++++++++--- lexer/lexer_test.go | 259 ++++++++++++++++++++++++++++++++++++++------ 2 files changed, 289 insertions(+), 46 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 828ac3c..c0e1948 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -22,6 +22,11 @@ type Lexer struct { // position state offset int lineOffset int + + // peek state + peeked bool + peekToken Token + peekErr error } func New(input io.Reader) *Lexer { @@ -33,40 +38,59 @@ func New(input io.Reader) *Lexer { } } -func (l *Lexer) Scan() (result bool) { +func (l *Lexer) scan() (result bool, token Token, err error) { + offset := l.offset + lineOffset := l.lineOffset + // Scan until next token. for { if !l.scanner.Scan() { - return false + return false, Token{}, nil } if err := l.scanner.Err(); err != nil { - l.err = err - return false + return false, Token{}, err } - result = true - txt := l.scanner.Text() r, _ := utf8.DecodeRune([]byte(txt)) if r == '\n' { - l.lineOffset++ - l.offset = 0 + lineOffset++ + offset = 0 continue } if isSpace(r) { - l.offset += len([]rune(txt)) + offset += len([]rune(txt)) continue } break } - txt := l.scanner.Text() pos := Position{ - Column: l.offset + 1, - Line: l.lineOffset + 1, + Column: offset + 1, + Line: lineOffset + 1, + } + + return true, NewToken(txt, pos), nil +} + +func (l *Lexer) Scan() (result bool) { + if l.peeked { + l.token = l.peekToken + l.err = l.peekErr + l.peeked = false + return true } - l.token = NewToken(txt, pos) - l.offset += len([]rune(txt)) + rslt, tkn, err := l.scan() + if err != nil { + l.err = err + return false + } + if !rslt { + return false + } + l.token = tkn + l.offset = tkn.Pos.Column - 1 + len([]rune(tkn.Text)) + l.lineOffset = tkn.Pos.Line - 1 return true } @@ -78,3 +102,27 @@ func (l Lexer) Token() Token { func (l Lexer) Error() error { return l.err } + +func (l *Lexer) Peek() bool { + if l.peeked { + return true + } + + rslt, tkn, err := l.scan() + if !rslt { + return false + } + + l.peeked = true + l.peekToken = tkn + l.peekErr = err + return true +} + +func (l Lexer) PeekToken() Token { + return l.peekToken +} + +func (l Lexer) PeekError() error { + return l.peekErr +} diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 8b62a9d..bd45f5a 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -1,52 +1,247 @@ -package lexer_test +package lexer import ( - "strings" + "errors" "testing" "github.com/google/go-cmp/cmp" - "github.com/uji/solparser/lexer" ) -func TestLexer_Scan(t *testing.T) { +type scanResult struct { + text string + err error +} + +type mockScanner struct { + calledCount int + results []scanResult +} + +func (s *mockScanner) Scan() bool { + if s.calledCount == len(s.results) { + return false + } + s.calledCount++ + return true +} +func (s *mockScanner) Text() string { + return s.results[s.calledCount-1].text +} +func (s *mockScanner) Err() error { + return s.results[s.calledCount-1].err +} + +func TestLexerScan(t *testing.T) { tests := []struct { - name string - input string - want []lexer.Token + name string + offset int + lineOffset int + scanResults []scanResult + result bool + token Token + err error }{ { - name: "normal case", - input: "pragma solidity ^0.8.13;\n\ncontract HelloWorld { ", - want: []lexer.Token{ - {lexer.Pragma, "pragma", lexer.Position{Column: 1, Line: 1}}, - {lexer.Unknown, "solidity", lexer.Position{Column: 8, Line: 1}}, - {lexer.Hat, "^", lexer.Position{Column: 17, Line: 1}}, - {lexer.Unknown, "0.8.13", lexer.Position{Column: 18, Line: 1}}, - {lexer.Semicolon, ";", lexer.Position{Column: 24, Line: 1}}, - {lexer.Contract, "contract", lexer.Position{Column: 1, Line: 3}}, - {lexer.Unknown, "HelloWorld", lexer.Position{Column: 10, Line: 3}}, - {lexer.BraceL, "{", lexer.Position{Column: 21, Line: 3}}, + name: "normal", + offset: 4, + lineOffset: 5, + scanResults: []scanResult{ + { + text: "pragma", + err: nil, + }, }, + result: true, + token: Token{ + TokenType: Pragma, + Text: "pragma", + Pos: Position{ + Column: 5, + Line: 6, + }, + }, + }, + { + name: "when scan space", + offset: 4, + lineOffset: 5, + scanResults: []scanResult{ + { + text: " ", + err: nil, + }, + { + text: "^", + err: nil, + }, + }, + result: true, + token: Token{ + TokenType: Hat, + Text: "^", + Pos: Position{ + Column: 7, + Line: 6, + }, + }, + }, + { + name: "when scan \\n", + offset: 4, + lineOffset: 5, + scanResults: []scanResult{ + { + text: " ", + err: nil, + }, + { + text: "\n", + err: nil, + }, + { + text: "^", + err: nil, + }, + }, + result: true, + token: Token{ + TokenType: Hat, + Text: "^", + Pos: Position{ + Column: 1, + Line: 7, + }, + }, + }, + { + name: "when scan is done", + offset: 4, + lineOffset: 5, + scanResults: []scanResult{}, + result: false, + token: Token{}, + }, + { + name: "when peeked", + offset: 4, + lineOffset: 5, + scanResults: []scanResult{}, + result: false, + token: Token{}, }, } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &mockScanner{ + results: tt.scanResults, + } + l := Lexer{ + offset: tt.offset, + lineOffset: tt.lineOffset, + scanner: s, + } + if rslt := l.Scan(); rslt != tt.result { + t.Errorf("result is wrong, want: %t, got: %t", tt.result, rslt) + } + if err := l.Error(); err != tt.err { + t.Errorf("error is wrong, want: %s, got: %s", tt.err, err) + } + if diff := cmp.Diff(tt.token, l.Token()); diff != "" { + t.Errorf(diff) + } + }) + } + + t.Run("when peeked", func(t *testing.T) { + s := &mockScanner{ + results: nil, + } + peekErr := errors.New("peekErr") + peekToken := Token{ + TokenType: Hat, + Text: "^", + Pos: Position{ + Column: 7, + Line: 6, + }, + } + l := Lexer{ + offset: 4, + lineOffset: 6, + scanner: s, + peeked: true, + peekToken: peekToken, + peekErr: peekErr, + } + if rslt := l.Scan(); !rslt { + t.Errorf("result is wrong, want: true, got: true") + } + if err := l.Error(); err != peekErr { + t.Errorf("error is wrong, want: %s, got: %s", peekErr, err) + } + if diff := cmp.Diff(peekToken, l.Token()); diff != "" { + t.Errorf(diff) + } + }) +} - for _, c := range tests { - t.Run(c.name, func(t *testing.T) { - buf := strings.NewReader(c.input) - l := lexer.New(buf) - got := make([]lexer.Token, 0, len(c.want)) - for i := 0; i < len(c.want); i++ { - l.Scan() - got = append(got, l.Token()) +func TestLexerPeek(t *testing.T) { + tests := []struct { + name string + offset int + lineOffset int + scanResults []scanResult + result bool + token Token + err error + }{ + { + name: "normal", + offset: 4, + lineOffset: 5, + scanResults: []scanResult{ + { + text: "pragma", + err: nil, + }, + }, + result: true, + token: Token{ + TokenType: Pragma, + Text: "pragma", + Pos: Position{ + Column: 5, + Line: 6, + }, + }, + }, + { + name: "when scan is done", + offset: 4, + lineOffset: 5, + scanResults: []scanResult{}, + result: false, + token: Token{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &mockScanner{ + results: tt.scanResults, + } + l := Lexer{ + offset: tt.offset, + lineOffset: tt.lineOffset, + scanner: s, } - if l.Scan() { - t.Errorf("scan ran too long, got %q", got) + if rslt := l.Peek(); tt.result != rslt { + t.Errorf("result is wrong, want: %t, got: %t", tt.result, rslt) } - if diff := cmp.Diff(c.want, got); diff != "" { - t.Errorf("%s", diff) + if err := l.PeekError(); err != tt.err { + t.Errorf("want: %s, got: %s", tt.err, err) } - if err := l.Error(); err != nil { - t.Errorf("%v", err) + if diff := cmp.Diff(tt.token, l.PeekToken()); diff != "" { + t.Errorf(diff) } }) } From 46aa78802723b6a3eb6455537240b34bdee45496 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 19 Jun 2022 16:42:55 +0900 Subject: [PATCH 035/138] solparser: split solparser.go by directives of source-unit --- ast/ast.go | 1 + contractDefinition.go | 46 +++++++++++++++++++++++ functionDefinition.go | 15 ++++++++ solparser.go | 85 +++++++++++++++---------------------------- 4 files changed, 91 insertions(+), 56 deletions(-) create mode 100644 contractDefinition.go create mode 100644 functionDefinition.go diff --git a/ast/ast.go b/ast/ast.go index ab197e9..1ae3910 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -56,4 +56,5 @@ type ContractDefinition struct { type SourceUnit struct { PragmaDirective *PragmaDirective ContractDefinition *ContractDefinition + FunctionDefinition *FunctionDefinition } diff --git a/contractDefinition.go b/contractDefinition.go new file mode 100644 index 0000000..b8e4690 --- /dev/null +++ b/contractDefinition.go @@ -0,0 +1,46 @@ +package solparser + +import ( + "errors" + + "github.com/uji/solparser/ast" + "github.com/uji/solparser/lexer" +) + +func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { + p.lexer.Scan() + keyward := p.lexer.Token() + if keyward.TokenType != lexer.Contract { + return nil, errors.New("not found contract definition") + } + + p.lexer.Scan() + if p.lexer.Token().TokenType != lexer.BraceL { + return nil, errors.New("not found left brace") + } + + part, err := p.ParseContractPart() + if err != nil { + return nil, err + } + + p.lexer.Scan() + if p.lexer.Token().TokenType != lexer.BraceR { + return nil, errors.New("not found right brace") + } + + return &ast.ContractDefinition{ + ContractPart: part, + }, nil +} + +func (p *Parser) ParseContractPart() (*ast.ContractPart, error) { + funcDef, err := p.ParseFunctionDefinition() + if err != nil { + return nil, err + } + + return &ast.ContractPart{ + FunctionDefinition: funcDef, + }, nil +} diff --git a/functionDefinition.go b/functionDefinition.go new file mode 100644 index 0000000..4268ca6 --- /dev/null +++ b/functionDefinition.go @@ -0,0 +1,15 @@ +package solparser + +import "github.com/uji/solparser/ast" + +func (p *Parser) ParseFunctionDefinition() (*ast.FunctionDefinition, error) { + return nil, nil +} + +func (p *Parser) ParseModirierList() (*ast.ModifierList, error) { + return nil, nil +} + +func (p *Parser) ParseReturnParameters() (*ast.ReturnParameters, error) { + return nil, nil +} diff --git a/solparser.go b/solparser.go index 991776b..09bca20 100644 --- a/solparser.go +++ b/solparser.go @@ -1,7 +1,6 @@ package solparser import ( - "errors" "io" "github.com/uji/solparser/ast" @@ -21,68 +20,42 @@ func New(input io.Reader) *Parser { } func (p *Parser) Parse(input io.Reader) (*ast.SourceUnit, error) { - pragmaDirective, err := p.ParsePragmaDirective() - if err != nil { - return nil, err + if !p.lexer.Peek() { + return nil, nil } - contractDefinition, err := p.ParseContractDefinition() - if err != nil { + if err := p.lexer.PeekError(); err != nil { return nil, err } + var pragmaDirective *ast.PragmaDirective + var contractDefinition *ast.ContractDefinition + var functionDefinition *ast.FunctionDefinition + + switch p.lexer.PeekToken().TokenType { + case lexer.Pragma: + prgm, err := p.ParsePragmaDirective() + if err != nil { + return nil, err + } + pragmaDirective = prgm + case lexer.Abstract, lexer.Contract: + cntrct, err := p.ParseContractDefinition() + if err != nil { + return nil, err + } + contractDefinition = cntrct + case lexer.Function: + fnc, err := p.ParseFunctionDefinition() + if err != nil { + return nil, err + } + functionDefinition = fnc + } + return &ast.SourceUnit{ PragmaDirective: pragmaDirective, ContractDefinition: contractDefinition, - }, nil -} - -func (p *Parser) ParseModirierList() (*ast.ModifierList, error) { - return nil, nil -} - -func (p *Parser) ParseReturnParameters() (*ast.ReturnParameters, error) { - return nil, nil -} - -func (p *Parser) ParseFunctionDefinition() (*ast.FunctionDefinition, error) { - return nil, nil -} - -func (p *Parser) ParseContractPart() (*ast.ContractPart, error) { - funcDef, err := p.ParseFunctionDefinition() - if err != nil { - return nil, err - } - - return &ast.ContractPart{ - FunctionDefinition: funcDef, - }, nil -} - -func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { - p.lexer.Scan() - keyward := p.lexer.Token() - if keyward.TokenType != lexer.Contract { - return nil, errors.New("not found contract definition") - } - - p.lexer.Scan() - if p.lexer.Token().TokenType != lexer.BraceL { - return nil, errors.New("not found left brace") - } - - part, err := p.ParseContractPart() - if err != nil { - return nil, err - } - - p.lexer.Scan() - if p.lexer.Token().TokenType != lexer.BraceR { - return nil, errors.New("not found right brace") - } - - return &ast.ContractDefinition{ - ContractPart: part, + FunctionDefinition: functionDefinition, }, nil } From 55f4995bad8849525821bf217342a5b192940794 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 2 Jul 2022 08:23:45 +0900 Subject: [PATCH 036/138] ast: add Abstract field to Contract Definition --- ast/ast.go | 10 +++++++--- contractDefinition.go | 29 +++++++++++------------------ 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 1ae3910..eed243e 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -44,12 +44,16 @@ type FunctionDefinition struct { ReturnParameters *ReturnParameters } -type ContractPart struct { - FunctionDefinition *FunctionDefinition +type InheritanceSpecifier struct { + IdentifierPath string + CallArgumentList CallArgumentList } +type CallArgumentList struct{} + type ContractDefinition struct { - ContractPart *ContractPart + Abstract bool + InheritanceSpecifiers []*InheritanceSpecifier } // A File node represents a Solidity source file. diff --git a/contractDefinition.go b/contractDefinition.go index b8e4690..b37e571 100644 --- a/contractDefinition.go +++ b/contractDefinition.go @@ -8,20 +8,24 @@ import ( ) func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { + var abstract bool p.lexer.Scan() - keyward := p.lexer.Token() - if keyward.TokenType != lexer.Contract { + tkn := p.lexer.Token() + if tkn.TokenType == lexer.Abstract { + abstract = true + p.lexer.Scan() + } + if tkn.TokenType != lexer.Contract { return nil, errors.New("not found contract definition") } p.lexer.Scan() - if p.lexer.Token().TokenType != lexer.BraceL { + if p.lexer.Token().TokenType != lexer.Unknown { return nil, errors.New("not found left brace") } - part, err := p.ParseContractPart() - if err != nil { - return nil, err + if p.lexer.Token().TokenType != lexer.BraceL { + return nil, errors.New("not found left brace") } p.lexer.Scan() @@ -30,17 +34,6 @@ func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { } return &ast.ContractDefinition{ - ContractPart: part, - }, nil -} - -func (p *Parser) ParseContractPart() (*ast.ContractPart, error) { - funcDef, err := p.ParseFunctionDefinition() - if err != nil { - return nil, err - } - - return &ast.ContractPart{ - FunctionDefinition: funcDef, + Abstract: abstract, }, nil } From 6d9f2216f41d39ac0f67ac4a3aca970b5dc2c66b Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 2 Jul 2022 21:22:12 +0900 Subject: [PATCH 037/138] token: split token package from lexer --- contractDefinition.go | 12 ++++++------ error.go | 6 +++--- error_test.go | 6 +++--- lexer/lexer.go | 20 ++++++++++--------- lexer/lexer_test.go | 41 ++++++++++++++++++++------------------- lexer/scanner.go | 8 ++++++-- lexer/token_test.go | 24 ----------------------- pragmaDirective.go | 12 ++++++------ pragmaDirective_test.go | 10 +++++----- solparser.go | 7 ++++--- {lexer => token}/token.go | 6 +++--- token/token_test.go | 24 +++++++++++++++++++++++ 12 files changed, 92 insertions(+), 84 deletions(-) delete mode 100644 lexer/token_test.go rename {lexer => token}/token.go (98%) create mode 100644 token/token_test.go diff --git a/contractDefinition.go b/contractDefinition.go index b37e571..045321d 100644 --- a/contractDefinition.go +++ b/contractDefinition.go @@ -4,32 +4,32 @@ import ( "errors" "github.com/uji/solparser/ast" - "github.com/uji/solparser/lexer" + "github.com/uji/solparser/token" ) func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { var abstract bool p.lexer.Scan() tkn := p.lexer.Token() - if tkn.TokenType == lexer.Abstract { + if tkn.TokenType == token.Abstract { abstract = true p.lexer.Scan() } - if tkn.TokenType != lexer.Contract { + if tkn.TokenType != token.Contract { return nil, errors.New("not found contract definition") } p.lexer.Scan() - if p.lexer.Token().TokenType != lexer.Unknown { + if p.lexer.Token().TokenType != token.Unknown { return nil, errors.New("not found left brace") } - if p.lexer.Token().TokenType != lexer.BraceL { + if p.lexer.Token().TokenType != token.BraceL { return nil, errors.New("not found left brace") } p.lexer.Scan() - if p.lexer.Token().TokenType != lexer.BraceR { + if p.lexer.Token().TokenType != token.BraceR { return nil, errors.New("not found right brace") } diff --git a/error.go b/error.go index 81a9084..b8ec82c 100644 --- a/error.go +++ b/error.go @@ -1,17 +1,17 @@ package solparser import ( - "github.com/uji/solparser/lexer" + "github.com/uji/solparser/token" ) type Error struct { - Pos lexer.Position + Pos token.Position Msg string } var _ error = &Error{} -func newError(pos lexer.Position, msg string) *Error { +func newError(pos token.Position, msg string) *Error { return &Error{ Pos: pos, Msg: msg, diff --git a/error_test.go b/error_test.go index dedefa9..ea3dba6 100644 --- a/error_test.go +++ b/error_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/uji/solparser" - "github.com/uji/solparser/lexer" + "github.com/uji/solparser/token" ) func TestErrorError(t *testing.T) { @@ -16,7 +16,7 @@ func TestErrorError(t *testing.T) { { name: "normal case", err: solparser.Error{ - Pos: lexer.Position{ + Pos: token.Position{ Column: 1, Line: 1, }, @@ -27,7 +27,7 @@ func TestErrorError(t *testing.T) { { name: "invalid position case", err: solparser.Error{ - Pos: lexer.Position{ + Pos: token.Position{ Column: 0, Line: 1, }, diff --git a/lexer/lexer.go b/lexer/lexer.go index c0e1948..2b58d46 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -4,6 +4,8 @@ import ( "bufio" "io" "unicode/utf8" + + "github.com/uji/solparser/token" ) type scanner interface { @@ -16,7 +18,7 @@ type Lexer struct { scanner scanner // scan result - token Token + token token.Token err error // position state @@ -25,7 +27,7 @@ type Lexer struct { // peek state peeked bool - peekToken Token + peekToken token.Token peekErr error } @@ -38,17 +40,17 @@ func New(input io.Reader) *Lexer { } } -func (l *Lexer) scan() (result bool, token Token, err error) { +func (l *Lexer) scan() (result bool, tkn token.Token, err error) { offset := l.offset lineOffset := l.lineOffset // Scan until next token. for { if !l.scanner.Scan() { - return false, Token{}, nil + return false, token.Token{}, nil } if err := l.scanner.Err(); err != nil { - return false, Token{}, err + return false, token.Token{}, err } txt := l.scanner.Text() r, _ := utf8.DecodeRune([]byte(txt)) @@ -64,12 +66,12 @@ func (l *Lexer) scan() (result bool, token Token, err error) { break } txt := l.scanner.Text() - pos := Position{ + pos := token.Position{ Column: offset + 1, Line: lineOffset + 1, } - return true, NewToken(txt, pos), nil + return true, token.NewToken(txt, pos), nil } func (l *Lexer) Scan() (result bool) { @@ -95,7 +97,7 @@ func (l *Lexer) Scan() (result bool) { return true } -func (l Lexer) Token() Token { +func (l Lexer) Token() token.Token { return l.token } @@ -119,7 +121,7 @@ func (l *Lexer) Peek() bool { return true } -func (l Lexer) PeekToken() Token { +func (l Lexer) PeekToken() token.Token { return l.peekToken } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index bd45f5a..cbec599 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/uji/solparser/token" ) type scanResult struct { @@ -38,7 +39,7 @@ func TestLexerScan(t *testing.T) { lineOffset int scanResults []scanResult result bool - token Token + token token.Token err error }{ { @@ -52,10 +53,10 @@ func TestLexerScan(t *testing.T) { }, }, result: true, - token: Token{ - TokenType: Pragma, + token: token.Token{ + TokenType: token.Pragma, Text: "pragma", - Pos: Position{ + Pos: token.Position{ Column: 5, Line: 6, }, @@ -76,10 +77,10 @@ func TestLexerScan(t *testing.T) { }, }, result: true, - token: Token{ - TokenType: Hat, + token: token.Token{ + TokenType: token.Hat, Text: "^", - Pos: Position{ + Pos: token.Position{ Column: 7, Line: 6, }, @@ -104,10 +105,10 @@ func TestLexerScan(t *testing.T) { }, }, result: true, - token: Token{ - TokenType: Hat, + token: token.Token{ + TokenType: token.Hat, Text: "^", - Pos: Position{ + Pos: token.Position{ Column: 1, Line: 7, }, @@ -119,7 +120,7 @@ func TestLexerScan(t *testing.T) { lineOffset: 5, scanResults: []scanResult{}, result: false, - token: Token{}, + token: token.Token{}, }, { name: "when peeked", @@ -127,7 +128,7 @@ func TestLexerScan(t *testing.T) { lineOffset: 5, scanResults: []scanResult{}, result: false, - token: Token{}, + token: token.Token{}, }, } for _, tt := range tests { @@ -157,10 +158,10 @@ func TestLexerScan(t *testing.T) { results: nil, } peekErr := errors.New("peekErr") - peekToken := Token{ - TokenType: Hat, + peekToken := token.Token{ + TokenType: token.Hat, Text: "^", - Pos: Position{ + Pos: token.Position{ Column: 7, Line: 6, }, @@ -192,7 +193,7 @@ func TestLexerPeek(t *testing.T) { lineOffset int scanResults []scanResult result bool - token Token + token token.Token err error }{ { @@ -206,10 +207,10 @@ func TestLexerPeek(t *testing.T) { }, }, result: true, - token: Token{ - TokenType: Pragma, + token: token.Token{ + TokenType: token.Pragma, Text: "pragma", - Pos: Position{ + Pos: token.Position{ Column: 5, Line: 6, }, @@ -221,7 +222,7 @@ func TestLexerPeek(t *testing.T) { lineOffset: 5, scanResults: []scanResult{}, result: false, - token: Token{}, + token: token.Token{}, }, } for _, tt := range tests { diff --git a/lexer/scanner.go b/lexer/scanner.go index 8e286b0..7559f9a 100644 --- a/lexer/scanner.go +++ b/lexer/scanner.go @@ -1,6 +1,10 @@ package lexer -import "unicode/utf8" +import ( + "unicode/utf8" + + "github.com/uji/solparser/token" +) // isSpace reports whether the character is a Unicode white space character. // We avoid dependency on the unicode package, but check validity of the implementation @@ -28,7 +32,7 @@ func isSpace(r rune) bool { } func isMiscToken(r rune) bool { - return asMiscToken(r) != Unknown + return token.AsMiscToken(r) != token.Unknown } func ScanTokens(data []byte, atEOF bool) (advance int, token []byte, err error) { diff --git a/lexer/token_test.go b/lexer/token_test.go deleted file mode 100644 index 9736811..0000000 --- a/lexer/token_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package lexer_test - -import ( - "testing" - - "github.com/uji/solparser/lexer" -) - -func TestPosition_String(t *testing.T) { - cases := []struct { - pos lexer.Position - want string - }{ - {lexer.Position{Column: 3, Line: 10}, "10:3"}, - {lexer.Position{Column: 0, Line: 10}, "-"}, - {lexer.Position{Column: 1, Line: 0}, "-"}, - } - - for n, c := range cases { - if c.pos.String() != c.want { - t.Errorf("#%d: got: %s, want: %s", n, c.pos.String(), c.want) - } - } -} diff --git a/pragmaDirective.go b/pragmaDirective.go index da602d1..8513331 100644 --- a/pragmaDirective.go +++ b/pragmaDirective.go @@ -2,35 +2,35 @@ package solparser import ( "github.com/uji/solparser/ast" - "github.com/uji/solparser/lexer" + "github.com/uji/solparser/token" ) func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { p.lexer.Scan() - if tkn := p.lexer.Token(); tkn.TokenType != lexer.Pragma { + if tkn := p.lexer.Token(); tkn.TokenType != token.Pragma { return nil, newError(tkn.Pos, "not found pragma") } p.lexer.Scan() pragmaName := p.lexer.Token() - if pragmaName.TokenType != lexer.Unknown { + if pragmaName.TokenType != token.Unknown { return nil, newError(pragmaName.Pos, "not found pragma name") } p.lexer.Scan() expression := p.lexer.Token() - if expression.TokenType != lexer.Hat { + if expression.TokenType != token.Hat { return nil, newError(expression.Pos, "not found Hat expression") } p.lexer.Scan() version := p.lexer.Token() - if version.TokenType != lexer.Unknown { + if version.TokenType != token.Unknown { return nil, newError(version.Pos, "not found version") } p.lexer.Scan() - if tkn := p.lexer.Token(); tkn.TokenType != lexer.Semicolon { + if tkn := p.lexer.Token(); tkn.TokenType != token.Semicolon { return nil, newError(tkn.Pos, "not found Semicolon") } diff --git a/pragmaDirective_test.go b/pragmaDirective_test.go index ea21ddf..84e1352 100644 --- a/pragmaDirective_test.go +++ b/pragmaDirective_test.go @@ -8,7 +8,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/uji/solparser" "github.com/uji/solparser/ast" - "github.com/uji/solparser/lexer" + "github.com/uji/solparser/token" ) func TestParserParsePragmaDirective(t *testing.T) { @@ -35,7 +35,7 @@ func TestParserParsePragmaDirective(t *testing.T) { input: "solidity ^0.8.13;", want: nil, err: &solparser.Error{ - Pos: lexer.Position{ + Pos: token.Position{ Column: 1, Line: 1, }, @@ -47,7 +47,7 @@ func TestParserParsePragmaDirective(t *testing.T) { input: "pragma ^0.8.13;", want: nil, err: &solparser.Error{ - Pos: lexer.Position{ + Pos: token.Position{ Column: 8, Line: 1, }, @@ -59,7 +59,7 @@ func TestParserParsePragmaDirective(t *testing.T) { input: "pragma solidity 0.8.13;", want: nil, err: &solparser.Error{ - Pos: lexer.Position{ + Pos: token.Position{ Column: 17, Line: 1, }, @@ -71,7 +71,7 @@ func TestParserParsePragmaDirective(t *testing.T) { input: "pragma solidity ^0.8.13", want: nil, err: &solparser.Error{ - Pos: lexer.Position{ + Pos: token.Position{ Column: 18, Line: 1, }, diff --git a/solparser.go b/solparser.go index 09bca20..f3c92f6 100644 --- a/solparser.go +++ b/solparser.go @@ -5,6 +5,7 @@ import ( "github.com/uji/solparser/ast" "github.com/uji/solparser/lexer" + "github.com/uji/solparser/token" ) type Parser struct { @@ -33,19 +34,19 @@ func (p *Parser) Parse(input io.Reader) (*ast.SourceUnit, error) { var functionDefinition *ast.FunctionDefinition switch p.lexer.PeekToken().TokenType { - case lexer.Pragma: + case token.Pragma: prgm, err := p.ParsePragmaDirective() if err != nil { return nil, err } pragmaDirective = prgm - case lexer.Abstract, lexer.Contract: + case token.Abstract, token.Contract: cntrct, err := p.ParseContractDefinition() if err != nil { return nil, err } contractDefinition = cntrct - case lexer.Function: + case token.Function: fnc, err := p.ParseFunctionDefinition() if err != nil { return nil, err diff --git a/lexer/token.go b/token/token.go similarity index 98% rename from lexer/token.go rename to token/token.go index 8733dee..e540767 100644 --- a/lexer/token.go +++ b/token/token.go @@ -1,4 +1,4 @@ -package lexer +package token import "fmt" @@ -118,7 +118,7 @@ const ( While ) -func asMiscToken(r rune) TokenType { +func AsMiscToken(r rune) TokenType { switch r { case '^': return Hat @@ -345,7 +345,7 @@ func asToken(str string) TokenType { if len(r) != 1 { return asKeyword(str) } - return asMiscToken(r[0]) + return AsMiscToken(r[0]) } type Token struct { diff --git a/token/token_test.go b/token/token_test.go new file mode 100644 index 0000000..f3916e8 --- /dev/null +++ b/token/token_test.go @@ -0,0 +1,24 @@ +package token_test + +import ( + "testing" + + "github.com/uji/solparser/token" +) + +func TestPosition_String(t *testing.T) { + cases := []struct { + pos token.Position + want string + }{ + {token.Position{Column: 3, Line: 10}, "10:3"}, + {token.Position{Column: 0, Line: 10}, "-"}, + {token.Position{Column: 1, Line: 0}, "-"}, + } + + for n, c := range cases { + if got := c.pos.String(); got != c.want { + t.Errorf("#%d: got: %s, want: %s", n, got, c.want) + } + } +} From 69a8fa7d11f1896dd6721eb6bad47f1e3c423ef0 Mon Sep 17 00:00:00 2001 From: uji Date: Tue, 5 Jul 2022 02:35:45 +0900 Subject: [PATCH 038/138] ast: implement literal rule --- ast/ast.go | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/ast/ast.go b/ast/ast.go index eed243e..6b8fe8f 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -1,5 +1,12 @@ package ast +import "github.com/uji/solparser/token" + +type Node interface { + Pos() token.Position + End() token.Position +} + type PragmaValue struct { Version string Expression string @@ -62,3 +69,66 @@ type SourceUnit struct { ContractDefinition *ContractDefinition FunctionDefinition *FunctionDefinition } + +type Literal Node // BooleanLiteral | StringLiteral | NumberLiteral | HexStringLiteral | UnicordStringLiteral + +type BooleanLiteral struct { + Pos token.Position + Bool bool +} + +type StringLiteral struct { + Pos token.Position + List []Node // EmptyStringLiteral | NonEmptyStringLiteral +} + +type HexStringLiteral []*HexString + +type UnicordStringLiteral []*UnicordStrings + +type NumberLiteral struct { + Number Node // DecimalNumber | HexNumber + NumberUnit *NumberUnit +} + +type NumberUnit struct { + Pos token.Position + Value string +} + +type EmptyStringLiteral struct { + Pos token.Position + SingleQuoted bool +} + +type NonEmptyStringLiteral struct { + Pos token.Position + List []Node // SingleQuotedPrintable | DoubleQuotedPrintable | EscapeSequence +} + +type SingleQuotedPrintable struct { + Pos token.Position + String string +} + +type DoubleQuotedPrintable struct { + Pos token.Position + String string +} + +type EscapeSequence struct { + Pos token.Position + String string +} + +// unicode-string-literal (https://github.com/ethereum/solidity/blob/develop/docs/grammar/SolidityParser.g4#L407) +type UnicordStrings struct { + Pos token.Position + List []Node // string | EscapeSequence +} + +type HexString struct { + Pos token.Position + SingleQuoted bool + String string +} From 3f9bf61d7fc825b2fbdfa3984e21fabf947d2637 Mon Sep 17 00:00:00 2001 From: uji Date: Tue, 5 Jul 2022 02:36:18 +0900 Subject: [PATCH 039/138] ast: implement ElementaryTypeName rule --- ast/ast.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ast/ast.go b/ast/ast.go index 6b8fe8f..bda5e3e 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -132,3 +132,23 @@ type HexString struct { SingleQuoted bool String string } + +type ElementaryTypeNameKind int + +const ( + ElementaryTypeName_Address ElementaryTypeNameKind = iota + 1 + ElementaryTypeName_AddressPayable + ElementaryTypeName_Bool + ElementaryTypeName_String + ElementaryTypeName_Bytes + ElementaryTypeName_SignedIntegerType + ElementaryTypeName_UnsignedIntegerType + ElementaryTypeName_FixedBytes + ElementaryTypeName_Fixed + ElementaryTypeName_Ufixed +) + +type ElementaryTypeName struct { + Pos token.Position + Kind ElementaryTypeNameKind +} From 4c854e8722d4dff131972b23b4ccef03c17d0d43 Mon Sep 17 00:00:00 2001 From: uji Date: Tue, 5 Jul 2022 03:05:25 +0900 Subject: [PATCH 040/138] .github/workflow: add ci.yml --- .github/workflows/ci.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..1a5082e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,28 @@ +name: ci + +on: [push] + +jobs: + setup: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version-file: 'go.mod' + + vet: + needs: setup + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: vet + run: go vet ./... + + test: + needs: setup + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: test + run: go test ./... -v From a124a4c7380ddc539b627335ddbb422bbcbcced1 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 9 Jul 2022 23:51:02 +0900 Subject: [PATCH 041/138] ast: implement methods of BooleanLiteral to meet Node interface --- ast/ast.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index bda5e3e..f644679 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -73,8 +73,18 @@ type SourceUnit struct { type Literal Node // BooleanLiteral | StringLiteral | NumberLiteral | HexStringLiteral | UnicordStringLiteral type BooleanLiteral struct { - Pos token.Position - Bool bool + Tkn token.Token +} + +func (b *BooleanLiteral) Pos() token.Position { + return b.Tkn.Pos +} + +func (b *BooleanLiteral) End() token.Position { + return token.Position{ + Column: b.Tkn.Pos.Column + len(b.Tkn.Text), + Line: b.Tkn.Pos.Line, + } } type StringLiteral struct { From 9c2df6b9d3ff967f1b681cc32eb5d3c6034c96c8 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 9 Jul 2022 23:51:44 +0900 Subject: [PATCH 042/138] solparser: implement ParseBooleanLiteral --- solparser.go | 19 +++++++++++ solparser_test.go | 84 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 solparser_test.go diff --git a/solparser.go b/solparser.go index f3c92f6..674a135 100644 --- a/solparser.go +++ b/solparser.go @@ -60,3 +60,22 @@ func (p *Parser) Parse(input io.Reader) (*ast.SourceUnit, error) { FunctionDefinition: functionDefinition, }, nil } + +func (p *Parser) ParseBooleanLiteral() (*ast.BooleanLiteral, error) { + if !p.lexer.Scan() { + return nil, nil + } + + if err := p.lexer.Error(); err != nil { + return nil, err + } + + tkn := p.lexer.Token() + if tkn.TokenType != token.True && tkn.TokenType != token.False { + return nil, newError(tkn.Pos, "not found keyword true or false") + } + + return &ast.BooleanLiteral{ + Tkn: p.lexer.Token(), + }, nil +} diff --git a/solparser_test.go b/solparser_test.go new file mode 100644 index 0000000..8e7a82a --- /dev/null +++ b/solparser_test.go @@ -0,0 +1,84 @@ +package solparser_test + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParserParseBooleanLiteral(t *testing.T) { + tests := []struct { + name string + input string + want *ast.BooleanLiteral + err *solparser.Error + }{ + { + name: "true case", + input: "true", + want: &ast.BooleanLiteral{ + Tkn: token.Token{ + TokenType: token.True, + Text: "true", + Pos: token.Position{ + Column: 1, + Line: 1, + }, + }, + }, + err: nil, + }, + { + name: "false case", + input: "false", + want: &ast.BooleanLiteral{ + Tkn: token.Token{ + TokenType: token.False, + Text: "false", + Pos: token.Position{ + Column: 1, + Line: 1, + }, + }, + }, + err: nil, + }, + { + name: "not true or false", + input: "solidity", + want: nil, + err: &solparser.Error{ + Pos: token.Position{ + Column: 1, + Line: 1, + }, + Msg: "not found keyword true or false", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseBooleanLiteral() + + var sErr *solparser.Error + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} From b630f67ede30207d2255bfe7036a463f223761d2 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 10 Jul 2022 00:05:20 +0900 Subject: [PATCH 043/138] lexer: rename miscToken to splitSymbol --- lexer/scanner.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lexer/scanner.go b/lexer/scanner.go index 7559f9a..173b0bb 100644 --- a/lexer/scanner.go +++ b/lexer/scanner.go @@ -2,8 +2,6 @@ package lexer import ( "unicode/utf8" - - "github.com/uji/solparser/token" ) // isSpace reports whether the character is a Unicode white space character. @@ -31,8 +29,12 @@ func isSpace(r rune) bool { return false } -func isMiscToken(r rune) bool { - return token.AsMiscToken(r) != token.Unknown +func isSplitSymbol(r rune) bool { + switch r { + case '^', '~', '<', '>', '=', ':', ';', '(', ')', '{', '}': + return true + } + return false } func ScanTokens(data []byte, atEOF bool) (advance int, token []byte, err error) { @@ -40,7 +42,7 @@ func ScanTokens(data []byte, atEOF bool) (advance int, token []byte, err error) // Return newline code or misk token. r, width := utf8.DecodeRune(data[start:]) - if r == '\n' || isMiscToken(r) { + if r == '\n' || isSplitSymbol(r) { return start + width, data[start : start+width], nil } @@ -49,7 +51,7 @@ func ScanTokens(data []byte, atEOF bool) (advance int, token []byte, err error) for width, i := 0, start; i < len(data); i += width { var r rune r, width = utf8.DecodeRune(data[i:]) - if r == '\n' || isMiscToken(r) || isSpace(r) != tokenIsSpace { + if r == '\n' || isSplitSymbol(r) || isSpace(r) != tokenIsSpace { return i, data[start:i], nil } } From 9bbb05e91988ac1c7968703733c6d82ec718cab2 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 10 Jul 2022 00:05:43 +0900 Subject: [PATCH 044/138] token: remove MistToken --- token/token.go | 41 +++++++++++++++-------------------------- 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/token/token.go b/token/token.go index e540767..a574a96 100644 --- a/token/token.go +++ b/token/token.go @@ -5,9 +5,9 @@ import "fmt" type TokenType int const ( - // Misc characters Invalid TokenType = iota Unknown + Hat Tilde Greater @@ -118,37 +118,30 @@ const ( While ) -func AsMiscToken(r rune) TokenType { - switch r { - case '^': +func asKeyword(str string) TokenType { + switch str { + case "^": return Hat - case '~': + case "~": return Tilde - case '<': + case "<": return Less - case '>': + case ">": return Greater - case '=': + case "=": return Equal - case ':': + case ":": return Colon - case ';': + case ";": return Semicolon - case '(': + case "(": return ParenL - case ')': + case ")": return ParenR - case '{': + case "{": return BraceL - case '}': + case "}": return BraceR - } - - return Unknown -} - -func asKeyword(str string) TokenType { - switch str { case "after": return After case "alias": @@ -341,11 +334,7 @@ func asKeyword(str string) TokenType { } func asToken(str string) TokenType { - r := []rune(str) - if len(r) != 1 { - return asKeyword(str) - } - return AsMiscToken(r[0]) + return asKeyword(str) } type Token struct { From b9e6ed007b4b4221c55e2d724489ee292670d139 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 10 Jul 2022 00:11:24 +0900 Subject: [PATCH 045/138] ast: rename BooleanLiretal field --- ast/ast.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index f644679..f53b5cb 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -73,17 +73,17 @@ type SourceUnit struct { type Literal Node // BooleanLiteral | StringLiteral | NumberLiteral | HexStringLiteral | UnicordStringLiteral type BooleanLiteral struct { - Tkn token.Token + Token token.Token } func (b *BooleanLiteral) Pos() token.Position { - return b.Tkn.Pos + return b.Token.Pos } func (b *BooleanLiteral) End() token.Position { return token.Position{ - Column: b.Tkn.Pos.Column + len(b.Tkn.Text), - Line: b.Tkn.Pos.Line, + Column: b.Token.Pos.Column + len(b.Token.Text), + Line: b.Token.Pos.Line, } } From cd8645540c368659e66dab40961a37fcccdd9ec2 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 10 Jul 2022 00:22:51 +0900 Subject: [PATCH 046/138] ast: implement ContractBodyElement --- ast/ast.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ast/ast.go b/ast/ast.go index f53b5cb..1894309 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -58,9 +58,23 @@ type InheritanceSpecifier struct { type CallArgumentList struct{} +type ContractBodyElement struct { + Node Node +} + +func (c *ContractBodyElement) Pos() token.Position { + return c.Node.Pos() +} + +func (c *ContractBodyElement) End() token.Position { + return c.Node.End() +} + type ContractDefinition struct { + Position token.Position Abstract bool InheritanceSpecifiers []*InheritanceSpecifier + ContractBodyElements []*ContractBodyElement } // A File node represents a Solidity source file. From 8b2b01af6b96cac6f962bafb362c69097203a920 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 10 Jul 2022 00:30:39 +0900 Subject: [PATCH 047/138] ast: change ContractBodyElement type definition --- ast/ast.go | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 1894309..3888d5e 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -58,17 +58,7 @@ type InheritanceSpecifier struct { type CallArgumentList struct{} -type ContractBodyElement struct { - Node Node -} - -func (c *ContractBodyElement) Pos() token.Position { - return c.Node.Pos() -} - -func (c *ContractBodyElement) End() token.Position { - return c.Node.End() -} +type ContractBodyElement Node type ContractDefinition struct { Position token.Position From fb27f3b2f0a4a200a0513d28166712db10a175be Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 10 Jul 2022 00:37:28 +0900 Subject: [PATCH 048/138] ast: add Expression type definition --- ast/ast.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ast/ast.go b/ast/ast.go index 3888d5e..00fbbe1 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -166,3 +166,5 @@ type ElementaryTypeName struct { Pos token.Position Kind ElementaryTypeNameKind } + +type Expression Node From 02f273041fb959021e44f3ba4f57ccc118af3546 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 10 Jul 2022 00:37:43 +0900 Subject: [PATCH 049/138] ast: implement ReturnStatement --- ast/ast.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ast/ast.go b/ast/ast.go index 00fbbe1..bfb9f1b 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -167,4 +167,19 @@ type ElementaryTypeName struct { Kind ElementaryTypeNameKind } +type ReturnStatement struct { + Position token.Position + Expression Expression +} + +func (r *ReturnStatement) Pos() token.Position { + return r.Position +} +func (r *ReturnStatement) End() token.Position { + return token.Position{ + Column: r.Expression.End().Column + 1, + Line: r.Expression.End().Line, + } +} + type Expression Node From 21657afca812d4861f168a0d43bdec665f5973f0 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 10 Jul 2022 14:30:13 +0900 Subject: [PATCH 050/138] .gitmodules: remove --- .gitmodules | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .gitmodules diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 20589ba..0000000 --- a/.gitmodules +++ /dev/null @@ -1,6 +0,0 @@ -[submodule "antlr4"] - path = antlr4 - url = https://github.com/antlr/antlr4.git -[submodule "solidity-antlr"] - path = solidity-antlr - url = https://github.com/solidity-parser/antlr.git From 31a1e6cdca0a749809d2df19ed02ea809088200c Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 16 Jul 2022 20:34:52 +0900 Subject: [PATCH 051/138] solparser: fix type field name --- solparser.go | 2 +- solparser_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/solparser.go b/solparser.go index 674a135..680742b 100644 --- a/solparser.go +++ b/solparser.go @@ -76,6 +76,6 @@ func (p *Parser) ParseBooleanLiteral() (*ast.BooleanLiteral, error) { } return &ast.BooleanLiteral{ - Tkn: p.lexer.Token(), + Token: p.lexer.Token(), }, nil } diff --git a/solparser_test.go b/solparser_test.go index 8e7a82a..0291d66 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -22,7 +22,7 @@ func TestParserParseBooleanLiteral(t *testing.T) { name: "true case", input: "true", want: &ast.BooleanLiteral{ - Tkn: token.Token{ + Token: token.Token{ TokenType: token.True, Text: "true", Pos: token.Position{ @@ -37,7 +37,7 @@ func TestParserParseBooleanLiteral(t *testing.T) { name: "false case", input: "false", want: &ast.BooleanLiteral{ - Tkn: token.Token{ + Token: token.Token{ TokenType: token.False, Text: "false", Pos: token.Position{ From 8865180bd32dcb772867bd1ffbe6b265114562d9 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 16 Jul 2022 20:43:18 +0900 Subject: [PATCH 052/138] token: rename Position to Pos --- ast/ast.go | 40 ++++++++++++++++++++-------------------- error.go | 4 ++-- error_test.go | 4 ++-- lexer/lexer.go | 2 +- lexer/lexer_test.go | 10 +++++----- pragmaDirective_test.go | 8 ++++---- solparser_test.go | 6 +++--- token/token.go | 14 +++++++------- token/token_test.go | 8 ++++---- 9 files changed, 48 insertions(+), 48 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index bfb9f1b..4e07994 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -3,8 +3,8 @@ package ast import "github.com/uji/solparser/token" type Node interface { - Pos() token.Position - End() token.Position + Pos() token.Pos + End() token.Pos } type PragmaValue struct { @@ -61,7 +61,7 @@ type CallArgumentList struct{} type ContractBodyElement Node type ContractDefinition struct { - Position token.Position + Position token.Pos Abstract bool InheritanceSpecifiers []*InheritanceSpecifier ContractBodyElements []*ContractBodyElement @@ -80,19 +80,19 @@ type BooleanLiteral struct { Token token.Token } -func (b *BooleanLiteral) Pos() token.Position { +func (b *BooleanLiteral) Pos() token.Pos { return b.Token.Pos } -func (b *BooleanLiteral) End() token.Position { - return token.Position{ +func (b *BooleanLiteral) End() token.Pos { + return token.Pos{ Column: b.Token.Pos.Column + len(b.Token.Text), Line: b.Token.Pos.Line, } } type StringLiteral struct { - Pos token.Position + Pos token.Pos List []Node // EmptyStringLiteral | NonEmptyStringLiteral } @@ -106,43 +106,43 @@ type NumberLiteral struct { } type NumberUnit struct { - Pos token.Position + Pos token.Pos Value string } type EmptyStringLiteral struct { - Pos token.Position + Pos token.Pos SingleQuoted bool } type NonEmptyStringLiteral struct { - Pos token.Position + Pos token.Pos List []Node // SingleQuotedPrintable | DoubleQuotedPrintable | EscapeSequence } type SingleQuotedPrintable struct { - Pos token.Position + Pos token.Pos String string } type DoubleQuotedPrintable struct { - Pos token.Position + Pos token.Pos String string } type EscapeSequence struct { - Pos token.Position + Pos token.Pos String string } // unicode-string-literal (https://github.com/ethereum/solidity/blob/develop/docs/grammar/SolidityParser.g4#L407) type UnicordStrings struct { - Pos token.Position + Pos token.Pos List []Node // string | EscapeSequence } type HexString struct { - Pos token.Position + Pos token.Pos SingleQuoted bool String string } @@ -163,20 +163,20 @@ const ( ) type ElementaryTypeName struct { - Pos token.Position + Pos token.Pos Kind ElementaryTypeNameKind } type ReturnStatement struct { - Position token.Position + Position token.Pos Expression Expression } -func (r *ReturnStatement) Pos() token.Position { +func (r *ReturnStatement) Pos() token.Pos { return r.Position } -func (r *ReturnStatement) End() token.Position { - return token.Position{ +func (r *ReturnStatement) End() token.Pos { + return token.Pos{ Column: r.Expression.End().Column + 1, Line: r.Expression.End().Line, } diff --git a/error.go b/error.go index b8ec82c..9fa3337 100644 --- a/error.go +++ b/error.go @@ -5,13 +5,13 @@ import ( ) type Error struct { - Pos token.Position + Pos token.Pos Msg string } var _ error = &Error{} -func newError(pos token.Position, msg string) *Error { +func newError(pos token.Pos, msg string) *Error { return &Error{ Pos: pos, Msg: msg, diff --git a/error_test.go b/error_test.go index ea3dba6..1d31ca9 100644 --- a/error_test.go +++ b/error_test.go @@ -16,7 +16,7 @@ func TestErrorError(t *testing.T) { { name: "normal case", err: solparser.Error{ - Pos: token.Position{ + Pos: token.Pos{ Column: 1, Line: 1, }, @@ -27,7 +27,7 @@ func TestErrorError(t *testing.T) { { name: "invalid position case", err: solparser.Error{ - Pos: token.Position{ + Pos: token.Pos{ Column: 0, Line: 1, }, diff --git a/lexer/lexer.go b/lexer/lexer.go index 2b58d46..524d882 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -66,7 +66,7 @@ func (l *Lexer) scan() (result bool, tkn token.Token, err error) { break } txt := l.scanner.Text() - pos := token.Position{ + pos := token.Pos{ Column: offset + 1, Line: lineOffset + 1, } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index cbec599..6de3de0 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -56,7 +56,7 @@ func TestLexerScan(t *testing.T) { token: token.Token{ TokenType: token.Pragma, Text: "pragma", - Pos: token.Position{ + Pos: token.Pos{ Column: 5, Line: 6, }, @@ -80,7 +80,7 @@ func TestLexerScan(t *testing.T) { token: token.Token{ TokenType: token.Hat, Text: "^", - Pos: token.Position{ + Pos: token.Pos{ Column: 7, Line: 6, }, @@ -108,7 +108,7 @@ func TestLexerScan(t *testing.T) { token: token.Token{ TokenType: token.Hat, Text: "^", - Pos: token.Position{ + Pos: token.Pos{ Column: 1, Line: 7, }, @@ -161,7 +161,7 @@ func TestLexerScan(t *testing.T) { peekToken := token.Token{ TokenType: token.Hat, Text: "^", - Pos: token.Position{ + Pos: token.Pos{ Column: 7, Line: 6, }, @@ -210,7 +210,7 @@ func TestLexerPeek(t *testing.T) { token: token.Token{ TokenType: token.Pragma, Text: "pragma", - Pos: token.Position{ + Pos: token.Pos{ Column: 5, Line: 6, }, diff --git a/pragmaDirective_test.go b/pragmaDirective_test.go index 84e1352..58b2d01 100644 --- a/pragmaDirective_test.go +++ b/pragmaDirective_test.go @@ -35,7 +35,7 @@ func TestParserParsePragmaDirective(t *testing.T) { input: "solidity ^0.8.13;", want: nil, err: &solparser.Error{ - Pos: token.Position{ + Pos: token.Pos{ Column: 1, Line: 1, }, @@ -47,7 +47,7 @@ func TestParserParsePragmaDirective(t *testing.T) { input: "pragma ^0.8.13;", want: nil, err: &solparser.Error{ - Pos: token.Position{ + Pos: token.Pos{ Column: 8, Line: 1, }, @@ -59,7 +59,7 @@ func TestParserParsePragmaDirective(t *testing.T) { input: "pragma solidity 0.8.13;", want: nil, err: &solparser.Error{ - Pos: token.Position{ + Pos: token.Pos{ Column: 17, Line: 1, }, @@ -71,7 +71,7 @@ func TestParserParsePragmaDirective(t *testing.T) { input: "pragma solidity ^0.8.13", want: nil, err: &solparser.Error{ - Pos: token.Position{ + Pos: token.Pos{ Column: 18, Line: 1, }, diff --git a/solparser_test.go b/solparser_test.go index 0291d66..ac14890 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -25,7 +25,7 @@ func TestParserParseBooleanLiteral(t *testing.T) { Token: token.Token{ TokenType: token.True, Text: "true", - Pos: token.Position{ + Pos: token.Pos{ Column: 1, Line: 1, }, @@ -40,7 +40,7 @@ func TestParserParseBooleanLiteral(t *testing.T) { Token: token.Token{ TokenType: token.False, Text: "false", - Pos: token.Position{ + Pos: token.Pos{ Column: 1, Line: 1, }, @@ -53,7 +53,7 @@ func TestParserParseBooleanLiteral(t *testing.T) { input: "solidity", want: nil, err: &solparser.Error{ - Pos: token.Position{ + Pos: token.Pos{ Column: 1, Line: 1, }, diff --git a/token/token.go b/token/token.go index a574a96..f5d563a 100644 --- a/token/token.go +++ b/token/token.go @@ -340,10 +340,10 @@ func asToken(str string) TokenType { type Token struct { TokenType TokenType Text string - Pos Position + Pos Pos } -func NewToken(ch string, pos Position) Token { +func NewToken(ch string, pos Pos) Token { return Token{ TokenType: asToken(ch), Text: ch, @@ -351,17 +351,17 @@ func NewToken(ch string, pos Position) Token { } } -type Position struct { +type Pos struct { Column int Line int } -func (pos Position) IsValid() bool { return pos.Line > 0 && pos.Column > 0 } +func (p Pos) IsValid() bool { return p.Line > 0 && p.Column > 0 } -func (pos Position) String() string { - if !pos.IsValid() { +func (p Pos) String() string { + if !p.IsValid() { return "-" } - return fmt.Sprintf("%d:%d", pos.Line, pos.Column) + return fmt.Sprintf("%d:%d", p.Line, p.Column) } diff --git a/token/token_test.go b/token/token_test.go index f3916e8..a9c99ef 100644 --- a/token/token_test.go +++ b/token/token_test.go @@ -8,12 +8,12 @@ import ( func TestPosition_String(t *testing.T) { cases := []struct { - pos token.Position + pos token.Pos want string }{ - {token.Position{Column: 3, Line: 10}, "10:3"}, - {token.Position{Column: 0, Line: 10}, "-"}, - {token.Position{Column: 1, Line: 0}, "-"}, + {token.Pos{Column: 3, Line: 10}, "10:3"}, + {token.Pos{Column: 0, Line: 10}, "-"}, + {token.Pos{Column: 1, Line: 0}, "-"}, } for n, c := range cases { From d2665d95bfa19176331137eb76e98ac8e1583dfa Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 16 Jul 2022 20:47:14 +0900 Subject: [PATCH 053/138] ast: rename returnStatement field --- ast/ast.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 4e07994..fdf1d04 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -168,12 +168,12 @@ type ElementaryTypeName struct { } type ReturnStatement struct { - Position token.Pos + Return token.Pos Expression Expression } func (r *ReturnStatement) Pos() token.Pos { - return r.Position + return r.Return } func (r *ReturnStatement) End() token.Pos { return token.Pos{ From 240b3d7f22736fe68eb700318b74612b92c314cc Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 16 Jul 2022 21:04:22 +0900 Subject: [PATCH 054/138] ast: organize Literal node --- ast/ast.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index fdf1d04..c77d303 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -74,7 +74,13 @@ type SourceUnit struct { FunctionDefinition *FunctionDefinition } -type Literal Node // BooleanLiteral | StringLiteral | NumberLiteral | HexStringLiteral | UnicordStringLiteral +// ---------------------------------------------------------------------------- +// Literal Nodes + +type Literal interface { + Node + literalNode() +} type BooleanLiteral struct { Token token.Token @@ -92,7 +98,6 @@ func (b *BooleanLiteral) End() token.Pos { } type StringLiteral struct { - Pos token.Pos List []Node // EmptyStringLiteral | NonEmptyStringLiteral } @@ -105,6 +110,14 @@ type NumberLiteral struct { NumberUnit *NumberUnit } +func (*BooleanLiteral) literalNode() {} +func (*StringLiteral) literalNode() {} +func (*HexStringLiteral) literalNode() {} +func (*UnicordStringLiteral) literalNode() {} +func (*NumberLiteral) literalNode() {} + +// ---------------------------------------------------------------------------- + type NumberUnit struct { Pos token.Pos Value string From 399154f7905c045ceb6336a6fc660672d225c475 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 17 Jul 2022 10:18:13 +0900 Subject: [PATCH 055/138] ast: implement Printable interface --- ast/ast.go | 55 ++++++++++++++++++++++++++++++++++++--- ast/ast_test.go | 68 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 4 deletions(-) create mode 100644 ast/ast_test.go diff --git a/ast/ast.go b/ast/ast.go index c77d303..2ba8777 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -130,24 +130,71 @@ type EmptyStringLiteral struct { type NonEmptyStringLiteral struct { Pos token.Pos - List []Node // SingleQuotedPrintable | DoubleQuotedPrintable | EscapeSequence + List []Printable +} + +// ---------------------------------------------------------------------------- +// Printable Nodes + +type Printable interface { + Node + printableNode() } type SingleQuotedPrintable struct { - Pos token.Pos + Begin token.Pos String string } +func (s *SingleQuotedPrintable) Pos() token.Pos { + return s.Begin +} + +func (s *SingleQuotedPrintable) End() token.Pos { + return token.Pos{ + Column: s.Begin.Column + len(s.String) + 1, + Line: s.Begin.Line, + } +} + type DoubleQuotedPrintable struct { - Pos token.Pos + Begin token.Pos String string } +func (d *DoubleQuotedPrintable) Pos() token.Pos { + return d.Begin +} + +func (d *DoubleQuotedPrintable) End() token.Pos { + return token.Pos{ + Column: d.Begin.Column + len(d.String) + 1, + Line: d.Begin.Line, + } +} + type EscapeSequence struct { - Pos token.Pos + Begin token.Pos String string } +func (e *EscapeSequence) Pos() token.Pos { + return e.Begin +} + +func (e *EscapeSequence) End() token.Pos { + return token.Pos{ + Column: e.Begin.Column + len(e.String), + Line: e.Begin.Line, + } +} + +func (*SingleQuotedPrintable) printableNode() {} +func (*DoubleQuotedPrintable) printableNode() {} +func (*EscapeSequence) printableNode() {} + +// ---------------------------------------------------------------------------- + // unicode-string-literal (https://github.com/ethereum/solidity/blob/develop/docs/grammar/SolidityParser.g4#L407) type UnicordStrings struct { Pos token.Pos diff --git a/ast/ast_test.go b/ast/ast_test.go new file mode 100644 index 0000000..1deb8e0 --- /dev/null +++ b/ast/ast_test.go @@ -0,0 +1,68 @@ +package ast_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestNode_End(t *testing.T) { + tests := []struct { + name string + node ast.Node + exptEnd token.Pos + }{ + { + name: "SingleQuotedPrintable", + node: &ast.SingleQuotedPrintable{ + Begin: token.Pos{ + Column: 4, + Line: 3, + }, + String: "test", + }, + exptEnd: token.Pos{ + Column: 9, + Line: 3, + }, + }, + { + name: "DoubleQuotedPrintable", + node: &ast.DoubleQuotedPrintable{ + Begin: token.Pos{ + Column: 4, + Line: 3, + }, + String: "test", + }, + exptEnd: token.Pos{ + Column: 9, + Line: 3, + }, + }, + { + name: "EscapeSequence", + node: &ast.EscapeSequence{ + Begin: token.Pos{ + Column: 4, + Line: 3, + }, + String: `\\u2000`, + }, + exptEnd: token.Pos{ + Column: 11, + Line: 3, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if diff := cmp.Diff(tt.node.End(), tt.exptEnd); diff != "" { + t.Error(diff) + } + }) + } +} From 8f7dc3b36a2794a34f0dbb9ca28d5c4e2e01e2d6 Mon Sep 17 00:00:00 2001 From: uji Date: Mon, 18 Jul 2022 18:19:05 +0900 Subject: [PATCH 056/138] lexer: rename scan function --- lexer/lexer.go | 2 +- lexer/scanner.go | 2 +- lexer/scanner_test.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 524d882..7cb5a44 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -33,7 +33,7 @@ type Lexer struct { func New(input io.Reader) *Lexer { s := bufio.NewScanner(input) - s.Split(ScanTokens) + s.Split(Scan) return &Lexer{ scanner: s, diff --git a/lexer/scanner.go b/lexer/scanner.go index 173b0bb..f78c5ec 100644 --- a/lexer/scanner.go +++ b/lexer/scanner.go @@ -37,7 +37,7 @@ func isSplitSymbol(r rune) bool { return false } -func ScanTokens(data []byte, atEOF bool) (advance int, token []byte, err error) { +func Scan(data []byte, atEOF bool) (advance int, token []byte, err error) { start := 0 // Return newline code or misk token. diff --git a/lexer/scanner_test.go b/lexer/scanner_test.go index 5625fd1..5456182 100644 --- a/lexer/scanner_test.go +++ b/lexer/scanner_test.go @@ -10,7 +10,7 @@ import ( ) // Test that the token splitter. -func TestScanTokens(t *testing.T) { +func TestScan(t *testing.T) { tests := []struct { input string want []string @@ -32,7 +32,7 @@ func TestScanTokens(t *testing.T) { for n, c := range tests { buf := strings.NewReader(c.input) s := bufio.NewScanner(buf) - s.Split(lexer.ScanTokens) + s.Split(lexer.Scan) got := make([]string, 0, len(c.want)) for i := 0; i < len(c.want); i++ { From 08db850b3d9ba781610478ec126da837d9ab7c20 Mon Sep 17 00:00:00 2001 From: uji Date: Mon, 18 Jul 2022 18:39:48 +0900 Subject: [PATCH 057/138] token: cover all operator --- contractDefinition.go | 4 +- lexer/lexer_test.go | 6 +- pragmaDirective.go | 4 +- pragmaDirective_test.go | 4 +- token/token.go | 177 ++++++++++++++++++++++++++++++++++------ 5 files changed, 159 insertions(+), 36 deletions(-) diff --git a/contractDefinition.go b/contractDefinition.go index 045321d..bad1e2b 100644 --- a/contractDefinition.go +++ b/contractDefinition.go @@ -24,12 +24,12 @@ func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { return nil, errors.New("not found left brace") } - if p.lexer.Token().TokenType != token.BraceL { + if p.lexer.Token().TokenType != token.LBrace { return nil, errors.New("not found left brace") } p.lexer.Scan() - if p.lexer.Token().TokenType != token.BraceR { + if p.lexer.Token().TokenType != token.RBrace { return nil, errors.New("not found right brace") } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 6de3de0..6cb69f4 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -78,7 +78,7 @@ func TestLexerScan(t *testing.T) { }, result: true, token: token.Token{ - TokenType: token.Hat, + TokenType: token.BitXor, Text: "^", Pos: token.Pos{ Column: 7, @@ -106,7 +106,7 @@ func TestLexerScan(t *testing.T) { }, result: true, token: token.Token{ - TokenType: token.Hat, + TokenType: token.BitXor, Text: "^", Pos: token.Pos{ Column: 1, @@ -159,7 +159,7 @@ func TestLexerScan(t *testing.T) { } peekErr := errors.New("peekErr") peekToken := token.Token{ - TokenType: token.Hat, + TokenType: token.BitXor, Text: "^", Pos: token.Pos{ Column: 7, diff --git a/pragmaDirective.go b/pragmaDirective.go index 8513331..2a9b689 100644 --- a/pragmaDirective.go +++ b/pragmaDirective.go @@ -19,8 +19,8 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { p.lexer.Scan() expression := p.lexer.Token() - if expression.TokenType != token.Hat { - return nil, newError(expression.Pos, "not found Hat expression") + if expression.TokenType != token.BitXor { + return nil, newError(expression.Pos, "not found BitXor expression") } p.lexer.Scan() diff --git a/pragmaDirective_test.go b/pragmaDirective_test.go index 58b2d01..08b38b0 100644 --- a/pragmaDirective_test.go +++ b/pragmaDirective_test.go @@ -55,7 +55,7 @@ func TestParserParsePragmaDirective(t *testing.T) { }, }, { - name: "there is no Hat expression", + name: "there is no BitXor expression", input: "pragma solidity 0.8.13;", want: nil, err: &solparser.Error{ @@ -63,7 +63,7 @@ func TestParserParsePragmaDirective(t *testing.T) { Column: 17, Line: 1, }, - Msg: "not found Hat expression", + Msg: "not found BitXor expression", }, }, { diff --git a/token/token.go b/token/token.go index f5d563a..ee8f5fb 100644 --- a/token/token.go +++ b/token/token.go @@ -8,17 +8,60 @@ const ( Invalid TokenType = iota Unknown - Hat - Tilde - Greater - Less - Equal + LParen + RParen + LBrack + RBrack + LBrace + RBrace Colon - ParenL - ParenR - BraceL - BraceR Semicolon + Period + Conditional + DoubleArrow + RightArrow + + Assign + AssignBitOr + AssignBitXor + AssignBitAnd + AssignShl + AssignSar + AssignShr + AssignAdd + AssignSub + AssignMul + AssignDiv + AssignMod + + Comma + Or + And + BitOr + BitXor + BitAnd + Shl + Sar + Shr + Add + Sub + Mul + Div + Mod + Exp + + Equal + NotEqual + LessThan + GreaterThan + LessThanOrEqual + GreaterThanOrEqual + Not + BitNot + Inc + Dec + DoubleQuote + SingleQuote // Reserved Keyword After @@ -120,28 +163,108 @@ const ( func asKeyword(str string) TokenType { switch str { - case "^": - return Hat - case "~": - return Tilde - case "<": - return Less - case ">": - return Greater - case "=": - return Equal - case ":": - return Colon - case ";": - return Semicolon case "(": - return ParenL + return LParen case ")": - return ParenR + return RParen + case "[": + return LBrack + case "]": + return RBrack case "{": - return BraceL + return LBrace case "}": - return BraceR + return RBrace + case ":": + return Colon + case ";": + return Semicolon + case ".": + return Period + case "?": + return Conditional + case "=>": + return DoubleArrow + case "->": + return RightArrow + case "=": + return Assign + case "|=": + return AssignBitOr + case "^=": + return AssignBitXor + case "&=": + return AssignBitAnd + case "<<=": + return AssignShl + case ">>=": + return AssignSar + case ">>>=": + return AssignShr + case "+=": + return AssignAdd + case "-=": + return AssignSub + case "*=": + return AssignMul + case "/=": + return AssignDiv + case "%=": + return AssignMod + case ",": + return Comma + case "||": + return Or + case "&&": + return And + case "|": + return BitOr + case "^": + return BitXor + case "&": + return BitAnd + case "<<": + return Shl + case ">>": + return Sar + case ">>>": + return Shr + case "+": + return Add + case "-": + return Sub + case "*": + return Mul + case "/": + return Div + case "%": + return Mod + case "**": + return Exp + case "==": + return Equal + case "!=": + return NotEqual + case "<": + return LessThan + case ">": + return GreaterThan + case "<=": + return LessThanOrEqual + case ">=": + return GreaterThanOrEqual + case "!": + return Not + case "~": + return BitNot + case "++": + return Inc + case "--": + return Dec + case `"`: + return DoubleQuote + case `\'`: + return SingleQuote case "after": return After case "alias": From 4077e1c3830b9d5366823e4be94c8c511dd44dc5 Mon Sep 17 00:00:00 2001 From: uji Date: Mon, 18 Jul 2022 21:45:16 +0900 Subject: [PATCH 058/138] lexer: cover split symbol --- lexer/scanner.go | 2 +- lexer/scanner_test.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lexer/scanner.go b/lexer/scanner.go index f78c5ec..975d12a 100644 --- a/lexer/scanner.go +++ b/lexer/scanner.go @@ -31,7 +31,7 @@ func isSpace(r rune) bool { func isSplitSymbol(r rune) bool { switch r { - case '^', '~', '<', '>', '=', ':', ';', '(', ')', '{', '}': + case '(', ')', '[', ']', '{', '}', ':', ';', '?', '=', '|', '^', '&', '<', '>', '+', '-', '*', '/', '%', ',', '!', '~', '"', '\'', '\\': return true } return false diff --git a/lexer/scanner_test.go b/lexer/scanner_test.go index 5456182..6a6ee6c 100644 --- a/lexer/scanner_test.go +++ b/lexer/scanner_test.go @@ -27,6 +27,7 @@ func TestScan(t *testing.T) { {"0.8.13;", []string{"0.8.13", ";"}}, {"pragma solidity ^0.8.13;", []string{"pragma", " ", "solidity", " ", "^", "0.8.13", ";"}}, {"contract HelloWorld {", []string{"contract", " ", "HelloWorld", " ", "{"}}, + {`\'test\'`, []string{`\`, "'", "test", `\`, "'"}}, } for n, c := range tests { From 6c08e846898f7e8b31b96dee6af7fe1817d3aa29 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 24 Jul 2022 18:58:06 +0900 Subject: [PATCH 059/138] lexer: refactor lexer_test --- lexer/lexer_test.go | 223 ++++++++++++++++---------------------------- 1 file changed, 79 insertions(+), 144 deletions(-) diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 6cb69f4..9905955 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -1,59 +1,34 @@ package lexer import ( - "errors" + "bufio" + "strings" "testing" "github.com/google/go-cmp/cmp" "github.com/uji/solparser/token" ) -type scanResult struct { - text string - err error -} - -type mockScanner struct { - calledCount int - results []scanResult -} - -func (s *mockScanner) Scan() bool { - if s.calledCount == len(s.results) { - return false - } - s.calledCount++ - return true -} -func (s *mockScanner) Text() string { - return s.results[s.calledCount-1].text -} -func (s *mockScanner) Err() error { - return s.results[s.calledCount-1].err -} - -func TestLexerScan(t *testing.T) { +func testLexerScan(t *testing.T) { tests := []struct { - name string - offset int - lineOffset int - scanResults []scanResult - result bool - token token.Token - err error + name string + offset int + lineOffset int + input string + peeked bool + peekToken token.Token + peekErr error + wantResult bool + wantErr error + wantToken token.Token }{ { name: "normal", offset: 4, lineOffset: 5, - scanResults: []scanResult{ - { - text: "pragma", - err: nil, - }, - }, - result: true, - token: token.Token{ + input: "pragma", + wantResult: true, + wantToken: token.Token{ TokenType: token.Pragma, Text: "pragma", Pos: token.Pos{ @@ -66,18 +41,9 @@ func TestLexerScan(t *testing.T) { name: "when scan space", offset: 4, lineOffset: 5, - scanResults: []scanResult{ - { - text: " ", - err: nil, - }, - { - text: "^", - err: nil, - }, - }, - result: true, - token: token.Token{ + input: " ^", + wantResult: true, + wantToken: token.Token{ TokenType: token.BitXor, Text: "^", Pos: token.Pos{ @@ -90,22 +56,9 @@ func TestLexerScan(t *testing.T) { name: "when scan \\n", offset: 4, lineOffset: 5, - scanResults: []scanResult{ - { - text: " ", - err: nil, - }, - { - text: "\n", - err: nil, - }, - { - text: "^", - err: nil, - }, - }, - result: true, - token: token.Token{ + input: " \n^", + wantResult: true, + wantToken: token.Token{ TokenType: token.BitXor, Text: "^", Pos: token.Pos{ @@ -115,98 +68,80 @@ func TestLexerScan(t *testing.T) { }, }, { - name: "when scan is done", - offset: 4, - lineOffset: 5, - scanResults: []scanResult{}, - result: false, - token: token.Token{}, + name: "when scan is done", + offset: 4, + lineOffset: 5, + input: "", + wantResult: false, + wantToken: token.Token{}, }, { - name: "when peeked", - offset: 4, - lineOffset: 5, - scanResults: []scanResult{}, - result: false, - token: token.Token{}, + name: "when peeked", + offset: 4, + lineOffset: 5, + input: "", + peeked: true, + peekToken: token.Token{ + TokenType: token.BitXor, + Text: "^", + Pos: token.Pos{ + Column: 4, + Line: 6, + }, + }, + wantResult: true, + wantToken: token.Token{ + TokenType: token.BitXor, + Text: "", + Pos: token.Pos{ + Column: 4, + Line: 6, + }, + }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := &mockScanner{ - results: tt.scanResults, - } + s := bufio.NewScanner(strings.NewReader(tt.input)) + s.Split(Scan) + l := Lexer{ offset: tt.offset, lineOffset: tt.lineOffset, scanner: s, + peeked: tt.peeked, + peekToken: tt.peekToken, + peekErr: tt.peekErr, } - if rslt := l.Scan(); rslt != tt.result { - t.Errorf("result is wrong, want: %t, got: %t", tt.result, rslt) + if rslt := l.Scan(); rslt != tt.wantResult { + t.Errorf("result is wrong, want: %t, got: %t", tt.wantResult, rslt) } - if err := l.Error(); err != tt.err { - t.Errorf("error is wrong, want: %s, got: %s", tt.err, err) + if err := l.Error(); err != tt.wantErr { + t.Errorf("error is wrong, want: %s, got: %s", tt.wantErr, err) } - if diff := cmp.Diff(tt.token, l.Token()); diff != "" { + if diff := cmp.Diff(tt.wantToken, l.Token()); diff != "" { t.Errorf(diff) } }) } - - t.Run("when peeked", func(t *testing.T) { - s := &mockScanner{ - results: nil, - } - peekErr := errors.New("peekErr") - peekToken := token.Token{ - TokenType: token.BitXor, - Text: "^", - Pos: token.Pos{ - Column: 7, - Line: 6, - }, - } - l := Lexer{ - offset: 4, - lineOffset: 6, - scanner: s, - peeked: true, - peekToken: peekToken, - peekErr: peekErr, - } - if rslt := l.Scan(); !rslt { - t.Errorf("result is wrong, want: true, got: true") - } - if err := l.Error(); err != peekErr { - t.Errorf("error is wrong, want: %s, got: %s", peekErr, err) - } - if diff := cmp.Diff(peekToken, l.Token()); diff != "" { - t.Errorf(diff) - } - }) } func TestLexerPeek(t *testing.T) { tests := []struct { - name string - offset int - lineOffset int - scanResults []scanResult - result bool - token token.Token - err error + name string + offset int + lineOffset int + input string + result bool + token token.Token + err error }{ { name: "normal", offset: 4, lineOffset: 5, - scanResults: []scanResult{ - { - text: "pragma", - err: nil, - }, - }, - result: true, + input: "pragma", + result: true, token: token.Token{ TokenType: token.Pragma, Text: "pragma", @@ -217,19 +152,19 @@ func TestLexerPeek(t *testing.T) { }, }, { - name: "when scan is done", - offset: 4, - lineOffset: 5, - scanResults: []scanResult{}, - result: false, - token: token.Token{}, + name: "when scan is done", + offset: 4, + lineOffset: 5, + input: "", + result: false, + token: token.Token{}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := &mockScanner{ - results: tt.scanResults, - } + s := bufio.NewScanner(strings.NewReader(tt.input)) + s.Split(Scan) + l := Lexer{ offset: tt.offset, lineOffset: tt.lineOffset, From c8719b4000f9a691e808b6d3017fc5b9e804f065 Mon Sep 17 00:00:00 2001 From: uji Date: Tue, 2 Aug 2022 22:48:37 +0900 Subject: [PATCH 060/138] ast: declare expression interface --- ast/ast.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 2ba8777..a869c6a 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -74,12 +74,21 @@ type SourceUnit struct { FunctionDefinition *FunctionDefinition } +// ---------------------------------------------------------------------------- +// Expression Nodes + +type Expression interface { + Node + expressionNode() +} + // ---------------------------------------------------------------------------- // Literal Nodes type Literal interface { Node literalNode() + expressionNode() } type BooleanLiteral struct { @@ -116,6 +125,12 @@ func (*HexStringLiteral) literalNode() {} func (*UnicordStringLiteral) literalNode() {} func (*NumberLiteral) literalNode() {} +func (*BooleanLiteral) expressionNode() {} +func (*StringLiteral) expressionNode() {} +func (*HexStringLiteral) expressionNode() {} +func (*UnicordStringLiteral) expressionNode() {} +func (*NumberLiteral) expressionNode() {} + // ---------------------------------------------------------------------------- type NumberUnit struct { @@ -241,5 +256,3 @@ func (r *ReturnStatement) End() token.Pos { Line: r.Expression.End().Line, } } - -type Expression Node From 669aa1675bea8364d120c3a9aac12b3be7f4fa7e Mon Sep 17 00:00:00 2001 From: uji Date: Thu, 4 Aug 2022 23:00:44 +0900 Subject: [PATCH 061/138] scanner: add scanner package to peek scan result --- scanner/scanner.go | 136 ++++++++++++++++++++++++++++++++++++++++ scanner/scanner_test.go | 114 +++++++++++++++++++++++++++++++++ 2 files changed, 250 insertions(+) create mode 100644 scanner/scanner.go create mode 100644 scanner/scanner_test.go diff --git a/scanner/scanner.go b/scanner/scanner.go new file mode 100644 index 0000000..f2c7f7c --- /dev/null +++ b/scanner/scanner.go @@ -0,0 +1,136 @@ +package scanner + +import ( + "bufio" + "io" + "unicode/utf8" +) + +// isSpace reports whether the character is a Unicode white space character. +// We avoid dependency on the unicode package, but check validity of the implementation +// in the tests. +func isSpace(r rune) bool { + if r <= '\u00FF' { + // Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs. + switch r { + case ' ', '\t', '\v', '\f', '\r': + return true + case '\u0085', '\u00A0': + return true + } + return false + } + // High-valued ones. + if '\u2000' <= r && r <= '\u200a' { + return true + } + switch r { + case '\u1680', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000': + return true + } + return false +} + +func isSplitSymbol(r rune) bool { + switch r { + case '(', ')', '[', ']', '{', '}', ':', ';', '?', '=', '|', '^', '&', '<', '>', '+', '-', '*', '/', '%', ',', '!', '~', '"', '\'', '\\': + return true + } + return false +} + +func Split(data []byte, atEOF bool) (advance int, token []byte, err error) { + start := 0 + + // Return newline code or misk token. + r, width := utf8.DecodeRune(data[start:]) + if r == '\n' || isSplitSymbol(r) { + return start + width, data[start : start+width], nil + } + + tokenIsSpace := isSpace(r) + // Scan until isSpace result changed, marking end of word. + for width, i := 0, start; i < len(data); i += width { + var r rune + r, width = utf8.DecodeRune(data[i:]) + if r == '\n' || isSplitSymbol(r) || isSpace(r) != tokenIsSpace { + return i, data[start:i], nil + } + } + // If we're at EOF, we have a final, non-empty, non-terminated word. Return it. + if atEOF && len(data) > start { + return len(data), data[start:], nil + } + // Request more data. + return start, nil, nil +} + +type Scanner struct { + scanner *bufio.Scanner + + // scan result + text string + err error + + // peek state + peeked bool + peekText string + peekErr error +} + +func New(input io.Reader) *Scanner { + s := bufio.NewScanner(input) + s.Split(Split) + + return &Scanner{ + scanner: s, + } +} + +func (s *Scanner) Scan() bool { + if s.peeked { + s.text = s.peekText + s.err = s.peekErr + s.peeked = false + return true + } + + if !s.scanner.Scan() { + return false + } + + s.text = s.scanner.Text() + s.err = s.scanner.Err() + return true +} + +func (s Scanner) Text() string { + return s.text +} + +func (s Scanner) Err() error { + return s.err +} + +func (s *Scanner) Peek() bool { + if s.peeked { + return true + } + + if !s.scanner.Scan() { + return false + } + + s.peeked = true + s.peekText = s.scanner.Text() + s.peekErr = s.scanner.Err() + return true +} + +func (s Scanner) PeekText() string { + return s.peekText +} + +func (s Scanner) PeekErr() error { + return s.peekErr +} diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go new file mode 100644 index 0000000..8cd03a0 --- /dev/null +++ b/scanner/scanner_test.go @@ -0,0 +1,114 @@ +package scanner + +import ( + "bufio" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser/lexer" +) + +func TestScan(t *testing.T) { + tests := []struct { + input string + want []string + }{ + {"", []string{}}, + {" ", []string{" "}}, + {"\n", []string{"\n"}}, + {"a", []string{"a"}}, + {" a ", []string{" ", "a", " "}}, + {"abc def ", []string{"abc", " ", "def", " "}}, + {"a\tb\nc\r\td\f", []string{"a", "\t", "b", "\n", "c", "\r\t", "d", "\f"}}, + {"e\vf\u0085g\u00a0\n", []string{"e", "\v", "f", "\u0085", "g", "\u00a0", "\n"}}, + {"^0.8.13", []string{"^", "0.8.13"}}, + {"0.8.13;", []string{"0.8.13", ";"}}, + {"pragma solidity ^0.8.13;", []string{"pragma", " ", "solidity", " ", "^", "0.8.13", ";"}}, + {"contract HelloWorld {", []string{"contract", " ", "HelloWorld", " ", "{"}}, + {`\'test\'`, []string{`\`, "'", "test", `\`, "'"}}, + } + + for n, c := range tests { + buf := strings.NewReader(c.input) + s := bufio.NewScanner(buf) + s.Split(lexer.Scan) + + got := make([]string, 0, len(c.want)) + for i := 0; i < len(c.want); i++ { + if !s.Scan() { + break + } + got = append(got, s.Text()) + } + if s.Scan() { + t.Errorf("#%d: scan ran too long, got %q", n, s.Text()) + } + if diff := cmp.Diff(c.want, got); diff != "" { + t.Errorf("#%d: %s", n, diff) + } + err := s.Err() + if err != nil { + t.Errorf("#%d: %v", n, err) + } + } +} + +func TestScannerScan(t *testing.T) { + tests := map[string]struct { + input string + peeked bool + peekText string + peekErr error + want bool + wantErr error + wantText string + }{ + "normal": { + input: "pragma solidity", + want: true, + wantErr: nil, + wantText: "pragma", + }, + "when scan is done": { + input: "", + want: false, + wantErr: nil, + wantText: "", + }, + "when peeked": { + input: "", + peeked: true, + peekText: "peekedText", + want: true, + wantErr: nil, + wantText: "peekedText", + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + scanner := bufio.NewScanner(strings.NewReader(tt.input)) + scanner.Split(Split) + + s := Scanner{ + scanner: scanner, + peeked: tt.peeked, + peekText: tt.peekText, + peekErr: tt.peekErr, + } + + if rslt := s.Scan(); rslt != tt.want { + t.Errorf("want: %t, got: %t", tt.want, rslt) + } + + if err := s.Err(); err != tt.wantErr { + t.Errorf("want: %s, got: %s", tt.wantErr, err) + } + + if text := s.Text(); text != tt.wantText { + t.Errorf("want: %s, got: %s", tt.wantText, text) + } + }) + } +} From 4961e1ccc90b8d2c30299e68b4ae654a0689ee35 Mon Sep 17 00:00:00 2001 From: uji Date: Fri, 5 Aug 2022 22:39:39 +0900 Subject: [PATCH 062/138] scanner: refactoring testcase --- scanner/scanner_test.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index 8cd03a0..b0c1754 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -55,7 +55,8 @@ func TestScan(t *testing.T) { } func TestScannerScan(t *testing.T) { - tests := map[string]struct { + tests := []struct { + name string input string peeked bool peekText string @@ -64,19 +65,22 @@ func TestScannerScan(t *testing.T) { wantErr error wantText string }{ - "normal": { + { + name: "normal", input: "pragma solidity", want: true, wantErr: nil, wantText: "pragma", }, - "when scan is done": { + { + name: "when scan is done", input: "", want: false, wantErr: nil, wantText: "", }, - "when peeked": { + { + name: "when peeked", input: "", peeked: true, peekText: "peekedText", @@ -86,8 +90,9 @@ func TestScannerScan(t *testing.T) { }, } - for name, tt := range tests { - t.Run(name, func(t *testing.T) { + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { scanner := bufio.NewScanner(strings.NewReader(tt.input)) scanner.Split(Split) From dd1b5c5c1038d317b630fb612f03eebd75cc34c4 Mon Sep 17 00:00:00 2001 From: uji Date: Fri, 5 Aug 2022 22:42:14 +0900 Subject: [PATCH 063/138] scanner: rename test function of Split --- scanner/scanner_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index b0c1754..b1e910c 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -9,7 +9,7 @@ import ( "github.com/uji/solparser/lexer" ) -func TestScan(t *testing.T) { +func TestSplit(t *testing.T) { tests := []struct { input string want []string From 9ba28ebeddf1562de30f78d7e8ce58eef8ec72b4 Mon Sep 17 00:00:00 2001 From: uji Date: Fri, 5 Aug 2022 22:42:38 +0900 Subject: [PATCH 064/138] scanner: implement Peek test --- scanner/scanner_test.go | 67 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index b1e910c..fa6a981 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/google/go-cmp/cmp" - "github.com/uji/solparser/lexer" ) func TestSplit(t *testing.T) { @@ -32,7 +31,7 @@ func TestSplit(t *testing.T) { for n, c := range tests { buf := strings.NewReader(c.input) s := bufio.NewScanner(buf) - s.Split(lexer.Scan) + s.Split(Split) got := make([]string, 0, len(c.want)) for i := 0; i < len(c.want); i++ { @@ -117,3 +116,67 @@ func TestScannerScan(t *testing.T) { }) } } + +func TestScannerPeek(t *testing.T) { + tests := []struct { + name string + input string + peeked bool + peekText string + peekErr error + want bool + wantErr error + wantText string + }{ + { + name: "when not peeked", + input: "pragma solidity", + want: true, + wantErr: nil, + wantText: "pragma", + }, + { + name: "when scan is done", + input: "", + want: false, + wantErr: nil, + wantText: "", + }, + { + name: "when peeked", + input: "", + peeked: true, + peekText: "peekedText", + want: true, + wantErr: nil, + wantText: "peekedText", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + scanner := bufio.NewScanner(strings.NewReader(tt.input)) + scanner.Split(Split) + + s := Scanner{ + scanner: scanner, + peeked: tt.peeked, + peekText: tt.peekText, + peekErr: tt.peekErr, + } + + if rslt := s.Peek(); rslt != tt.want { + t.Errorf("want: %t, got: %t", tt.want, rslt) + } + + if err := s.PeekErr(); err != tt.wantErr { + t.Errorf("want: %s, got: %s", tt.wantErr, err) + } + + if text := s.PeekText(); text != tt.wantText { + t.Errorf("want: %s, got: %s", tt.wantText, text) + } + }) + } +} From d11fd361048bee716d3c9a14577154bfc772aeef Mon Sep 17 00:00:00 2001 From: uji Date: Fri, 5 Aug 2022 22:50:23 +0900 Subject: [PATCH 065/138] lexer: replace scanner from bufio to solparser/scanner --- lexer/lexer.go | 35 +++++++++++++++++------ lexer/lexer_test.go | 10 +++---- lexer/scanner.go | 64 ------------------------------------------- lexer/scanner_test.go | 56 ------------------------------------- 4 files changed, 32 insertions(+), 133 deletions(-) delete mode 100644 lexer/scanner.go delete mode 100644 lexer/scanner_test.go diff --git a/lexer/lexer.go b/lexer/lexer.go index 7cb5a44..867182f 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -1,21 +1,41 @@ package lexer import ( - "bufio" "io" "unicode/utf8" + "github.com/uji/solparser/scanner" "github.com/uji/solparser/token" ) -type scanner interface { - Scan() bool - Text() string - Err() error +// isSpace reports whether the character is a Unicode white space character. +// We avoid dependency on the unicode package, but check validity of the implementation +// in the tests. +// TODO: remove function because scanner package has same function. +func isSpace(r rune) bool { + if r <= '\u00FF' { + // Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs. + switch r { + case ' ', '\t', '\v', '\f', '\r': + return true + case '\u0085', '\u00A0': + return true + } + return false + } + // High-valued ones. + if '\u2000' <= r && r <= '\u200a' { + return true + } + switch r { + case '\u1680', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000': + return true + } + return false } type Lexer struct { - scanner scanner + scanner *scanner.Scanner // scan result token token.Token @@ -32,8 +52,7 @@ type Lexer struct { } func New(input io.Reader) *Lexer { - s := bufio.NewScanner(input) - s.Split(Scan) + s := scanner.New(input) return &Lexer{ scanner: s, diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 9905955..8e870e5 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -1,11 +1,11 @@ package lexer import ( - "bufio" "strings" "testing" "github.com/google/go-cmp/cmp" + "github.com/uji/solparser/scanner" "github.com/uji/solparser/token" ) @@ -101,9 +101,9 @@ func testLexerScan(t *testing.T) { }, } for _, tt := range tests { + tt := tt t.Run(tt.name, func(t *testing.T) { - s := bufio.NewScanner(strings.NewReader(tt.input)) - s.Split(Scan) + s := scanner.New(strings.NewReader(tt.input)) l := Lexer{ offset: tt.offset, @@ -161,9 +161,9 @@ func TestLexerPeek(t *testing.T) { }, } for _, tt := range tests { + tt := tt t.Run(tt.name, func(t *testing.T) { - s := bufio.NewScanner(strings.NewReader(tt.input)) - s.Split(Scan) + s := scanner.New(strings.NewReader(tt.input)) l := Lexer{ offset: tt.offset, diff --git a/lexer/scanner.go b/lexer/scanner.go deleted file mode 100644 index 975d12a..0000000 --- a/lexer/scanner.go +++ /dev/null @@ -1,64 +0,0 @@ -package lexer - -import ( - "unicode/utf8" -) - -// isSpace reports whether the character is a Unicode white space character. -// We avoid dependency on the unicode package, but check validity of the implementation -// in the tests. -func isSpace(r rune) bool { - if r <= '\u00FF' { - // Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs. - switch r { - case ' ', '\t', '\v', '\f', '\r': - return true - case '\u0085', '\u00A0': - return true - } - return false - } - // High-valued ones. - if '\u2000' <= r && r <= '\u200a' { - return true - } - switch r { - case '\u1680', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000': - return true - } - return false -} - -func isSplitSymbol(r rune) bool { - switch r { - case '(', ')', '[', ']', '{', '}', ':', ';', '?', '=', '|', '^', '&', '<', '>', '+', '-', '*', '/', '%', ',', '!', '~', '"', '\'', '\\': - return true - } - return false -} - -func Scan(data []byte, atEOF bool) (advance int, token []byte, err error) { - start := 0 - - // Return newline code or misk token. - r, width := utf8.DecodeRune(data[start:]) - if r == '\n' || isSplitSymbol(r) { - return start + width, data[start : start+width], nil - } - - tokenIsSpace := isSpace(r) - // Scan until isSpace result changed, marking end of word. - for width, i := 0, start; i < len(data); i += width { - var r rune - r, width = utf8.DecodeRune(data[i:]) - if r == '\n' || isSplitSymbol(r) || isSpace(r) != tokenIsSpace { - return i, data[start:i], nil - } - } - // If we're at EOF, we have a final, non-empty, non-terminated word. Return it. - if atEOF && len(data) > start { - return len(data), data[start:], nil - } - // Request more data. - return start, nil, nil -} diff --git a/lexer/scanner_test.go b/lexer/scanner_test.go deleted file mode 100644 index 6a6ee6c..0000000 --- a/lexer/scanner_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package lexer_test - -import ( - "bufio" - "strings" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/uji/solparser/lexer" -) - -// Test that the token splitter. -func TestScan(t *testing.T) { - tests := []struct { - input string - want []string - }{ - {"", []string{}}, - {" ", []string{" "}}, - {"\n", []string{"\n"}}, - {"a", []string{"a"}}, - {" a ", []string{" ", "a", " "}}, - {"abc def ", []string{"abc", " ", "def", " "}}, - {"a\tb\nc\r\td\f", []string{"a", "\t", "b", "\n", "c", "\r\t", "d", "\f"}}, - {"e\vf\u0085g\u00a0\n", []string{"e", "\v", "f", "\u0085", "g", "\u00a0", "\n"}}, - {"^0.8.13", []string{"^", "0.8.13"}}, - {"0.8.13;", []string{"0.8.13", ";"}}, - {"pragma solidity ^0.8.13;", []string{"pragma", " ", "solidity", " ", "^", "0.8.13", ";"}}, - {"contract HelloWorld {", []string{"contract", " ", "HelloWorld", " ", "{"}}, - {`\'test\'`, []string{`\`, "'", "test", `\`, "'"}}, - } - - for n, c := range tests { - buf := strings.NewReader(c.input) - s := bufio.NewScanner(buf) - s.Split(lexer.Scan) - - got := make([]string, 0, len(c.want)) - for i := 0; i < len(c.want); i++ { - if !s.Scan() { - break - } - got = append(got, s.Text()) - } - if s.Scan() { - t.Errorf("#%d: scan ran too long, got %q", n, s.Text()) - } - if diff := cmp.Diff(c.want, got); diff != "" { - t.Errorf("#%d: %s", n, diff) - } - err := s.Err() - if err != nil { - t.Errorf("#%d: %v", n, err) - } - } -} From 05fdd57ab923fc39b9b97b4d05eec558cac1a77d Mon Sep 17 00:00:00 2001 From: uji Date: Thu, 18 Aug 2022 20:35:11 +0900 Subject: [PATCH 066/138] scanner: publish IsSpace funcsion. --- scanner/scanner.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scanner/scanner.go b/scanner/scanner.go index f2c7f7c..7fa5bca 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -6,10 +6,10 @@ import ( "unicode/utf8" ) -// isSpace reports whether the character is a Unicode white space character. +// IsSpace reports whether the character is a Unicode white space character. // We avoid dependency on the unicode package, but check validity of the implementation // in the tests. -func isSpace(r rune) bool { +func IsSpace(r rune) bool { if r <= '\u00FF' { // Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs. switch r { @@ -48,12 +48,12 @@ func Split(data []byte, atEOF bool) (advance int, token []byte, err error) { return start + width, data[start : start+width], nil } - tokenIsSpace := isSpace(r) + tokenIsSpace := IsSpace(r) // Scan until isSpace result changed, marking end of word. for width, i := 0, start; i < len(data); i += width { var r rune r, width = utf8.DecodeRune(data[i:]) - if r == '\n' || isSplitSymbol(r) || isSpace(r) != tokenIsSpace { + if r == '\n' || isSplitSymbol(r) || IsSpace(r) != tokenIsSpace { return i, data[start:i], nil } } From fa4653262f8da536478e760e33f8b58a3fe0e625 Mon Sep 17 00:00:00 2001 From: uji Date: Thu, 18 Aug 2022 20:43:11 +0900 Subject: [PATCH 067/138] lexer: replace isSpace to scanner.IsSpace. --- lexer/lexer.go | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 867182f..7c3fb96 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -8,32 +8,6 @@ import ( "github.com/uji/solparser/token" ) -// isSpace reports whether the character is a Unicode white space character. -// We avoid dependency on the unicode package, but check validity of the implementation -// in the tests. -// TODO: remove function because scanner package has same function. -func isSpace(r rune) bool { - if r <= '\u00FF' { - // Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs. - switch r { - case ' ', '\t', '\v', '\f', '\r': - return true - case '\u0085', '\u00A0': - return true - } - return false - } - // High-valued ones. - if '\u2000' <= r && r <= '\u200a' { - return true - } - switch r { - case '\u1680', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000': - return true - } - return false -} - type Lexer struct { scanner *scanner.Scanner @@ -78,7 +52,7 @@ func (l *Lexer) scan() (result bool, tkn token.Token, err error) { offset = 0 continue } - if isSpace(r) { + if scanner.IsSpace(r) { offset += len([]rune(txt)) continue } From 5b285d7fcd82315473c9451a3f38fc86e1f34682 Mon Sep 17 00:00:00 2001 From: uji Date: Thu, 18 Aug 2022 21:06:37 +0900 Subject: [PATCH 068/138] scanner: add test for two length operator. --- scanner/scanner_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index fa6a981..7f12285 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -26,6 +26,7 @@ func TestSplit(t *testing.T) { {"pragma solidity ^0.8.13;", []string{"pragma", " ", "solidity", " ", "^", "0.8.13", ";"}}, {"contract HelloWorld {", []string{"contract", " ", "HelloWorld", " ", "{"}}, {`\'test\'`, []string{`\`, "'", "test", `\`, "'"}}, + {"a ++ b", []string{"a", " ", "+", "+", " ", "b"}}, } for n, c := range tests { From dd776c6c755b086016626991f579f08c96c78525 Mon Sep 17 00:00:00 2001 From: uji Date: Thu, 18 Aug 2022 21:12:28 +0900 Subject: [PATCH 069/138] lexer: fix testLexerScan. --- lexer/lexer_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 8e870e5..4cbb4ef 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -9,7 +9,7 @@ import ( "github.com/uji/solparser/token" ) -func testLexerScan(t *testing.T) { +func TestLexerScan(t *testing.T) { tests := []struct { name string offset int @@ -92,7 +92,7 @@ func testLexerScan(t *testing.T) { wantResult: true, wantToken: token.Token{ TokenType: token.BitXor, - Text: "", + Text: "^", Pos: token.Pos{ Column: 4, Line: 6, From fb3baf093c650b30a8bc9c5bd15788ded69b4e75 Mon Sep 17 00:00:00 2001 From: uji Date: Thu, 18 Aug 2022 21:21:52 +0900 Subject: [PATCH 070/138] token: add comment to operator. --- token/token.go | 102 ++++++++++++++++++++++++------------------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/token/token.go b/token/token.go index ee8f5fb..83d1ff9 100644 --- a/token/token.go +++ b/token/token.go @@ -8,60 +8,60 @@ const ( Invalid TokenType = iota Unknown - LParen - RParen - LBrack - RBrack - LBrace - RBrace - Colon - Semicolon - Period - Conditional - DoubleArrow - RightArrow + LParen // ( + RParen // ) + LBrack // [ + RBrack // ] + LBrace // { + RBrace // } + Colon // : + Semicolon // ; + Period // . + Conditional // ? + DoubleArrow // => + RightArrow // -> - Assign - AssignBitOr - AssignBitXor - AssignBitAnd - AssignShl - AssignSar - AssignShr - AssignAdd - AssignSub - AssignMul - AssignDiv - AssignMod + Assign // = + AssignBitOr // |= + AssignBitXor // ^= + AssignBitAnd // &= + AssignShl // <<= + AssignSar // >>= + AssignShr // >>>= + AssignAdd // += + AssignSub // -= + AssignMul // *= + AssignDiv // /= + AssignMod // %= - Comma - Or - And - BitOr - BitXor - BitAnd - Shl - Sar - Shr - Add - Sub - Mul - Div - Mod - Exp + Comma // , + Or // || + And // && + BitOr // | + BitXor // ^ + BitAnd // & + Shl // << + Sar // >> + Shr // >>> + Add // + + Sub // - + Mul // * + Div // / + Mod // % + Exp // ** - Equal - NotEqual - LessThan - GreaterThan - LessThanOrEqual - GreaterThanOrEqual - Not - BitNot - Inc - Dec - DoubleQuote - SingleQuote + Equal // = + NotEqual // != + LessThan // < + GreaterThan // > + LessThanOrEqual // <= + GreaterThanOrEqual // >= + Not // ! + BitNot // ~ + Inc // ++ + Dec // -- + DoubleQuote // " + SingleQuote // ' // Reserved Keyword After From cb834031966faa322c3f1d3a63867cf54db597dc Mon Sep 17 00:00:00 2001 From: uji Date: Thu, 18 Aug 2022 23:09:06 +0900 Subject: [PATCH 071/138] scanner: scan multi length operator. --- scanner/scanner.go | 61 +++++++++++++++++++++++++++++++++++++++-- scanner/scanner_test.go | 4 ++- 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/scanner/scanner.go b/scanner/scanner.go index 7fa5bca..1f27727 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -31,6 +31,14 @@ func IsSpace(r rune) bool { return false } +func isMultiLengthOperatorSymbol(r rune) bool { + switch r { + case '=', '|', '^', '&', '+', '-', '*', '/', '%', '<', '>', '!': + return true + } + return false +} + func isSplitSymbol(r rune) bool { switch r { case '(', ')', '[', ']', '{', '}', ':', ';', '?', '=', '|', '^', '&', '<', '>', '+', '-', '*', '/', '%', ',', '!', '~', '"', '\'', '\\': @@ -39,12 +47,61 @@ func isSplitSymbol(r rune) bool { return false } +// TODO: cover all patterns. +// scanOperatorLength return operator length at the prefix of data. +func scanOperatorLength(data []byte) (length int) { + if len(data) < 1 { + return 0 + } + + r1, _ := utf8.DecodeRune(data) + if !isSplitSymbol(r1) { + return 0 + } + if len(data) < 2 || !isMultiLengthOperatorSymbol(r1) { + return 1 + } + + r2, _ := utf8.DecodeRune(data[1:]) + switch r1 { + case '=': + if r2 == '>' { + return 2 + } + case '|': + if r2 == '=' || r2 == '|' { + return 2 + } + case '+': + if r2 == '=' || r2 == '+' { + return 2 + } + case '<': + switch r2 { + case '=': + return 2 + case '<': + r3, _ := utf8.DecodeRune(data[2:]) + if r3 == '=' { + return 3 + } + return 2 + } + } + return 1 +} + func Split(data []byte, atEOF bool) (advance int, token []byte, err error) { start := 0 - // Return newline code or misk token. r, width := utf8.DecodeRune(data[start:]) - if r == '\n' || isSplitSymbol(r) { + // Return operator. + if isSplitSymbol(r) { + w := scanOperatorLength(data[start:]) + return start + w, data[start : start+w], nil + } + // Return newline code. + if r == '\n' { return start + width, data[start : start+width], nil } diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index 7f12285..f1e0fcc 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -26,7 +26,9 @@ func TestSplit(t *testing.T) { {"pragma solidity ^0.8.13;", []string{"pragma", " ", "solidity", " ", "^", "0.8.13", ";"}}, {"contract HelloWorld {", []string{"contract", " ", "HelloWorld", " ", "{"}}, {`\'test\'`, []string{`\`, "'", "test", `\`, "'"}}, - {"a ++ b", []string{"a", " ", "+", "+", " ", "b"}}, + {"a ++ b", []string{"a", " ", "++", " ", "b"}}, + {"a =>b", []string{"a", " ", "=>", "b"}}, + {"a<<= b", []string{"a", "<<=", " ", "b"}}, } for n, c := range tests { From 4ecb8d8f7fc032466aaee352c930dfb5fd94b128 Mon Sep 17 00:00:00 2001 From: uji Date: Tue, 6 Sep 2022 03:21:02 +0900 Subject: [PATCH 072/138] scanner2: define scanner sigunature --- scanner2/scanner.go | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 scanner2/scanner.go diff --git a/scanner2/scanner.go b/scanner2/scanner.go new file mode 100644 index 0000000..2fab881 --- /dev/null +++ b/scanner2/scanner.go @@ -0,0 +1,39 @@ +package scanner + +import ( + "io" + + "github.com/uji/solparser/token" +) + +type Scanner struct { + // source + r io.Reader + + // position state + offset int + lineOffset int +} + +func New(reader io.Reader) *Scanner { + return &Scanner{ + r: reader, + } +} + +func (s *Scanner) readRune() (rune, error) { + char := make([]byte, 2) + + if _, err := s.r.Read(char); err != nil { + return 0, err + } + return ' ', nil +} + +func (s *Scanner) Scan() (pos token.Pos, lit string, err error) { + return token.Pos{}, "", nil +} + +func (s *Scanner) Peek() (pos token.Pos, lit string, err error) { + return token.Pos{}, "", nil +} From 35b5e02c507f212a4453f946f4c4a917072c4924 Mon Sep 17 00:00:00 2001 From: uji Date: Tue, 6 Sep 2022 03:31:15 +0900 Subject: [PATCH 073/138] lexer: use new scanner package --- lexer/lexer.go | 42 ++------------- lexer/lexer_test.go | 129 ++++++++++++++------------------------------ 2 files changed, 46 insertions(+), 125 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 7c3fb96..b2e71cb 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -2,9 +2,8 @@ package lexer import ( "io" - "unicode/utf8" - "github.com/uji/solparser/scanner" + scanner "github.com/uji/solparser/scanner2" "github.com/uji/solparser/token" ) @@ -15,10 +14,6 @@ type Lexer struct { token token.Token err error - // position state - offset int - lineOffset int - // peek state peeked bool peekToken token.Token @@ -34,37 +29,12 @@ func New(input io.Reader) *Lexer { } func (l *Lexer) scan() (result bool, tkn token.Token, err error) { - offset := l.offset - lineOffset := l.lineOffset - - // Scan until next token. - for { - if !l.scanner.Scan() { - return false, token.Token{}, nil - } - if err := l.scanner.Err(); err != nil { - return false, token.Token{}, err - } - txt := l.scanner.Text() - r, _ := utf8.DecodeRune([]byte(txt)) - if r == '\n' { - lineOffset++ - offset = 0 - continue - } - if scanner.IsSpace(r) { - offset += len([]rune(txt)) - continue - } - break - } - txt := l.scanner.Text() - pos := token.Pos{ - Column: offset + 1, - Line: lineOffset + 1, + pos, lit, err := l.scanner.Scan() + if err != nil { + return false, token.Token{}, err } - return true, token.NewToken(txt, pos), nil + return true, token.NewToken(lit, pos), nil } func (l *Lexer) Scan() (result bool) { @@ -84,8 +54,6 @@ func (l *Lexer) Scan() (result bool) { return false } l.token = tkn - l.offset = tkn.Pos.Column - 1 + len([]rune(tkn.Text)) - l.lineOffset = tkn.Pos.Line - 1 return true } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 4cbb4ef..1c1d8b2 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -5,15 +5,13 @@ import ( "testing" "github.com/google/go-cmp/cmp" - "github.com/uji/solparser/scanner" + scanner "github.com/uji/solparser/scanner2" "github.com/uji/solparser/token" ) func TestLexerScan(t *testing.T) { tests := []struct { name string - offset int - lineOffset int input string peeked bool peekToken token.Token @@ -22,65 +20,30 @@ func TestLexerScan(t *testing.T) { wantErr error wantToken token.Token }{ + // TODO: fix test after implementing new scanner. + // { + // name: "normal", + // input: "pragma", + // wantResult: true, + // wantToken: token.Token{ + // TokenType: token.Pragma, + // Text: "pragma", + // Pos: token.Pos{ + // Column: 5, + // Line: 6, + // }, + // }, + // }, + // { + // name: "when scan is done", + // input: "", + // wantResult: false, + // wantToken: token.Token{}, + // }, { - name: "normal", - offset: 4, - lineOffset: 5, - input: "pragma", - wantResult: true, - wantToken: token.Token{ - TokenType: token.Pragma, - Text: "pragma", - Pos: token.Pos{ - Column: 5, - Line: 6, - }, - }, - }, - { - name: "when scan space", - offset: 4, - lineOffset: 5, - input: " ^", - wantResult: true, - wantToken: token.Token{ - TokenType: token.BitXor, - Text: "^", - Pos: token.Pos{ - Column: 7, - Line: 6, - }, - }, - }, - { - name: "when scan \\n", - offset: 4, - lineOffset: 5, - input: " \n^", - wantResult: true, - wantToken: token.Token{ - TokenType: token.BitXor, - Text: "^", - Pos: token.Pos{ - Column: 1, - Line: 7, - }, - }, - }, - { - name: "when scan is done", - offset: 4, - lineOffset: 5, - input: "", - wantResult: false, - wantToken: token.Token{}, - }, - { - name: "when peeked", - offset: 4, - lineOffset: 5, - input: "", - peeked: true, + name: "when peeked", + input: "", + peeked: true, peekToken: token.Token{ TokenType: token.BitXor, Text: "^", @@ -106,12 +69,10 @@ func TestLexerScan(t *testing.T) { s := scanner.New(strings.NewReader(tt.input)) l := Lexer{ - offset: tt.offset, - lineOffset: tt.lineOffset, - scanner: s, - peeked: tt.peeked, - peekToken: tt.peekToken, - peekErr: tt.peekErr, + scanner: s, + peeked: tt.peeked, + peekToken: tt.peekToken, + peekErr: tt.peekErr, } if rslt := l.Scan(); rslt != tt.wantResult { t.Errorf("result is wrong, want: %t, got: %t", tt.wantResult, rslt) @@ -128,20 +89,16 @@ func TestLexerScan(t *testing.T) { func TestLexerPeek(t *testing.T) { tests := []struct { - name string - offset int - lineOffset int - input string - result bool - token token.Token - err error + name string + input string + result bool + token token.Token + err error }{ { - name: "normal", - offset: 4, - lineOffset: 5, - input: "pragma", - result: true, + name: "normal", + input: "pragma", + result: true, token: token.Token{ TokenType: token.Pragma, Text: "pragma", @@ -152,12 +109,10 @@ func TestLexerPeek(t *testing.T) { }, }, { - name: "when scan is done", - offset: 4, - lineOffset: 5, - input: "", - result: false, - token: token.Token{}, + name: "when scan is done", + input: "", + result: false, + token: token.Token{}, }, } for _, tt := range tests { @@ -166,9 +121,7 @@ func TestLexerPeek(t *testing.T) { s := scanner.New(strings.NewReader(tt.input)) l := Lexer{ - offset: tt.offset, - lineOffset: tt.lineOffset, - scanner: s, + scanner: s, } if rslt := l.Peek(); tt.result != rslt { t.Errorf("result is wrong, want: %t, got: %t", tt.result, rslt) From f2c6e220f727a65f61416cae7e9c7794a234e6d2 Mon Sep 17 00:00:00 2001 From: uji Date: Tue, 13 Sep 2022 00:26:36 +0900 Subject: [PATCH 074/138] scanner2: implement base of scan --- go.mod | 5 +- go.sum | 2 + scanner2/scanner.go | 104 +++++++++++++++++++++++++++++++++++---- scanner2/scanner_test.go | 82 ++++++++++++++++++++++++++++++ 4 files changed, 183 insertions(+), 10 deletions(-) create mode 100644 scanner2/scanner_test.go diff --git a/go.mod b/go.mod index 7884694..5096641 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,7 @@ module github.com/uji/solparser go 1.16 -require github.com/google/go-cmp v0.5.8 +require ( + github.com/SteelSeries/bufrr v0.0.0-20161129220322-72103137aa3c + github.com/google/go-cmp v0.5.8 +) diff --git a/go.sum b/go.sum index e9b099c..b1690b2 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,4 @@ +github.com/SteelSeries/bufrr v0.0.0-20161129220322-72103137aa3c h1:SlXEIQaashOxRg92emqhRAAUma1GmJW4t1y49om03Ck= +github.com/SteelSeries/bufrr v0.0.0-20161129220322-72103137aa3c/go.mod h1:B80GUChy+pCjfTRNgr9KKXfMOGNhVwl2cQqyMjoyQ5s= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= diff --git a/scanner2/scanner.go b/scanner2/scanner.go index 2fab881..5f6c4a9 100644 --- a/scanner2/scanner.go +++ b/scanner2/scanner.go @@ -3,12 +3,13 @@ package scanner import ( "io" + "github.com/SteelSeries/bufrr" "github.com/uji/solparser/token" ) type Scanner struct { // source - r io.Reader + r *bufrr.Reader // position state offset int @@ -16,22 +17,107 @@ type Scanner struct { } func New(reader io.Reader) *Scanner { + r := bufrr.NewReader(reader) return &Scanner{ - r: reader, + r: r, } } -func (s *Scanner) readRune() (rune, error) { - char := make([]byte, 2) +// isSpace reports whether the character is a Unicode white space character. +// We avoid dependency on the unicode package, but check validity of the implementation +// in the tests. +func isSpace(r rune) bool { + if r <= '\u00FF' { + // Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs. + switch r { + case ' ', '\t', '\v', '\f', '\r': + return true + case '\u0085', '\u00A0': + return true + } + return false + } + // High-valued ones. + if '\u2000' <= r && r <= '\u200a' { + return true + } + switch r { + case '\u1680', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000': + return true + } + return false +} - if _, err := s.r.Read(char); err != nil { - return 0, err +func isMultiLengthOperatorSymbol(r rune) bool { + switch r { + case '=', '|', '^', '&', '+', '-', '*', '/', '%', '<', '>', '!': + return true } - return ' ', nil + return false } -func (s *Scanner) Scan() (pos token.Pos, lit string, err error) { - return token.Pos{}, "", nil +func isSplitSymbol(r rune) bool { + switch r { + case '(', ')', '[', ']', '{', '}', ':', ';', '?', '=', '|', '^', '&', '<', '>', '+', '-', '*', '/', '%', ',', '!', '~', '"', '\'', '\\': + return true + } + return false +} + +const invalidRune = -1 + +func (s *Scanner) readRune() (rune, error) { + r, _, err := s.r.ReadRune() + if err != nil { + return invalidRune, err + } + if r == '\n' { + s.offset = 0 + s.lineOffset++ + } else { + s.offset++ + } + return r, nil +} + +func (s *Scanner) Scan() (token.Pos, string, error) { + pos := token.Pos{ + Column: s.offset + 1, + Line: s.lineOffset + 1, + } + + var char rune + for { + ch, err := s.readRune() + if err != nil { + return token.Pos{}, "", err + } + if ch == bufrr.EOF { + return token.Pos{}, "", io.EOF + } + if !isSpace(ch) { + char = ch + break + } + } + runes := []rune{char} + + for { + ch, err := s.readRune() + if err != nil { + return token.Pos{}, "", err + } + if !isSpace(ch) && !isSplitSymbol(ch) && ch != bufrr.EOF { + runes = append(runes, ch) + continue + } + if err := s.r.UnreadRune(); err != nil { + return token.Pos{}, "", err + } + break + } + + return pos, string(runes), nil } func (s *Scanner) Peek() (pos token.Pos, lit string, err error) { diff --git a/scanner2/scanner_test.go b/scanner2/scanner_test.go new file mode 100644 index 0000000..5adb596 --- /dev/null +++ b/scanner2/scanner_test.go @@ -0,0 +1,82 @@ +package scanner + +import ( + "io" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser/token" +) + +func TestScannerScan(t *testing.T) { + tests := []struct { + name string + input string + wantPoss []token.Pos + wantLits []string + }{ + { + name: "normal", + input: "pragma solidity ^0.8.13;", + wantPoss: []token.Pos{ + { + Column: 1, + Line: 1, + }, + { + Column: 8, + Line: 1, + }, + { + Column: 17, + Line: 1, + }, + { + Column: 18, + Line: 1, + }, + { + Column: 24, + Line: 1, + }, + }, + wantLits: []string{ + "pragma", + "solidity", + "^", + "0.8.13", + ";", + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + s := New(strings.NewReader(tt.input)) + + poss := make([]token.Pos, 0, len(tt.wantPoss)) + lits := make([]string, 0, len(tt.wantLits)) + + for { + pos, lit, err := s.Scan() + if err == io.EOF { + break + } + if err != nil { + t.Errorf("got unexpected error: %v", err) + } + poss = append(poss, pos) + lits = append(lits, lit) + } + + if diff := cmp.Diff(poss, tt.wantPoss); diff != "" { + t.Errorf(diff) + } + if diff := cmp.Diff(lits, tt.wantLits); diff != "" { + t.Errorf(diff) + } + }) + } +} From 83d32277433fd004c7dde889d83a9a56d835e677 Mon Sep 17 00:00:00 2001 From: uji Date: Thu, 3 Nov 2022 22:09:56 +0900 Subject: [PATCH 075/138] scanner2: fix scanner logic --- scanner2/scanner.go | 62 ++++++++++++++++++++++++++-------------- scanner2/scanner_test.go | 4 +-- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/scanner2/scanner.go b/scanner2/scanner.go index 5f6c4a9..1cd9e23 100644 --- a/scanner2/scanner.go +++ b/scanner2/scanner.go @@ -74,47 +74,65 @@ func (s *Scanner) readRune() (rune, error) { if r == '\n' { s.offset = 0 s.lineOffset++ - } else { - s.offset++ + return r, nil } + s.offset++ return r, nil } +func (s *Scanner) unreadRune() error { + if err := s.r.UnreadRune(); err != nil { + return err + } + s.offset-- + return nil +} + func (s *Scanner) Scan() (token.Pos, string, error) { - pos := token.Pos{ - Column: s.offset + 1, - Line: s.lineOffset + 1, + ch, err := s.readRune() + if err != nil { + return token.Pos{}, "", err + } + if ch == bufrr.EOF { + return token.Pos{}, "", io.EOF + } + if isSplitSymbol(ch) { + return token.Pos{ + Column: s.offset, + Line: s.lineOffset + 1, + }, string(ch), nil } - var char rune - for { - ch, err := s.readRune() + // skip space. + for isSpace(ch) { + ch, err = s.readRune() if err != nil { return token.Pos{}, "", err } - if ch == bufrr.EOF { - return token.Pos{}, "", io.EOF - } - if !isSpace(ch) { - char = ch - break - } } - runes := []rune{char} + + runes := []rune{ch} + pos := token.Pos{ + Column: s.offset, + Line: s.lineOffset + 1, + } for { ch, err := s.readRune() if err != nil { return token.Pos{}, "", err } - if !isSpace(ch) && !isSplitSymbol(ch) && ch != bufrr.EOF { - runes = append(runes, ch) - continue + if isSpace(ch) || ch == bufrr.EOF { + break } - if err := s.r.UnreadRune(); err != nil { - return token.Pos{}, "", err + if isSplitSymbol(ch) { + // unread to be read in the next scan. + if err := s.unreadRune(); err != nil { + return token.Pos{}, "", err + } + break } - break + runes = append(runes, ch) } return pos, string(runes), nil diff --git a/scanner2/scanner_test.go b/scanner2/scanner_test.go index 5adb596..183432d 100644 --- a/scanner2/scanner_test.go +++ b/scanner2/scanner_test.go @@ -71,10 +71,10 @@ func TestScannerScan(t *testing.T) { lits = append(lits, lit) } - if diff := cmp.Diff(poss, tt.wantPoss); diff != "" { + if diff := cmp.Diff(tt.wantPoss, poss); diff != "" { t.Errorf(diff) } - if diff := cmp.Diff(lits, tt.wantLits); diff != "" { + if diff := cmp.Diff(tt.wantLits, lits); diff != "" { t.Errorf(diff) } }) From 9dd8e829708498f29df1f6e06c6ee740b8ad63fb Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 6 Nov 2022 22:20:41 +0900 Subject: [PATCH 076/138] scanner2: fix \n logic and refactoring --- scanner2/scanner.go | 60 +++++++++++++++++----------------------- scanner2/scanner_test.go | 25 ++++++++++++++++- 2 files changed, 49 insertions(+), 36 deletions(-) diff --git a/scanner2/scanner.go b/scanner2/scanner.go index 1cd9e23..ba84528 100644 --- a/scanner2/scanner.go +++ b/scanner2/scanner.go @@ -80,61 +80,51 @@ func (s *Scanner) readRune() (rune, error) { return r, nil } -func (s *Scanner) unreadRune() error { - if err := s.r.UnreadRune(); err != nil { - return err - } - s.offset-- - return nil -} - func (s *Scanner) Scan() (token.Pos, string, error) { - ch, err := s.readRune() - if err != nil { - return token.Pos{}, "", err - } - if ch == bufrr.EOF { - return token.Pos{}, "", io.EOF - } - if isSplitSymbol(ch) { - return token.Pos{ - Column: s.offset, - Line: s.lineOffset + 1, - }, string(ch), nil - } + runes := make([]rune, 1) + for { + // Retain the state before reading. + c := s.offset + 1 + l := s.lineOffset + 1 - // skip space. - for isSpace(ch) { - ch, err = s.readRune() + ch, err := s.readRune() if err != nil { return token.Pos{}, "", err } + if ch == bufrr.EOF { + return token.Pos{}, "", io.EOF + } + if isSplitSymbol(ch) || ch == '\n' { + return token.Pos{ + Column: c, + Line: l, + }, string(ch), nil + } + if !isSpace(ch) { + runes[0] = ch + break + } } - runes := []rune{ch} pos := token.Pos{ Column: s.offset, Line: s.lineOffset + 1, } for { - ch, err := s.readRune() + ch, _, err := s.r.PeekRune() if err != nil { return token.Pos{}, "", err } - if isSpace(ch) || ch == bufrr.EOF { - break + if isSpace(ch) || isSplitSymbol(ch) || ch == '\n' || ch == bufrr.EOF { + return pos, string(runes), nil } - if isSplitSymbol(ch) { - // unread to be read in the next scan. - if err := s.unreadRune(); err != nil { - return token.Pos{}, "", err - } - break + ch, err = s.readRune() + if err != nil { + return token.Pos{}, "", err } runes = append(runes, ch) } - return pos, string(runes), nil } diff --git a/scanner2/scanner_test.go b/scanner2/scanner_test.go index 183432d..c270afc 100644 --- a/scanner2/scanner_test.go +++ b/scanner2/scanner_test.go @@ -49,6 +49,29 @@ func TestScannerScan(t *testing.T) { ";", }, }, + { + name: "normal", + input: "pragma\n solidity", + wantPoss: []token.Pos{ + { + Column: 1, + Line: 1, + }, + { + Column: 7, + Line: 1, + }, + { + Column: 2, + Line: 2, + }, + }, + wantLits: []string{ + "pragma", + "\n", + "solidity", + }, + }, } for _, tt := range tests { @@ -65,7 +88,7 @@ func TestScannerScan(t *testing.T) { break } if err != nil { - t.Errorf("got unexpected error: %v", err) + t.Fatalf("got unexpected error: %s", err) } poss = append(poss, pos) lits = append(lits, lit) From 2b8ce9e33df6a29b344b6351c6dde765603b1f7f Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 6 Nov 2022 22:59:45 +0900 Subject: [PATCH 077/138] lexer: fix test for scanner2 --- lexer/lexer_test.go | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 1c1d8b2..7a4ad1e 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -1,6 +1,7 @@ package lexer import ( + "io" "strings" "testing" @@ -20,26 +21,26 @@ func TestLexerScan(t *testing.T) { wantErr error wantToken token.Token }{ - // TODO: fix test after implementing new scanner. - // { - // name: "normal", - // input: "pragma", - // wantResult: true, - // wantToken: token.Token{ - // TokenType: token.Pragma, - // Text: "pragma", - // Pos: token.Pos{ - // Column: 5, - // Line: 6, - // }, - // }, - // }, - // { - // name: "when scan is done", - // input: "", - // wantResult: false, - // wantToken: token.Token{}, - // }, + { + name: "normal", + input: "pragma", + wantResult: true, + wantToken: token.Token{ + TokenType: token.Pragma, + Text: "pragma", + Pos: token.Pos{ + Column: 1, + Line: 1, + }, + }, + }, + { + name: "when scan is done", + input: "", + wantResult: false, + wantToken: token.Token{}, + wantErr: io.EOF, + }, { name: "when peeked", input: "", @@ -103,8 +104,8 @@ func TestLexerPeek(t *testing.T) { TokenType: token.Pragma, Text: "pragma", Pos: token.Pos{ - Column: 5, - Line: 6, + Column: 1, + Line: 1, }, }, }, From 9b6e624f25e4075059dfb385d724fe934d90b5d6 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 6 Nov 2022 23:00:41 +0900 Subject: [PATCH 078/138] scanner: remove old scanner package --- scanner/scanner.go | 193 ---------------------------------------- scanner/scanner_test.go | 185 -------------------------------------- 2 files changed, 378 deletions(-) delete mode 100644 scanner/scanner.go delete mode 100644 scanner/scanner_test.go diff --git a/scanner/scanner.go b/scanner/scanner.go deleted file mode 100644 index 1f27727..0000000 --- a/scanner/scanner.go +++ /dev/null @@ -1,193 +0,0 @@ -package scanner - -import ( - "bufio" - "io" - "unicode/utf8" -) - -// IsSpace reports whether the character is a Unicode white space character. -// We avoid dependency on the unicode package, but check validity of the implementation -// in the tests. -func IsSpace(r rune) bool { - if r <= '\u00FF' { - // Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs. - switch r { - case ' ', '\t', '\v', '\f', '\r': - return true - case '\u0085', '\u00A0': - return true - } - return false - } - // High-valued ones. - if '\u2000' <= r && r <= '\u200a' { - return true - } - switch r { - case '\u1680', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000': - return true - } - return false -} - -func isMultiLengthOperatorSymbol(r rune) bool { - switch r { - case '=', '|', '^', '&', '+', '-', '*', '/', '%', '<', '>', '!': - return true - } - return false -} - -func isSplitSymbol(r rune) bool { - switch r { - case '(', ')', '[', ']', '{', '}', ':', ';', '?', '=', '|', '^', '&', '<', '>', '+', '-', '*', '/', '%', ',', '!', '~', '"', '\'', '\\': - return true - } - return false -} - -// TODO: cover all patterns. -// scanOperatorLength return operator length at the prefix of data. -func scanOperatorLength(data []byte) (length int) { - if len(data) < 1 { - return 0 - } - - r1, _ := utf8.DecodeRune(data) - if !isSplitSymbol(r1) { - return 0 - } - if len(data) < 2 || !isMultiLengthOperatorSymbol(r1) { - return 1 - } - - r2, _ := utf8.DecodeRune(data[1:]) - switch r1 { - case '=': - if r2 == '>' { - return 2 - } - case '|': - if r2 == '=' || r2 == '|' { - return 2 - } - case '+': - if r2 == '=' || r2 == '+' { - return 2 - } - case '<': - switch r2 { - case '=': - return 2 - case '<': - r3, _ := utf8.DecodeRune(data[2:]) - if r3 == '=' { - return 3 - } - return 2 - } - } - return 1 -} - -func Split(data []byte, atEOF bool) (advance int, token []byte, err error) { - start := 0 - - r, width := utf8.DecodeRune(data[start:]) - // Return operator. - if isSplitSymbol(r) { - w := scanOperatorLength(data[start:]) - return start + w, data[start : start+w], nil - } - // Return newline code. - if r == '\n' { - return start + width, data[start : start+width], nil - } - - tokenIsSpace := IsSpace(r) - // Scan until isSpace result changed, marking end of word. - for width, i := 0, start; i < len(data); i += width { - var r rune - r, width = utf8.DecodeRune(data[i:]) - if r == '\n' || isSplitSymbol(r) || IsSpace(r) != tokenIsSpace { - return i, data[start:i], nil - } - } - // If we're at EOF, we have a final, non-empty, non-terminated word. Return it. - if atEOF && len(data) > start { - return len(data), data[start:], nil - } - // Request more data. - return start, nil, nil -} - -type Scanner struct { - scanner *bufio.Scanner - - // scan result - text string - err error - - // peek state - peeked bool - peekText string - peekErr error -} - -func New(input io.Reader) *Scanner { - s := bufio.NewScanner(input) - s.Split(Split) - - return &Scanner{ - scanner: s, - } -} - -func (s *Scanner) Scan() bool { - if s.peeked { - s.text = s.peekText - s.err = s.peekErr - s.peeked = false - return true - } - - if !s.scanner.Scan() { - return false - } - - s.text = s.scanner.Text() - s.err = s.scanner.Err() - return true -} - -func (s Scanner) Text() string { - return s.text -} - -func (s Scanner) Err() error { - return s.err -} - -func (s *Scanner) Peek() bool { - if s.peeked { - return true - } - - if !s.scanner.Scan() { - return false - } - - s.peeked = true - s.peekText = s.scanner.Text() - s.peekErr = s.scanner.Err() - return true -} - -func (s Scanner) PeekText() string { - return s.peekText -} - -func (s Scanner) PeekErr() error { - return s.peekErr -} diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go deleted file mode 100644 index f1e0fcc..0000000 --- a/scanner/scanner_test.go +++ /dev/null @@ -1,185 +0,0 @@ -package scanner - -import ( - "bufio" - "strings" - "testing" - - "github.com/google/go-cmp/cmp" -) - -func TestSplit(t *testing.T) { - tests := []struct { - input string - want []string - }{ - {"", []string{}}, - {" ", []string{" "}}, - {"\n", []string{"\n"}}, - {"a", []string{"a"}}, - {" a ", []string{" ", "a", " "}}, - {"abc def ", []string{"abc", " ", "def", " "}}, - {"a\tb\nc\r\td\f", []string{"a", "\t", "b", "\n", "c", "\r\t", "d", "\f"}}, - {"e\vf\u0085g\u00a0\n", []string{"e", "\v", "f", "\u0085", "g", "\u00a0", "\n"}}, - {"^0.8.13", []string{"^", "0.8.13"}}, - {"0.8.13;", []string{"0.8.13", ";"}}, - {"pragma solidity ^0.8.13;", []string{"pragma", " ", "solidity", " ", "^", "0.8.13", ";"}}, - {"contract HelloWorld {", []string{"contract", " ", "HelloWorld", " ", "{"}}, - {`\'test\'`, []string{`\`, "'", "test", `\`, "'"}}, - {"a ++ b", []string{"a", " ", "++", " ", "b"}}, - {"a =>b", []string{"a", " ", "=>", "b"}}, - {"a<<= b", []string{"a", "<<=", " ", "b"}}, - } - - for n, c := range tests { - buf := strings.NewReader(c.input) - s := bufio.NewScanner(buf) - s.Split(Split) - - got := make([]string, 0, len(c.want)) - for i := 0; i < len(c.want); i++ { - if !s.Scan() { - break - } - got = append(got, s.Text()) - } - if s.Scan() { - t.Errorf("#%d: scan ran too long, got %q", n, s.Text()) - } - if diff := cmp.Diff(c.want, got); diff != "" { - t.Errorf("#%d: %s", n, diff) - } - err := s.Err() - if err != nil { - t.Errorf("#%d: %v", n, err) - } - } -} - -func TestScannerScan(t *testing.T) { - tests := []struct { - name string - input string - peeked bool - peekText string - peekErr error - want bool - wantErr error - wantText string - }{ - { - name: "normal", - input: "pragma solidity", - want: true, - wantErr: nil, - wantText: "pragma", - }, - { - name: "when scan is done", - input: "", - want: false, - wantErr: nil, - wantText: "", - }, - { - name: "when peeked", - input: "", - peeked: true, - peekText: "peekedText", - want: true, - wantErr: nil, - wantText: "peekedText", - }, - } - - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - scanner := bufio.NewScanner(strings.NewReader(tt.input)) - scanner.Split(Split) - - s := Scanner{ - scanner: scanner, - peeked: tt.peeked, - peekText: tt.peekText, - peekErr: tt.peekErr, - } - - if rslt := s.Scan(); rslt != tt.want { - t.Errorf("want: %t, got: %t", tt.want, rslt) - } - - if err := s.Err(); err != tt.wantErr { - t.Errorf("want: %s, got: %s", tt.wantErr, err) - } - - if text := s.Text(); text != tt.wantText { - t.Errorf("want: %s, got: %s", tt.wantText, text) - } - }) - } -} - -func TestScannerPeek(t *testing.T) { - tests := []struct { - name string - input string - peeked bool - peekText string - peekErr error - want bool - wantErr error - wantText string - }{ - { - name: "when not peeked", - input: "pragma solidity", - want: true, - wantErr: nil, - wantText: "pragma", - }, - { - name: "when scan is done", - input: "", - want: false, - wantErr: nil, - wantText: "", - }, - { - name: "when peeked", - input: "", - peeked: true, - peekText: "peekedText", - want: true, - wantErr: nil, - wantText: "peekedText", - }, - } - - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - scanner := bufio.NewScanner(strings.NewReader(tt.input)) - scanner.Split(Split) - - s := Scanner{ - scanner: scanner, - peeked: tt.peeked, - peekText: tt.peekText, - peekErr: tt.peekErr, - } - - if rslt := s.Peek(); rslt != tt.want { - t.Errorf("want: %t, got: %t", tt.want, rslt) - } - - if err := s.PeekErr(); err != tt.wantErr { - t.Errorf("want: %s, got: %s", tt.wantErr, err) - } - - if text := s.PeekText(); text != tt.wantText { - t.Errorf("want: %s, got: %s", tt.wantText, text) - } - }) - } -} From 81ed11592805c89e04a7c3efd2682939ab5f0bd5 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 6 Nov 2022 23:03:09 +0900 Subject: [PATCH 079/138] scanner2: rename package scanner2 -> scanner --- lexer/lexer.go | 2 +- lexer/lexer_test.go | 2 +- {scanner2 => scanner}/scanner.go | 0 {scanner2 => scanner}/scanner_test.go | 0 4 files changed, 2 insertions(+), 2 deletions(-) rename {scanner2 => scanner}/scanner.go (100%) rename {scanner2 => scanner}/scanner_test.go (100%) diff --git a/lexer/lexer.go b/lexer/lexer.go index b2e71cb..970a8e6 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -3,7 +3,7 @@ package lexer import ( "io" - scanner "github.com/uji/solparser/scanner2" + "github.com/uji/solparser/scanner" "github.com/uji/solparser/token" ) diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 7a4ad1e..06a9e35 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -6,7 +6,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" - scanner "github.com/uji/solparser/scanner2" + "github.com/uji/solparser/scanner" "github.com/uji/solparser/token" ) diff --git a/scanner2/scanner.go b/scanner/scanner.go similarity index 100% rename from scanner2/scanner.go rename to scanner/scanner.go diff --git a/scanner2/scanner_test.go b/scanner/scanner_test.go similarity index 100% rename from scanner2/scanner_test.go rename to scanner/scanner_test.go From 907d917b8dde5cd8746b456691a3c68269d724aa Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 6 Nov 2022 23:06:58 +0900 Subject: [PATCH 080/138] scanner: remove unreachable code --- scanner/scanner.go | 1 - 1 file changed, 1 deletion(-) diff --git a/scanner/scanner.go b/scanner/scanner.go index ba84528..589e414 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -125,7 +125,6 @@ func (s *Scanner) Scan() (token.Pos, string, error) { } runes = append(runes, ch) } - return pos, string(runes), nil } func (s *Scanner) Peek() (pos token.Pos, lit string, err error) { From f627bdaa027edc96fb8ab9c564671eacce64a511 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 12 Nov 2022 21:41:42 +0900 Subject: [PATCH 081/138] .github/workflows: fix test options --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a5082e..66301f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,4 +25,4 @@ jobs: steps: - uses: actions/checkout@v3 - name: test - run: go test ./... -v + run: go test -race -cover -shuffle=on ./... From d86e97a7a2b9020e2a28aa0c6a1f93ec9ebcf1f7 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 12 Nov 2022 22:18:49 +0900 Subject: [PATCH 082/138] .gitignore: init --- .gitignore | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f9a726a --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Temporary file directory +/tmp/ + +# Go workspace file +go.work + +# Dependency directory +/vendor/ From fdb499e5ea0aabc13fc867a0b52699a4f9368cbc Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 12 Nov 2022 22:19:16 +0900 Subject: [PATCH 083/138] Makefile: init, define coverage command. --- Makefile | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6b5d723 --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +.PHONY: help coverage +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +coverage: ## generate test coverage profile to cover.html + go test -cover -coverprofile=cover.out ./... + go tool cover -html=cover.out -o cover.html From 98f035bb4b9598ca86057c0800344ba6abb8f9b8 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 12 Nov 2022 22:28:43 +0900 Subject: [PATCH 084/138] .gitignore: remove tmp directory --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index f9a726a..808b277 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ # Output of the go coverage tool, specifically when used with LiteIDE *.out -# Temporary file directory -/tmp/ +# Test coverage file +cover.html # Go workspace file go.work From 8ab745b8005e9b87dc98b3ad92c7ece76da57484 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 12 Nov 2022 23:07:08 +0900 Subject: [PATCH 085/138] lexer: remove result return value from Lexer.Scan() --- contractDefinition.go | 26 +++++++++++++++++++------- lexer/lexer.go | 31 ++++--------------------------- lexer/lexer_test.go | 38 ++++++++++++++++---------------------- pragmaDirective.go | 32 ++++++++++++++++++++++---------- solparser.go | 10 +++------- 5 files changed, 64 insertions(+), 73 deletions(-) diff --git a/contractDefinition.go b/contractDefinition.go index bad1e2b..af04f35 100644 --- a/contractDefinition.go +++ b/contractDefinition.go @@ -9,8 +9,10 @@ import ( func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { var abstract bool - p.lexer.Scan() - tkn := p.lexer.Token() + tkn, err := p.lexer.Scan() + if err != nil { + return nil, err + } if tkn.TokenType == token.Abstract { abstract = true p.lexer.Scan() @@ -19,17 +21,27 @@ func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { return nil, errors.New("not found contract definition") } - p.lexer.Scan() - if p.lexer.Token().TokenType != token.Unknown { + tkn, err = p.lexer.Scan() + if err != nil { + return nil, err + } + if tkn.TokenType != token.Unknown { return nil, errors.New("not found left brace") } - if p.lexer.Token().TokenType != token.LBrace { + tkn, err = p.lexer.Scan() + if err != nil { + return nil, err + } + if tkn.TokenType != token.LBrace { return nil, errors.New("not found left brace") } - p.lexer.Scan() - if p.lexer.Token().TokenType != token.RBrace { + tkn, err = p.lexer.Scan() + if err != nil { + return nil, err + } + if tkn.TokenType != token.RBrace { return nil, errors.New("not found right brace") } diff --git a/lexer/lexer.go b/lexer/lexer.go index 970a8e6..99869d8 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -10,10 +10,6 @@ import ( type Lexer struct { scanner *scanner.Scanner - // scan result - token token.Token - err error - // peek state peeked bool peekToken token.Token @@ -37,33 +33,14 @@ func (l *Lexer) scan() (result bool, tkn token.Token, err error) { return true, token.NewToken(lit, pos), nil } -func (l *Lexer) Scan() (result bool) { +func (l *Lexer) Scan() (token.Token, error) { if l.peeked { - l.token = l.peekToken - l.err = l.peekErr l.peeked = false - return true - } - - rslt, tkn, err := l.scan() - if err != nil { - l.err = err - return false + return l.peekToken, l.peekErr } - if !rslt { - return false - } - l.token = tkn - - return true -} - -func (l Lexer) Token() token.Token { - return l.token -} -func (l Lexer) Error() error { - return l.err + _, tkn, err := l.scan() + return tkn, err } func (l *Lexer) Peek() bool { diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 06a9e35..5754253 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -12,19 +12,17 @@ import ( func TestLexerScan(t *testing.T) { tests := []struct { - name string - input string - peeked bool - peekToken token.Token - peekErr error - wantResult bool - wantErr error - wantToken token.Token + name string + input string + peeked bool + peekToken token.Token + peekErr error + wantErr error + wantToken token.Token }{ { - name: "normal", - input: "pragma", - wantResult: true, + name: "normal", + input: "pragma", wantToken: token.Token{ TokenType: token.Pragma, Text: "pragma", @@ -35,11 +33,10 @@ func TestLexerScan(t *testing.T) { }, }, { - name: "when scan is done", - input: "", - wantResult: false, - wantToken: token.Token{}, - wantErr: io.EOF, + name: "when scan is done", + input: "", + wantToken: token.Token{}, + wantErr: io.EOF, }, { name: "when peeked", @@ -53,7 +50,6 @@ func TestLexerScan(t *testing.T) { Line: 6, }, }, - wantResult: true, wantToken: token.Token{ TokenType: token.BitXor, Text: "^", @@ -75,13 +71,11 @@ func TestLexerScan(t *testing.T) { peekToken: tt.peekToken, peekErr: tt.peekErr, } - if rslt := l.Scan(); rslt != tt.wantResult { - t.Errorf("result is wrong, want: %t, got: %t", tt.wantResult, rslt) - } - if err := l.Error(); err != tt.wantErr { + tkn, err := l.Scan() + if err != tt.wantErr { t.Errorf("error is wrong, want: %s, got: %s", tt.wantErr, err) } - if diff := cmp.Diff(tt.wantToken, l.Token()); diff != "" { + if diff := cmp.Diff(tt.wantToken, tkn); diff != "" { t.Errorf(diff) } }) diff --git a/pragmaDirective.go b/pragmaDirective.go index 2a9b689..4cfb897 100644 --- a/pragmaDirective.go +++ b/pragmaDirective.go @@ -6,31 +6,43 @@ import ( ) func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { - p.lexer.Scan() - if tkn := p.lexer.Token(); tkn.TokenType != token.Pragma { + tkn, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if tkn.TokenType != token.Pragma { return nil, newError(tkn.Pos, "not found pragma") } - p.lexer.Scan() - pragmaName := p.lexer.Token() + pragmaName, err := p.lexer.Scan() + if err != nil { + return nil, err + } if pragmaName.TokenType != token.Unknown { return nil, newError(pragmaName.Pos, "not found pragma name") } - p.lexer.Scan() - expression := p.lexer.Token() + expression, err := p.lexer.Scan() + if err != nil { + return nil, err + } if expression.TokenType != token.BitXor { return nil, newError(expression.Pos, "not found BitXor expression") } - p.lexer.Scan() - version := p.lexer.Token() + version, err := p.lexer.Scan() + if err != nil { + return nil, err + } if version.TokenType != token.Unknown { return nil, newError(version.Pos, "not found version") } - p.lexer.Scan() - if tkn := p.lexer.Token(); tkn.TokenType != token.Semicolon { + tkn, err = p.lexer.Scan() + if err != nil { + return nil, err + } + if tkn.TokenType != token.Semicolon { return nil, newError(tkn.Pos, "not found Semicolon") } diff --git a/solparser.go b/solparser.go index 680742b..294f844 100644 --- a/solparser.go +++ b/solparser.go @@ -62,20 +62,16 @@ func (p *Parser) Parse(input io.Reader) (*ast.SourceUnit, error) { } func (p *Parser) ParseBooleanLiteral() (*ast.BooleanLiteral, error) { - if !p.lexer.Scan() { - return nil, nil - } - - if err := p.lexer.Error(); err != nil { + tkn, err := p.lexer.Scan() + if err != nil { return nil, err } - tkn := p.lexer.Token() if tkn.TokenType != token.True && tkn.TokenType != token.False { return nil, newError(tkn.Pos, "not found keyword true or false") } return &ast.BooleanLiteral{ - Token: p.lexer.Token(), + Token: tkn, }, nil } From 1a29decafe9a9f7349dd05e43b0e0ef1f822e74c Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 12 Nov 2022 23:20:11 +0900 Subject: [PATCH 086/138] lexer: remove result return value from Lexer.Peek() --- lexer/lexer.go | 20 ++++---------------- lexer/lexer_test.go | 30 +++++++++++++----------------- solparser.go | 8 ++++---- 3 files changed, 21 insertions(+), 37 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 99869d8..8873484 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -43,26 +43,14 @@ func (l *Lexer) Scan() (token.Token, error) { return tkn, err } -func (l *Lexer) Peek() bool { +func (l *Lexer) Peek() (token.Token, error) { if l.peeked { - return true - } - - rslt, tkn, err := l.scan() - if !rslt { - return false + return l.peekToken, l.peekErr } + _, tkn, err := l.scan() l.peeked = true l.peekToken = tkn l.peekErr = err - return true -} - -func (l Lexer) PeekToken() token.Token { - return l.peekToken -} - -func (l Lexer) PeekError() error { - return l.peekErr + return l.peekToken, l.peekErr } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 5754253..a4e2a97 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -84,16 +84,14 @@ func TestLexerScan(t *testing.T) { func TestLexerPeek(t *testing.T) { tests := []struct { - name string - input string - result bool - token token.Token - err error + name string + input string + token token.Token + err error }{ { - name: "normal", - input: "pragma", - result: true, + name: "normal", + input: "pragma", token: token.Token{ TokenType: token.Pragma, Text: "pragma", @@ -104,10 +102,10 @@ func TestLexerPeek(t *testing.T) { }, }, { - name: "when scan is done", - input: "", - result: false, - token: token.Token{}, + name: "when scan is done", + input: "", + token: token.Token{}, + err: io.EOF, }, } for _, tt := range tests { @@ -118,13 +116,11 @@ func TestLexerPeek(t *testing.T) { l := Lexer{ scanner: s, } - if rslt := l.Peek(); tt.result != rslt { - t.Errorf("result is wrong, want: %t, got: %t", tt.result, rslt) - } - if err := l.PeekError(); err != tt.err { + tkn, err := l.Peek() + if err != tt.err { t.Errorf("want: %s, got: %s", tt.err, err) } - if diff := cmp.Diff(tt.token, l.PeekToken()); diff != "" { + if diff := cmp.Diff(tt.token, tkn); diff != "" { t.Errorf(diff) } }) diff --git a/solparser.go b/solparser.go index 294f844..4d51f67 100644 --- a/solparser.go +++ b/solparser.go @@ -21,11 +21,11 @@ func New(input io.Reader) *Parser { } func (p *Parser) Parse(input io.Reader) (*ast.SourceUnit, error) { - if !p.lexer.Peek() { + tkn, err := p.lexer.Peek() + if err == io.EOF { return nil, nil } - - if err := p.lexer.PeekError(); err != nil { + if err != nil { return nil, err } @@ -33,7 +33,7 @@ func (p *Parser) Parse(input io.Reader) (*ast.SourceUnit, error) { var contractDefinition *ast.ContractDefinition var functionDefinition *ast.FunctionDefinition - switch p.lexer.PeekToken().TokenType { + switch tkn.TokenType { case token.Pragma: prgm, err := p.ParsePragmaDirective() if err != nil { From c02cc7bd1bc35d0aeed3a8b3942187c4e121b60d Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 12 Nov 2022 23:21:42 +0900 Subject: [PATCH 087/138] lexer: remove result return value from Lexer.scan() --- lexer/lexer.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 8873484..a862b32 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -24,13 +24,13 @@ func New(input io.Reader) *Lexer { } } -func (l *Lexer) scan() (result bool, tkn token.Token, err error) { +func (l *Lexer) scan() (tkn token.Token, err error) { pos, lit, err := l.scanner.Scan() if err != nil { - return false, token.Token{}, err + return token.Token{}, err } - return true, token.NewToken(lit, pos), nil + return token.NewToken(lit, pos), nil } func (l *Lexer) Scan() (token.Token, error) { @@ -39,8 +39,7 @@ func (l *Lexer) Scan() (token.Token, error) { return l.peekToken, l.peekErr } - _, tkn, err := l.scan() - return tkn, err + return l.scan() } func (l *Lexer) Peek() (token.Token, error) { @@ -48,7 +47,7 @@ func (l *Lexer) Peek() (token.Token, error) { return l.peekToken, l.peekErr } - _, tkn, err := l.scan() + tkn, err := l.scan() l.peeked = true l.peekToken = tkn l.peekErr = err From 9aa28c4309037d13f868087888e4d75fb70d0cda Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 13 Nov 2022 11:50:07 +0900 Subject: [PATCH 088/138] lexer: refactor lexer test case --- lexer/lexer_test.go | 84 ++++++++++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index a4e2a97..0f3a030 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -10,14 +10,10 @@ import ( "github.com/uji/solparser/token" ) -func TestLexerScan(t *testing.T) { +func TestLexer_Scan(t *testing.T) { tests := []struct { name string input string - peeked bool - peekToken token.Token - peekErr error - wantErr error wantToken token.Token }{ { @@ -32,57 +28,65 @@ func TestLexerScan(t *testing.T) { }, }, }, - { - name: "when scan is done", - input: "", - wantToken: token.Token{}, - wantErr: io.EOF, - }, - { - name: "when peeked", - input: "", - peeked: true, - peekToken: token.Token{ - TokenType: token.BitXor, - Text: "^", - Pos: token.Pos{ - Column: 4, - Line: 6, - }, - }, - wantToken: token.Token{ - TokenType: token.BitXor, - Text: "^", - Pos: token.Pos{ - Column: 4, - Line: 6, - }, - }, - }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { s := scanner.New(strings.NewReader(tt.input)) - l := Lexer{ - scanner: s, - peeked: tt.peeked, - peekToken: tt.peekToken, - peekErr: tt.peekErr, + scanner: s, } + tkn, err := l.Scan() - if err != tt.wantErr { - t.Errorf("error is wrong, want: %s, got: %s", tt.wantErr, err) + if err != nil { + t.Errorf("error is not nil, got: %s", err) } if diff := cmp.Diff(tt.wantToken, tkn); diff != "" { t.Errorf(diff) } }) } + + t.Run("when scan is done", func(t *testing.T) { + s := scanner.New(strings.NewReader("")) + l := Lexer{ + scanner: s, + } + + _, err := l.Scan() + if err != io.EOF { + t.Errorf("error is not io.EOF, got: %s", err) + } + }) + + t.Run("when peeked", func(t *testing.T) { + peekToken := token.Token{ + TokenType: token.BitXor, + Text: "^", + Pos: token.Pos{ + Column: 4, + Line: 6, + }, + } + s := scanner.New(strings.NewReader("")) + l := Lexer{ + scanner: s, + peeked: true, + peekToken: peekToken, + peekErr: nil, + } + + tkn, err := l.Scan() + if err != nil { + t.Errorf("error is not nil, got: %s", err) + } + if diff := cmp.Diff(peekToken, tkn); diff != "" { + t.Errorf(diff) + } + }) } -func TestLexerPeek(t *testing.T) { +func TestLexer_Peek(t *testing.T) { tests := []struct { name string input string From ea0a72ccb6af330a18a9dc0a273be3be6ddb3721 Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 16 Nov 2022 22:23:15 +0900 Subject: [PATCH 089/138] token: add literals to token list --- token/token.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/token/token.go b/token/token.go index 83d1ff9..03ba866 100644 --- a/token/token.go +++ b/token/token.go @@ -159,6 +159,15 @@ const ( View Virtual While + + // Literal + TrueLiteral + FalseLiteral + Number + StringLiteral + UnicodeStringLiteral + HexStringLiteral + CommentLiteral ) func asKeyword(str string) TokenType { From 252628899d9c2772b69ab3da5349aadcb102c90d Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 16 Nov 2022 22:38:37 +0900 Subject: [PATCH 090/138] scanner: do not ignore space literal --- scanner/scanner.go | 45 ++++++++++++++++------------------------- scanner/scanner_test.go | 15 ++++++++++++++ 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/scanner/scanner.go b/scanner/scanner.go index 589e414..a32ff65 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -81,43 +81,32 @@ func (s *Scanner) readRune() (rune, error) { } func (s *Scanner) Scan() (token.Pos, string, error) { - runes := make([]rune, 1) - for { - // Retain the state before reading. - c := s.offset + 1 - l := s.lineOffset + 1 - - ch, err := s.readRune() - if err != nil { - return token.Pos{}, "", err - } - if ch == bufrr.EOF { - return token.Pos{}, "", io.EOF - } - if isSplitSymbol(ch) || ch == '\n' { - return token.Pos{ - Column: c, - Line: l, - }, string(ch), nil - } - if !isSpace(ch) { - runes[0] = ch - break - } + startPos := token.Pos{ + Column: s.offset + 1, + Line: s.lineOffset + 1, } - pos := token.Pos{ - Column: s.offset, - Line: s.lineOffset + 1, + ch, err := s.readRune() + if err != nil { + return token.Pos{}, "", err } + if ch == bufrr.EOF { + return token.Pos{}, "", io.EOF + } + if isSplitSymbol(ch) || ch == '\n' { + return startPos, string(ch), nil + } + + readingSpace := isSpace(ch) + runes := []rune{ch} for { ch, _, err := s.r.PeekRune() if err != nil { return token.Pos{}, "", err } - if isSpace(ch) || isSplitSymbol(ch) || ch == '\n' || ch == bufrr.EOF { - return pos, string(runes), nil + if isSpace(ch) != readingSpace || isSplitSymbol(ch) || ch == '\n' || ch == bufrr.EOF { + return startPos, string(runes), nil } ch, err = s.readRune() if err != nil { diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index c270afc..b093403 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -24,10 +24,18 @@ func TestScannerScan(t *testing.T) { Column: 1, Line: 1, }, + { + Column: 7, + Line: 1, + }, { Column: 8, Line: 1, }, + { + Column: 16, + Line: 1, + }, { Column: 17, Line: 1, @@ -43,7 +51,9 @@ func TestScannerScan(t *testing.T) { }, wantLits: []string{ "pragma", + " ", "solidity", + " ", "^", "0.8.13", ";", @@ -61,6 +71,10 @@ func TestScannerScan(t *testing.T) { Column: 7, Line: 1, }, + { + Column: 1, + Line: 2, + }, { Column: 2, Line: 2, @@ -69,6 +83,7 @@ func TestScannerScan(t *testing.T) { wantLits: []string{ "pragma", "\n", + " ", "solidity", }, }, From 533a93578d7b25c9846548aacc8e2a8c1a46cb87 Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 16 Nov 2022 22:43:51 +0900 Subject: [PATCH 091/138] lexer: implement ScanStringLiteral() scanning double-quoted-printable string literal --- lexer/lexer.go | 27 +++++++++++++++++++++++++++ lexer/lexer_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/lexer/lexer.go b/lexer/lexer.go index a862b32..62a2eff 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -1,6 +1,7 @@ package lexer import ( + "errors" "io" "github.com/uji/solparser/scanner" @@ -53,3 +54,29 @@ func (l *Lexer) Peek() (token.Token, error) { l.peekErr = err return l.peekToken, l.peekErr } + +func (l *Lexer) ScanStringLiteral() (token.Token, error) { + firstPos, lit, err := l.scanner.Scan() + if err != nil { + return token.Token{}, err + } + if lit != `"` { + return token.Token{}, errors.New("test") // TODO: include Pos in error + } + + txt := lit + for { + _, lit, err := l.scanner.Scan() + if err != nil { + return token.Token{}, err + } + txt = txt + lit + if lit == `"` { + return token.Token{ + TokenType: token.StringLiteral, + Text: txt, + Pos: firstPos, + }, nil + } + } +} diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 0f3a030..1dd7492 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -130,3 +130,34 @@ func TestLexer_Peek(t *testing.T) { }) } } + +func TestLexer_ScanStringLiteral(t *testing.T) { + tests := []struct { + name string + input string + wantText string + }{ + { + name: "double-quoted-printable", + input: `"Hello world!!";`, + wantText: `"Hello world!!"`, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + s := scanner.New(strings.NewReader(tt.input)) + l := Lexer{ + scanner: s, + } + + tkn, err := l.ScanStringLiteral() + if err != nil { + t.Errorf("error is not nil, got: %s", err) + } + if diff := cmp.Diff(tt.wantText, tkn.Text); diff != "" { + t.Errorf(diff) + } + }) + } +} From c2f9520d6ddf593e72c816507b017d8fbdbc112c Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 16 Nov 2022 23:06:37 +0900 Subject: [PATCH 092/138] main, token: move Pos Error from main package to token package --- error.go | 26 -------------------------- pragmaDirective.go | 10 +++++----- pragmaDirective_test.go | 12 ++++++------ solparser.go | 2 +- solparser_test.go | 6 +++--- token/error.go | 22 ++++++++++++++++++++++ error_test.go => token/error_test.go | 11 +++++------ 7 files changed, 42 insertions(+), 47 deletions(-) delete mode 100644 error.go create mode 100644 token/error.go rename error_test.go => token/error_test.go (79%) diff --git a/error.go b/error.go deleted file mode 100644 index 9fa3337..0000000 --- a/error.go +++ /dev/null @@ -1,26 +0,0 @@ -package solparser - -import ( - "github.com/uji/solparser/token" -) - -type Error struct { - Pos token.Pos - Msg string -} - -var _ error = &Error{} - -func newError(pos token.Pos, msg string) *Error { - return &Error{ - Pos: pos, - Msg: msg, - } -} - -func (e *Error) Error() string { - if e.Pos.IsValid() { - return e.Pos.String() + ": " + e.Msg - } - return e.Msg -} diff --git a/pragmaDirective.go b/pragmaDirective.go index 4cfb897..7a3b76a 100644 --- a/pragmaDirective.go +++ b/pragmaDirective.go @@ -11,7 +11,7 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { return nil, err } if tkn.TokenType != token.Pragma { - return nil, newError(tkn.Pos, "not found pragma") + return nil, token.NewPosError(tkn.Pos, "not found pragma") } pragmaName, err := p.lexer.Scan() @@ -19,7 +19,7 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { return nil, err } if pragmaName.TokenType != token.Unknown { - return nil, newError(pragmaName.Pos, "not found pragma name") + return nil, token.NewPosError(pragmaName.Pos, "not found pragma name") } expression, err := p.lexer.Scan() @@ -27,7 +27,7 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { return nil, err } if expression.TokenType != token.BitXor { - return nil, newError(expression.Pos, "not found BitXor expression") + return nil, token.NewPosError(expression.Pos, "not found BitXor expression") } version, err := p.lexer.Scan() @@ -35,7 +35,7 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { return nil, err } if version.TokenType != token.Unknown { - return nil, newError(version.Pos, "not found version") + return nil, token.NewPosError(version.Pos, "not found version") } tkn, err = p.lexer.Scan() @@ -43,7 +43,7 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { return nil, err } if tkn.TokenType != token.Semicolon { - return nil, newError(tkn.Pos, "not found Semicolon") + return nil, token.NewPosError(tkn.Pos, "not found Semicolon") } return &ast.PragmaDirective{ diff --git a/pragmaDirective_test.go b/pragmaDirective_test.go index 08b38b0..a1ad8a7 100644 --- a/pragmaDirective_test.go +++ b/pragmaDirective_test.go @@ -16,7 +16,7 @@ func TestParserParsePragmaDirective(t *testing.T) { name string input string want *ast.PragmaDirective - err *solparser.Error + err *token.PosError }{ { name: "normal case", @@ -34,7 +34,7 @@ func TestParserParsePragmaDirective(t *testing.T) { name: "there is no pragma keyword", input: "solidity ^0.8.13;", want: nil, - err: &solparser.Error{ + err: &token.PosError{ Pos: token.Pos{ Column: 1, Line: 1, @@ -46,7 +46,7 @@ func TestParserParsePragmaDirective(t *testing.T) { name: "there is no pragma name", input: "pragma ^0.8.13;", want: nil, - err: &solparser.Error{ + err: &token.PosError{ Pos: token.Pos{ Column: 8, Line: 1, @@ -58,7 +58,7 @@ func TestParserParsePragmaDirective(t *testing.T) { name: "there is no BitXor expression", input: "pragma solidity 0.8.13;", want: nil, - err: &solparser.Error{ + err: &token.PosError{ Pos: token.Pos{ Column: 17, Line: 1, @@ -70,7 +70,7 @@ func TestParserParsePragmaDirective(t *testing.T) { name: "there is no Semicolon", input: "pragma solidity ^0.8.13", want: nil, - err: &solparser.Error{ + err: &token.PosError{ Pos: token.Pos{ Column: 18, Line: 1, @@ -87,7 +87,7 @@ func TestParserParsePragmaDirective(t *testing.T) { got, err := p.ParsePragmaDirective() - var sErr *solparser.Error + var sErr *token.PosError if errors.As(err, &sErr) { if diff := cmp.Diff(tt.err, sErr); diff != "" { t.Errorf("%s", diff) diff --git a/solparser.go b/solparser.go index 4d51f67..052db18 100644 --- a/solparser.go +++ b/solparser.go @@ -68,7 +68,7 @@ func (p *Parser) ParseBooleanLiteral() (*ast.BooleanLiteral, error) { } if tkn.TokenType != token.True && tkn.TokenType != token.False { - return nil, newError(tkn.Pos, "not found keyword true or false") + return nil, token.NewPosError(tkn.Pos, "not found keyword true or false") } return &ast.BooleanLiteral{ diff --git a/solparser_test.go b/solparser_test.go index ac14890..e944bd5 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -16,7 +16,7 @@ func TestParserParseBooleanLiteral(t *testing.T) { name string input string want *ast.BooleanLiteral - err *solparser.Error + err *token.PosError }{ { name: "true case", @@ -52,7 +52,7 @@ func TestParserParseBooleanLiteral(t *testing.T) { name: "not true or false", input: "solidity", want: nil, - err: &solparser.Error{ + err: &token.PosError{ Pos: token.Pos{ Column: 1, Line: 1, @@ -69,7 +69,7 @@ func TestParserParseBooleanLiteral(t *testing.T) { got, err := p.ParseBooleanLiteral() - var sErr *solparser.Error + var sErr *token.PosError if errors.As(err, &sErr) { if diff := cmp.Diff(tt.err, sErr); diff != "" { t.Errorf("%s", diff) diff --git a/token/error.go b/token/error.go new file mode 100644 index 0000000..9994b95 --- /dev/null +++ b/token/error.go @@ -0,0 +1,22 @@ +package token + +type PosError struct { + Pos Pos + Msg string +} + +var _ error = &PosError{} + +func NewPosError(pos Pos, msg string) *PosError { + return &PosError{ + Pos: pos, + Msg: msg, + } +} + +func (e *PosError) Error() string { + if e.Pos.IsValid() { + return e.Pos.String() + ": " + e.Msg + } + return e.Msg +} diff --git a/error_test.go b/token/error_test.go similarity index 79% rename from error_test.go rename to token/error_test.go index 1d31ca9..5da487d 100644 --- a/error_test.go +++ b/token/error_test.go @@ -1,21 +1,20 @@ -package solparser_test +package token_test import ( "testing" - "github.com/uji/solparser" "github.com/uji/solparser/token" ) -func TestErrorError(t *testing.T) { +func TestError_PosError(t *testing.T) { tests := []struct { name string - err solparser.Error + err token.PosError want string }{ { name: "normal case", - err: solparser.Error{ + err: token.PosError{ Pos: token.Pos{ Column: 1, Line: 1, @@ -26,7 +25,7 @@ func TestErrorError(t *testing.T) { }, { name: "invalid position case", - err: solparser.Error{ + err: token.PosError{ Pos: token.Pos{ Column: 0, Line: 1, From 1c496621362b6f3041a3b59793dfbc8f462eb417 Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 16 Nov 2022 23:08:07 +0900 Subject: [PATCH 093/138] lexer: correspond to String Literal containing a space --- lexer/lexer.go | 3 +-- lexer/lexer_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 62a2eff..386d9d2 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -1,7 +1,6 @@ package lexer import ( - "errors" "io" "github.com/uji/solparser/scanner" @@ -61,7 +60,7 @@ func (l *Lexer) ScanStringLiteral() (token.Token, error) { return token.Token{}, err } if lit != `"` { - return token.Token{}, errors.New("test") // TODO: include Pos in error + return token.Token{}, token.NewPosError(firstPos, `not found "`) } txt := lit diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 1dd7492..86d15bb 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -1,6 +1,7 @@ package lexer import ( + "errors" "io" "strings" "testing" @@ -160,4 +161,27 @@ func TestLexer_ScanStringLiteral(t *testing.T) { } }) } + + t.Run(`not found "`, func(t *testing.T) { + s := scanner.New(strings.NewReader("a")) + l := Lexer{ + scanner: s, + } + exptErr := &token.PosError{ + Pos: token.Pos{ + Column: 1, + Line: 1, + }, + Msg: `not found "`, + } + + _, err := l.ScanStringLiteral() + var pErr *token.PosError + if !errors.As(err, &pErr) { + t.Errorf("error is unexpected, got: %s", err) + } + if diff := cmp.Diff(pErr, exptErr); diff != "" { + t.Errorf(diff) + } + }) } From fd25c4060bb619079daec1504eadeafc0a42a175 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 4 Feb 2023 12:47:03 +0900 Subject: [PATCH 094/138] lexer, scanner: Ignore spaces in lexer.scan() --- lexer/lexer.go | 4 ++++ lexer/lexer_test.go | 12 ++++++++++++ scanner/scanner.go | 8 ++++---- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 386d9d2..5298762 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -30,6 +30,10 @@ func (l *Lexer) scan() (tkn token.Token, err error) { return token.Token{}, err } + if scanner.IsSpace([]rune(lit)[0]) { + return l.scan() + } + return token.NewToken(lit, pos), nil } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 86d15bb..aa11632 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -29,6 +29,18 @@ func TestLexer_Scan(t *testing.T) { }, }, }, + { + name: "There is a space at the beginning", + input: " pragma", + wantToken: token.Token{ + TokenType: token.Pragma, + Text: "pragma", + Pos: token.Pos{ + Column: 2, + Line: 1, + }, + }, + }, } for _, tt := range tests { tt := tt diff --git a/scanner/scanner.go b/scanner/scanner.go index a32ff65..4a70288 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -23,10 +23,10 @@ func New(reader io.Reader) *Scanner { } } -// isSpace reports whether the character is a Unicode white space character. +// IsSpace reports whether the character is a Unicode white space character. // We avoid dependency on the unicode package, but check validity of the implementation // in the tests. -func isSpace(r rune) bool { +func IsSpace(r rune) bool { if r <= '\u00FF' { // Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs. switch r { @@ -97,7 +97,7 @@ func (s *Scanner) Scan() (token.Pos, string, error) { return startPos, string(ch), nil } - readingSpace := isSpace(ch) + readingSpace := IsSpace(ch) runes := []rune{ch} for { @@ -105,7 +105,7 @@ func (s *Scanner) Scan() (token.Pos, string, error) { if err != nil { return token.Pos{}, "", err } - if isSpace(ch) != readingSpace || isSplitSymbol(ch) || ch == '\n' || ch == bufrr.EOF { + if IsSpace(ch) != readingSpace || isSplitSymbol(ch) || ch == '\n' || ch == bufrr.EOF { return startPos, string(runes), nil } ch, err = s.readRune() From 97cb8b290f90f7ce121a472833fd763f1f6451f2 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 4 Feb 2023 14:23:30 +0900 Subject: [PATCH 095/138] all: move IsSpace() from scanner package to token package --- lexer/lexer.go | 2 +- scanner/scanner.go | 29 ++--------------------------- token/token.go | 25 +++++++++++++++++++++++++ 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 5298762..c210d90 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -30,7 +30,7 @@ func (l *Lexer) scan() (tkn token.Token, err error) { return token.Token{}, err } - if scanner.IsSpace([]rune(lit)[0]) { + if token.IsSpace([]rune(lit)[0]) { return l.scan() } diff --git a/scanner/scanner.go b/scanner/scanner.go index 4a70288..a35fc15 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -23,31 +23,6 @@ func New(reader io.Reader) *Scanner { } } -// IsSpace reports whether the character is a Unicode white space character. -// We avoid dependency on the unicode package, but check validity of the implementation -// in the tests. -func IsSpace(r rune) bool { - if r <= '\u00FF' { - // Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs. - switch r { - case ' ', '\t', '\v', '\f', '\r': - return true - case '\u0085', '\u00A0': - return true - } - return false - } - // High-valued ones. - if '\u2000' <= r && r <= '\u200a' { - return true - } - switch r { - case '\u1680', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000': - return true - } - return false -} - func isMultiLengthOperatorSymbol(r rune) bool { switch r { case '=', '|', '^', '&', '+', '-', '*', '/', '%', '<', '>', '!': @@ -97,7 +72,7 @@ func (s *Scanner) Scan() (token.Pos, string, error) { return startPos, string(ch), nil } - readingSpace := IsSpace(ch) + readingSpace := token.IsSpace(ch) runes := []rune{ch} for { @@ -105,7 +80,7 @@ func (s *Scanner) Scan() (token.Pos, string, error) { if err != nil { return token.Pos{}, "", err } - if IsSpace(ch) != readingSpace || isSplitSymbol(ch) || ch == '\n' || ch == bufrr.EOF { + if token.IsSpace(ch) != readingSpace || isSplitSymbol(ch) || ch == bufrr.EOF { return startPos, string(runes), nil } ch, err = s.readRune() diff --git a/token/token.go b/token/token.go index 03ba866..fb32cae 100644 --- a/token/token.go +++ b/token/token.go @@ -497,3 +497,28 @@ func (p Pos) String() string { return fmt.Sprintf("%d:%d", p.Line, p.Column) } + +// IsSpace reports whether the character is a Unicode white space character. +// We avoid dependency on the unicode package, but check validity of the implementation +// in the tests. +func IsSpace(r rune) bool { + if r <= '\u00FF' { + // Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs. + switch r { + case ' ', '\n', '\t', '\v', '\f', '\r': + return true + case '\u0085', '\u00A0': + return true + } + return false + } + // High-valued ones. + if '\u2000' <= r && r <= '\u200a' { + return true + } + switch r { + case '\u1680', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000': + return true + } + return false +} From 523ffbffc0b9a4da5a9358a79761f591aa0900dd Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 4 Feb 2023 22:43:30 +0900 Subject: [PATCH 096/138] lexer: fix string literal scan description --- lexer/lexer.go | 11 +++++++---- lexer/lexer_test.go | 8 +++++++- scanner/scanner.go | 2 +- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index c210d90..e085623 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -1,6 +1,7 @@ package lexer import ( + "fmt" "io" "github.com/uji/solparser/scanner" @@ -58,23 +59,25 @@ func (l *Lexer) Peek() (token.Token, error) { return l.peekToken, l.peekErr } +// ScanStringLiteral parse NonEmptyStringLiteral or EmptyStringLiteral then return StringLiteral token. func (l *Lexer) ScanStringLiteral() (token.Token, error) { firstPos, lit, err := l.scanner.Scan() if err != nil { return token.Token{}, err } - if lit != `"` { - return token.Token{}, token.NewPosError(firstPos, `not found "`) + fmt.Println(firstPos, lit) + if lit != `"` && lit != `\'` { + return token.Token{}, token.NewPosError(firstPos, `not found " or \'`) } - txt := lit + quote, txt := lit, lit for { _, lit, err := l.scanner.Scan() if err != nil { return token.Token{}, err } txt = txt + lit - if lit == `"` { + if lit == quote { return token.Token{ TokenType: token.StringLiteral, Text: txt, diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index aa11632..3e22e75 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -155,6 +155,12 @@ func TestLexer_ScanStringLiteral(t *testing.T) { input: `"Hello world!!";`, wantText: `"Hello world!!"`, }, + // TODO + // { + // name: "single-quoted-printable", + // input: `\'Hello world!!\';`, + // wantText: `\'Hello world!!\'`, + // }, } for _, tt := range tests { tt := tt @@ -184,7 +190,7 @@ func TestLexer_ScanStringLiteral(t *testing.T) { Column: 1, Line: 1, }, - Msg: `not found "`, + Msg: `not found " or \'`, } _, err := l.ScanStringLiteral() diff --git a/scanner/scanner.go b/scanner/scanner.go index a35fc15..c31fcf2 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -25,7 +25,7 @@ func New(reader io.Reader) *Scanner { func isMultiLengthOperatorSymbol(r rune) bool { switch r { - case '=', '|', '^', '&', '+', '-', '*', '/', '%', '<', '>', '!': + case '=', '|', '^', '&', '+', '-', '*', '/', '\\', '%', '<', '>', '!': return true } return false From 8bf6c037a75c84e747aa41510fc8be517fe25668 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 5 Feb 2023 15:19:18 +0900 Subject: [PATCH 097/138] scanner: scan multi length operator --- lexer/lexer_test.go | 11 +++--- scanner/scanner.go | 80 ++++++++++++++++++++++++++++++++++++++-- scanner/scanner_test.go | 81 +++++++++++++++++------------------------ token/token.go | 2 +- 4 files changed, 117 insertions(+), 57 deletions(-) diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 3e22e75..72290d6 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -155,12 +155,11 @@ func TestLexer_ScanStringLiteral(t *testing.T) { input: `"Hello world!!";`, wantText: `"Hello world!!"`, }, - // TODO - // { - // name: "single-quoted-printable", - // input: `\'Hello world!!\';`, - // wantText: `\'Hello world!!\'`, - // }, + { + name: "single-quoted-printable", + input: `\'Hello world!!\';`, + wantText: `\'Hello world!!\'`, + }, } for _, tt := range tests { tt := tt diff --git a/scanner/scanner.go b/scanner/scanner.go index c31fcf2..364121d 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -1,6 +1,7 @@ package scanner import ( + "errors" "io" "github.com/SteelSeries/bufrr" @@ -55,23 +56,96 @@ func (s *Scanner) readRune() (rune, error) { return r, nil } +func (s *Scanner) scanOperator() (string, error) { + ch, _, err := s.r.PeekRune() + if err != nil { + return "", err + } + if !isSplitSymbol(ch) { + return "", errors.New("Not operator") + } + s.readRune() + if !isMultiLengthOperatorSymbol(ch) { + return string(ch), nil + } + ch2, _, err := s.r.PeekRune() + if err != nil { + return string(ch), nil + } + oprt := string([]rune{ch, ch2}) + switch oprt { + case "=>", "->", "|=", "^=", "&=", "+=", "-=", "*=", "/=", "%=", "==", "||", "&&", "**", "!=", "<-", ">-", "++", "--", `\'`: + s.readRune() + return oprt, nil + case "<<": + s.readRune() + ch, _, err := s.r.PeekRune() + if err != nil { + return oprt, nil + } + if ch == '=' { + return "<<=", nil + } + return oprt, nil + case ">>": + s.readRune() + ch3, _, err := s.r.PeekRune() + if err != nil { + return oprt, nil + } + if ch3 == '=' { + return ">>=", nil + } + if ch3 == '>' { + s.readRune() + oprt = ">>>" + ch4, _, err := s.r.PeekRune() + if err != nil { + return oprt, nil + } + if ch4 == '=' { + s.readRune() + return ">>>=", nil + } + return oprt, nil + } + return oprt, nil + } + return string(ch), nil +} + +// Scan divides a string into the smallest units from the bufrr.Reader and returns them one by one. +// +// - If first rune is an error or EOF, return an error and exit. +// - If first rune is operator character, scan operator. +// - If first rune is a space character, scan until the end of the blank character. +// - Else scan to next space or operator string. func (s *Scanner) Scan() (token.Pos, string, error) { startPos := token.Pos{ Column: s.offset + 1, Line: s.lineOffset + 1, } - ch, err := s.readRune() + ch, _, err := s.r.PeekRune() if err != nil { + s.readRune() return token.Pos{}, "", err } if ch == bufrr.EOF { + s.readRune() return token.Pos{}, "", io.EOF } - if isSplitSymbol(ch) || ch == '\n' { - return startPos, string(ch), nil + if isSplitSymbol(ch) { + // scan operator. + oprt, err := s.scanOperator() + if err != nil { + return token.Pos{}, "", err + } + return startPos, oprt, nil } + s.readRune() + readingSpace := token.IsSpace(ch) runes := []rune{ch} diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index b093403..2cc9ca8 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -20,34 +20,13 @@ func TestScannerScan(t *testing.T) { name: "normal", input: "pragma solidity ^0.8.13;", wantPoss: []token.Pos{ - { - Column: 1, - Line: 1, - }, - { - Column: 7, - Line: 1, - }, - { - Column: 8, - Line: 1, - }, - { - Column: 16, - Line: 1, - }, - { - Column: 17, - Line: 1, - }, - { - Column: 18, - Line: 1, - }, - { - Column: 24, - Line: 1, - }, + {Column: 1, Line: 1}, + {Column: 7, Line: 1}, + {Column: 8, Line: 1}, + {Column: 16, Line: 1}, + {Column: 17, Line: 1}, + {Column: 18, Line: 1}, + {Column: 24, Line: 1}, }, wantLits: []string{ "pragma", @@ -60,33 +39,41 @@ func TestScannerScan(t *testing.T) { }, }, { - name: "normal", + name: "there is newline character", input: "pragma\n solidity", wantPoss: []token.Pos{ - { - Column: 1, - Line: 1, - }, - { - Column: 7, - Line: 1, - }, - { - Column: 1, - Line: 2, - }, - { - Column: 2, - Line: 2, - }, + {Column: 1, Line: 1}, + {Column: 7, Line: 1}, + {Column: 2, Line: 2}, }, wantLits: []string{ "pragma", - "\n", - " ", + "\n ", "solidity", }, }, + { + name: "there are multi length operators", + input: `a >> \'test\'`, + wantPoss: []token.Pos{ + {Column: 1, Line: 1}, + {Column: 2, Line: 1}, + {Column: 3, Line: 1}, + {Column: 5, Line: 1}, + {Column: 6, Line: 1}, + {Column: 8, Line: 1}, + {Column: 12, Line: 1}, + }, + wantLits: []string{ + "a", + " ", + ">>", + " ", + `\'`, + "test", + `\'`, + }, + }, } for _, tt := range tests { diff --git a/token/token.go b/token/token.go index fb32cae..898c941 100644 --- a/token/token.go +++ b/token/token.go @@ -61,7 +61,7 @@ const ( Inc // ++ Dec // -- DoubleQuote // " - SingleQuote // ' + SingleQuote // ' or \' // Reserved Keyword After From fb305f5f025ae058466a202975c9abd49d0d8486 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 5 Feb 2023 20:59:04 +0900 Subject: [PATCH 098/138] scanner: refactor Scan() --- scanner/scanner.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/scanner/scanner.go b/scanner/scanner.go index 364121d..a160547 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -128,11 +128,9 @@ func (s *Scanner) Scan() (token.Pos, string, error) { ch, _, err := s.r.PeekRune() if err != nil { - s.readRune() return token.Pos{}, "", err } if ch == bufrr.EOF { - s.readRune() return token.Pos{}, "", io.EOF } if isSplitSymbol(ch) { @@ -144,10 +142,12 @@ func (s *Scanner) Scan() (token.Pos, string, error) { return startPos, oprt, nil } - s.readRune() + if _, err := s.readRune(); err != nil { + return token.Pos{}, "", err + } readingSpace := token.IsSpace(ch) - runes := []rune{ch} + rslt := []rune{ch} for { ch, _, err := s.r.PeekRune() @@ -155,13 +155,12 @@ func (s *Scanner) Scan() (token.Pos, string, error) { return token.Pos{}, "", err } if token.IsSpace(ch) != readingSpace || isSplitSymbol(ch) || ch == bufrr.EOF { - return startPos, string(runes), nil + return startPos, string(rslt), nil } - ch, err = s.readRune() - if err != nil { + if _, err = s.readRune(); err != nil { return token.Pos{}, "", err } - runes = append(runes, ch) + rslt = append(rslt, ch) } } From 7ce29b72d07c181b69571d10e67f28d14e2aafa6 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 5 Feb 2023 21:28:09 +0900 Subject: [PATCH 099/138] scanner: add scanOperator test and fix logic. --- scanner/scanner.go | 65 ++++++++++++++++++++++++----------------- scanner/scanner_test.go | 40 ++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 28 deletions(-) diff --git a/scanner/scanner.go b/scanner/scanner.go index a160547..edeb008 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -24,15 +24,7 @@ func New(reader io.Reader) *Scanner { } } -func isMultiLengthOperatorSymbol(r rune) bool { - switch r { - case '=', '|', '^', '&', '+', '-', '*', '/', '\\', '%', '<', '>', '!': - return true - } - return false -} - -func isSplitSymbol(r rune) bool { +func isOperatorRune(r rune) bool { switch r { case '(', ')', '[', ']', '{', '}', ':', ';', '?', '=', '|', '^', '&', '<', '>', '+', '-', '*', '/', '%', ',', '!', '~', '"', '\'', '\\': return true @@ -56,62 +48,81 @@ func (s *Scanner) readRune() (rune, error) { return r, nil } +var ( + errNotOperator = errors.New("Not operator.") +) + func (s *Scanner) scanOperator() (string, error) { - ch, _, err := s.r.PeekRune() + ch1, _, err := s.r.PeekRune() if err != nil { return "", err } - if !isSplitSymbol(ch) { - return "", errors.New("Not operator") + if !isOperatorRune(ch1) { + return "", errNotOperator } - s.readRune() - if !isMultiLengthOperatorSymbol(ch) { - return string(ch), nil + if _, err := s.readRune(); err != nil { + return "", err } ch2, _, err := s.r.PeekRune() if err != nil { - return string(ch), nil + return string(ch1), nil } - oprt := string([]rune{ch, ch2}) + oprt := string([]rune{ch1, ch2}) switch oprt { case "=>", "->", "|=", "^=", "&=", "+=", "-=", "*=", "/=", "%=", "==", "||", "&&", "**", "!=", "<-", ">-", "++", "--", `\'`: - s.readRune() + if _, err := s.readRune(); err != nil { + return "", err + } return oprt, nil case "<<": - s.readRune() - ch, _, err := s.r.PeekRune() + if _, err := s.readRune(); err != nil { + return "", err + } + ch3, _, err := s.r.PeekRune() if err != nil { return oprt, nil } - if ch == '=' { + if ch3 == '=' { + if _, err := s.readRune(); err != nil { + return "", err + } return "<<=", nil } return oprt, nil case ">>": - s.readRune() + if _, err := s.readRune(); err != nil { + return "", err + } ch3, _, err := s.r.PeekRune() if err != nil { return oprt, nil } if ch3 == '=' { + if _, err := s.readRune(); err != nil { + return "", err + } return ">>=", nil } if ch3 == '>' { - s.readRune() + if _, err := s.readRune(); err != nil { + return "", err + } oprt = ">>>" ch4, _, err := s.r.PeekRune() if err != nil { return oprt, nil } if ch4 == '=' { - s.readRune() + if _, err := s.readRune(); err != nil { + return "", err + } return ">>>=", nil } return oprt, nil } return oprt, nil } - return string(ch), nil + return string(ch1), nil } // Scan divides a string into the smallest units from the bufrr.Reader and returns them one by one. @@ -133,7 +144,7 @@ func (s *Scanner) Scan() (token.Pos, string, error) { if ch == bufrr.EOF { return token.Pos{}, "", io.EOF } - if isSplitSymbol(ch) { + if isOperatorRune(ch) { // scan operator. oprt, err := s.scanOperator() if err != nil { @@ -154,7 +165,7 @@ func (s *Scanner) Scan() (token.Pos, string, error) { if err != nil { return token.Pos{}, "", err } - if token.IsSpace(ch) != readingSpace || isSplitSymbol(ch) || ch == bufrr.EOF { + if token.IsSpace(ch) != readingSpace || isOperatorRune(ch) || ch == bufrr.EOF { return startPos, string(rslt), nil } if _, err = s.readRune(); err != nil { diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index 2cc9ca8..1c86d46 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -1,6 +1,7 @@ package scanner import ( + "errors" "io" "strings" "testing" @@ -53,7 +54,7 @@ func TestScannerScan(t *testing.T) { }, }, { - name: "there are multi length operators", + name: "there are operators", input: `a >> \'test\'`, wantPoss: []token.Pos{ {Column: 1, Line: 1}, @@ -105,3 +106,40 @@ func TestScannerScan(t *testing.T) { }) } } + +func TestScanner_scanOperator(t *testing.T) { + tests := []struct { + input string + want string + wantErr error + }{ + {input: "^0.8.13", want: "^"}, + {input: "=>>", want: "=>"}, + {input: "<< ", want: "<<"}, + {input: "<<=a", want: "<<="}, + {input: ">>1", want: ">>"}, + {input: ">>=x", want: ">>="}, + {input: ">>>>", want: ">>>"}, + {input: ">>>==", want: ">>>="}, + {input: "pragma", wantErr: errNotOperator}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.input, func(t *testing.T) { + s := New(strings.NewReader(tt.input)) + + got, err := s.scanOperator() + if !errors.Is(err, tt.wantErr) { + t.Error(err) + } + if got != tt.want { + t.Errorf("got: %s, want: %s", got, tt.want) + } + expectedOffset := len(tt.want) + if s.offset != expectedOffset { + t.Errorf("offset is %d, expected is %d", s.offset, expectedOffset) + } + }) + } +} From f19d4b89015067eb028fb7b248abb722668c0e47 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 11 Feb 2023 22:58:32 +0900 Subject: [PATCH 100/138] scanner: imeplement Peek --- scanner/scanner.go | 36 +++++++++++-- scanner/scanner_test.go | 114 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 137 insertions(+), 13 deletions(-) diff --git a/scanner/scanner.go b/scanner/scanner.go index edeb008..92b402d 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -15,6 +15,12 @@ type Scanner struct { // position state offset int lineOffset int + + // peek state + peeked bool + peekStr string + peekPos token.Pos + peekErr error } func New(reader io.Reader) *Scanner { @@ -125,13 +131,13 @@ func (s *Scanner) scanOperator() (string, error) { return string(ch1), nil } -// Scan divides a string into the smallest units from the bufrr.Reader and returns them one by one. +// scan flow // // - If first rune is an error or EOF, return an error and exit. // - If first rune is operator character, scan operator. // - If first rune is a space character, scan until the end of the blank character. // - Else scan to next space or operator string. -func (s *Scanner) Scan() (token.Pos, string, error) { +func (s *Scanner) scan() (token.Pos, string, error) { startPos := token.Pos{ Column: s.offset + 1, Line: s.lineOffset + 1, @@ -175,6 +181,28 @@ func (s *Scanner) Scan() (token.Pos, string, error) { } } -func (s *Scanner) Peek() (pos token.Pos, lit string, err error) { - return token.Pos{}, "", nil +// Scan divides a string into the smallest units from the bufrr.Reader and returns them one by one. +func (s *Scanner) Scan() (token.Pos, string, error) { + if s.peeked { + s.peeked = false + return s.peekPos, s.peekStr, s.peekErr + } + + return s.scan() +} + +// Peek reads ahead and returns the result of Scan. +// The offset is unchanged. +func (s *Scanner) Peek() (pos token.Pos, str string, err error) { + if s.peeked { + return s.peekPos, s.peekStr, s.peekErr + } + + s.peeked = true + + pos, str, err = s.scan() + s.peekPos = pos + s.peekStr = str + s.peekErr = err + return } diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index 1c86d46..1aad218 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -6,16 +6,17 @@ import ( "strings" "testing" + "github.com/SteelSeries/bufrr" "github.com/google/go-cmp/cmp" "github.com/uji/solparser/token" ) -func TestScannerScan(t *testing.T) { +func TestScanner_Scan(t *testing.T) { tests := []struct { name string input string wantPoss []token.Pos - wantLits []string + wantStrs []string }{ { name: "normal", @@ -29,7 +30,7 @@ func TestScannerScan(t *testing.T) { {Column: 18, Line: 1}, {Column: 24, Line: 1}, }, - wantLits: []string{ + wantStrs: []string{ "pragma", " ", "solidity", @@ -47,7 +48,7 @@ func TestScannerScan(t *testing.T) { {Column: 7, Line: 1}, {Column: 2, Line: 2}, }, - wantLits: []string{ + wantStrs: []string{ "pragma", "\n ", "solidity", @@ -65,7 +66,7 @@ func TestScannerScan(t *testing.T) { {Column: 8, Line: 1}, {Column: 12, Line: 1}, }, - wantLits: []string{ + wantStrs: []string{ "a", " ", ">>", @@ -83,10 +84,10 @@ func TestScannerScan(t *testing.T) { s := New(strings.NewReader(tt.input)) poss := make([]token.Pos, 0, len(tt.wantPoss)) - lits := make([]string, 0, len(tt.wantLits)) + strs := make([]string, 0, len(tt.wantStrs)) for { - pos, lit, err := s.Scan() + pos, str, err := s.Scan() if err == io.EOF { break } @@ -94,17 +95,112 @@ func TestScannerScan(t *testing.T) { t.Fatalf("got unexpected error: %s", err) } poss = append(poss, pos) - lits = append(lits, lit) + strs = append(strs, str) } if diff := cmp.Diff(tt.wantPoss, poss); diff != "" { t.Errorf(diff) } - if diff := cmp.Diff(tt.wantLits, lits); diff != "" { + if diff := cmp.Diff(tt.wantStrs, strs); diff != "" { t.Errorf(diff) } }) } + + t.Run("when peeked", func(t *testing.T) { + wantStr := "str" + wantPos := token.Pos{ + Column: 3, + Line: 4, + } + wantErr := errors.New(t.Name()) + s := &Scanner{ + r: bufrr.NewReader(strings.NewReader("input will not scand")), + peeked: true, + peekStr: wantStr, + peekPos: wantPos, + peekErr: wantErr, + } + + pos, str, err := s.Scan() + if !errors.Is(err, wantErr) { + t.Fatalf("got unexpected error: %s", err) + } + if diff := cmp.Diff(wantPos, pos); diff != "" { + t.Errorf(diff) + } + if str != wantStr { + t.Errorf("got %s, want %s", str, wantStr) + } + if s.peeked != false { + t.Error("peeked status was not cleared") + } + }) +} + +func TestScanner_Peek(t *testing.T) { + terr := errors.New(t.Name()) + tests := []struct { + name string + input string + peeked bool + peekedPos token.Pos + peekedStr string + peekedErr error + wantPos token.Pos + wantStr string + wantErr error + }{ + { + name: "not peeked", + input: "pragma solidity ^0.8.13;", + peeked: false, + wantPos: token.Pos{ + Column: 1, + Line: 1, + }, + wantStr: "pragma", + }, + { + name: "peeked", + peeked: true, + peekedStr: "peekedStr", + peekedPos: token.Pos{ + Column: 2, + Line: 2, + }, + wantPos: token.Pos{ + Column: 2, + Line: 2, + }, + wantErr: terr, + wantStr: "peekedStr", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + s := &Scanner{ + r: bufrr.NewReader(strings.NewReader(tt.input)), + peeked: tt.peeked, + peekStr: tt.peekedStr, + peekPos: tt.peekedPos, + peekErr: tt.peekedErr, + } + + pos, str, err := s.Peek() + if !errors.Is(err, tt.peekedErr) { + t.Fatalf("got unexpected error: %s", err) + } + if diff := cmp.Diff(tt.wantPos, pos); diff != "" { + t.Errorf(diff) + } + if str != tt.wantStr { + t.Errorf("got %s, want %s", str, tt.wantStr) + } + }) + } } func TestScanner_scanOperator(t *testing.T) { From 4bc910f314c2de50baf79d9ea4928adfe45729c8 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 11 Feb 2023 23:07:45 +0900 Subject: [PATCH 101/138] lexer: implement Scan StringLiteral into scan. --- lexer/lexer.go | 31 ++++++++++++++++++++----------- lexer/lexer_test.go | 12 ++++++++++++ 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index e085623..90ae859 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -1,7 +1,7 @@ package lexer import ( - "fmt" + "errors" "io" "github.com/uji/solparser/scanner" @@ -26,16 +26,26 @@ func New(input io.Reader) *Lexer { } func (l *Lexer) scan() (tkn token.Token, err error) { - pos, lit, err := l.scanner.Scan() + pos, str, err := l.scanner.Peek() if err != nil { return token.Token{}, err } + if str == "" { + return token.Token{}, errors.New("Empty character scanned.") + } + + if str == `"` || str == `\"` { + return l.ScanStringLiteral() + } + + l.scanner.Scan() - if token.IsSpace([]rune(lit)[0]) { + // If space, scan for the next token + if token.IsSpace([]rune(str)[0]) { return l.scan() } - return token.NewToken(lit, pos), nil + return token.NewToken(str, pos), nil } func (l *Lexer) Scan() (token.Token, error) { @@ -61,23 +71,22 @@ func (l *Lexer) Peek() (token.Token, error) { // ScanStringLiteral parse NonEmptyStringLiteral or EmptyStringLiteral then return StringLiteral token. func (l *Lexer) ScanStringLiteral() (token.Token, error) { - firstPos, lit, err := l.scanner.Scan() + firstPos, str, err := l.scanner.Scan() if err != nil { return token.Token{}, err } - fmt.Println(firstPos, lit) - if lit != `"` && lit != `\'` { + if str != `"` && str != `\'` { return token.Token{}, token.NewPosError(firstPos, `not found " or \'`) } - quote, txt := lit, lit + quote, txt := str, str for { - _, lit, err := l.scanner.Scan() + _, str, err := l.scanner.Scan() if err != nil { return token.Token{}, err } - txt = txt + lit - if lit == quote { + txt = txt + str + if str == quote { return token.Token{ TokenType: token.StringLiteral, Text: txt, diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 72290d6..e07a7ef 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -41,6 +41,18 @@ func TestLexer_Scan(t *testing.T) { }, }, }, + { + name: "There is a StringLiteral", + input: `"pragma"`, + wantToken: token.Token{ + TokenType: token.StringLiteral, + Text: `"pragma"`, + Pos: token.Pos{ + Column: 1, + Line: 1, + }, + }, + }, } for _, tt := range tests { tt := tt From 03dd73e58fe1058610698c0b704da971b1d49f91 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 12 Feb 2023 15:09:01 +0900 Subject: [PATCH 102/138] token: remove token.True, token.False and use XxxLiteral --- solparser.go | 4 ++-- solparser_test.go | 6 +++--- token/token.go | 6 ++---- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/solparser.go b/solparser.go index 052db18..778fad8 100644 --- a/solparser.go +++ b/solparser.go @@ -20,7 +20,7 @@ func New(input io.Reader) *Parser { } } -func (p *Parser) Parse(input io.Reader) (*ast.SourceUnit, error) { +func (p *Parser) Parse() (*ast.SourceUnit, error) { tkn, err := p.lexer.Peek() if err == io.EOF { return nil, nil @@ -67,7 +67,7 @@ func (p *Parser) ParseBooleanLiteral() (*ast.BooleanLiteral, error) { return nil, err } - if tkn.TokenType != token.True && tkn.TokenType != token.False { + if tkn.TokenType != token.TrueLiteral && tkn.TokenType != token.FalseLiteral { return nil, token.NewPosError(tkn.Pos, "not found keyword true or false") } diff --git a/solparser_test.go b/solparser_test.go index e944bd5..d127fc3 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -11,7 +11,7 @@ import ( "github.com/uji/solparser/token" ) -func TestParserParseBooleanLiteral(t *testing.T) { +func TestParser_ParseBooleanLiteral(t *testing.T) { tests := []struct { name string input string @@ -23,7 +23,7 @@ func TestParserParseBooleanLiteral(t *testing.T) { input: "true", want: &ast.BooleanLiteral{ Token: token.Token{ - TokenType: token.True, + TokenType: token.TrueLiteral, Text: "true", Pos: token.Pos{ Column: 1, @@ -38,7 +38,7 @@ func TestParserParseBooleanLiteral(t *testing.T) { input: "false", want: &ast.BooleanLiteral{ Token: token.Token{ - TokenType: token.False, + TokenType: token.FalseLiteral, Text: "false", Pos: token.Pos{ Column: 1, diff --git a/token/token.go b/token/token.go index 898c941..83df08a 100644 --- a/token/token.go +++ b/token/token.go @@ -120,7 +120,6 @@ const ( Event External Fallback - False Fixed For From @@ -152,7 +151,6 @@ const ( Storage String Struct - True Try Type Using @@ -383,7 +381,7 @@ func asKeyword(str string) TokenType { case "fallback": return Fallback case "false": - return False + return FalseLiteral case "fixed": return Fixed case "for": @@ -447,7 +445,7 @@ func asKeyword(str string) TokenType { case "struct": return Struct case "true": - return True + return TrueLiteral case "try": return Try case "type": From 2fbb4aee0d66cb20ddbf580972b2e1a1d49b9134 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 12 Feb 2023 16:52:53 +0900 Subject: [PATCH 103/138] solparser: parse StringLiteral as Literal --- ast/ast.go | 13 +++++++++- literal.go | 26 +++++++++++++++++++ literal_test.go | 69 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 literal.go create mode 100644 literal_test.go diff --git a/ast/ast.go b/ast/ast.go index a869c6a..02c2bdc 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -106,8 +106,19 @@ func (b *BooleanLiteral) End() token.Pos { } } +// EmptyStringLiteral | NonEmptyStringLiteral type StringLiteral struct { - List []Node // EmptyStringLiteral | NonEmptyStringLiteral + Str string + Position token.Pos + EndPos token.Pos +} + +func (s StringLiteral) Pos() token.Pos { + return s.Position +} + +func (s StringLiteral) End() token.Pos { + return s.EndPos } type HexStringLiteral []*HexString diff --git a/literal.go b/literal.go new file mode 100644 index 0000000..60d43d2 --- /dev/null +++ b/literal.go @@ -0,0 +1,26 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseLiteral() (ast.Literal, error) { + tkn, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + if tkn.TokenType != token.StringLiteral { + return nil, token.NewPosError(tkn.Pos, "not found string literal quote") + } + + return &ast.StringLiteral{ + Str: tkn.Text, + Position: tkn.Pos, + EndPos: token.Pos{ + Column: tkn.Pos.Column + len(tkn.Text) - 1, + Line: tkn.Pos.Line, // TODO: support new line code + }, + }, nil +} diff --git a/literal_test.go b/literal_test.go new file mode 100644 index 0000000..c7f9e62 --- /dev/null +++ b/literal_test.go @@ -0,0 +1,69 @@ +package solparser_test + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseLiteral(t *testing.T) { + tests := []struct { + name string + input string + want ast.Literal + err *token.PosError + }{ + { + name: "StringLiteral", + input: `"Hello World!!";`, + want: &ast.StringLiteral{ + Str: `"Hello World!!"`, + Position: token.Pos{ + Column: 1, + Line: 1, + }, + EndPos: token.Pos{ + Column: 15, + Line: 1, + }, + }, + err: nil, + }, + { + name: "Not Literal", + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{ + Column: 1, + Line: 1, + }, + Msg: "not found string literal quote", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseLiteral() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} From 4fa9750ee451df9bed7d6fe50fdb7ef64941cc85 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 18 Feb 2023 15:01:27 +0900 Subject: [PATCH 104/138] all: parse EOF as tokentype `EOS` --- lexer/lexer_test.go | 28 ++++++++++++++++++++++------ pragmaDirective_test.go | 2 +- scanner/scanner.go | 3 +-- scanner/scanner_test.go | 7 +++---- token/token.go | 12 +++++++++++- 5 files changed, 38 insertions(+), 14 deletions(-) diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index e07a7ef..b69f5a5 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -2,7 +2,6 @@ package lexer import ( "errors" - "io" "strings" "testing" @@ -78,9 +77,20 @@ func TestLexer_Scan(t *testing.T) { scanner: s, } - _, err := l.Scan() - if err != io.EOF { - t.Errorf("error is not io.EOF, got: %s", err) + got, err := l.Scan() + if err != nil { + t.Errorf("got error: %s", err) + } + want := token.Token{ + TokenType: token.EOS, + Text: token.EOSString, + Pos: token.Pos{ + Column: 1, + Line: 1, + }, + } + if diff := cmp.Diff(want, got); diff != "" { + t.Error(diff) } }) @@ -133,8 +143,14 @@ func TestLexer_Peek(t *testing.T) { { name: "when scan is done", input: "", - token: token.Token{}, - err: io.EOF, + token: token.Token{ + TokenType: token.EOS, + Text: token.EOSString, + Pos: token.Pos{ + Column: 1, + Line: 1, + }, + }, }, } for _, tt := range tests { diff --git a/pragmaDirective_test.go b/pragmaDirective_test.go index a1ad8a7..f2a65da 100644 --- a/pragmaDirective_test.go +++ b/pragmaDirective_test.go @@ -72,7 +72,7 @@ func TestParserParsePragmaDirective(t *testing.T) { want: nil, err: &token.PosError{ Pos: token.Pos{ - Column: 18, + Column: 24, Line: 1, }, Msg: "not found Semicolon", diff --git a/scanner/scanner.go b/scanner/scanner.go index 92b402d..ee68301 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -133,7 +133,6 @@ func (s *Scanner) scanOperator() (string, error) { // scan flow // -// - If first rune is an error or EOF, return an error and exit. // - If first rune is operator character, scan operator. // - If first rune is a space character, scan until the end of the blank character. // - Else scan to next space or operator string. @@ -148,7 +147,7 @@ func (s *Scanner) scan() (token.Pos, string, error) { return token.Pos{}, "", err } if ch == bufrr.EOF { - return token.Pos{}, "", io.EOF + return startPos, token.EOSString, nil } if isOperatorRune(ch) { // scan operator. diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index 1aad218..85bedb2 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -2,7 +2,6 @@ package scanner import ( "errors" - "io" "strings" "testing" @@ -88,12 +87,12 @@ func TestScanner_Scan(t *testing.T) { for { pos, str, err := s.Scan() - if err == io.EOF { - break - } if err != nil { t.Fatalf("got unexpected error: %s", err) } + if str == token.EOSString { + break + } poss = append(poss, pos) strs = append(strs, str) } diff --git a/token/token.go b/token/token.go index 83df08a..16d4f1f 100644 --- a/token/token.go +++ b/token/token.go @@ -1,6 +1,10 @@ package token -import "fmt" +import ( + "fmt" + + "github.com/SteelSeries/bufrr" +) type TokenType int @@ -8,6 +12,8 @@ const ( Invalid TokenType = iota Unknown + EOS // End of source indicator. + LParen // ( RParen // ) LBrack // [ @@ -168,8 +174,12 @@ const ( CommentLiteral ) +var EOSString string = string([]rune{bufrr.EOF}) + func asKeyword(str string) TokenType { switch str { + case EOSString: + return EOS case "(": return LParen case ")": From 57b84f05461560aa62fa2a1c8999ea20c92f4d65 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 18 Feb 2023 23:12:06 +0900 Subject: [PATCH 105/138] main: parse literal --- literal.go | 10 ++++++++-- literal_test.go | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/literal.go b/literal.go index 60d43d2..d7c2a21 100644 --- a/literal.go +++ b/literal.go @@ -15,12 +15,18 @@ func (p *Parser) ParseLiteral() (ast.Literal, error) { return nil, token.NewPosError(tkn.Pos, "not found string literal quote") } + // peek to find literal end position. + peek, err := p.lexer.Peek() + if err != nil { + return nil, err + } + return &ast.StringLiteral{ Str: tkn.Text, Position: tkn.Pos, EndPos: token.Pos{ - Column: tkn.Pos.Column + len(tkn.Text) - 1, - Line: tkn.Pos.Line, // TODO: support new line code + Column: peek.Pos.Column - 1, + Line: peek.Pos.Line, }, }, nil } diff --git a/literal_test.go b/literal_test.go index c7f9e62..02dda01 100644 --- a/literal_test.go +++ b/literal_test.go @@ -19,7 +19,7 @@ func TestParser_ParseLiteral(t *testing.T) { err *token.PosError }{ { - name: "StringLiteral", + name: "normal", input: `"Hello World!!";`, want: &ast.StringLiteral{ Str: `"Hello World!!"`, @@ -34,6 +34,39 @@ func TestParser_ParseLiteral(t *testing.T) { }, err: nil, }, + { + name: `Including \n`, + input: "\"Hello \nWorld!!\";", + want: &ast.StringLiteral{ + Str: "\"Hello \nWorld!!\"", + Position: token.Pos{ + Column: 1, + Line: 1, + }, + EndPos: token.Pos{ + Column: 8, + Line: 2, + }, + }, + err: nil, + }, + // TODO + // { + // name: `Next token is \n`, + // input: "\"Hello World!!\"\n", + // want: &ast.StringLiteral{ + // Str: `"Hello World!!"`, + // Position: token.Pos{ + // Column: 1, + // Line: 1, + // }, + // EndPos: token.Pos{ + // Column: 15, + // Line: 1, + // }, + // }, + // err: nil, + // }, { name: "Not Literal", input: "pragma", From 9ae80170a9cb624725ed539384bbc9dce3c74225 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 18 Feb 2023 23:33:54 +0900 Subject: [PATCH 106/138] main: parse expression (support literal only) --- expression.go | 19 +++++++++++++ expression_test.go | 69 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 expression.go create mode 100644 expression_test.go diff --git a/expression.go b/expression.go new file mode 100644 index 0000000..8a8b11b --- /dev/null +++ b/expression.go @@ -0,0 +1,19 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseExpression() (ast.Expression, error) { + tkn, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + if tkn.TokenType == token.StringLiteral { + return p.ParseLiteral() + } + + return nil, token.NewPosError(tkn.Pos, "not found expression.") +} diff --git a/expression_test.go b/expression_test.go new file mode 100644 index 0000000..72bdb1e --- /dev/null +++ b/expression_test.go @@ -0,0 +1,69 @@ +package solparser_test + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseExpression(t *testing.T) { + tests := []struct { + name string + input string + want ast.Expression + err *token.PosError + }{ + { + name: "Literal", + input: `"Hello World!!";`, + want: &ast.StringLiteral{ + Str: `"Hello World!!"`, + Position: token.Pos{ + Column: 1, + Line: 1, + }, + EndPos: token.Pos{ + Column: 15, + Line: 1, + }, + }, + err: nil, + }, + { + name: "Not expression", + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{ + Column: 1, + Line: 1, + }, + Msg: "not found expression.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseExpression() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} From f5f3f6d2bbb2994dea6a236544c4594f0999512a Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 19 Feb 2023 12:15:48 +0900 Subject: [PATCH 107/138] main, ast: rename StringLiteral fields --- ast/ast.go | 9 ++++----- expression_test.go | 6 +++--- literal.go | 6 +++--- literal_test.go | 12 ++++++------ 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 02c2bdc..c2d2bd8 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -108,17 +108,16 @@ func (b *BooleanLiteral) End() token.Pos { // EmptyStringLiteral | NonEmptyStringLiteral type StringLiteral struct { - Str string - Position token.Pos - EndPos token.Pos + From, To token.Pos + Value string } func (s StringLiteral) Pos() token.Pos { - return s.Position + return s.From } func (s StringLiteral) End() token.Pos { - return s.EndPos + return s.To } type HexStringLiteral []*HexString diff --git a/expression_test.go b/expression_test.go index 72bdb1e..0fee217 100644 --- a/expression_test.go +++ b/expression_test.go @@ -22,12 +22,12 @@ func TestParser_ParseExpression(t *testing.T) { name: "Literal", input: `"Hello World!!";`, want: &ast.StringLiteral{ - Str: `"Hello World!!"`, - Position: token.Pos{ + Value: `"Hello World!!"`, + From: token.Pos{ Column: 1, Line: 1, }, - EndPos: token.Pos{ + To: token.Pos{ Column: 15, Line: 1, }, diff --git a/literal.go b/literal.go index d7c2a21..9cdf99b 100644 --- a/literal.go +++ b/literal.go @@ -22,9 +22,9 @@ func (p *Parser) ParseLiteral() (ast.Literal, error) { } return &ast.StringLiteral{ - Str: tkn.Text, - Position: tkn.Pos, - EndPos: token.Pos{ + Value: tkn.Text, + From: tkn.Pos, + To: token.Pos{ Column: peek.Pos.Column - 1, Line: peek.Pos.Line, }, diff --git a/literal_test.go b/literal_test.go index 02dda01..1f74cf0 100644 --- a/literal_test.go +++ b/literal_test.go @@ -22,12 +22,12 @@ func TestParser_ParseLiteral(t *testing.T) { name: "normal", input: `"Hello World!!";`, want: &ast.StringLiteral{ - Str: `"Hello World!!"`, - Position: token.Pos{ + Value: `"Hello World!!"`, + From: token.Pos{ Column: 1, Line: 1, }, - EndPos: token.Pos{ + To: token.Pos{ Column: 15, Line: 1, }, @@ -38,12 +38,12 @@ func TestParser_ParseLiteral(t *testing.T) { name: `Including \n`, input: "\"Hello \nWorld!!\";", want: &ast.StringLiteral{ - Str: "\"Hello \nWorld!!\"", - Position: token.Pos{ + Value: "\"Hello \nWorld!!\"", + From: token.Pos{ Column: 1, Line: 1, }, - EndPos: token.Pos{ + To: token.Pos{ Column: 8, Line: 2, }, From 35b8fde80c21099f0b74ba0c536cd33629c0b9ea Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 19 Feb 2023 12:55:16 +0900 Subject: [PATCH 108/138] main, ast: parse statement (only returnStatement) --- ast/ast.go | 20 +++++-- statement.go | 50 +++++++++++++++++ statement_test.go | 140 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 204 insertions(+), 6 deletions(-) create mode 100644 statement.go create mode 100644 statement_test.go diff --git a/ast/ast.go b/ast/ast.go index c2d2bd8..2fa3f5d 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -252,17 +252,25 @@ type ElementaryTypeName struct { Kind ElementaryTypeNameKind } +// ---------------------------------------------------------------------------- +// Statement Nodes + +type Statement interface { + Node + statementNode() +} + type ReturnStatement struct { - Return token.Pos + From token.Pos + SemiPos token.Pos Expression Expression } func (r *ReturnStatement) Pos() token.Pos { - return r.Return + return r.From } func (r *ReturnStatement) End() token.Pos { - return token.Pos{ - Column: r.Expression.End().Column + 1, - Line: r.Expression.End().Line, - } + return r.SemiPos } + +func (s *ReturnStatement) statementNode() {} diff --git a/statement.go b/statement.go new file mode 100644 index 0000000..b9905a7 --- /dev/null +++ b/statement.go @@ -0,0 +1,50 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseStatement() (ast.Statement, error) { + tkn, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + if tkn.TokenType == token.Return { + return p.ParseReturnStatement() + } + + return nil, token.NewPosError(tkn.Pos, "not found statement.") +} + +func (p *Parser) ParseReturnStatement() (ast.Statement, error) { + rtn, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + if rtn.TokenType != token.Return { + return nil, token.NewPosError(rtn.Pos, "not found return keyword.") + } + + exp, err := p.ParseExpression() + if err != nil { + return nil, err + } + + semi, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + if semi.TokenType != token.Semicolon { + return nil, token.NewPosError(semi.Pos, "not found semicolon.") + } + + return &ast.ReturnStatement{ + From: rtn.Pos, + SemiPos: semi.Pos, + Expression: exp, + }, nil +} diff --git a/statement_test.go b/statement_test.go new file mode 100644 index 0000000..a63c1ec --- /dev/null +++ b/statement_test.go @@ -0,0 +1,140 @@ +package solparser_test + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseStatement(t *testing.T) { + tests := []struct { + name string + input string + want ast.Statement + err *token.PosError + }{ + { + name: "ReturnStatement", + input: `return "Hello World!!";`, + want: &ast.ReturnStatement{ + From: token.Pos{Column: 1, Line: 1}, + SemiPos: token.Pos{Column: 23, Line: 1}, + Expression: &ast.StringLiteral{ + From: token.Pos{Column: 8, Line: 1}, + To: token.Pos{Column: 22, Line: 1}, + Value: `"Hello World!!"`, + }, + }, + err: nil, + }, + { + name: "Not Statement", + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{Column: 1, Line: 1}, + Msg: "not found statement.", + }, + }, + { + name: "Found broken return statement", + input: "return pragma", + err: &token.PosError{ + Pos: token.Pos{Column: 8, Line: 1}, + Msg: "not found expression.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseStatement() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} + +func TestParser_ParseReturnStatement(t *testing.T) { + tests := []struct { + name string + input string + want ast.Statement + err *token.PosError + }{ + { + name: "normal", + input: `return "Hello World!!";`, + want: &ast.ReturnStatement{ + From: token.Pos{Column: 1, Line: 1}, + SemiPos: token.Pos{Column: 23, Line: 1}, + Expression: &ast.StringLiteral{ + From: token.Pos{Column: 8, Line: 1}, + To: token.Pos{Column: 22, Line: 1}, + Value: `"Hello World!!"`, + }, + }, + err: nil, + }, + { + name: "Not found return keyword", + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{Column: 1, Line: 1}, + Msg: "not found return keyword.", + }, + }, + { + name: "Not found expression", + input: "return pragma", + err: &token.PosError{ + Pos: token.Pos{Column: 8, Line: 1}, + Msg: "not found expression.", + }, + }, + { + name: "Not found semicolon.", + input: `return "test" pragma`, + err: &token.PosError{ + Pos: token.Pos{Column: 15, Line: 1}, + Msg: "not found semicolon.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseReturnStatement() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} From 8ce442a44077fbed0bf44dbb8b440c0ffbb00cb1 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 19 Feb 2023 13:15:31 +0900 Subject: [PATCH 109/138] main, ast: parse block (supported only one statement) --- ast/ast.go | 16 +++++++++++ block.go | 37 ++++++++++++++++++++++++ block_test.go | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 block.go create mode 100644 block_test.go diff --git a/ast/ast.go b/ast/ast.go index 2fa3f5d..48b8a37 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -252,6 +252,22 @@ type ElementaryTypeName struct { Kind ElementaryTypeNameKind } +// ---------------------------------------------------------------------------- + +type Block struct { + LBracePos token.Pos + RBracePos token.Pos + Nodes []Node // statement | unblocked-block +} + +func (b Block) Pos() token.Pos { + return b.LBracePos +} + +func (b Block) End() token.Pos { + return b.RBracePos +} + // ---------------------------------------------------------------------------- // Statement Nodes diff --git a/block.go b/block.go new file mode 100644 index 0000000..e1399e3 --- /dev/null +++ b/block.go @@ -0,0 +1,37 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseBlock() (*ast.Block, error) { + lblace, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + if lblace.TokenType != token.LBrace { + return nil, token.NewPosError(lblace.Pos, "not found LBrace.") + } + + stmt, err := p.ParseStatement() + if err != nil { + return nil, err + } + + rblace, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + if rblace.TokenType != token.RBrace { + return nil, token.NewPosError(rblace.Pos, "not found RBrace.") + } + + return &ast.Block{ + LBracePos: lblace.Pos, + RBracePos: rblace.Pos, + Nodes: []ast.Node{stmt}, + }, nil +} diff --git a/block_test.go b/block_test.go new file mode 100644 index 0000000..fa82134 --- /dev/null +++ b/block_test.go @@ -0,0 +1,78 @@ +package solparser_test + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseBlock(t *testing.T) { + tests := []struct { + name string + input string + want *ast.Block + err *token.PosError + }{ + { + name: "One Statement", + input: "{ \nreturn \"Hello World!!\";\n }", + want: &ast.Block{ + LBracePos: token.Pos{Column: 1, Line: 1}, + RBracePos: token.Pos{Column: 2, Line: 3}, + Nodes: []ast.Node{ + &ast.ReturnStatement{ + From: token.Pos{Column: 1, Line: 2}, + SemiPos: token.Pos{Column: 23, Line: 2}, + Expression: &ast.StringLiteral{ + From: token.Pos{Column: 8, Line: 2}, + To: token.Pos{Column: 22, Line: 2}, + Value: "\"Hello World!!\"", + }, + }, + }, + }, + err: nil, + }, + { + name: "Not found LBrace", + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{Column: 1, Line: 1}, + Msg: "not found LBrace.", + }, + }, + { + name: "Not found RBrace", + input: "{ \nreturn \"Hello World!!\";", + err: &token.PosError{ + Pos: token.Pos{Column: 24, Line: 2}, + Msg: "not found RBrace.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseBlock() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} From 2b8589e2804195e607452a575935faecf4d97ce6 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 19 Feb 2023 15:30:04 +0900 Subject: [PATCH 110/138] main, ast: parse typeName (only elementaryTypeNode) --- ast/ast.go | 48 ++++++++--------- typeName.go | 46 ++++++++++++++++ typeName_test.go | 135 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 205 insertions(+), 24 deletions(-) create mode 100644 typeName.go create mode 100644 typeName_test.go diff --git a/ast/ast.go b/ast/ast.go index 48b8a37..ef85e2b 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -29,10 +29,6 @@ type ModifierList struct { StateMutability *StateMutability } -type TypeName struct { - ElementalyTypeName string -} - type EventParameter struct { TypeName *TypeName } @@ -74,6 +70,30 @@ type SourceUnit struct { FunctionDefinition *FunctionDefinition } +// ---------------------------------------------------------------------------- +// TypeName Nodes + +type TypeName interface { + Node + typeNameNode() +} + +type ElementaryTypeName []token.Token + +func (e ElementaryTypeName) Pos() token.Pos { + return e[0].Pos +} + +func (e ElementaryTypeName) End() token.Pos { + // TODO + return token.Pos{ + Column: 1, + Line: 1, + } +} + +func (e ElementaryTypeName) typeNameNode() {} + // ---------------------------------------------------------------------------- // Expression Nodes @@ -232,26 +252,6 @@ type HexString struct { String string } -type ElementaryTypeNameKind int - -const ( - ElementaryTypeName_Address ElementaryTypeNameKind = iota + 1 - ElementaryTypeName_AddressPayable - ElementaryTypeName_Bool - ElementaryTypeName_String - ElementaryTypeName_Bytes - ElementaryTypeName_SignedIntegerType - ElementaryTypeName_UnsignedIntegerType - ElementaryTypeName_FixedBytes - ElementaryTypeName_Fixed - ElementaryTypeName_Ufixed -) - -type ElementaryTypeName struct { - Pos token.Pos - Kind ElementaryTypeNameKind -} - // ---------------------------------------------------------------------------- type Block struct { diff --git a/typeName.go b/typeName.go new file mode 100644 index 0000000..96d5ea2 --- /dev/null +++ b/typeName.go @@ -0,0 +1,46 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseTypeName() (ast.TypeName, error) { + tkn, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + switch tkn.TokenType { + case token.Address, token.String, token.Bytes, token.Fixed, token.Bool: + return p.ParseElementaryTypeName() + } + + return nil, token.NewPosError(tkn.Pos, "not found type-name.") +} + +func (p *Parser) ParseElementaryTypeName() (ast.TypeName, error) { + tkn, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + if tkn.TokenType == token.Address { + tkn2, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + if tkn2.TokenType == token.Payable { + p.lexer.Scan() + return ast.ElementaryTypeName{tkn, tkn2}, nil + } + } + + switch tkn.TokenType { + case token.Address, token.String, token.Bytes, token.Fixed, token.Bool: + return ast.ElementaryTypeName{tkn}, nil + } + + return nil, token.NewPosError(tkn.Pos, "not found elementary type name keyword.") +} diff --git a/typeName_test.go b/typeName_test.go new file mode 100644 index 0000000..b0f5730 --- /dev/null +++ b/typeName_test.go @@ -0,0 +1,135 @@ +package solparser_test + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseTypeName(t *testing.T) { + tests := []struct { + name string + input string + want ast.TypeName + err *token.PosError + }{ + { + name: "ElementaryTypeName", + input: "string)", + want: ast.ElementaryTypeName{ + { + TokenType: token.String, + Text: "string", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + }, + { + name: "Not TypeName", + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{Column: 1, Line: 1}, + Msg: "not found type-name.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseTypeName() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} + +func TestParser_ParseElementaryTypeName(t *testing.T) { + tests := []struct { + input string + want ast.TypeName + err *token.PosError + }{ + { + input: "address", + want: ast.ElementaryTypeName{ + { + TokenType: token.Address, + Text: "address", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + }, + { + input: "bool", + want: ast.ElementaryTypeName{ + { + TokenType: token.Bool, + Text: "bool", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + }, + { + input: "string", + want: ast.ElementaryTypeName{ + { + TokenType: token.String, + Text: "string", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + }, + { + input: "address payable", + want: ast.ElementaryTypeName{ + { + TokenType: token.Address, + Text: "address", + Pos: token.Pos{Column: 1, Line: 1}, + }, + { + TokenType: token.Payable, + Text: "payable", + Pos: token.Pos{Column: 9, Line: 1}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.input, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseElementaryTypeName() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} From dc95af2165bb776b90a19d35c22cf981f3b358f3 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 19 Feb 2023 16:07:56 +0900 Subject: [PATCH 111/138] main, ast: parse parameter-list --- ast/ast.go | 9 ++- parameterList.go | 40 ++++++++++++ parameterList_test.go | 149 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 193 insertions(+), 5 deletions(-) create mode 100644 parameterList.go create mode 100644 parameterList_test.go diff --git a/ast/ast.go b/ast/ast.go index ef85e2b..d67e7f4 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -29,13 +29,12 @@ type ModifierList struct { StateMutability *StateMutability } -type EventParameter struct { - TypeName *TypeName +// Parameter is type of ParameterList elements +type Parameter struct { + TypeName TypeName } -type ParameterList struct { - EventParameter *EventParameter -} +type ParameterList []*Parameter type ReturnParameters struct { ParameterList *ParameterList diff --git a/parameterList.go b/parameterList.go new file mode 100644 index 0000000..a1dd76e --- /dev/null +++ b/parameterList.go @@ -0,0 +1,40 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseParameter() (*ast.Parameter, error) { + tn, err := p.ParseTypeName() + if err != nil { + return nil, err + } + + return &ast.Parameter{ + TypeName: tn, + }, nil +} + +func (p *Parser) ParseParameterList() (ast.ParameterList, error) { + prms := make(ast.ParameterList, 0, 1) + for { + prm, err := p.ParseParameter() + if err != nil { + return nil, err + } + + prms = append(prms, prm) + + comma, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + if comma.TokenType != token.Comma { + return prms, nil + } + + p.lexer.Scan() + } +} diff --git a/parameterList_test.go b/parameterList_test.go new file mode 100644 index 0000000..366fdc3 --- /dev/null +++ b/parameterList_test.go @@ -0,0 +1,149 @@ +package solparser_test + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseParameter(t *testing.T) { + tests := []struct { + name string + input string + want *ast.Parameter + err *token.PosError + }{ + { + name: "TypeName only", + input: "string", + want: &ast.Parameter{ + TypeName: ast.ElementaryTypeName{ + { + TokenType: token.String, + Text: "string", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + }, + err: nil, + }, + { + name: "Not ParameterList", + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{ + Column: 1, + Line: 1, + }, + Msg: "not found type-name.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseParameter() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} + +func TestParser_ParseParameterList(t *testing.T) { + tests := []struct { + name string + input string + want ast.ParameterList + err *token.PosError + }{ + { + name: "One Parameter", + input: "string", + want: ast.ParameterList{ + { + TypeName: ast.ElementaryTypeName{ + { + TokenType: token.String, + Text: "string", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + }, + }, + err: nil, + }, + { + name: "Some Parameter", + input: "string, bool", + want: ast.ParameterList{ + { + TypeName: ast.ElementaryTypeName{ + { + TokenType: token.String, + Text: "string", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + }, + { + TypeName: ast.ElementaryTypeName{ + { + TokenType: token.Bool, + Text: "bool", + Pos: token.Pos{Column: 9, Line: 1}, + }, + }, + }, + }, + err: nil, + }, + { + name: "Not ParameterList", + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{ + Column: 1, + Line: 1, + }, + Msg: "not found type-name.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseParameterList() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} From dfd3d35671c54522bc0c97e9b98a5d4d3aae0537 Mon Sep 17 00:00:00 2001 From: uji Date: Thu, 23 Feb 2023 15:13:58 +0900 Subject: [PATCH 112/138] main, ast: parse visibility and state-mutability. --- ast/ast.go | 8 +-- functionDefinition.go | 35 ++++++++++--- functionDefinition_test.go | 100 +++++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 12 deletions(-) create mode 100644 functionDefinition_test.go diff --git a/ast/ast.go b/ast/ast.go index d67e7f4..390ffd5 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -21,14 +21,12 @@ type FunctionDescriptor struct { Name string } -type StateMutability struct { - Pure bool -} - type ModifierList struct { StateMutability *StateMutability } +type Visibility = token.Token + // Parameter is type of ParameterList elements type Parameter struct { TypeName TypeName @@ -36,6 +34,8 @@ type Parameter struct { type ParameterList []*Parameter +type StateMutability = token.Token + type ReturnParameters struct { ParameterList *ParameterList } diff --git a/functionDefinition.go b/functionDefinition.go index 4268ca6..bfddd4e 100644 --- a/functionDefinition.go +++ b/functionDefinition.go @@ -1,15 +1,34 @@ package solparser -import "github.com/uji/solparser/ast" +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) -func (p *Parser) ParseFunctionDefinition() (*ast.FunctionDefinition, error) { - return nil, nil -} +func (p *Parser) ParseVisibility() (ast.Visibility, error) { + tkn, err := p.lexer.Scan() + if err != nil { + return token.Token{}, err + } + + switch tkn.TokenType { + case token.Internal, token.External, token.Public, token.Private: + return tkn, nil + } -func (p *Parser) ParseModirierList() (*ast.ModifierList, error) { - return nil, nil + return token.Token{}, token.NewPosError(tkn.Pos, "not found visibility keyword.") } -func (p *Parser) ParseReturnParameters() (*ast.ReturnParameters, error) { - return nil, nil +func (p *Parser) ParseStateMutability() (ast.StateMutability, error) { + tkn, err := p.lexer.Scan() + if err != nil { + return token.Token{}, err + } + + switch tkn.TokenType { + case token.Pure, token.View, token.Payable: + return tkn, nil + } + + return token.Token{}, token.NewPosError(tkn.Pos, "not found state-mutability keyword.") } diff --git a/functionDefinition_test.go b/functionDefinition_test.go new file mode 100644 index 0000000..a258263 --- /dev/null +++ b/functionDefinition_test.go @@ -0,0 +1,100 @@ +package solparser_test + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseVisibility(t *testing.T) { + tests := []struct { + input string + want ast.Visibility + err *token.PosError + }{ + { + input: "internal", + want: ast.Visibility{ + TokenType: token.Internal, + Text: "internal", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + { + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{Column: 1, Line: 1}, + Msg: "not found visibility keyword.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.input, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseVisibility() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} + +func TestParser_ParseStateMutability(t *testing.T) { + tests := []struct { + input string + want ast.StateMutability + err *token.PosError + }{ + { + input: "pure", + want: ast.StateMutability{ + TokenType: token.Pure, + Text: "pure", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + { + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{Column: 1, Line: 1}, + Msg: "not found state-mutability keyword.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.input, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseStateMutability() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} From 4e51f4ddaa6194e5ccb30f44e2e2d8da0cbefcc3 Mon Sep 17 00:00:00 2001 From: uji Date: Thu, 23 Feb 2023 15:27:42 +0900 Subject: [PATCH 113/138] main, ast, token: implement identifier --- ast/ast.go | 4 +++ contractDefinition.go | 2 +- identifier.go | 20 +++++++++++ identifier_test.go | 80 +++++++++++++++++++++++++++++++++++++++++ pragmaDirective.go | 4 +-- pragmaDirective_test.go | 4 +-- token/token.go | 6 ++-- 7 files changed, 113 insertions(+), 7 deletions(-) create mode 100644 identifier.go create mode 100644 identifier_test.go diff --git a/ast/ast.go b/ast/ast.go index 390ffd5..0db342a 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -101,6 +101,10 @@ type Expression interface { expressionNode() } +// ---------------------------------------------------------------------------- + +type Identifier = token.Token + // ---------------------------------------------------------------------------- // Literal Nodes diff --git a/contractDefinition.go b/contractDefinition.go index af04f35..edbe354 100644 --- a/contractDefinition.go +++ b/contractDefinition.go @@ -25,7 +25,7 @@ func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { if err != nil { return nil, err } - if tkn.TokenType != token.Unknown { + if tkn.TokenType != token.Identifier { return nil, errors.New("not found left brace") } diff --git a/identifier.go b/identifier.go new file mode 100644 index 0000000..f908f3d --- /dev/null +++ b/identifier.go @@ -0,0 +1,20 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseIdentifier() (ast.Identifier, error) { + tkn, err := p.lexer.Scan() + if err != nil { + return token.Token{}, err + } + + switch tkn.TokenType { + case token.Identifier, token.From, token.Error, token.Revert, token.Global: + return tkn, nil + } + + return token.Token{}, token.NewPosError(tkn.Pos, "keyword is not available as identifier.") +} diff --git a/identifier_test.go b/identifier_test.go new file mode 100644 index 0000000..b6f658a --- /dev/null +++ b/identifier_test.go @@ -0,0 +1,80 @@ +package solparser_test + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseIdentifier(t *testing.T) { + tests := []struct { + input string + want ast.Identifier + err *token.PosError + }{ + { + input: "identifier", + want: token.Token{ + TokenType: token.Identifier, + Text: "identifier", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + { + input: "from", + want: token.Token{ + TokenType: token.From, + Text: "from", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + { + input: "error", + want: token.Token{ + TokenType: token.Error, + Text: "error", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + { + input: "global", + want: token.Token{ + TokenType: token.Global, + Text: "global", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + { + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{Column: 1, Line: 1}, + Msg: "keyword is not available as identifier.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.input, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseIdentifier() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} diff --git a/pragmaDirective.go b/pragmaDirective.go index 7a3b76a..8d3b457 100644 --- a/pragmaDirective.go +++ b/pragmaDirective.go @@ -18,7 +18,7 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { if err != nil { return nil, err } - if pragmaName.TokenType != token.Unknown { + if pragmaName.TokenType == token.Semicolon { return nil, token.NewPosError(pragmaName.Pos, "not found pragma name") } @@ -34,7 +34,7 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { if err != nil { return nil, err } - if version.TokenType != token.Unknown { + if version.TokenType != token.Identifier { return nil, token.NewPosError(version.Pos, "not found version") } diff --git a/pragmaDirective_test.go b/pragmaDirective_test.go index f2a65da..6c482d6 100644 --- a/pragmaDirective_test.go +++ b/pragmaDirective_test.go @@ -48,10 +48,10 @@ func TestParserParsePragmaDirective(t *testing.T) { want: nil, err: &token.PosError{ Pos: token.Pos{ - Column: 8, + Column: 9, Line: 1, }, - Msg: "not found pragma name", + Msg: "not found BitXor expression", }, }, { diff --git a/token/token.go b/token/token.go index 16d4f1f..2c82937 100644 --- a/token/token.go +++ b/token/token.go @@ -10,7 +10,6 @@ type TokenType int const ( Invalid TokenType = iota - Unknown EOS // End of source indicator. @@ -172,6 +171,9 @@ const ( UnicodeStringLiteral HexStringLiteral CommentLiteral + + // Not keywords or future reserved words + Identifier ) var EOSString string = string([]rune{bufrr.EOF}) @@ -470,7 +472,7 @@ func asKeyword(str string) TokenType { return While } - return Unknown + return Identifier } func asToken(str string) TokenType { From 9d2d5f0f52aabfaabb211f9398818af72194f7fa Mon Sep 17 00:00:00 2001 From: uji Date: Thu, 23 Feb 2023 17:28:31 +0900 Subject: [PATCH 114/138] main, ast: parse functionDefinition --- ast/ast.go | 23 +++-- functionDefinition.go | 129 ++++++++++++++++++++++++++ functionDefinition_test.go | 185 +++++++++++++++++++++++++++++++++++++ 3 files changed, 329 insertions(+), 8 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 0db342a..8c8a7ec 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -17,11 +17,8 @@ type PragmaDirective struct { PragmaValue PragmaValue } -type FunctionDescriptor struct { - Name string -} - type ModifierList struct { + Visibility *Visibility StateMutability *StateMutability } @@ -36,14 +33,24 @@ type ParameterList []*Parameter type StateMutability = token.Token -type ReturnParameters struct { - ParameterList *ParameterList +// identifier | fallback | recevie +type FunctionDescriptor = token.Token + +type FunctionDefinitionReturns struct { + From token.Pos + LParen token.Pos + ParameterList ParameterList + RParen token.Pos } type FunctionDefinition struct { - FunctionDescriptor *FunctionDescriptor + From token.Pos + FunctionDescriptor FunctionDescriptor + LParen token.Pos + RParen token.Pos ModifierList *ModifierList - ReturnParameters *ReturnParameters + Returns *FunctionDefinitionReturns + Block *Block } type InheritanceSpecifier struct { diff --git a/functionDefinition.go b/functionDefinition.go index bfddd4e..7c4b906 100644 --- a/functionDefinition.go +++ b/functionDefinition.go @@ -32,3 +32,132 @@ func (p *Parser) ParseStateMutability() (ast.StateMutability, error) { return token.Token{}, token.NewPosError(tkn.Pos, "not found state-mutability keyword.") } + +func (p *Parser) ParseFunctionDefinitionReturns() (*ast.FunctionDefinitionReturns, error) { + tkn, err := p.lexer.Peek() + if err != nil { + return nil, err + } + if tkn.TokenType != token.Returns { + return nil, nil + } + if _, err := p.lexer.Scan(); err != nil { + return nil, err + } + + lparen, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if lparen.TokenType != token.LParen { + return nil, token.NewPosError(lparen.Pos, "not found arguments LParen.") + } + + pl, err := p.ParseParameterList() + if err != nil { + return nil, err + } + + rparen, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if rparen.TokenType != token.RParen { + return nil, token.NewPosError(rparen.Pos, "not found arguments RParen.") + } + + return &ast.FunctionDefinitionReturns{ + From: tkn.Pos, + LParen: lparen.Pos, + ParameterList: pl, + RParen: rparen.Pos, + }, nil +} + +func (p *Parser) ParseFunctionDefinition() (*ast.FunctionDefinition, error) { + from, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + if from.TokenType != token.Function { + return nil, token.NewPosError(from.Pos, "not found function keyword.") + } + + dsc, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + switch dsc.TokenType { + case token.Identifier, token.From, token.Error, token.Revert, token.Global, token.Fallback, token.Receive: + default: + return nil, token.NewPosError(dsc.Pos, "not found function description.") + } + + lparen, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if lparen.TokenType != token.LParen { + return nil, token.NewPosError(lparen.Pos, "not found arguments LParen.") + } + + rparen, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if rparen.TokenType != token.RParen { + return nil, token.NewPosError(rparen.Pos, "not found arguments RParen.") + } + + modifierList := &ast.ModifierList{ + Visibility: nil, + StateMutability: nil, + } + + for { + tkn, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + switch tkn.TokenType { + case token.Internal, token.External, token.Public, token.Private: + vs, err := p.ParseVisibility() + if err != nil { + return nil, err + } + modifierList.Visibility = &vs + continue + case token.Pure, token.View, token.Payable: + sm, err := p.ParseStateMutability() + if err != nil { + return nil, err + } + modifierList.StateMutability = &sm + continue + } + break + } + + r, err := p.ParseFunctionDefinitionReturns() + if err != nil { + return nil, err + } + + b, err := p.ParseBlock() + if err != nil { + return nil, err + } + + return &ast.FunctionDefinition{ + From: from.Pos, + FunctionDescriptor: dsc, + LParen: lparen.Pos, + RParen: rparen.Pos, + ModifierList: modifierList, + Returns: r, + Block: b, + }, nil +} diff --git a/functionDefinition_test.go b/functionDefinition_test.go index a258263..6fed4a6 100644 --- a/functionDefinition_test.go +++ b/functionDefinition_test.go @@ -98,3 +98,188 @@ func TestParser_ParseStateMutability(t *testing.T) { }) } } + +func TestParser_ParseFunctionDefinitionReturns(t *testing.T) { + tests := []struct { + input string + want *ast.FunctionDefinitionReturns + err *token.PosError + }{ + { + input: "returns (string)", + want: &ast.FunctionDefinitionReturns{ + From: token.Pos{Column: 1, Line: 1}, + LParen: token.Pos{Column: 9, Line: 1}, + ParameterList: []*ast.Parameter{ + { + TypeName: ast.ElementaryTypeName{ + { + TokenType: token.String, + Text: "string", + Pos: token.Pos{Column: 10, Line: 1}, + }, + }, + }, + }, + RParen: token.Pos{Column: 16, Line: 1}, + }, + }, + { + input: ";", + want: nil, + }, + { + input: "returns string)", + err: &token.PosError{ + Pos: token.Pos{Column: 9, Line: 1}, + Msg: "not found arguments LParen.", + }, + }, + { + input: "returns (string {", + err: &token.PosError{ + Pos: token.Pos{Column: 17, Line: 1}, + Msg: "not found arguments RParen.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.input, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseFunctionDefinitionReturns() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} + +func TestParser_ParseFunctionDefinition(t *testing.T) { + tests := []struct { + name string + input string + want *ast.FunctionDefinition + err *token.PosError + }{ + { + name: "normal case", + input: `function hello() public pure returns (string) { + return "Hello World!!"; + }`, + want: &ast.FunctionDefinition{ + From: token.Pos{Column: 1, Line: 1}, + FunctionDescriptor: token.Token{ + TokenType: token.Identifier, + Text: "hello", + Pos: token.Pos{Column: 10, Line: 1}, + }, + LParen: token.Pos{Column: 15, Line: 1}, + RParen: token.Pos{Column: 16, Line: 1}, + ModifierList: &ast.ModifierList{ + Visibility: &token.Token{ + TokenType: token.Public, + Text: "public", + Pos: token.Pos{Column: 18, Line: 1}, + }, + StateMutability: &token.Token{ + TokenType: token.Pure, + Text: "pure", + Pos: token.Pos{Column: 25, Line: 1}, + }, + }, + Returns: &ast.FunctionDefinitionReturns{ + From: token.Pos{Column: 30, Line: 1}, + LParen: token.Pos{Column: 38, Line: 1}, + ParameterList: []*ast.Parameter{ + { + TypeName: ast.ElementaryTypeName{ + { + TokenType: token.String, + Text: "string", + Pos: token.Pos{Column: 39, Line: 1}, + }, + }, + }, + }, + RParen: token.Pos{Column: 45, Line: 1}, + }, + Block: &ast.Block{ + LBracePos: token.Pos{Column: 47, Line: 1}, + RBracePos: token.Pos{Column: 5, Line: 3}, + Nodes: []ast.Node{ + &ast.ReturnStatement{ + From: token.Pos{Column: 9, Line: 2}, + SemiPos: token.Pos{Column: 31, Line: 2}, + Expression: &ast.StringLiteral{ + From: token.Pos{Column: 16, Line: 2}, + To: token.Pos{Column: 30, Line: 2}, + Value: "\"Hello World!!\"", + }, + }, + }, + }, + }, + }, + { + name: "function description is invalid", + input: `function pragma() public pure returns (string) { + return "Hello World!!"; + }`, + err: &token.PosError{ + Pos: token.Pos{Column: 10, Line: 1}, + Msg: "not found function description.", + }, + }, + { + name: "not found lparen", + input: `function hello) public pure returns (string) { + return "Hello World!!"; + }`, + err: &token.PosError{ + Pos: token.Pos{Column: 15, Line: 1}, + Msg: "not found arguments LParen.", + }, + }, + { + name: "not found rparen", + input: `function hello( public pure returns (string) { + return "Hello World!!"; + }`, + err: &token.PosError{ + Pos: token.Pos{Column: 17, Line: 1}, + Msg: "not found arguments RParen.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseFunctionDefinition() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} From 68454fff5d282ab1a42328d530141aedb6efc8ec Mon Sep 17 00:00:00 2001 From: uji Date: Thu, 23 Feb 2023 21:30:29 +0900 Subject: [PATCH 115/138] main, ast: parse contractDefinition. --- ast/ast.go | 27 ++++- contractDefinition.go | 49 ++++---- contractDefinition_test.go | 240 +++++++++++++++++++++++++++++++++++++ 3 files changed, 290 insertions(+), 26 deletions(-) create mode 100644 contractDefinition_test.go diff --git a/ast/ast.go b/ast/ast.go index 8c8a7ec..9ad3766 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -53,20 +53,35 @@ type FunctionDefinition struct { Block *Block } +func (f FunctionDefinition) Pos() token.Pos { + return f.From +} + +func (f FunctionDefinition) End() token.Pos { + return f.Block.End() +} + type InheritanceSpecifier struct { IdentifierPath string CallArgumentList CallArgumentList } -type CallArgumentList struct{} +type ContractBodyElement interface { + Node + contractBodyElementNode() +} -type ContractBodyElement Node +func (f *FunctionDefinition) contractBodyElementNode() {} + +type CallArgumentList struct{} type ContractDefinition struct { - Position token.Pos - Abstract bool - InheritanceSpecifiers []*InheritanceSpecifier - ContractBodyElements []*ContractBodyElement + Abstract *token.Pos + Contract token.Pos + Identifier Identifier + LBrace token.Pos + ContractBodyElements []ContractBodyElement + RBrace token.Pos } // A File node represents a Solidity source file. diff --git a/contractDefinition.go b/contractDefinition.go index edbe354..503ba63 100644 --- a/contractDefinition.go +++ b/contractDefinition.go @@ -1,51 +1,60 @@ package solparser import ( - "errors" - "github.com/uji/solparser/ast" "github.com/uji/solparser/token" ) func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { - var abstract bool - tkn, err := p.lexer.Scan() + cntr, err := p.lexer.Scan() if err != nil { return nil, err } - if tkn.TokenType == token.Abstract { - abstract = true - p.lexer.Scan() + var abstractPos *token.Pos + if cntr.TokenType == token.Abstract { + pos := cntr.Pos + abstractPos = &pos + cntr, err = p.lexer.Scan() + if err != nil { + return nil, err + } } - if tkn.TokenType != token.Contract { - return nil, errors.New("not found contract definition") + if cntr.TokenType != token.Contract { + return nil, token.NewPosError(cntr.Pos, "not found contract keyword.") } - tkn, err = p.lexer.Scan() + i, err := p.ParseIdentifier() if err != nil { return nil, err } - if tkn.TokenType != token.Identifier { - return nil, errors.New("not found left brace") - } - tkn, err = p.lexer.Scan() + lbrace, err := p.lexer.Scan() if err != nil { return nil, err } - if tkn.TokenType != token.LBrace { - return nil, errors.New("not found left brace") + if lbrace.TokenType != token.LBrace { + return nil, token.NewPosError(lbrace.Pos, "not found left brace.") + } + + fn, err := p.ParseFunctionDefinition() + if err != nil { + return nil, err } - tkn, err = p.lexer.Scan() + rbrace, err := p.lexer.Scan() if err != nil { return nil, err } - if tkn.TokenType != token.RBrace { - return nil, errors.New("not found right brace") + if rbrace.TokenType != token.RBrace { + return nil, token.NewPosError(rbrace.Pos, "not found right brace.") } return &ast.ContractDefinition{ - Abstract: abstract, + Abstract: abstractPos, + Contract: cntr.Pos, + Identifier: i, + LBrace: lbrace.Pos, + ContractBodyElements: []ast.ContractBodyElement{fn}, + RBrace: rbrace.Pos, }, nil } diff --git a/contractDefinition_test.go b/contractDefinition_test.go new file mode 100644 index 0000000..15d7378 --- /dev/null +++ b/contractDefinition_test.go @@ -0,0 +1,240 @@ +package solparser_test + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseContractDefinition(t *testing.T) { + tests := []struct { + name string + input string + want *ast.ContractDefinition + err *token.PosError + }{ + { + name: "normal case", + input: `contract HelloWorld { + function hello() public pure returns (string) { + return "Hello World!!"; + } +}`, + want: &ast.ContractDefinition{ + Contract: token.Pos{Column: 1, Line: 1}, + Identifier: token.Token{ + TokenType: token.Identifier, + Text: "HelloWorld", + Pos: token.Pos{Column: 10, Line: 1}, + }, + LBrace: token.Pos{Column: 21, Line: 1}, + ContractBodyElements: []ast.ContractBodyElement{ + &ast.FunctionDefinition{ + From: token.Pos{Column: 5, Line: 2}, + FunctionDescriptor: token.Token{ + TokenType: token.Identifier, + Text: "hello", + Pos: token.Pos{Column: 14, Line: 2}, + }, + LParen: token.Pos{Column: 19, Line: 2}, + RParen: token.Pos{Column: 20, Line: 2}, + ModifierList: &ast.ModifierList{ + Visibility: &token.Token{ + TokenType: token.Public, + Text: "public", + Pos: token.Pos{Column: 22, Line: 2}, + }, + StateMutability: &token.Token{ + TokenType: token.Pure, + Text: "pure", + Pos: token.Pos{Column: 29, Line: 2}, + }, + }, + Returns: &ast.FunctionDefinitionReturns{ + From: token.Pos{Column: 34, Line: 2}, + LParen: token.Pos{Column: 42, Line: 2}, + ParameterList: []*ast.Parameter{ + { + TypeName: ast.ElementaryTypeName{ + { + TokenType: token.String, + Text: "string", + Pos: token.Pos{Column: 43, Line: 2}, + }, + }, + }, + }, + RParen: token.Pos{Column: 49, Line: 2}, + }, + Block: &ast.Block{ + LBracePos: token.Pos{Column: 51, Line: 2}, + RBracePos: token.Pos{Column: 5, Line: 4}, + Nodes: []ast.Node{ + &ast.ReturnStatement{ + From: token.Pos{Column: 9, Line: 3}, + SemiPos: token.Pos{Column: 31, Line: 3}, + Expression: &ast.StringLiteral{ + From: token.Pos{Column: 16, Line: 3}, + To: token.Pos{Column: 30, Line: 3}, + Value: "\"Hello World!!\"", + }, + }, + }, + }, + }, + }, + RBrace: token.Pos{Column: 1, Line: 5}, + }, + }, + { + name: "with abstract", + input: `abstract contract HelloWorld { + function hello() public pure returns (string) { + return "Hello World!!"; + } +}`, + want: &ast.ContractDefinition{ + Abstract: &token.Pos{Column: 1, Line: 1}, + Contract: token.Pos{Column: 10, Line: 1}, + Identifier: token.Token{ + TokenType: token.Identifier, + Text: "HelloWorld", + Pos: token.Pos{Column: 19, Line: 1}, + }, + LBrace: token.Pos{Column: 30, Line: 1}, + ContractBodyElements: []ast.ContractBodyElement{ + &ast.FunctionDefinition{ + From: token.Pos{Column: 5, Line: 2}, + FunctionDescriptor: token.Token{ + TokenType: token.Identifier, + Text: "hello", + Pos: token.Pos{Column: 14, Line: 2}, + }, + LParen: token.Pos{Column: 19, Line: 2}, + RParen: token.Pos{Column: 20, Line: 2}, + ModifierList: &ast.ModifierList{ + Visibility: &token.Token{ + TokenType: token.Public, + Text: "public", + Pos: token.Pos{Column: 22, Line: 2}, + }, + StateMutability: &token.Token{ + TokenType: token.Pure, + Text: "pure", + Pos: token.Pos{Column: 29, Line: 2}, + }, + }, + Returns: &ast.FunctionDefinitionReturns{ + From: token.Pos{Column: 34, Line: 2}, + LParen: token.Pos{Column: 42, Line: 2}, + ParameterList: []*ast.Parameter{ + { + TypeName: ast.ElementaryTypeName{ + { + TokenType: token.String, + Text: "string", + Pos: token.Pos{Column: 43, Line: 2}, + }, + }, + }, + }, + RParen: token.Pos{Column: 49, Line: 2}, + }, + Block: &ast.Block{ + LBracePos: token.Pos{Column: 51, Line: 2}, + RBracePos: token.Pos{Column: 5, Line: 4}, + Nodes: []ast.Node{ + &ast.ReturnStatement{ + From: token.Pos{Column: 9, Line: 3}, + SemiPos: token.Pos{Column: 31, Line: 3}, + Expression: &ast.StringLiteral{ + From: token.Pos{Column: 16, Line: 3}, + To: token.Pos{Column: 30, Line: 3}, + Value: "\"Hello World!!\"", + }, + }, + }, + }, + }, + }, + RBrace: token.Pos{Column: 1, Line: 5}, + }, + }, + { + name: "not found contract keyword", + input: `HelloWorld { + function hello() public pure returns (string) { + return "Hello World!!"; + } +}`, + err: &token.PosError{ + Pos: token.Pos{Column: 1, Line: 1}, + Msg: "not found contract keyword.", + }, + }, + { + name: "not found identifier", + input: `contract { + function hello() public pure returns (string) { + return "Hello World!!"; + } +}`, + err: &token.PosError{ + Pos: token.Pos{Column: 10, Line: 1}, + Msg: "keyword is not available as identifier.", + }, + }, + { + name: "not found LBrace", + input: "contract HelloWorld function", + err: &token.PosError{ + Pos: token.Pos{Column: 21, Line: 1}, + Msg: "not found left brace.", + }, + }, + { + name: "not found FunctionDefinition", + input: "contract HelloWorld { hello() }", + err: &token.PosError{ + Pos: token.Pos{Column: 23, Line: 1}, + Msg: "not found function keyword.", + }, + }, + { + name: "not found RBrace", + input: `contract HelloWorld { + function hello() public pure returns (string) { + return "Hello World!!"; + }`, + err: &token.PosError{ + Pos: token.Pos{Column: 6, Line: 4}, + Msg: "not found right brace.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseContractDefinition() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} From f41607ed7d38d61f380953c2902b5c3279501d45 Mon Sep 17 00:00:00 2001 From: uji Date: Fri, 24 Feb 2023 23:19:36 +0900 Subject: [PATCH 116/138] main: Parse test --- solparser.go | 60 ++++++++++++------------ solparser_test.go | 114 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 30 deletions(-) diff --git a/solparser.go b/solparser.go index 778fad8..452195d 100644 --- a/solparser.go +++ b/solparser.go @@ -21,44 +21,44 @@ func New(input io.Reader) *Parser { } func (p *Parser) Parse() (*ast.SourceUnit, error) { - tkn, err := p.lexer.Peek() - if err == io.EOF { - return nil, nil - } - if err != nil { - return nil, err - } - var pragmaDirective *ast.PragmaDirective var contractDefinition *ast.ContractDefinition var functionDefinition *ast.FunctionDefinition - - switch tkn.TokenType { - case token.Pragma: - prgm, err := p.ParsePragmaDirective() - if err != nil { - return nil, err - } - pragmaDirective = prgm - case token.Abstract, token.Contract: - cntrct, err := p.ParseContractDefinition() + for { + tkn, err := p.lexer.Peek() if err != nil { return nil, err } - contractDefinition = cntrct - case token.Function: - fnc, err := p.ParseFunctionDefinition() - if err != nil { - return nil, err + + switch tkn.TokenType { + case token.Pragma: + prgm, err := p.ParsePragmaDirective() + if err != nil { + return nil, err + } + pragmaDirective = prgm + case token.Abstract, token.Contract: + cntrct, err := p.ParseContractDefinition() + if err != nil { + return nil, err + } + contractDefinition = cntrct + case token.Function: + fnc, err := p.ParseFunctionDefinition() + if err != nil { + return nil, err + } + functionDefinition = fnc + case token.EOS: + return &ast.SourceUnit{ + PragmaDirective: pragmaDirective, + ContractDefinition: contractDefinition, + FunctionDefinition: functionDefinition, + }, nil + default: + return nil, token.NewPosError(tkn.Pos, "invalid") } - functionDefinition = fnc } - - return &ast.SourceUnit{ - PragmaDirective: pragmaDirective, - ContractDefinition: contractDefinition, - FunctionDefinition: functionDefinition, - }, nil } func (p *Parser) ParseBooleanLiteral() (*ast.BooleanLiteral, error) { diff --git a/solparser_test.go b/solparser_test.go index d127fc3..78cd1e8 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -11,6 +11,120 @@ import ( "github.com/uji/solparser/token" ) +func TestParser_Parse(t *testing.T) { + tests := []struct { + name string + input string + want *ast.SourceUnit + err *token.PosError + }{ + { + name: "normal", + input: `pragma solidity ^0.8.13; + +contract HelloWorld { + function hello() public pure returns (string) { + return "Hello World!!"; + } +}`, + want: &ast.SourceUnit{ + PragmaDirective: &ast.PragmaDirective{ + PragmaName: "solidity", + PragmaValue: ast.PragmaValue{ + Version: "0.8.13", + Expression: "^", + }, + }, + ContractDefinition: &ast.ContractDefinition{ + Contract: token.Pos{Column: 1, Line: 3}, + Identifier: token.Token{ + TokenType: token.Identifier, + Text: "HelloWorld", + Pos: token.Pos{Column: 10, Line: 3}, + }, + LBrace: token.Pos{Column: 21, Line: 3}, + ContractBodyElements: []ast.ContractBodyElement{ + &ast.FunctionDefinition{ + From: token.Pos{Column: 5, Line: 4}, + FunctionDescriptor: token.Token{ + TokenType: token.Identifier, + Text: "hello", + Pos: token.Pos{Column: 14, Line: 4}, + }, + LParen: token.Pos{Column: 19, Line: 4}, + RParen: token.Pos{Column: 20, Line: 4}, + ModifierList: &ast.ModifierList{ + Visibility: &token.Token{ + TokenType: token.Public, + Text: "public", + Pos: token.Pos{Column: 22, Line: 4}, + }, + StateMutability: &token.Token{ + TokenType: token.Pure, + Text: "pure", + Pos: token.Pos{Column: 29, Line: 4}, + }, + }, + Returns: &ast.FunctionDefinitionReturns{ + From: token.Pos{Column: 34, Line: 4}, + LParen: token.Pos{Column: 42, Line: 4}, + ParameterList: []*ast.Parameter{ + { + TypeName: ast.ElementaryTypeName{ + { + TokenType: token.String, + Text: "string", + Pos: token.Pos{Column: 43, Line: 4}, + }, + }, + }, + }, + RParen: token.Pos{Column: 49, Line: 4}, + }, + Block: &ast.Block{ + LBracePos: token.Pos{Column: 51, Line: 4}, + RBracePos: token.Pos{Column: 5, Line: 6}, + Nodes: []ast.Node{ + &ast.ReturnStatement{ + From: token.Pos{Column: 9, Line: 5}, + SemiPos: token.Pos{Column: 31, Line: 5}, + Expression: &ast.StringLiteral{ + From: token.Pos{Column: 16, Line: 5}, + To: token.Pos{Column: 30, Line: 5}, + Value: "\"Hello World!!\"", + }, + }, + }, + }, + }, + }, + RBrace: token.Pos{Column: 1, Line: 7}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.Parse() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} + func TestParser_ParseBooleanLiteral(t *testing.T) { tests := []struct { name string From 5de6da14dffde7caa4db084295c3cbfd6f6c4d74 Mon Sep 17 00:00:00 2001 From: uji Date: Fri, 24 Feb 2023 23:35:44 +0900 Subject: [PATCH 117/138] main, ast: fix pragmaDirective ast --- ast/ast.go | 10 ++----- pragmaDirective.go | 64 +++++++++++++++-------------------------- pragmaDirective_test.go | 47 +++++++++++++++--------------- solparser_test.go | 22 +++++++++++--- 4 files changed, 68 insertions(+), 75 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 9ad3766..8238d3f 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -7,14 +7,10 @@ type Node interface { End() token.Pos } -type PragmaValue struct { - Version string - Expression string -} - type PragmaDirective struct { - PragmaName string - PragmaValue PragmaValue + Pragma token.Pos + PragmaTokens []*token.Token + Semicolon token.Pos } type ModifierList struct { diff --git a/pragmaDirective.go b/pragmaDirective.go index 8d3b457..d234ed8 100644 --- a/pragmaDirective.go +++ b/pragmaDirective.go @@ -6,51 +6,33 @@ import ( ) func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { - tkn, err := p.lexer.Scan() + prgm, err := p.lexer.Scan() if err != nil { return nil, err } - if tkn.TokenType != token.Pragma { - return nil, token.NewPosError(tkn.Pos, "not found pragma") + if prgm.TokenType != token.Pragma { + return nil, token.NewPosError(prgm.Pos, "not found pragma") } - pragmaName, err := p.lexer.Scan() - if err != nil { - return nil, err - } - if pragmaName.TokenType == token.Semicolon { - return nil, token.NewPosError(pragmaName.Pos, "not found pragma name") - } - - expression, err := p.lexer.Scan() - if err != nil { - return nil, err - } - if expression.TokenType != token.BitXor { - return nil, token.NewPosError(expression.Pos, "not found BitXor expression") + tkns := make([]*token.Token, 0, 1) + for { + tkn, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if tkn.TokenType == token.EOS { + return nil, token.NewPosError(tkn.Pos, "not found Semicolon.") + } + if tkn.TokenType == token.Semicolon { + if len(tkns) == 0 { + return nil, token.NewPosError(tkn.Pos, "not found pragma tokens.") + } + return &ast.PragmaDirective{ + Pragma: prgm.Pos, + PragmaTokens: tkns, + Semicolon: tkn.Pos, + }, nil + } + tkns = append(tkns, &tkn) } - - version, err := p.lexer.Scan() - if err != nil { - return nil, err - } - if version.TokenType != token.Identifier { - return nil, token.NewPosError(version.Pos, "not found version") - } - - tkn, err = p.lexer.Scan() - if err != nil { - return nil, err - } - if tkn.TokenType != token.Semicolon { - return nil, token.NewPosError(tkn.Pos, "not found Semicolon") - } - - return &ast.PragmaDirective{ - PragmaName: pragmaName.Text, - PragmaValue: ast.PragmaValue{ - Version: version.Text, - Expression: expression.Text, - }, - }, nil } diff --git a/pragmaDirective_test.go b/pragmaDirective_test.go index 6c482d6..8daa561 100644 --- a/pragmaDirective_test.go +++ b/pragmaDirective_test.go @@ -11,7 +11,7 @@ import ( "github.com/uji/solparser/token" ) -func TestParserParsePragmaDirective(t *testing.T) { +func TestParser_ParsePragmaDirective(t *testing.T) { tests := []struct { name string input string @@ -22,13 +22,26 @@ func TestParserParsePragmaDirective(t *testing.T) { name: "normal case", input: "pragma solidity ^0.8.13;", want: &ast.PragmaDirective{ - PragmaName: "solidity", - PragmaValue: ast.PragmaValue{ - Version: "0.8.13", - Expression: "^", + Pragma: token.Pos{Column: 1, Line: 1}, + PragmaTokens: []*token.Token{ + { + TokenType: token.Identifier, + Text: "solidity", + Pos: token.Pos{Column: 8, Line: 1}, + }, + { + TokenType: token.BitXor, + Text: "^", + Pos: token.Pos{Column: 17, Line: 1}, + }, + { + TokenType: token.Identifier, + Text: "0.8.13", + Pos: token.Pos{Column: 18, Line: 1}, + }, }, + Semicolon: token.Pos{Column: 24, Line: 1}, }, - err: nil, }, { name: "there is no pragma keyword", @@ -43,27 +56,15 @@ func TestParserParsePragmaDirective(t *testing.T) { }, }, { - name: "there is no pragma name", - input: "pragma ^0.8.13;", + name: "there is no pragma tokens", + input: "pragma ;", want: nil, err: &token.PosError{ Pos: token.Pos{ - Column: 9, + Column: 8, Line: 1, }, - Msg: "not found BitXor expression", - }, - }, - { - name: "there is no BitXor expression", - input: "pragma solidity 0.8.13;", - want: nil, - err: &token.PosError{ - Pos: token.Pos{ - Column: 17, - Line: 1, - }, - Msg: "not found BitXor expression", + Msg: "not found pragma tokens.", }, }, { @@ -75,7 +76,7 @@ func TestParserParsePragmaDirective(t *testing.T) { Column: 24, Line: 1, }, - Msg: "not found Semicolon", + Msg: "not found Semicolon.", }, }, } diff --git a/solparser_test.go b/solparser_test.go index 78cd1e8..8bbb09d 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -29,11 +29,25 @@ contract HelloWorld { }`, want: &ast.SourceUnit{ PragmaDirective: &ast.PragmaDirective{ - PragmaName: "solidity", - PragmaValue: ast.PragmaValue{ - Version: "0.8.13", - Expression: "^", + Pragma: token.Pos{Column: 1, Line: 1}, + PragmaTokens: []*token.Token{ + { + TokenType: token.Identifier, + Text: "solidity", + Pos: token.Pos{Column: 8, Line: 1}, + }, + { + TokenType: token.BitXor, + Text: "^", + Pos: token.Pos{Column: 17, Line: 1}, + }, + { + TokenType: token.Identifier, + Text: "0.8.13", + Pos: token.Pos{Column: 18, Line: 1}, + }, }, + Semicolon: token.Pos{Column: 24, Line: 1}, }, ContractDefinition: &ast.ContractDefinition{ Contract: token.Pos{Column: 1, Line: 3}, From b7c229562a463376e438a72d9c5e6132d36d9ddb Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 25 Feb 2023 12:15:31 +0900 Subject: [PATCH 118/138] main: add test helper functions --- helper_test.go | 42 ++++++++++++++++++++++ pragmaDirective.go | 2 +- pragmaDirective_test.go | 79 ++++++----------------------------------- 3 files changed, 53 insertions(+), 70 deletions(-) create mode 100644 helper_test.go diff --git a/helper_test.go b/helper_test.go new file mode 100644 index 0000000..de6cd65 --- /dev/null +++ b/helper_test.go @@ -0,0 +1,42 @@ +package solparser_test + +import ( + "errors" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser/token" +) + +func pos(c, l int) token.Pos { + return token.Pos{Column: c, Line: l} +} + +func tkn(tp token.TokenType, text string, pos token.Pos) token.Token { + return token.Token{ + TokenType: tp, + Text: text, + Pos: pos, + } +} + +func perr(pos token.Pos, msg string) *token.PosError { + return &token.PosError{ + Pos: pos, + Msg: msg, + } +} + +func assert(t *testing.T, got, want interface{}, gotErr, wantErr error) { + t.Helper() + var sErr *token.PosError + if errors.As(gotErr, &sErr) { + if diff := cmp.Diff(wantErr, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("%s", diff) + } +} diff --git a/pragmaDirective.go b/pragmaDirective.go index d234ed8..aed0f54 100644 --- a/pragmaDirective.go +++ b/pragmaDirective.go @@ -11,7 +11,7 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { return nil, err } if prgm.TokenType != token.Pragma { - return nil, token.NewPosError(prgm.Pos, "not found pragma") + return nil, token.NewPosError(prgm.Pos, "not found pragma.") } tkns := make([]*token.Token, 0, 1) diff --git a/pragmaDirective_test.go b/pragmaDirective_test.go index 8daa561..dbd37e9 100644 --- a/pragmaDirective_test.go +++ b/pragmaDirective_test.go @@ -1,11 +1,9 @@ package solparser_test import ( - "errors" "strings" "testing" - "github.com/google/go-cmp/cmp" "github.com/uji/solparser" "github.com/uji/solparser/ast" "github.com/uji/solparser/token" @@ -13,91 +11,34 @@ import ( func TestParser_ParsePragmaDirective(t *testing.T) { tests := []struct { - name string input string want *ast.PragmaDirective err *token.PosError }{ { - name: "normal case", input: "pragma solidity ^0.8.13;", want: &ast.PragmaDirective{ - Pragma: token.Pos{Column: 1, Line: 1}, + Pragma: pos(1, 1), PragmaTokens: []*token.Token{ - { - TokenType: token.Identifier, - Text: "solidity", - Pos: token.Pos{Column: 8, Line: 1}, - }, - { - TokenType: token.BitXor, - Text: "^", - Pos: token.Pos{Column: 17, Line: 1}, - }, - { - TokenType: token.Identifier, - Text: "0.8.13", - Pos: token.Pos{Column: 18, Line: 1}, - }, + {TokenType: token.Identifier, Text: "solidity", Pos: pos(8, 1)}, + {TokenType: token.BitXor, Text: "^", Pos: pos(17, 1)}, + {TokenType: token.Identifier, Text: "0.8.13", Pos: pos(18, 1)}, }, - Semicolon: token.Pos{Column: 24, Line: 1}, - }, - }, - { - name: "there is no pragma keyword", - input: "solidity ^0.8.13;", - want: nil, - err: &token.PosError{ - Pos: token.Pos{ - Column: 1, - Line: 1, - }, - Msg: "not found pragma", - }, - }, - { - name: "there is no pragma tokens", - input: "pragma ;", - want: nil, - err: &token.PosError{ - Pos: token.Pos{ - Column: 8, - Line: 1, - }, - Msg: "not found pragma tokens.", - }, - }, - { - name: "there is no Semicolon", - input: "pragma solidity ^0.8.13", - want: nil, - err: &token.PosError{ - Pos: token.Pos{ - Column: 24, - Line: 1, - }, - Msg: "not found Semicolon.", + Semicolon: pos(24, 1), }, }, + {input: "solidity ^0.8.13;", err: perr(pos(1, 1), "not found pragma.")}, + {input: "pragma ;", err: perr(pos(8, 1), "not found pragma tokens.")}, + {input: "pragma solidity ^0.8.13", err: perr(pos(24, 1), "not found Semicolon.")}, } for _, tt := range tests { - t.Run(tt.name, func(*testing.T) { + t.Run(tt.input, func(*testing.T) { r := strings.NewReader(tt.input) p := solparser.New(r) got, err := p.ParsePragmaDirective() - - var sErr *token.PosError - if errors.As(err, &sErr) { - if diff := cmp.Diff(tt.err, sErr); diff != "" { - t.Errorf("%s", diff) - } - } - - if diff := cmp.Diff(tt.want, got); diff != "" { - t.Errorf("%s", diff) - } + assert(t, got, tt.want, err, tt.err) }) } } From 8e5ebf0f14f4ede9e8a92781359b1e62393c26f3 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 26 Feb 2023 15:57:56 +0900 Subject: [PATCH 119/138] main, ast: refactor elementaryTypeName --- ast/ast.go | 8 ++++---- typeName.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 8238d3f..f52c9fb 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -95,17 +95,17 @@ type TypeName interface { typeNameNode() } -type ElementaryTypeName []token.Token +type ElementaryTypeName []*token.Token func (e ElementaryTypeName) Pos() token.Pos { return e[0].Pos } func (e ElementaryTypeName) End() token.Pos { - // TODO + eTkn := e[len(e)-1] return token.Pos{ - Column: 1, - Line: 1, + Column: eTkn.Pos.Column + len(eTkn.Text), + Line: eTkn.Pos.Line, } } diff --git a/typeName.go b/typeName.go index 96d5ea2..fbf983a 100644 --- a/typeName.go +++ b/typeName.go @@ -33,13 +33,13 @@ func (p *Parser) ParseElementaryTypeName() (ast.TypeName, error) { if tkn2.TokenType == token.Payable { p.lexer.Scan() - return ast.ElementaryTypeName{tkn, tkn2}, nil + return ast.ElementaryTypeName{&tkn, &tkn2}, nil } } switch tkn.TokenType { case token.Address, token.String, token.Bytes, token.Fixed, token.Bool: - return ast.ElementaryTypeName{tkn}, nil + return ast.ElementaryTypeName{&tkn}, nil } return nil, token.NewPosError(tkn.Pos, "not found elementary type name keyword.") From 1934c4a74fadc9bc75042aac8acf71a9cd41c219 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 26 Feb 2023 16:22:46 +0900 Subject: [PATCH 120/138] all: rename token field --- ast/ast.go | 12 ++++---- block.go | 12 ++++---- contractDefinition.go | 22 +++++++------- contractDefinition_test.go | 60 +++++++++++++++++++------------------- expression.go | 4 +-- functionDefinition.go | 48 +++++++++++++++--------------- functionDefinition_test.go | 42 +++++++++++++------------- helper_test.go | 6 ++-- identifier.go | 4 +-- identifier_test.go | 24 +++++++-------- lexer/lexer.go | 6 ++-- lexer/lexer_test.go | 44 ++++++++++++++-------------- literal.go | 12 ++++---- literal_test.go | 7 ++--- parameterList.go | 2 +- parameterList_test.go | 24 +++++++-------- pragmaDirective.go | 16 +++++----- pragmaDirective_test.go | 6 ++-- solparser.go | 8 ++--- solparser_test.go | 60 +++++++++++++++++++------------------- statement.go | 16 +++++----- token/token.go | 12 ++++---- typeName.go | 12 ++++---- typeName_test.go | 36 +++++++++++------------ 24 files changed, 247 insertions(+), 248 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index f52c9fb..3776ef2 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -98,14 +98,14 @@ type TypeName interface { type ElementaryTypeName []*token.Token func (e ElementaryTypeName) Pos() token.Pos { - return e[0].Pos + return e[0].Position } func (e ElementaryTypeName) End() token.Pos { eTkn := e[len(e)-1] return token.Pos{ - Column: eTkn.Pos.Column + len(eTkn.Text), - Line: eTkn.Pos.Line, + Column: eTkn.Position.Column + len(eTkn.Value), + Line: eTkn.Position.Line, } } @@ -137,13 +137,13 @@ type BooleanLiteral struct { } func (b *BooleanLiteral) Pos() token.Pos { - return b.Token.Pos + return b.Token.Position } func (b *BooleanLiteral) End() token.Pos { return token.Pos{ - Column: b.Token.Pos.Column + len(b.Token.Text), - Line: b.Token.Pos.Line, + Column: b.Token.Position.Column + len(b.Token.Value), + Line: b.Token.Position.Line, } } diff --git a/block.go b/block.go index e1399e3..2166456 100644 --- a/block.go +++ b/block.go @@ -11,8 +11,8 @@ func (p *Parser) ParseBlock() (*ast.Block, error) { return nil, err } - if lblace.TokenType != token.LBrace { - return nil, token.NewPosError(lblace.Pos, "not found LBrace.") + if lblace.Type != token.LBrace { + return nil, token.NewPosError(lblace.Position, "not found LBrace.") } stmt, err := p.ParseStatement() @@ -25,13 +25,13 @@ func (p *Parser) ParseBlock() (*ast.Block, error) { return nil, err } - if rblace.TokenType != token.RBrace { - return nil, token.NewPosError(rblace.Pos, "not found RBrace.") + if rblace.Type != token.RBrace { + return nil, token.NewPosError(rblace.Position, "not found RBrace.") } return &ast.Block{ - LBracePos: lblace.Pos, - RBracePos: rblace.Pos, + LBracePos: lblace.Position, + RBracePos: rblace.Position, Nodes: []ast.Node{stmt}, }, nil } diff --git a/contractDefinition.go b/contractDefinition.go index 503ba63..47db899 100644 --- a/contractDefinition.go +++ b/contractDefinition.go @@ -11,16 +11,16 @@ func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { return nil, err } var abstractPos *token.Pos - if cntr.TokenType == token.Abstract { - pos := cntr.Pos + if cntr.Type == token.Abstract { + pos := cntr.Position abstractPos = &pos cntr, err = p.lexer.Scan() if err != nil { return nil, err } } - if cntr.TokenType != token.Contract { - return nil, token.NewPosError(cntr.Pos, "not found contract keyword.") + if cntr.Type != token.Contract { + return nil, token.NewPosError(cntr.Position, "not found contract keyword.") } i, err := p.ParseIdentifier() @@ -32,8 +32,8 @@ func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { if err != nil { return nil, err } - if lbrace.TokenType != token.LBrace { - return nil, token.NewPosError(lbrace.Pos, "not found left brace.") + if lbrace.Type != token.LBrace { + return nil, token.NewPosError(lbrace.Position, "not found left brace.") } fn, err := p.ParseFunctionDefinition() @@ -45,16 +45,16 @@ func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { if err != nil { return nil, err } - if rbrace.TokenType != token.RBrace { - return nil, token.NewPosError(rbrace.Pos, "not found right brace.") + if rbrace.Type != token.RBrace { + return nil, token.NewPosError(rbrace.Position, "not found right brace.") } return &ast.ContractDefinition{ Abstract: abstractPos, - Contract: cntr.Pos, + Contract: cntr.Position, Identifier: i, - LBrace: lbrace.Pos, + LBrace: lbrace.Position, ContractBodyElements: []ast.ContractBodyElement{fn}, - RBrace: rbrace.Pos, + RBrace: rbrace.Position, }, nil } diff --git a/contractDefinition_test.go b/contractDefinition_test.go index 15d7378..5abd512 100644 --- a/contractDefinition_test.go +++ b/contractDefinition_test.go @@ -28,31 +28,31 @@ func TestParser_ParseContractDefinition(t *testing.T) { want: &ast.ContractDefinition{ Contract: token.Pos{Column: 1, Line: 1}, Identifier: token.Token{ - TokenType: token.Identifier, - Text: "HelloWorld", - Pos: token.Pos{Column: 10, Line: 1}, + Type: token.Identifier, + Value: "HelloWorld", + Position: token.Pos{Column: 10, Line: 1}, }, LBrace: token.Pos{Column: 21, Line: 1}, ContractBodyElements: []ast.ContractBodyElement{ &ast.FunctionDefinition{ From: token.Pos{Column: 5, Line: 2}, FunctionDescriptor: token.Token{ - TokenType: token.Identifier, - Text: "hello", - Pos: token.Pos{Column: 14, Line: 2}, + Type: token.Identifier, + Value: "hello", + Position: token.Pos{Column: 14, Line: 2}, }, LParen: token.Pos{Column: 19, Line: 2}, RParen: token.Pos{Column: 20, Line: 2}, ModifierList: &ast.ModifierList{ Visibility: &token.Token{ - TokenType: token.Public, - Text: "public", - Pos: token.Pos{Column: 22, Line: 2}, + Type: token.Public, + Value: "public", + Position: token.Pos{Column: 22, Line: 2}, }, StateMutability: &token.Token{ - TokenType: token.Pure, - Text: "pure", - Pos: token.Pos{Column: 29, Line: 2}, + Type: token.Pure, + Value: "pure", + Position: token.Pos{Column: 29, Line: 2}, }, }, Returns: &ast.FunctionDefinitionReturns{ @@ -62,9 +62,9 @@ func TestParser_ParseContractDefinition(t *testing.T) { { TypeName: ast.ElementaryTypeName{ { - TokenType: token.String, - Text: "string", - Pos: token.Pos{Column: 43, Line: 2}, + Type: token.String, + Value: "string", + Position: token.Pos{Column: 43, Line: 2}, }, }, }, @@ -102,31 +102,31 @@ func TestParser_ParseContractDefinition(t *testing.T) { Abstract: &token.Pos{Column: 1, Line: 1}, Contract: token.Pos{Column: 10, Line: 1}, Identifier: token.Token{ - TokenType: token.Identifier, - Text: "HelloWorld", - Pos: token.Pos{Column: 19, Line: 1}, + Type: token.Identifier, + Value: "HelloWorld", + Position: token.Pos{Column: 19, Line: 1}, }, LBrace: token.Pos{Column: 30, Line: 1}, ContractBodyElements: []ast.ContractBodyElement{ &ast.FunctionDefinition{ From: token.Pos{Column: 5, Line: 2}, FunctionDescriptor: token.Token{ - TokenType: token.Identifier, - Text: "hello", - Pos: token.Pos{Column: 14, Line: 2}, + Type: token.Identifier, + Value: "hello", + Position: token.Pos{Column: 14, Line: 2}, }, LParen: token.Pos{Column: 19, Line: 2}, RParen: token.Pos{Column: 20, Line: 2}, ModifierList: &ast.ModifierList{ Visibility: &token.Token{ - TokenType: token.Public, - Text: "public", - Pos: token.Pos{Column: 22, Line: 2}, + Type: token.Public, + Value: "public", + Position: token.Pos{Column: 22, Line: 2}, }, StateMutability: &token.Token{ - TokenType: token.Pure, - Text: "pure", - Pos: token.Pos{Column: 29, Line: 2}, + Type: token.Pure, + Value: "pure", + Position: token.Pos{Column: 29, Line: 2}, }, }, Returns: &ast.FunctionDefinitionReturns{ @@ -136,9 +136,9 @@ func TestParser_ParseContractDefinition(t *testing.T) { { TypeName: ast.ElementaryTypeName{ { - TokenType: token.String, - Text: "string", - Pos: token.Pos{Column: 43, Line: 2}, + Type: token.String, + Value: "string", + Position: token.Pos{Column: 43, Line: 2}, }, }, }, diff --git a/expression.go b/expression.go index 8a8b11b..f9a8d40 100644 --- a/expression.go +++ b/expression.go @@ -11,9 +11,9 @@ func (p *Parser) ParseExpression() (ast.Expression, error) { return nil, err } - if tkn.TokenType == token.StringLiteral { + if tkn.Type == token.StringLiteral { return p.ParseLiteral() } - return nil, token.NewPosError(tkn.Pos, "not found expression.") + return nil, token.NewPosError(tkn.Position, "not found expression.") } diff --git a/functionDefinition.go b/functionDefinition.go index 7c4b906..4c9ca81 100644 --- a/functionDefinition.go +++ b/functionDefinition.go @@ -11,12 +11,12 @@ func (p *Parser) ParseVisibility() (ast.Visibility, error) { return token.Token{}, err } - switch tkn.TokenType { + switch tkn.Type { case token.Internal, token.External, token.Public, token.Private: return tkn, nil } - return token.Token{}, token.NewPosError(tkn.Pos, "not found visibility keyword.") + return token.Token{}, token.NewPosError(tkn.Position, "not found visibility keyword.") } func (p *Parser) ParseStateMutability() (ast.StateMutability, error) { @@ -25,12 +25,12 @@ func (p *Parser) ParseStateMutability() (ast.StateMutability, error) { return token.Token{}, err } - switch tkn.TokenType { + switch tkn.Type { case token.Pure, token.View, token.Payable: return tkn, nil } - return token.Token{}, token.NewPosError(tkn.Pos, "not found state-mutability keyword.") + return token.Token{}, token.NewPosError(tkn.Position, "not found state-mutability keyword.") } func (p *Parser) ParseFunctionDefinitionReturns() (*ast.FunctionDefinitionReturns, error) { @@ -38,7 +38,7 @@ func (p *Parser) ParseFunctionDefinitionReturns() (*ast.FunctionDefinitionReturn if err != nil { return nil, err } - if tkn.TokenType != token.Returns { + if tkn.Type != token.Returns { return nil, nil } if _, err := p.lexer.Scan(); err != nil { @@ -49,8 +49,8 @@ func (p *Parser) ParseFunctionDefinitionReturns() (*ast.FunctionDefinitionReturn if err != nil { return nil, err } - if lparen.TokenType != token.LParen { - return nil, token.NewPosError(lparen.Pos, "not found arguments LParen.") + if lparen.Type != token.LParen { + return nil, token.NewPosError(lparen.Position, "not found arguments LParen.") } pl, err := p.ParseParameterList() @@ -62,15 +62,15 @@ func (p *Parser) ParseFunctionDefinitionReturns() (*ast.FunctionDefinitionReturn if err != nil { return nil, err } - if rparen.TokenType != token.RParen { - return nil, token.NewPosError(rparen.Pos, "not found arguments RParen.") + if rparen.Type != token.RParen { + return nil, token.NewPosError(rparen.Position, "not found arguments RParen.") } return &ast.FunctionDefinitionReturns{ - From: tkn.Pos, - LParen: lparen.Pos, + From: tkn.Position, + LParen: lparen.Position, ParameterList: pl, - RParen: rparen.Pos, + RParen: rparen.Position, }, nil } @@ -80,8 +80,8 @@ func (p *Parser) ParseFunctionDefinition() (*ast.FunctionDefinition, error) { return nil, err } - if from.TokenType != token.Function { - return nil, token.NewPosError(from.Pos, "not found function keyword.") + if from.Type != token.Function { + return nil, token.NewPosError(from.Position, "not found function keyword.") } dsc, err := p.lexer.Scan() @@ -89,26 +89,26 @@ func (p *Parser) ParseFunctionDefinition() (*ast.FunctionDefinition, error) { return nil, err } - switch dsc.TokenType { + switch dsc.Type { case token.Identifier, token.From, token.Error, token.Revert, token.Global, token.Fallback, token.Receive: default: - return nil, token.NewPosError(dsc.Pos, "not found function description.") + return nil, token.NewPosError(dsc.Position, "not found function description.") } lparen, err := p.lexer.Scan() if err != nil { return nil, err } - if lparen.TokenType != token.LParen { - return nil, token.NewPosError(lparen.Pos, "not found arguments LParen.") + if lparen.Type != token.LParen { + return nil, token.NewPosError(lparen.Position, "not found arguments LParen.") } rparen, err := p.lexer.Scan() if err != nil { return nil, err } - if rparen.TokenType != token.RParen { - return nil, token.NewPosError(rparen.Pos, "not found arguments RParen.") + if rparen.Type != token.RParen { + return nil, token.NewPosError(rparen.Position, "not found arguments RParen.") } modifierList := &ast.ModifierList{ @@ -122,7 +122,7 @@ func (p *Parser) ParseFunctionDefinition() (*ast.FunctionDefinition, error) { return nil, err } - switch tkn.TokenType { + switch tkn.Type { case token.Internal, token.External, token.Public, token.Private: vs, err := p.ParseVisibility() if err != nil { @@ -152,10 +152,10 @@ func (p *Parser) ParseFunctionDefinition() (*ast.FunctionDefinition, error) { } return &ast.FunctionDefinition{ - From: from.Pos, + From: from.Position, FunctionDescriptor: dsc, - LParen: lparen.Pos, - RParen: rparen.Pos, + LParen: lparen.Position, + RParen: rparen.Position, ModifierList: modifierList, Returns: r, Block: b, diff --git a/functionDefinition_test.go b/functionDefinition_test.go index 6fed4a6..308341e 100644 --- a/functionDefinition_test.go +++ b/functionDefinition_test.go @@ -20,9 +20,9 @@ func TestParser_ParseVisibility(t *testing.T) { { input: "internal", want: ast.Visibility{ - TokenType: token.Internal, - Text: "internal", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.Internal, + Value: "internal", + Position: token.Pos{Column: 1, Line: 1}, }, }, { @@ -64,9 +64,9 @@ func TestParser_ParseStateMutability(t *testing.T) { { input: "pure", want: ast.StateMutability{ - TokenType: token.Pure, - Text: "pure", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.Pure, + Value: "pure", + Position: token.Pos{Column: 1, Line: 1}, }, }, { @@ -114,9 +114,9 @@ func TestParser_ParseFunctionDefinitionReturns(t *testing.T) { { TypeName: ast.ElementaryTypeName{ { - TokenType: token.String, - Text: "string", - Pos: token.Pos{Column: 10, Line: 1}, + Type: token.String, + Value: "string", + Position: token.Pos{Column: 10, Line: 1}, }, }, }, @@ -180,22 +180,22 @@ func TestParser_ParseFunctionDefinition(t *testing.T) { want: &ast.FunctionDefinition{ From: token.Pos{Column: 1, Line: 1}, FunctionDescriptor: token.Token{ - TokenType: token.Identifier, - Text: "hello", - Pos: token.Pos{Column: 10, Line: 1}, + Type: token.Identifier, + Value: "hello", + Position: token.Pos{Column: 10, Line: 1}, }, LParen: token.Pos{Column: 15, Line: 1}, RParen: token.Pos{Column: 16, Line: 1}, ModifierList: &ast.ModifierList{ Visibility: &token.Token{ - TokenType: token.Public, - Text: "public", - Pos: token.Pos{Column: 18, Line: 1}, + Type: token.Public, + Value: "public", + Position: token.Pos{Column: 18, Line: 1}, }, StateMutability: &token.Token{ - TokenType: token.Pure, - Text: "pure", - Pos: token.Pos{Column: 25, Line: 1}, + Type: token.Pure, + Value: "pure", + Position: token.Pos{Column: 25, Line: 1}, }, }, Returns: &ast.FunctionDefinitionReturns{ @@ -205,9 +205,9 @@ func TestParser_ParseFunctionDefinition(t *testing.T) { { TypeName: ast.ElementaryTypeName{ { - TokenType: token.String, - Text: "string", - Pos: token.Pos{Column: 39, Line: 1}, + Type: token.String, + Value: "string", + Position: token.Pos{Column: 39, Line: 1}, }, }, }, diff --git a/helper_test.go b/helper_test.go index de6cd65..d80c961 100644 --- a/helper_test.go +++ b/helper_test.go @@ -14,9 +14,9 @@ func pos(c, l int) token.Pos { func tkn(tp token.TokenType, text string, pos token.Pos) token.Token { return token.Token{ - TokenType: tp, - Text: text, - Pos: pos, + Type: tp, + Value: text, + Position: pos, } } diff --git a/identifier.go b/identifier.go index f908f3d..335996d 100644 --- a/identifier.go +++ b/identifier.go @@ -11,10 +11,10 @@ func (p *Parser) ParseIdentifier() (ast.Identifier, error) { return token.Token{}, err } - switch tkn.TokenType { + switch tkn.Type { case token.Identifier, token.From, token.Error, token.Revert, token.Global: return tkn, nil } - return token.Token{}, token.NewPosError(tkn.Pos, "keyword is not available as identifier.") + return token.Token{}, token.NewPosError(tkn.Position, "keyword is not available as identifier.") } diff --git a/identifier_test.go b/identifier_test.go index b6f658a..793116e 100644 --- a/identifier_test.go +++ b/identifier_test.go @@ -20,33 +20,33 @@ func TestParser_ParseIdentifier(t *testing.T) { { input: "identifier", want: token.Token{ - TokenType: token.Identifier, - Text: "identifier", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.Identifier, + Value: "identifier", + Position: token.Pos{Column: 1, Line: 1}, }, }, { input: "from", want: token.Token{ - TokenType: token.From, - Text: "from", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.From, + Value: "from", + Position: token.Pos{Column: 1, Line: 1}, }, }, { input: "error", want: token.Token{ - TokenType: token.Error, - Text: "error", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.Error, + Value: "error", + Position: token.Pos{Column: 1, Line: 1}, }, }, { input: "global", want: token.Token{ - TokenType: token.Global, - Text: "global", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.Global, + Value: "global", + Position: token.Pos{Column: 1, Line: 1}, }, }, { diff --git a/lexer/lexer.go b/lexer/lexer.go index 90ae859..14eea5b 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -88,9 +88,9 @@ func (l *Lexer) ScanStringLiteral() (token.Token, error) { txt = txt + str if str == quote { return token.Token{ - TokenType: token.StringLiteral, - Text: txt, - Pos: firstPos, + Type: token.StringLiteral, + Value: txt, + Position: firstPos, }, nil } } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index b69f5a5..66f735a 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -20,9 +20,9 @@ func TestLexer_Scan(t *testing.T) { name: "normal", input: "pragma", wantToken: token.Token{ - TokenType: token.Pragma, - Text: "pragma", - Pos: token.Pos{ + Type: token.Pragma, + Value: "pragma", + Position: token.Pos{ Column: 1, Line: 1, }, @@ -32,9 +32,9 @@ func TestLexer_Scan(t *testing.T) { name: "There is a space at the beginning", input: " pragma", wantToken: token.Token{ - TokenType: token.Pragma, - Text: "pragma", - Pos: token.Pos{ + Type: token.Pragma, + Value: "pragma", + Position: token.Pos{ Column: 2, Line: 1, }, @@ -44,9 +44,9 @@ func TestLexer_Scan(t *testing.T) { name: "There is a StringLiteral", input: `"pragma"`, wantToken: token.Token{ - TokenType: token.StringLiteral, - Text: `"pragma"`, - Pos: token.Pos{ + Type: token.StringLiteral, + Value: `"pragma"`, + Position: token.Pos{ Column: 1, Line: 1, }, @@ -82,9 +82,9 @@ func TestLexer_Scan(t *testing.T) { t.Errorf("got error: %s", err) } want := token.Token{ - TokenType: token.EOS, - Text: token.EOSString, - Pos: token.Pos{ + Type: token.EOS, + Value: token.EOSString, + Position: token.Pos{ Column: 1, Line: 1, }, @@ -96,9 +96,9 @@ func TestLexer_Scan(t *testing.T) { t.Run("when peeked", func(t *testing.T) { peekToken := token.Token{ - TokenType: token.BitXor, - Text: "^", - Pos: token.Pos{ + Type: token.BitXor, + Value: "^", + Position: token.Pos{ Column: 4, Line: 6, }, @@ -132,9 +132,9 @@ func TestLexer_Peek(t *testing.T) { name: "normal", input: "pragma", token: token.Token{ - TokenType: token.Pragma, - Text: "pragma", - Pos: token.Pos{ + Type: token.Pragma, + Value: "pragma", + Position: token.Pos{ Column: 1, Line: 1, }, @@ -144,9 +144,9 @@ func TestLexer_Peek(t *testing.T) { name: "when scan is done", input: "", token: token.Token{ - TokenType: token.EOS, - Text: token.EOSString, - Pos: token.Pos{ + Type: token.EOS, + Value: token.EOSString, + Position: token.Pos{ Column: 1, Line: 1, }, @@ -201,7 +201,7 @@ func TestLexer_ScanStringLiteral(t *testing.T) { if err != nil { t.Errorf("error is not nil, got: %s", err) } - if diff := cmp.Diff(tt.wantText, tkn.Text); diff != "" { + if diff := cmp.Diff(tt.wantText, tkn.Value); diff != "" { t.Errorf(diff) } }) diff --git a/literal.go b/literal.go index 9cdf99b..681c2f4 100644 --- a/literal.go +++ b/literal.go @@ -11,8 +11,8 @@ func (p *Parser) ParseLiteral() (ast.Literal, error) { return nil, err } - if tkn.TokenType != token.StringLiteral { - return nil, token.NewPosError(tkn.Pos, "not found string literal quote") + if tkn.Type != token.StringLiteral { + return nil, token.NewPosError(tkn.Position, "not found string literal quote") } // peek to find literal end position. @@ -22,11 +22,11 @@ func (p *Parser) ParseLiteral() (ast.Literal, error) { } return &ast.StringLiteral{ - Value: tkn.Text, - From: tkn.Pos, + Value: tkn.Value, + From: tkn.Position, To: token.Pos{ - Column: peek.Pos.Column - 1, - Line: peek.Pos.Line, + Column: peek.Position.Column - 1, + Line: peek.Position.Line, }, }, nil } diff --git a/literal_test.go b/literal_test.go index 1f74cf0..1483795 100644 --- a/literal_test.go +++ b/literal_test.go @@ -50,17 +50,16 @@ func TestParser_ParseLiteral(t *testing.T) { }, err: nil, }, - // TODO // { // name: `Next token is \n`, // input: "\"Hello World!!\"\n", // want: &ast.StringLiteral{ - // Str: `"Hello World!!"`, - // Position: token.Pos{ + // Value: `"Hello World!!"`, + // From: token.Pos{ // Column: 1, // Line: 1, // }, - // EndPos: token.Pos{ + // To: token.Pos{ // Column: 15, // Line: 1, // }, diff --git a/parameterList.go b/parameterList.go index a1dd76e..b97bd8d 100644 --- a/parameterList.go +++ b/parameterList.go @@ -31,7 +31,7 @@ func (p *Parser) ParseParameterList() (ast.ParameterList, error) { return nil, err } - if comma.TokenType != token.Comma { + if comma.Type != token.Comma { return prms, nil } diff --git a/parameterList_test.go b/parameterList_test.go index 366fdc3..809bc45 100644 --- a/parameterList_test.go +++ b/parameterList_test.go @@ -24,9 +24,9 @@ func TestParser_ParseParameter(t *testing.T) { want: &ast.Parameter{ TypeName: ast.ElementaryTypeName{ { - TokenType: token.String, - Text: "string", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.String, + Value: "string", + Position: token.Pos{Column: 1, Line: 1}, }, }, }, @@ -80,9 +80,9 @@ func TestParser_ParseParameterList(t *testing.T) { { TypeName: ast.ElementaryTypeName{ { - TokenType: token.String, - Text: "string", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.String, + Value: "string", + Position: token.Pos{Column: 1, Line: 1}, }, }, }, @@ -96,18 +96,18 @@ func TestParser_ParseParameterList(t *testing.T) { { TypeName: ast.ElementaryTypeName{ { - TokenType: token.String, - Text: "string", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.String, + Value: "string", + Position: token.Pos{Column: 1, Line: 1}, }, }, }, { TypeName: ast.ElementaryTypeName{ { - TokenType: token.Bool, - Text: "bool", - Pos: token.Pos{Column: 9, Line: 1}, + Type: token.Bool, + Value: "bool", + Position: token.Pos{Column: 9, Line: 1}, }, }, }, diff --git a/pragmaDirective.go b/pragmaDirective.go index aed0f54..43cda5f 100644 --- a/pragmaDirective.go +++ b/pragmaDirective.go @@ -10,8 +10,8 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { if err != nil { return nil, err } - if prgm.TokenType != token.Pragma { - return nil, token.NewPosError(prgm.Pos, "not found pragma.") + if prgm.Type != token.Pragma { + return nil, token.NewPosError(prgm.Position, "not found pragma.") } tkns := make([]*token.Token, 0, 1) @@ -20,17 +20,17 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { if err != nil { return nil, err } - if tkn.TokenType == token.EOS { - return nil, token.NewPosError(tkn.Pos, "not found Semicolon.") + if tkn.Type == token.EOS { + return nil, token.NewPosError(tkn.Position, "not found Semicolon.") } - if tkn.TokenType == token.Semicolon { + if tkn.Type == token.Semicolon { if len(tkns) == 0 { - return nil, token.NewPosError(tkn.Pos, "not found pragma tokens.") + return nil, token.NewPosError(tkn.Position, "not found pragma tokens.") } return &ast.PragmaDirective{ - Pragma: prgm.Pos, + Pragma: prgm.Position, PragmaTokens: tkns, - Semicolon: tkn.Pos, + Semicolon: tkn.Position, }, nil } tkns = append(tkns, &tkn) diff --git a/pragmaDirective_test.go b/pragmaDirective_test.go index dbd37e9..192f270 100644 --- a/pragmaDirective_test.go +++ b/pragmaDirective_test.go @@ -20,9 +20,9 @@ func TestParser_ParsePragmaDirective(t *testing.T) { want: &ast.PragmaDirective{ Pragma: pos(1, 1), PragmaTokens: []*token.Token{ - {TokenType: token.Identifier, Text: "solidity", Pos: pos(8, 1)}, - {TokenType: token.BitXor, Text: "^", Pos: pos(17, 1)}, - {TokenType: token.Identifier, Text: "0.8.13", Pos: pos(18, 1)}, + {Type: token.Identifier, Value: "solidity", Position: pos(8, 1)}, + {Type: token.BitXor, Value: "^", Position: pos(17, 1)}, + {Type: token.Identifier, Value: "0.8.13", Position: pos(18, 1)}, }, Semicolon: pos(24, 1), }, diff --git a/solparser.go b/solparser.go index 452195d..4bdb977 100644 --- a/solparser.go +++ b/solparser.go @@ -30,7 +30,7 @@ func (p *Parser) Parse() (*ast.SourceUnit, error) { return nil, err } - switch tkn.TokenType { + switch tkn.Type { case token.Pragma: prgm, err := p.ParsePragmaDirective() if err != nil { @@ -56,7 +56,7 @@ func (p *Parser) Parse() (*ast.SourceUnit, error) { FunctionDefinition: functionDefinition, }, nil default: - return nil, token.NewPosError(tkn.Pos, "invalid") + return nil, token.NewPosError(tkn.Position, "invalid") } } } @@ -67,8 +67,8 @@ func (p *Parser) ParseBooleanLiteral() (*ast.BooleanLiteral, error) { return nil, err } - if tkn.TokenType != token.TrueLiteral && tkn.TokenType != token.FalseLiteral { - return nil, token.NewPosError(tkn.Pos, "not found keyword true or false") + if tkn.Type != token.TrueLiteral && tkn.Type != token.FalseLiteral { + return nil, token.NewPosError(tkn.Position, "not found keyword true or false") } return &ast.BooleanLiteral{ diff --git a/solparser_test.go b/solparser_test.go index 8bbb09d..4bf968e 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -32,19 +32,19 @@ contract HelloWorld { Pragma: token.Pos{Column: 1, Line: 1}, PragmaTokens: []*token.Token{ { - TokenType: token.Identifier, - Text: "solidity", - Pos: token.Pos{Column: 8, Line: 1}, + Type: token.Identifier, + Value: "solidity", + Position: token.Pos{Column: 8, Line: 1}, }, { - TokenType: token.BitXor, - Text: "^", - Pos: token.Pos{Column: 17, Line: 1}, + Type: token.BitXor, + Value: "^", + Position: token.Pos{Column: 17, Line: 1}, }, { - TokenType: token.Identifier, - Text: "0.8.13", - Pos: token.Pos{Column: 18, Line: 1}, + Type: token.Identifier, + Value: "0.8.13", + Position: token.Pos{Column: 18, Line: 1}, }, }, Semicolon: token.Pos{Column: 24, Line: 1}, @@ -52,31 +52,31 @@ contract HelloWorld { ContractDefinition: &ast.ContractDefinition{ Contract: token.Pos{Column: 1, Line: 3}, Identifier: token.Token{ - TokenType: token.Identifier, - Text: "HelloWorld", - Pos: token.Pos{Column: 10, Line: 3}, + Type: token.Identifier, + Value: "HelloWorld", + Position: token.Pos{Column: 10, Line: 3}, }, LBrace: token.Pos{Column: 21, Line: 3}, ContractBodyElements: []ast.ContractBodyElement{ &ast.FunctionDefinition{ From: token.Pos{Column: 5, Line: 4}, FunctionDescriptor: token.Token{ - TokenType: token.Identifier, - Text: "hello", - Pos: token.Pos{Column: 14, Line: 4}, + Type: token.Identifier, + Value: "hello", + Position: token.Pos{Column: 14, Line: 4}, }, LParen: token.Pos{Column: 19, Line: 4}, RParen: token.Pos{Column: 20, Line: 4}, ModifierList: &ast.ModifierList{ Visibility: &token.Token{ - TokenType: token.Public, - Text: "public", - Pos: token.Pos{Column: 22, Line: 4}, + Type: token.Public, + Value: "public", + Position: token.Pos{Column: 22, Line: 4}, }, StateMutability: &token.Token{ - TokenType: token.Pure, - Text: "pure", - Pos: token.Pos{Column: 29, Line: 4}, + Type: token.Pure, + Value: "pure", + Position: token.Pos{Column: 29, Line: 4}, }, }, Returns: &ast.FunctionDefinitionReturns{ @@ -86,9 +86,9 @@ contract HelloWorld { { TypeName: ast.ElementaryTypeName{ { - TokenType: token.String, - Text: "string", - Pos: token.Pos{Column: 43, Line: 4}, + Type: token.String, + Value: "string", + Position: token.Pos{Column: 43, Line: 4}, }, }, }, @@ -151,9 +151,9 @@ func TestParser_ParseBooleanLiteral(t *testing.T) { input: "true", want: &ast.BooleanLiteral{ Token: token.Token{ - TokenType: token.TrueLiteral, - Text: "true", - Pos: token.Pos{ + Type: token.TrueLiteral, + Value: "true", + Position: token.Pos{ Column: 1, Line: 1, }, @@ -166,9 +166,9 @@ func TestParser_ParseBooleanLiteral(t *testing.T) { input: "false", want: &ast.BooleanLiteral{ Token: token.Token{ - TokenType: token.FalseLiteral, - Text: "false", - Pos: token.Pos{ + Type: token.FalseLiteral, + Value: "false", + Position: token.Pos{ Column: 1, Line: 1, }, diff --git a/statement.go b/statement.go index b9905a7..0184548 100644 --- a/statement.go +++ b/statement.go @@ -11,11 +11,11 @@ func (p *Parser) ParseStatement() (ast.Statement, error) { return nil, err } - if tkn.TokenType == token.Return { + if tkn.Type == token.Return { return p.ParseReturnStatement() } - return nil, token.NewPosError(tkn.Pos, "not found statement.") + return nil, token.NewPosError(tkn.Position, "not found statement.") } func (p *Parser) ParseReturnStatement() (ast.Statement, error) { @@ -24,8 +24,8 @@ func (p *Parser) ParseReturnStatement() (ast.Statement, error) { return nil, err } - if rtn.TokenType != token.Return { - return nil, token.NewPosError(rtn.Pos, "not found return keyword.") + if rtn.Type != token.Return { + return nil, token.NewPosError(rtn.Position, "not found return keyword.") } exp, err := p.ParseExpression() @@ -38,13 +38,13 @@ func (p *Parser) ParseReturnStatement() (ast.Statement, error) { return nil, err } - if semi.TokenType != token.Semicolon { - return nil, token.NewPosError(semi.Pos, "not found semicolon.") + if semi.Type != token.Semicolon { + return nil, token.NewPosError(semi.Position, "not found semicolon.") } return &ast.ReturnStatement{ - From: rtn.Pos, - SemiPos: semi.Pos, + From: rtn.Position, + SemiPos: semi.Position, Expression: exp, }, nil } diff --git a/token/token.go b/token/token.go index 2c82937..8b3952f 100644 --- a/token/token.go +++ b/token/token.go @@ -480,16 +480,16 @@ func asToken(str string) TokenType { } type Token struct { - TokenType TokenType - Text string - Pos Pos + Type TokenType + Value string + Position Pos } func NewToken(ch string, pos Pos) Token { return Token{ - TokenType: asToken(ch), - Text: ch, - Pos: pos, + Type: asToken(ch), + Value: ch, + Position: pos, } } diff --git a/typeName.go b/typeName.go index fbf983a..704017c 100644 --- a/typeName.go +++ b/typeName.go @@ -11,12 +11,12 @@ func (p *Parser) ParseTypeName() (ast.TypeName, error) { return nil, err } - switch tkn.TokenType { + switch tkn.Type { case token.Address, token.String, token.Bytes, token.Fixed, token.Bool: return p.ParseElementaryTypeName() } - return nil, token.NewPosError(tkn.Pos, "not found type-name.") + return nil, token.NewPosError(tkn.Position, "not found type-name.") } func (p *Parser) ParseElementaryTypeName() (ast.TypeName, error) { @@ -25,22 +25,22 @@ func (p *Parser) ParseElementaryTypeName() (ast.TypeName, error) { return nil, err } - if tkn.TokenType == token.Address { + if tkn.Type == token.Address { tkn2, err := p.lexer.Peek() if err != nil { return nil, err } - if tkn2.TokenType == token.Payable { + if tkn2.Type == token.Payable { p.lexer.Scan() return ast.ElementaryTypeName{&tkn, &tkn2}, nil } } - switch tkn.TokenType { + switch tkn.Type { case token.Address, token.String, token.Bytes, token.Fixed, token.Bool: return ast.ElementaryTypeName{&tkn}, nil } - return nil, token.NewPosError(tkn.Pos, "not found elementary type name keyword.") + return nil, token.NewPosError(tkn.Position, "not found elementary type name keyword.") } diff --git a/typeName_test.go b/typeName_test.go index b0f5730..20d4e3f 100644 --- a/typeName_test.go +++ b/typeName_test.go @@ -23,9 +23,9 @@ func TestParser_ParseTypeName(t *testing.T) { input: "string)", want: ast.ElementaryTypeName{ { - TokenType: token.String, - Text: "string", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.String, + Value: "string", + Position: token.Pos{Column: 1, Line: 1}, }, }, }, @@ -70,9 +70,9 @@ func TestParser_ParseElementaryTypeName(t *testing.T) { input: "address", want: ast.ElementaryTypeName{ { - TokenType: token.Address, - Text: "address", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.Address, + Value: "address", + Position: token.Pos{Column: 1, Line: 1}, }, }, }, @@ -80,9 +80,9 @@ func TestParser_ParseElementaryTypeName(t *testing.T) { input: "bool", want: ast.ElementaryTypeName{ { - TokenType: token.Bool, - Text: "bool", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.Bool, + Value: "bool", + Position: token.Pos{Column: 1, Line: 1}, }, }, }, @@ -90,9 +90,9 @@ func TestParser_ParseElementaryTypeName(t *testing.T) { input: "string", want: ast.ElementaryTypeName{ { - TokenType: token.String, - Text: "string", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.String, + Value: "string", + Position: token.Pos{Column: 1, Line: 1}, }, }, }, @@ -100,14 +100,14 @@ func TestParser_ParseElementaryTypeName(t *testing.T) { input: "address payable", want: ast.ElementaryTypeName{ { - TokenType: token.Address, - Text: "address", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.Address, + Value: "address", + Position: token.Pos{Column: 1, Line: 1}, }, { - TokenType: token.Payable, - Text: "payable", - Pos: token.Pos{Column: 9, Line: 1}, + Type: token.Payable, + Value: "payable", + Position: token.Pos{Column: 9, Line: 1}, }, }, }, From 1a2ec4edd7528940240dc65416d9a9c1fd7bf4c1 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 26 Feb 2023 17:08:53 +0900 Subject: [PATCH 121/138] ast: refactor StringLiteral --- ast/ast.go | 27 +++++++++++++++++++-------- ast/ast_test.go | 30 ++++++++++++++++++++++++++++++ block_test.go | 6 +++--- contractDefinition_test.go | 12 ++++++------ expression_test.go | 12 +++--------- functionDefinition_test.go | 6 +++--- literal.go | 16 ++-------------- literal_test.go | 26 ++++++-------------------- solparser_test.go | 6 +++--- statement_test.go | 12 ++++++------ 10 files changed, 81 insertions(+), 72 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 3776ef2..86af812 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -1,6 +1,10 @@ package ast -import "github.com/uji/solparser/token" +import ( + "strings" + + "github.com/uji/solparser/token" +) type Node interface { Pos() token.Pos @@ -148,17 +152,24 @@ func (b *BooleanLiteral) End() token.Pos { } // EmptyStringLiteral | NonEmptyStringLiteral -type StringLiteral struct { - From, To token.Pos - Value string -} +type StringLiteral token.Token func (s StringLiteral) Pos() token.Pos { - return s.From + return s.Position } -func (s StringLiteral) End() token.Pos { - return s.To +func (s StringLiteral) End() (pos token.Pos) { + // Calculate Line and Offset by referring to the number of new line codes + nc := strings.Count(s.Value, "\n") + + pos.Line = s.Position.Line + nc + if nc == 0 { + pos.Column = s.Position.Column + len(s.Value) - 1 + return + } + + pos.Column = len(s.Value) - strings.LastIndexByte(s.Value, '\n') - 1 + return } type HexStringLiteral []*HexString diff --git a/ast/ast_test.go b/ast/ast_test.go index 1deb8e0..dbc6252 100644 --- a/ast/ast_test.go +++ b/ast/ast_test.go @@ -56,6 +56,36 @@ func TestNode_End(t *testing.T) { Line: 3, }, }, + { + name: "StringLiteral", + node: &ast.StringLiteral{ + Type: token.StringLiteral, + Value: "\"Hello world!!\"", + Position: token.Pos{ + Column: 4, + Line: 3, + }, + }, + exptEnd: token.Pos{ + Column: 18, + Line: 3, + }, + }, + { + name: "StringLiteral with new line codes", + node: &ast.StringLiteral{ + Type: token.StringLiteral, + Value: "\"Hello \n new \nworld!!\"", + Position: token.Pos{ + Column: 4, + Line: 3, + }, + }, + exptEnd: token.Pos{ + Column: 8, + Line: 5, + }, + }, } for _, tt := range tests { diff --git a/block_test.go b/block_test.go index fa82134..4d4e0bd 100644 --- a/block_test.go +++ b/block_test.go @@ -29,9 +29,9 @@ func TestParser_ParseBlock(t *testing.T) { From: token.Pos{Column: 1, Line: 2}, SemiPos: token.Pos{Column: 23, Line: 2}, Expression: &ast.StringLiteral{ - From: token.Pos{Column: 8, Line: 2}, - To: token.Pos{Column: 22, Line: 2}, - Value: "\"Hello World!!\"", + Type: token.StringLiteral, + Position: token.Pos{Column: 8, Line: 2}, + Value: "\"Hello World!!\"", }, }, }, diff --git a/contractDefinition_test.go b/contractDefinition_test.go index 5abd512..41cacbd 100644 --- a/contractDefinition_test.go +++ b/contractDefinition_test.go @@ -79,9 +79,9 @@ func TestParser_ParseContractDefinition(t *testing.T) { From: token.Pos{Column: 9, Line: 3}, SemiPos: token.Pos{Column: 31, Line: 3}, Expression: &ast.StringLiteral{ - From: token.Pos{Column: 16, Line: 3}, - To: token.Pos{Column: 30, Line: 3}, - Value: "\"Hello World!!\"", + Type: token.StringLiteral, + Position: token.Pos{Column: 16, Line: 3}, + Value: "\"Hello World!!\"", }, }, }, @@ -153,9 +153,9 @@ func TestParser_ParseContractDefinition(t *testing.T) { From: token.Pos{Column: 9, Line: 3}, SemiPos: token.Pos{Column: 31, Line: 3}, Expression: &ast.StringLiteral{ - From: token.Pos{Column: 16, Line: 3}, - To: token.Pos{Column: 30, Line: 3}, - Value: "\"Hello World!!\"", + Type: token.StringLiteral, + Position: token.Pos{Column: 16, Line: 3}, + Value: "\"Hello World!!\"", }, }, }, diff --git a/expression_test.go b/expression_test.go index 0fee217..64fd674 100644 --- a/expression_test.go +++ b/expression_test.go @@ -22,15 +22,9 @@ func TestParser_ParseExpression(t *testing.T) { name: "Literal", input: `"Hello World!!";`, want: &ast.StringLiteral{ - Value: `"Hello World!!"`, - From: token.Pos{ - Column: 1, - Line: 1, - }, - To: token.Pos{ - Column: 15, - Line: 1, - }, + Type: token.StringLiteral, + Value: `"Hello World!!"`, + Position: pos(1, 1), }, err: nil, }, diff --git a/functionDefinition_test.go b/functionDefinition_test.go index 308341e..cdfda0d 100644 --- a/functionDefinition_test.go +++ b/functionDefinition_test.go @@ -222,9 +222,9 @@ func TestParser_ParseFunctionDefinition(t *testing.T) { From: token.Pos{Column: 9, Line: 2}, SemiPos: token.Pos{Column: 31, Line: 2}, Expression: &ast.StringLiteral{ - From: token.Pos{Column: 16, Line: 2}, - To: token.Pos{Column: 30, Line: 2}, - Value: "\"Hello World!!\"", + Type: token.StringLiteral, + Position: token.Pos{Column: 16, Line: 2}, + Value: "\"Hello World!!\"", }, }, }, diff --git a/literal.go b/literal.go index 681c2f4..0e814e9 100644 --- a/literal.go +++ b/literal.go @@ -15,18 +15,6 @@ func (p *Parser) ParseLiteral() (ast.Literal, error) { return nil, token.NewPosError(tkn.Position, "not found string literal quote") } - // peek to find literal end position. - peek, err := p.lexer.Peek() - if err != nil { - return nil, err - } - - return &ast.StringLiteral{ - Value: tkn.Value, - From: tkn.Position, - To: token.Pos{ - Column: peek.Position.Column - 1, - Line: peek.Position.Line, - }, - }, nil + lit := ast.StringLiteral(tkn) + return &lit, nil } diff --git a/literal_test.go b/literal_test.go index 1483795..3509bf6 100644 --- a/literal_test.go +++ b/literal_test.go @@ -22,33 +22,19 @@ func TestParser_ParseLiteral(t *testing.T) { name: "normal", input: `"Hello World!!";`, want: &ast.StringLiteral{ - Value: `"Hello World!!"`, - From: token.Pos{ - Column: 1, - Line: 1, - }, - To: token.Pos{ - Column: 15, - Line: 1, - }, + Type: token.StringLiteral, + Value: `"Hello World!!"`, + Position: pos(1, 1), }, - err: nil, }, { name: `Including \n`, input: "\"Hello \nWorld!!\";", want: &ast.StringLiteral{ - Value: "\"Hello \nWorld!!\"", - From: token.Pos{ - Column: 1, - Line: 1, - }, - To: token.Pos{ - Column: 8, - Line: 2, - }, + Type: token.StringLiteral, + Value: "\"Hello \nWorld!!\"", + Position: pos(1, 1), }, - err: nil, }, // { // name: `Next token is \n`, diff --git a/solparser_test.go b/solparser_test.go index 4bf968e..3d5e43f 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -103,9 +103,9 @@ contract HelloWorld { From: token.Pos{Column: 9, Line: 5}, SemiPos: token.Pos{Column: 31, Line: 5}, Expression: &ast.StringLiteral{ - From: token.Pos{Column: 16, Line: 5}, - To: token.Pos{Column: 30, Line: 5}, - Value: "\"Hello World!!\"", + Type: token.StringLiteral, + Position: token.Pos{Column: 16, Line: 5}, + Value: "\"Hello World!!\"", }, }, }, diff --git a/statement_test.go b/statement_test.go index a63c1ec..a85023f 100644 --- a/statement_test.go +++ b/statement_test.go @@ -25,9 +25,9 @@ func TestParser_ParseStatement(t *testing.T) { From: token.Pos{Column: 1, Line: 1}, SemiPos: token.Pos{Column: 23, Line: 1}, Expression: &ast.StringLiteral{ - From: token.Pos{Column: 8, Line: 1}, - To: token.Pos{Column: 22, Line: 1}, - Value: `"Hello World!!"`, + Type: token.StringLiteral, + Position: token.Pos{Column: 8, Line: 1}, + Value: `"Hello World!!"`, }, }, err: nil, @@ -85,9 +85,9 @@ func TestParser_ParseReturnStatement(t *testing.T) { From: token.Pos{Column: 1, Line: 1}, SemiPos: token.Pos{Column: 23, Line: 1}, Expression: &ast.StringLiteral{ - From: token.Pos{Column: 8, Line: 1}, - To: token.Pos{Column: 22, Line: 1}, - Value: `"Hello World!!"`, + Type: token.StringLiteral, + Position: token.Pos{Column: 8, Line: 1}, + Value: `"Hello World!!"`, }, }, err: nil, From bb73c42bf9ae8acf6b115607b2eaf5bd789f6610 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 26 Feb 2023 18:04:05 +0900 Subject: [PATCH 122/138] main: parser example --- example_test.go | 35 +++++++++++++++++++++++++++++++++++ solparser.go | 1 + 2 files changed, 36 insertions(+) create mode 100644 example_test.go diff --git a/example_test.go b/example_test.go new file mode 100644 index 0000000..22bdb95 --- /dev/null +++ b/example_test.go @@ -0,0 +1,35 @@ +package solparser_test + +import ( + "fmt" + "strings" + + "github.com/uji/solparser" + "github.com/uji/solparser/ast" +) + +func Example() { + input := + `pragma solidity ^0.8.13; + +contract HelloWorld { + function hello() public pure returns (string) { + return "Hello World!!"; + } +}` + + parser := solparser.New(strings.NewReader(input)) + + got, err := parser.Parse() + if err != nil { + fmt.Println(err) + return + } + + fmt.Println(got.ContractDefinition.Identifier.Value) + fmt.Println(got.ContractDefinition.ContractBodyElements[0].(*ast.FunctionDefinition).FunctionDescriptor.Value) + + // Output: + // HelloWorld + // hello +} diff --git a/solparser.go b/solparser.go index 4bdb977..2c11763 100644 --- a/solparser.go +++ b/solparser.go @@ -8,6 +8,7 @@ import ( "github.com/uji/solparser/token" ) +// Parser parses "Solidity" code and outputs ASTs. type Parser struct { input io.Reader lexer *lexer.Lexer From 89618c37b1a3db593787c46f66ab40fcdb37fb20 Mon Sep 17 00:00:00 2001 From: uji Date: Mon, 27 Feb 2023 00:11:09 +0900 Subject: [PATCH 123/138] main, ast: parse symbolAliases --- ast/ast.go | 13 +++++ helper_test.go | 12 ++++ symbolAliases.go | 112 +++++++++++++++++++++++++++++++++++ symbolAliases_test.go | 132 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 269 insertions(+) create mode 100644 symbolAliases.go create mode 100644 symbolAliases_test.go diff --git a/ast/ast.go b/ast/ast.go index 86af812..5613761 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -11,6 +11,19 @@ type Node interface { End() token.Pos } +type SymbolAlias struct { + Symbol Identifier + As *token.Pos + Alias *Identifier +} + +type SymbolAliases struct { + LBrace token.Pos + Aliases []*SymbolAlias + Commas []*token.Pos + RBrace token.Pos +} + type PragmaDirective struct { Pragma token.Pos PragmaTokens []*token.Token diff --git a/helper_test.go b/helper_test.go index d80c961..de20174 100644 --- a/helper_test.go +++ b/helper_test.go @@ -12,6 +12,10 @@ func pos(c, l int) token.Pos { return token.Pos{Column: c, Line: l} } +func posPtr(c, l int) *token.Pos { + return &token.Pos{Column: c, Line: l} +} + func tkn(tp token.TokenType, text string, pos token.Pos) token.Token { return token.Token{ Type: tp, @@ -20,6 +24,14 @@ func tkn(tp token.TokenType, text string, pos token.Pos) token.Token { } } +func tknPtr(tp token.TokenType, text string, pos token.Pos) *token.Token { + return &token.Token{ + Type: tp, + Value: text, + Position: pos, + } +} + func perr(pos token.Pos, msg string) *token.PosError { return &token.PosError{ Pos: pos, diff --git a/symbolAliases.go b/symbolAliases.go new file mode 100644 index 0000000..f71c71d --- /dev/null +++ b/symbolAliases.go @@ -0,0 +1,112 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseSymbolAlias() (*ast.SymbolAlias, error) { + smbl, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + if smbl.Type != token.Identifier { + return nil, token.NewPosError(smbl.Position, "not found identifier.") + } + + if _, err := p.lexer.Scan(); err != nil { + return nil, err + } + + as, err := p.lexer.Peek() + if err != nil { + return nil, err + } + if as.Type != token.As { + return &ast.SymbolAlias{Symbol: smbl}, nil + } + if _, err := p.lexer.Scan(); err != nil { + return nil, err + } + + alias, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + if alias.Type != token.Identifier { + return nil, token.NewPosError(alias.Position, "not found identifier.") + } + + return &ast.SymbolAlias{ + Symbol: smbl, + As: &as.Position, + Alias: &alias, + }, nil +} + +func (p *Parser) ParseSymbolAliases() (*ast.SymbolAliases, error) { + lbrace, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + if lbrace.Type != token.LBrace { + return nil, token.NewPosError(lbrace.Position, "not found LBrace.") + } + + if _, err := p.lexer.Scan(); err != nil { + return nil, err + } + + alias, err := p.ParseSymbolAlias() + if err != nil { + return nil, err + } + + aliases := []*ast.SymbolAlias{alias} + commas := make([]*token.Pos, 0) + + for { + comma, err := p.lexer.Peek() + if err != nil { + return nil, err + } + if comma.Type != token.Comma { + break + } + + if _, err := p.lexer.Scan(); err != nil { + return nil, err + } + + alias, err := p.ParseSymbolAlias() + if err != nil { + return nil, err + } + + aliases = append(aliases, alias) + commas = append(commas, &comma.Position) + } + + rbrace, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + if rbrace.Type != token.RBrace { + return nil, token.NewPosError(rbrace.Position, "not found RBrace.") + } + + if _, err := p.lexer.Scan(); err != nil { + return nil, err + } + + return &ast.SymbolAliases{ + LBrace: lbrace.Position, + Aliases: aliases, + Commas: commas, + RBrace: rbrace.Position, + }, nil +} diff --git a/symbolAliases_test.go b/symbolAliases_test.go new file mode 100644 index 0000000..fc37cc3 --- /dev/null +++ b/symbolAliases_test.go @@ -0,0 +1,132 @@ +package solparser_test + +import ( + "strings" + "testing" + + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseSymbolAlias(t *testing.T) { + tests := []struct { + input string + want *ast.SymbolAlias + err *token.PosError + }{ + { + input: "symbol as alias1}", + want: &ast.SymbolAlias{ + Symbol: tkn(token.Identifier, "symbol", pos(1, 1)), + As: posPtr(8, 1), + Alias: tknPtr(token.Identifier, "alias1", pos(11, 1)), + }, + }, + { + input: "symbol}", + want: &ast.SymbolAlias{ + Symbol: tkn(token.Identifier, "symbol", pos(1, 1)), + As: nil, + Alias: nil, + }, + }, + { + input: "pragma", + err: perr(pos(1, 1), "not found identifier."), + }, + { + input: "symbol as pragma", + err: perr(pos(11, 1), "not found identifier."), + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.input, func(t *testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseSymbolAlias() + assert(t, got, tt.want, err, tt.err) + }) + } +} + +func TestParser_ParseSymbolAliases(t *testing.T) { + tests := []struct { + input string + want *ast.SymbolAliases + err *token.PosError + }{ + { + input: "{symbol1}", + want: &ast.SymbolAliases{ + LBrace: pos(1, 1), + Aliases: []*ast.SymbolAlias{ + { + Symbol: tkn(token.Identifier, "symbol1", pos(2, 1)), + }, + }, + Commas: []*token.Pos{}, + RBrace: pos(9, 1), + }, + }, + { + input: "{symbol2 as alias1}", + want: &ast.SymbolAliases{ + LBrace: pos(1, 1), + Aliases: []*ast.SymbolAlias{ + { + Symbol: tkn(token.Identifier, "symbol2", pos(2, 1)), + As: posPtr(10, 1), + Alias: tknPtr(token.Identifier, "alias1", pos(13, 1)), + }, + }, + Commas: []*token.Pos{}, + RBrace: pos(19, 1), + }, + }, + { + input: "{symbol1, symbol2 as alias1}", + want: &ast.SymbolAliases{ + LBrace: pos(1, 1), + Aliases: []*ast.SymbolAlias{ + { + Symbol: tkn(token.Identifier, "symbol1", pos(2, 1)), + }, + { + Symbol: tkn(token.Identifier, "symbol2", pos(11, 1)), + As: posPtr(19, 1), + Alias: tknPtr(token.Identifier, "alias1", pos(22, 1)), + }, + }, + Commas: []*token.Pos{posPtr(9, 1)}, + RBrace: pos(28, 1), + }, + }, + { + input: "symbol1, symbol2 as alian1}", + err: perr(pos(1, 1), "not found LBrace."), + }, + { + input: "{symbol1, as alian1", + err: perr(pos(11, 1), "not found identifier."), + }, + { + input: "{symbol1, symbol2 as alian1", + err: perr(pos(28, 1), "not found RBrace."), + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.input, func(t *testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseSymbolAliases() + assert(t, got, tt.want, err, tt.err) + }) + } +} From 202f304aa62563561f91a97645defa6392e9adf4 Mon Sep 17 00:00:00 2001 From: uji Date: Tue, 28 Feb 2023 22:39:09 +0900 Subject: [PATCH 124/138] main: refactor ParseSymbolAlias --- symbolAliases.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/symbolAliases.go b/symbolAliases.go index f71c71d..7dd4849 100644 --- a/symbolAliases.go +++ b/symbolAliases.go @@ -90,7 +90,7 @@ func (p *Parser) ParseSymbolAliases() (*ast.SymbolAliases, error) { commas = append(commas, &comma.Position) } - rbrace, err := p.lexer.Peek() + rbrace, err := p.lexer.Scan() if err != nil { return nil, err } @@ -99,10 +99,6 @@ func (p *Parser) ParseSymbolAliases() (*ast.SymbolAliases, error) { return nil, token.NewPosError(rbrace.Position, "not found RBrace.") } - if _, err := p.lexer.Scan(); err != nil { - return nil, err - } - return &ast.SymbolAliases{ LBrace: lbrace.Position, Aliases: aliases, From eea44cec43389852a27357353820460753aa7ea3 Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 1 Mar 2023 11:54:55 +0900 Subject: [PATCH 125/138] all: split token.StringLiteral to Empty and NoEmpty --- ast/ast_test.go | 4 ++-- block_test.go | 2 +- contractDefinition_test.go | 4 ++-- expression.go | 2 +- expression_test.go | 2 +- functionDefinition_test.go | 2 +- lexer/lexer.go | 4 +++- lexer/lexer_test.go | 45 ++++++++++++++++++++++++++++---------- literal.go | 2 +- literal_test.go | 4 ++-- solparser_test.go | 2 +- statement_test.go | 4 ++-- token/token.go | 3 ++- 13 files changed, 52 insertions(+), 28 deletions(-) diff --git a/ast/ast_test.go b/ast/ast_test.go index dbc6252..ce3d381 100644 --- a/ast/ast_test.go +++ b/ast/ast_test.go @@ -59,7 +59,7 @@ func TestNode_End(t *testing.T) { { name: "StringLiteral", node: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Value: "\"Hello world!!\"", Position: token.Pos{ Column: 4, @@ -74,7 +74,7 @@ func TestNode_End(t *testing.T) { { name: "StringLiteral with new line codes", node: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Value: "\"Hello \n new \nworld!!\"", Position: token.Pos{ Column: 4, diff --git a/block_test.go b/block_test.go index 4d4e0bd..403991d 100644 --- a/block_test.go +++ b/block_test.go @@ -29,7 +29,7 @@ func TestParser_ParseBlock(t *testing.T) { From: token.Pos{Column: 1, Line: 2}, SemiPos: token.Pos{Column: 23, Line: 2}, Expression: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Position: token.Pos{Column: 8, Line: 2}, Value: "\"Hello World!!\"", }, diff --git a/contractDefinition_test.go b/contractDefinition_test.go index 41cacbd..b1928c2 100644 --- a/contractDefinition_test.go +++ b/contractDefinition_test.go @@ -79,7 +79,7 @@ func TestParser_ParseContractDefinition(t *testing.T) { From: token.Pos{Column: 9, Line: 3}, SemiPos: token.Pos{Column: 31, Line: 3}, Expression: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Position: token.Pos{Column: 16, Line: 3}, Value: "\"Hello World!!\"", }, @@ -153,7 +153,7 @@ func TestParser_ParseContractDefinition(t *testing.T) { From: token.Pos{Column: 9, Line: 3}, SemiPos: token.Pos{Column: 31, Line: 3}, Expression: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Position: token.Pos{Column: 16, Line: 3}, Value: "\"Hello World!!\"", }, diff --git a/expression.go b/expression.go index f9a8d40..5ad23a8 100644 --- a/expression.go +++ b/expression.go @@ -11,7 +11,7 @@ func (p *Parser) ParseExpression() (ast.Expression, error) { return nil, err } - if tkn.Type == token.StringLiteral { + if tkn.Type == token.NonEmptyStringLiteral { return p.ParseLiteral() } diff --git a/expression_test.go b/expression_test.go index 64fd674..a9683f5 100644 --- a/expression_test.go +++ b/expression_test.go @@ -22,7 +22,7 @@ func TestParser_ParseExpression(t *testing.T) { name: "Literal", input: `"Hello World!!";`, want: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Value: `"Hello World!!"`, Position: pos(1, 1), }, diff --git a/functionDefinition_test.go b/functionDefinition_test.go index cdfda0d..34aa549 100644 --- a/functionDefinition_test.go +++ b/functionDefinition_test.go @@ -222,7 +222,7 @@ func TestParser_ParseFunctionDefinition(t *testing.T) { From: token.Pos{Column: 9, Line: 2}, SemiPos: token.Pos{Column: 31, Line: 2}, Expression: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Position: token.Pos{Column: 16, Line: 2}, Value: "\"Hello World!!\"", }, diff --git a/lexer/lexer.go b/lexer/lexer.go index 14eea5b..459e503 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -80,6 +80,7 @@ func (l *Lexer) ScanStringLiteral() (token.Token, error) { } quote, txt := str, str + tokenType := token.EmptyStringLiteral for { _, str, err := l.scanner.Scan() if err != nil { @@ -88,10 +89,11 @@ func (l *Lexer) ScanStringLiteral() (token.Token, error) { txt = txt + str if str == quote { return token.Token{ - Type: token.StringLiteral, + Type: tokenType, Value: txt, Position: firstPos, }, nil } + tokenType = token.NonEmptyStringLiteral } } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 66f735a..a5beb05 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -44,7 +44,7 @@ func TestLexer_Scan(t *testing.T) { name: "There is a StringLiteral", input: `"pragma"`, wantToken: token.Token{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Value: `"pragma"`, Position: token.Pos{ Column: 1, @@ -174,24 +174,45 @@ func TestLexer_Peek(t *testing.T) { func TestLexer_ScanStringLiteral(t *testing.T) { tests := []struct { - name string - input string - wantText string + input string + want token.Token }{ { - name: "double-quoted-printable", - input: `"Hello world!!";`, - wantText: `"Hello world!!"`, + input: `"Hello world!!";`, + want: token.Token{ + Type: token.NonEmptyStringLiteral, + Value: `"Hello world!!"`, + Position: token.Pos{Column: 1, Line: 1}, + }, + }, + { + input: `\'Hello world!!\';`, + want: token.Token{ + Type: token.NonEmptyStringLiteral, + Value: `\'Hello world!!\'`, + Position: token.Pos{Column: 1, Line: 1}, + }, + }, + { + input: `"";`, + want: token.Token{ + Type: token.EmptyStringLiteral, + Value: `""`, + Position: token.Pos{Column: 1, Line: 1}, + }, }, { - name: "single-quoted-printable", - input: `\'Hello world!!\';`, - wantText: `\'Hello world!!\'`, + input: `\'\';`, + want: token.Token{ + Type: token.EmptyStringLiteral, + Value: `\'\'`, + Position: token.Pos{Column: 1, Line: 1}, + }, }, } for _, tt := range tests { tt := tt - t.Run(tt.name, func(t *testing.T) { + t.Run(tt.input, func(t *testing.T) { s := scanner.New(strings.NewReader(tt.input)) l := Lexer{ scanner: s, @@ -201,7 +222,7 @@ func TestLexer_ScanStringLiteral(t *testing.T) { if err != nil { t.Errorf("error is not nil, got: %s", err) } - if diff := cmp.Diff(tt.wantText, tkn.Value); diff != "" { + if diff := cmp.Diff(tt.want, tkn); diff != "" { t.Errorf(diff) } }) diff --git a/literal.go b/literal.go index 0e814e9..b2117d9 100644 --- a/literal.go +++ b/literal.go @@ -11,7 +11,7 @@ func (p *Parser) ParseLiteral() (ast.Literal, error) { return nil, err } - if tkn.Type != token.StringLiteral { + if tkn.Type != token.NonEmptyStringLiteral { return nil, token.NewPosError(tkn.Position, "not found string literal quote") } diff --git a/literal_test.go b/literal_test.go index 3509bf6..0082803 100644 --- a/literal_test.go +++ b/literal_test.go @@ -22,7 +22,7 @@ func TestParser_ParseLiteral(t *testing.T) { name: "normal", input: `"Hello World!!";`, want: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Value: `"Hello World!!"`, Position: pos(1, 1), }, @@ -31,7 +31,7 @@ func TestParser_ParseLiteral(t *testing.T) { name: `Including \n`, input: "\"Hello \nWorld!!\";", want: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Value: "\"Hello \nWorld!!\"", Position: pos(1, 1), }, diff --git a/solparser_test.go b/solparser_test.go index 3d5e43f..ef1060e 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -103,7 +103,7 @@ contract HelloWorld { From: token.Pos{Column: 9, Line: 5}, SemiPos: token.Pos{Column: 31, Line: 5}, Expression: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Position: token.Pos{Column: 16, Line: 5}, Value: "\"Hello World!!\"", }, diff --git a/statement_test.go b/statement_test.go index a85023f..ff0cb66 100644 --- a/statement_test.go +++ b/statement_test.go @@ -25,7 +25,7 @@ func TestParser_ParseStatement(t *testing.T) { From: token.Pos{Column: 1, Line: 1}, SemiPos: token.Pos{Column: 23, Line: 1}, Expression: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Position: token.Pos{Column: 8, Line: 1}, Value: `"Hello World!!"`, }, @@ -85,7 +85,7 @@ func TestParser_ParseReturnStatement(t *testing.T) { From: token.Pos{Column: 1, Line: 1}, SemiPos: token.Pos{Column: 23, Line: 1}, Expression: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Position: token.Pos{Column: 8, Line: 1}, Value: `"Hello World!!"`, }, diff --git a/token/token.go b/token/token.go index 8b3952f..783aec6 100644 --- a/token/token.go +++ b/token/token.go @@ -167,7 +167,8 @@ const ( TrueLiteral FalseLiteral Number - StringLiteral + NonEmptyStringLiteral + EmptyStringLiteral UnicodeStringLiteral HexStringLiteral CommentLiteral From 3ba67460850e3bf59d98cab0a9d91fb642711d42 Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 1 Mar 2023 13:06:40 +0900 Subject: [PATCH 126/138] main, ast: refactor identifier type --- ast/ast.go | 10 +++++++++- contractDefinition_test.go | 4 ++-- identifier.go | 6 +++--- identifier_test.go | 8 ++++---- solparser_test.go | 2 +- symbolAliases.go | 16 ++-------------- symbolAliases_test.go | 36 ++++++++++++++++++++++++------------ 7 files changed, 45 insertions(+), 37 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 5613761..34a3039 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -138,7 +138,15 @@ type Expression interface { // ---------------------------------------------------------------------------- -type Identifier = token.Token +type Identifier token.Token + +func (i Identifier) Pos() token.Pos { return i.Position } +func (i Identifier) End() token.Pos { + return token.Pos{ + Column: i.Position.Column + len(i.Value), + Line: i.Position.Line, + } +} // ---------------------------------------------------------------------------- // Literal Nodes diff --git a/contractDefinition_test.go b/contractDefinition_test.go index b1928c2..3fd6a1c 100644 --- a/contractDefinition_test.go +++ b/contractDefinition_test.go @@ -27,7 +27,7 @@ func TestParser_ParseContractDefinition(t *testing.T) { }`, want: &ast.ContractDefinition{ Contract: token.Pos{Column: 1, Line: 1}, - Identifier: token.Token{ + Identifier: ast.Identifier{ Type: token.Identifier, Value: "HelloWorld", Position: token.Pos{Column: 10, Line: 1}, @@ -101,7 +101,7 @@ func TestParser_ParseContractDefinition(t *testing.T) { want: &ast.ContractDefinition{ Abstract: &token.Pos{Column: 1, Line: 1}, Contract: token.Pos{Column: 10, Line: 1}, - Identifier: token.Token{ + Identifier: ast.Identifier{ Type: token.Identifier, Value: "HelloWorld", Position: token.Pos{Column: 19, Line: 1}, diff --git a/identifier.go b/identifier.go index 335996d..aa39106 100644 --- a/identifier.go +++ b/identifier.go @@ -8,13 +8,13 @@ import ( func (p *Parser) ParseIdentifier() (ast.Identifier, error) { tkn, err := p.lexer.Scan() if err != nil { - return token.Token{}, err + return ast.Identifier{}, err } switch tkn.Type { case token.Identifier, token.From, token.Error, token.Revert, token.Global: - return tkn, nil + return ast.Identifier(tkn), nil } - return token.Token{}, token.NewPosError(tkn.Position, "keyword is not available as identifier.") + return ast.Identifier{}, token.NewPosError(tkn.Position, "keyword is not available as identifier.") } diff --git a/identifier_test.go b/identifier_test.go index 793116e..826c2ed 100644 --- a/identifier_test.go +++ b/identifier_test.go @@ -19,7 +19,7 @@ func TestParser_ParseIdentifier(t *testing.T) { }{ { input: "identifier", - want: token.Token{ + want: ast.Identifier{ Type: token.Identifier, Value: "identifier", Position: token.Pos{Column: 1, Line: 1}, @@ -27,7 +27,7 @@ func TestParser_ParseIdentifier(t *testing.T) { }, { input: "from", - want: token.Token{ + want: ast.Identifier{ Type: token.From, Value: "from", Position: token.Pos{Column: 1, Line: 1}, @@ -35,7 +35,7 @@ func TestParser_ParseIdentifier(t *testing.T) { }, { input: "error", - want: token.Token{ + want: ast.Identifier{ Type: token.Error, Value: "error", Position: token.Pos{Column: 1, Line: 1}, @@ -43,7 +43,7 @@ func TestParser_ParseIdentifier(t *testing.T) { }, { input: "global", - want: token.Token{ + want: ast.Identifier{ Type: token.Global, Value: "global", Position: token.Pos{Column: 1, Line: 1}, diff --git a/solparser_test.go b/solparser_test.go index ef1060e..b2f12ea 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -51,7 +51,7 @@ contract HelloWorld { }, ContractDefinition: &ast.ContractDefinition{ Contract: token.Pos{Column: 1, Line: 3}, - Identifier: token.Token{ + Identifier: ast.Identifier{ Type: token.Identifier, Value: "HelloWorld", Position: token.Pos{Column: 10, Line: 3}, diff --git a/symbolAliases.go b/symbolAliases.go index 7dd4849..d9a96ad 100644 --- a/symbolAliases.go +++ b/symbolAliases.go @@ -6,19 +6,11 @@ import ( ) func (p *Parser) ParseSymbolAlias() (*ast.SymbolAlias, error) { - smbl, err := p.lexer.Peek() + smbl, err := p.ParseIdentifier() if err != nil { return nil, err } - if smbl.Type != token.Identifier { - return nil, token.NewPosError(smbl.Position, "not found identifier.") - } - - if _, err := p.lexer.Scan(); err != nil { - return nil, err - } - as, err := p.lexer.Peek() if err != nil { return nil, err @@ -30,15 +22,11 @@ func (p *Parser) ParseSymbolAlias() (*ast.SymbolAlias, error) { return nil, err } - alias, err := p.lexer.Scan() + alias, err := p.ParseIdentifier() if err != nil { return nil, err } - if alias.Type != token.Identifier { - return nil, token.NewPosError(alias.Position, "not found identifier.") - } - return &ast.SymbolAlias{ Symbol: smbl, As: &as.Position, diff --git a/symbolAliases_test.go b/symbolAliases_test.go index fc37cc3..67a9d71 100644 --- a/symbolAliases_test.go +++ b/symbolAliases_test.go @@ -18,26 +18,30 @@ func TestParser_ParseSymbolAlias(t *testing.T) { { input: "symbol as alias1}", want: &ast.SymbolAlias{ - Symbol: tkn(token.Identifier, "symbol", pos(1, 1)), + Symbol: ast.Identifier(tkn(token.Identifier, "symbol", pos(1, 1))), As: posPtr(8, 1), - Alias: tknPtr(token.Identifier, "alias1", pos(11, 1)), + Alias: &ast.Identifier{ + Type: token.Identifier, + Value: "alias1", + Position: pos(11, 1), + }, }, }, { input: "symbol}", want: &ast.SymbolAlias{ - Symbol: tkn(token.Identifier, "symbol", pos(1, 1)), + Symbol: ast.Identifier(tkn(token.Identifier, "symbol", pos(1, 1))), As: nil, Alias: nil, }, }, { input: "pragma", - err: perr(pos(1, 1), "not found identifier."), + err: perr(pos(1, 1), "keyword is not available as identifier."), }, { input: "symbol as pragma", - err: perr(pos(11, 1), "not found identifier."), + err: perr(pos(11, 1), "keyword is not available as identifier."), }, } @@ -65,7 +69,7 @@ func TestParser_ParseSymbolAliases(t *testing.T) { LBrace: pos(1, 1), Aliases: []*ast.SymbolAlias{ { - Symbol: tkn(token.Identifier, "symbol1", pos(2, 1)), + Symbol: ast.Identifier(tkn(token.Identifier, "symbol1", pos(2, 1))), }, }, Commas: []*token.Pos{}, @@ -78,9 +82,13 @@ func TestParser_ParseSymbolAliases(t *testing.T) { LBrace: pos(1, 1), Aliases: []*ast.SymbolAlias{ { - Symbol: tkn(token.Identifier, "symbol2", pos(2, 1)), + Symbol: ast.Identifier(tkn(token.Identifier, "symbol2", pos(2, 1))), As: posPtr(10, 1), - Alias: tknPtr(token.Identifier, "alias1", pos(13, 1)), + Alias: &ast.Identifier{ + Type: token.Identifier, + Value: "alias1", + Position: pos(13, 1), + }, }, }, Commas: []*token.Pos{}, @@ -93,12 +101,16 @@ func TestParser_ParseSymbolAliases(t *testing.T) { LBrace: pos(1, 1), Aliases: []*ast.SymbolAlias{ { - Symbol: tkn(token.Identifier, "symbol1", pos(2, 1)), + Symbol: ast.Identifier(tkn(token.Identifier, "symbol1", pos(2, 1))), }, { - Symbol: tkn(token.Identifier, "symbol2", pos(11, 1)), + Symbol: ast.Identifier(tkn(token.Identifier, "symbol2", pos(11, 1))), As: posPtr(19, 1), - Alias: tknPtr(token.Identifier, "alias1", pos(22, 1)), + Alias: &ast.Identifier{ + Type: token.Identifier, + Value: "alias1", + Position: pos(22, 1), + }, }, }, Commas: []*token.Pos{posPtr(9, 1)}, @@ -111,7 +123,7 @@ func TestParser_ParseSymbolAliases(t *testing.T) { }, { input: "{symbol1, as alian1", - err: perr(pos(11, 1), "not found identifier."), + err: perr(pos(11, 1), "keyword is not available as identifier."), }, { input: "{symbol1, symbol2 as alian1", From 163d45c9a78be24b20935f7c4b7e5ebbd9402f0e Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 1 Mar 2023 14:41:40 +0900 Subject: [PATCH 127/138] main, ast: implement import-directive --- ast/ast.go | 63 ++++++++++ importDirective.go | 169 +++++++++++++++++++++++++++ importDirective_test.go | 248 ++++++++++++++++++++++++++++++++++++++++ solparser.go | 8 ++ 4 files changed, 488 insertions(+) create mode 100644 importDirective.go create mode 100644 importDirective_test.go diff --git a/ast/ast.go b/ast/ast.go index 34a3039..4889e4f 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -11,6 +11,65 @@ type Node interface { End() token.Pos } +// ---------------------------------------------------------------------------- +// ImportDirectiveElement Nodes + +type ImportDirectiveElement interface { + Node + importDirectiveElementNode() +} + +type ImportDirectivePathElement struct { + Path Path + As *token.Pos + Identifier *Identifier +} + +func (i ImportDirectivePathElement) Pos() token.Pos { return i.Path.Pos() } +func (i ImportDirectivePathElement) End() token.Pos { return i.Identifier.End() } + +type ImportDirectiveSymbolAliasesElement struct { + SymbolAliases *SymbolAliases + From token.Pos + Path Path +} + +func (i ImportDirectiveSymbolAliasesElement) Pos() token.Pos { return i.SymbolAliases.Pos() } +func (i ImportDirectiveSymbolAliasesElement) End() token.Pos { return i.Path.End() } + +type ImportDirectiveMulElement struct { + Mul token.Pos + As token.Pos + Identifier Identifier + From token.Pos + Path Path +} + +func (i ImportDirectiveMulElement) Pos() token.Pos { return i.Mul } +func (i ImportDirectiveMulElement) End() token.Pos { return i.Path.End() } + +func (i *ImportDirectivePathElement) importDirectiveElementNode() {} +func (i *ImportDirectiveSymbolAliasesElement) importDirectiveElementNode() {} +func (i *ImportDirectiveMulElement) importDirectiveElementNode() {} + +// ---------------------------------------------------------------------------- + +type ImportDirective struct { + Import token.Pos + Element ImportDirectiveElement + Semicolon token.Pos +} + +type Path token.Token + +func (p Path) Pos() token.Pos { return p.Position } +func (p Path) End() token.Pos { + return token.Pos{ + Column: p.Position.Column + len(p.Value), // Path does not include newline code. + Line: p.Position.Line, + } +} + type SymbolAlias struct { Symbol Identifier As *token.Pos @@ -24,6 +83,9 @@ type SymbolAliases struct { RBrace token.Pos } +func (s SymbolAliases) Pos() token.Pos { return s.LBrace } +func (s SymbolAliases) End() token.Pos { return s.RBrace } + type PragmaDirective struct { Pragma token.Pos PragmaTokens []*token.Token @@ -100,6 +162,7 @@ type ContractDefinition struct { // A File node represents a Solidity source file. type SourceUnit struct { PragmaDirective *PragmaDirective + ImportDirective *ImportDirective ContractDefinition *ContractDefinition FunctionDefinition *FunctionDefinition } diff --git a/importDirective.go b/importDirective.go new file mode 100644 index 0000000..2fff313 --- /dev/null +++ b/importDirective.go @@ -0,0 +1,169 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParsePath() (ast.Path, error) { + lit, err := p.lexer.Peek() + if err != nil { + return ast.Path{}, err + } + if lit.Type != token.NonEmptyStringLiteral { + return ast.Path{}, token.NewPosError(lit.Position, "not found non-empty-string-literal.") + } + if _, err := p.lexer.Scan(); err != nil { + return ast.Path{}, err + } + return ast.Path(lit), nil +} + +func (p *Parser) ParseImportDirectivePathElement() (*ast.ImportDirectivePathElement, error) { + path, err := p.ParsePath() + if err != nil { + return nil, err + } + + as, err := p.lexer.Peek() + if err != nil { + return nil, err + } + if as.Type != token.As { + return &ast.ImportDirectivePathElement{Path: path}, nil + } + if _, err := p.lexer.Scan(); err != nil { + return nil, err + } + + id, err := p.ParseIdentifier() + if err != nil { + return nil, err + } + + return &ast.ImportDirectivePathElement{ + Path: path, + As: &as.Position, + Identifier: &id, + }, nil +} + +func (p *Parser) ParseImportDirectiveSymbolAliasesElement() (*ast.ImportDirectiveSymbolAliasesElement, error) { + aliases, err := p.ParseSymbolAliases() + if err != nil { + return nil, err + } + + from, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if from.Type != token.From { + return nil, token.NewPosError(from.Position, "not found from keyword.") + } + + path, err := p.ParsePath() + if err != nil { + return nil, err + } + + return &ast.ImportDirectiveSymbolAliasesElement{ + SymbolAliases: aliases, + From: from.Position, + Path: path, + }, nil +} + +func (p *Parser) ParseImportDirectiveMulElement() (*ast.ImportDirectiveMulElement, error) { + mul, err := p.lexer.Peek() + if err != nil { + return nil, err + } + if mul.Type != token.Mul { + return nil, token.NewPosError(mul.Position, "not found mul keyword.") + } + if _, err := p.lexer.Scan(); err != nil { + return nil, err + } + + as, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if as.Type != token.As { + return nil, token.NewPosError(as.Position, "not found as keyword.") + } + + id, err := p.ParseIdentifier() + if err != nil { + return nil, err + } + + from, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if from.Type != token.From { + return nil, token.NewPosError(from.Position, "not found from keyword.") + } + + path, err := p.ParsePath() + if err != nil { + return nil, err + } + + return &ast.ImportDirectiveMulElement{ + Mul: mul.Position, + As: as.Position, + Identifier: id, + From: from.Position, + Path: path, + }, nil +} + +func (p *Parser) ParseImportDirective() (*ast.ImportDirective, error) { + impt, err := p.lexer.Peek() + if err != nil { + return nil, err + } + if impt.Type != token.Import { + return nil, token.NewPosError(impt.Position, "not found import keyword.") + } + if _, err := p.lexer.Scan(); err != nil { + return nil, err + } + + tkn, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + var el ast.ImportDirectiveElement + switch tkn.Type { + case token.NonEmptyStringLiteral: + el, err = p.ParseImportDirectivePathElement() + case token.LBrace: + el, err = p.ParseImportDirectiveSymbolAliasesElement() + case token.Mul: + el, err = p.ParseImportDirectiveMulElement() + default: + return nil, token.NewPosError(tkn.Position, "not found import-directive element.") + } + if err != nil { + return nil, err + } + + semi, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if semi.Type != token.Semicolon { + return nil, token.NewPosError(semi.Position, "not found semicolon.") + } + + return &ast.ImportDirective{ + Import: impt.Position, + Element: el, + Semicolon: semi.Position, + }, nil +} diff --git a/importDirective_test.go b/importDirective_test.go new file mode 100644 index 0000000..dea4854 --- /dev/null +++ b/importDirective_test.go @@ -0,0 +1,248 @@ +package solparser_test + +import ( + "strings" + "testing" + + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParsePath(t *testing.T) { + tests := []struct { + input string + want ast.Path + err *token.PosError + }{ + { + input: `"test.sol"`, + want: ast.Path(tkn(token.NonEmptyStringLiteral, `"test.sol"`, pos(1, 1))), + }, + { + input: "pragma", + err: perr(pos(1, 1), "not found non-empty-string-literal."), + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.input, func(t *testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParsePath() + assert(t, got, tt.want, err, tt.err) + }) + } +} + +func TestParser_ParseImportDirectivePathElement(t *testing.T) { + tests := []struct { + input string + want *ast.ImportDirectivePathElement + err *token.PosError + }{ + { + input: `"test.sol";`, + want: &ast.ImportDirectivePathElement{ + Path: ast.Path(tkn(token.NonEmptyStringLiteral, `"test.sol"`, pos(1, 1))), + }, + }, + { + input: `"test.sol" as alias1 ;`, + want: &ast.ImportDirectivePathElement{ + Path: ast.Path(tkn(token.NonEmptyStringLiteral, `"test.sol"`, pos(1, 1))), + As: posPtr(12, 1), + Identifier: &ast.Identifier{ + Type: token.Identifier, + Value: "alias1", + Position: pos(15, 1), + }, + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.input, func(t *testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseImportDirectivePathElement() + assert(t, got, tt.want, err, tt.err) + }) + } +} + +func TestParser_ParseImportDirectiveSymbolAliasesElement(t *testing.T) { + tests := []struct { + input string + want *ast.ImportDirectiveSymbolAliasesElement + err *token.PosError + }{ + { + input: `{symbol1} from "test.sol"`, + want: &ast.ImportDirectiveSymbolAliasesElement{ + SymbolAliases: &ast.SymbolAliases{ + LBrace: pos(1, 1), + Aliases: []*ast.SymbolAlias{ + {Symbol: ast.Identifier{ + Type: token.Identifier, + Value: "symbol1", + Position: pos(2, 1), + }}, + }, + Commas: []*token.Pos{}, + RBrace: pos(9, 1), + }, + From: pos(11, 1), + Path: ast.Path(tkn(token.NonEmptyStringLiteral, `"test.sol"`, pos(16, 1))), + }, + }, + { + input: `{symbol1} "test.sol"`, + err: perr(pos(11, 1), "not found from keyword."), + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.input, func(t *testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseImportDirectiveSymbolAliasesElement() + assert(t, got, tt.want, err, tt.err) + }) + } +} + +func TestParser_ParseImportDirectiveMulElement(t *testing.T) { + tests := []struct { + input string + want *ast.ImportDirectiveMulElement + err *token.PosError + }{ + { + input: `* as alias1 from "test.sol"`, + want: &ast.ImportDirectiveMulElement{ + Mul: pos(1, 1), + As: pos(3, 1), + Identifier: ast.Identifier{ + Type: token.Identifier, + Value: "alias1", + Position: pos(6, 1), + }, + From: pos(13, 1), + Path: ast.Path(tkn(token.NonEmptyStringLiteral, `"test.sol"`, pos(18, 1))), + }, + }, + { + input: `{symbol1} "test.sol"`, + err: perr(pos(1, 1), "not found mul keyword."), + }, + { + input: `* alias1 from "test.sol"`, + err: perr(pos(3, 1), "not found as keyword."), + }, + { + input: `* as alias1 "test.sol"`, + err: perr(pos(13, 1), "not found from keyword."), + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.input, func(t *testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseImportDirectiveMulElement() + assert(t, got, tt.want, err, tt.err) + }) + } +} + +func TestParser_ParseImportDirective(t *testing.T) { + tests := []struct { + input string + want *ast.ImportDirective + err *token.PosError + }{ + { + input: `import "test.sol";`, + want: &ast.ImportDirective{ + Import: pos(1, 1), + Element: &ast.ImportDirectivePathElement{ + Path: ast.Path(tkn(token.NonEmptyStringLiteral, `"test.sol"`, pos(8, 1))), + }, + Semicolon: pos(18, 1), + }, + }, + { + input: `import {symbol1} from "test.sol";`, + want: &ast.ImportDirective{ + Import: pos(1, 1), + Element: &ast.ImportDirectiveSymbolAliasesElement{ + SymbolAliases: &ast.SymbolAliases{ + LBrace: pos(8, 1), + Aliases: []*ast.SymbolAlias{ + {Symbol: ast.Identifier{ + Type: token.Identifier, + Value: "symbol1", + Position: pos(9, 1), + }}, + }, + Commas: []*token.Pos{}, + RBrace: pos(16, 1), + }, + From: pos(18, 1), + Path: ast.Path(tkn(token.NonEmptyStringLiteral, `"test.sol"`, pos(23, 1))), + }, + Semicolon: pos(33, 1), + }, + }, + { + input: `import * as alias1 from "test.sol";`, + want: &ast.ImportDirective{ + Import: pos(1, 1), + Element: &ast.ImportDirectiveMulElement{ + Mul: pos(8, 1), + As: pos(10, 1), + Identifier: ast.Identifier{ + Type: token.Identifier, + Value: "alias1", + Position: pos(13, 1), + }, + From: pos(20, 1), + Path: ast.Path(tkn(token.NonEmptyStringLiteral, `"test.sol"`, pos(25, 1))), + }, + Semicolon: pos(35, 1), + }, + }, + { + input: "symbol as pragma", + err: perr(pos(1, 1), "not found import keyword."), + }, + { + input: `import alias1 from "test.sol";`, + err: perr(pos(8, 1), "not found import-directive element."), + }, + { + input: `import * as alias1 from "test.sol"`, + err: perr(pos(35, 1), "not found semicolon."), + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.input, func(t *testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseImportDirective() + assert(t, got, tt.want, err, tt.err) + }) + } +} diff --git a/solparser.go b/solparser.go index 2c11763..a2a02fd 100644 --- a/solparser.go +++ b/solparser.go @@ -23,6 +23,7 @@ func New(input io.Reader) *Parser { func (p *Parser) Parse() (*ast.SourceUnit, error) { var pragmaDirective *ast.PragmaDirective + var importDirective *ast.ImportDirective var contractDefinition *ast.ContractDefinition var functionDefinition *ast.FunctionDefinition for { @@ -38,6 +39,12 @@ func (p *Parser) Parse() (*ast.SourceUnit, error) { return nil, err } pragmaDirective = prgm + case token.Import: + imp, err := p.ParseImportDirective() + if err != nil { + return nil, err + } + importDirective = imp case token.Abstract, token.Contract: cntrct, err := p.ParseContractDefinition() if err != nil { @@ -53,6 +60,7 @@ func (p *Parser) Parse() (*ast.SourceUnit, error) { case token.EOS: return &ast.SourceUnit{ PragmaDirective: pragmaDirective, + ImportDirective: importDirective, ContractDefinition: contractDefinition, FunctionDefinition: functionDefinition, }, nil From 567a618fbb1959eb224f8e5cfc0f5cb622e7abc7 Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 1 Mar 2023 15:15:37 +0900 Subject: [PATCH 128/138] all: add generics test helper struct --- go.mod | 2 +- helper_test.go | 20 +++++++++ importDirective_test.go | 96 +++++++++-------------------------------- 3 files changed, 41 insertions(+), 77 deletions(-) diff --git a/go.mod b/go.mod index 5096641..c51c2e3 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/uji/solparser -go 1.16 +go 1.18 require ( github.com/SteelSeries/bufrr v0.0.0-20161129220322-72103137aa3c diff --git a/helper_test.go b/helper_test.go index de20174..43008db 100644 --- a/helper_test.go +++ b/helper_test.go @@ -2,9 +2,11 @@ package solparser_test import ( "errors" + "strings" "testing" "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" "github.com/uji/solparser/token" ) @@ -52,3 +54,21 @@ func assert(t *testing.T, got, want interface{}, gotErr, wantErr error) { t.Errorf("%s", diff) } } + +type TestData[T any] []struct { + input string + want T + err error +} + +func (ts TestData[T]) Test(t *testing.T, target func(p *solparser.Parser) (T, error)) { + for _, tt := range ts { + tt := tt + t.Run(tt.input, func(t *testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + got, err := target(p) + assert(t, got, tt.want, err, tt.err) + }) + } +} diff --git a/importDirective_test.go b/importDirective_test.go index dea4854..aa91d4d 100644 --- a/importDirective_test.go +++ b/importDirective_test.go @@ -1,7 +1,6 @@ package solparser_test import ( - "strings" "testing" "github.com/uji/solparser" @@ -10,11 +9,7 @@ import ( ) func TestParser_ParsePath(t *testing.T) { - tests := []struct { - input string - want ast.Path - err *token.PosError - }{ + tests := TestData[ast.Path]{ { input: `"test.sol"`, want: ast.Path(tkn(token.NonEmptyStringLiteral, `"test.sol"`, pos(1, 1))), @@ -25,24 +20,13 @@ func TestParser_ParsePath(t *testing.T) { }, } - for _, tt := range tests { - tt := tt - t.Run(tt.input, func(t *testing.T) { - r := strings.NewReader(tt.input) - p := solparser.New(r) - - got, err := p.ParsePath() - assert(t, got, tt.want, err, tt.err) - }) - } + tests.Test(t, func(p *solparser.Parser) (ast.Path, error) { + return p.ParsePath() + }) } func TestParser_ParseImportDirectivePathElement(t *testing.T) { - tests := []struct { - input string - want *ast.ImportDirectivePathElement - err *token.PosError - }{ + tests := TestData[*ast.ImportDirectivePathElement]{ { input: `"test.sol";`, want: &ast.ImportDirectivePathElement{ @@ -63,24 +47,13 @@ func TestParser_ParseImportDirectivePathElement(t *testing.T) { }, } - for _, tt := range tests { - tt := tt - t.Run(tt.input, func(t *testing.T) { - r := strings.NewReader(tt.input) - p := solparser.New(r) - - got, err := p.ParseImportDirectivePathElement() - assert(t, got, tt.want, err, tt.err) - }) - } + tests.Test(t, func(p *solparser.Parser) (*ast.ImportDirectivePathElement, error) { + return p.ParseImportDirectivePathElement() + }) } func TestParser_ParseImportDirectiveSymbolAliasesElement(t *testing.T) { - tests := []struct { - input string - want *ast.ImportDirectiveSymbolAliasesElement - err *token.PosError - }{ + tests := TestData[*ast.ImportDirectiveSymbolAliasesElement]{ { input: `{symbol1} from "test.sol"`, want: &ast.ImportDirectiveSymbolAliasesElement{ @@ -106,24 +79,13 @@ func TestParser_ParseImportDirectiveSymbolAliasesElement(t *testing.T) { }, } - for _, tt := range tests { - tt := tt - t.Run(tt.input, func(t *testing.T) { - r := strings.NewReader(tt.input) - p := solparser.New(r) - - got, err := p.ParseImportDirectiveSymbolAliasesElement() - assert(t, got, tt.want, err, tt.err) - }) - } + tests.Test(t, func(p *solparser.Parser) (*ast.ImportDirectiveSymbolAliasesElement, error) { + return p.ParseImportDirectiveSymbolAliasesElement() + }) } func TestParser_ParseImportDirectiveMulElement(t *testing.T) { - tests := []struct { - input string - want *ast.ImportDirectiveMulElement - err *token.PosError - }{ + tests := TestData[*ast.ImportDirectiveMulElement]{ { input: `* as alias1 from "test.sol"`, want: &ast.ImportDirectiveMulElement{ @@ -152,24 +114,13 @@ func TestParser_ParseImportDirectiveMulElement(t *testing.T) { }, } - for _, tt := range tests { - tt := tt - t.Run(tt.input, func(t *testing.T) { - r := strings.NewReader(tt.input) - p := solparser.New(r) - - got, err := p.ParseImportDirectiveMulElement() - assert(t, got, tt.want, err, tt.err) - }) - } + tests.Test(t, func(p *solparser.Parser) (*ast.ImportDirectiveMulElement, error) { + return p.ParseImportDirectiveMulElement() + }) } func TestParser_ParseImportDirective(t *testing.T) { - tests := []struct { - input string - want *ast.ImportDirective - err *token.PosError - }{ + tests := TestData[*ast.ImportDirective]{ { input: `import "test.sol";`, want: &ast.ImportDirective{ @@ -235,14 +186,7 @@ func TestParser_ParseImportDirective(t *testing.T) { }, } - for _, tt := range tests { - tt := tt - t.Run(tt.input, func(t *testing.T) { - r := strings.NewReader(tt.input) - p := solparser.New(r) - - got, err := p.ParseImportDirective() - assert(t, got, tt.want, err, tt.err) - }) - } + tests.Test(t, func(p *solparser.Parser) (*ast.ImportDirective, error) { + return p.ParseImportDirective() + }) } From 83d92cad83fcb36f4fbd3c8bf36e597e75c5128d Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 1 Mar 2023 20:32:15 +0900 Subject: [PATCH 129/138] ast: refactor ast Nodes --- ast/ast.go | 115 +++++++++++++++++++++++++++--------------------- ast/ast_test.go | 18 ++++++++ 2 files changed, 84 insertions(+), 49 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 4889e4f..e224820 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -6,19 +6,52 @@ import ( "github.com/uji/solparser/token" ) +// All node types implement the Node interface. type Node interface { Pos() token.Pos End() token.Pos } -// ---------------------------------------------------------------------------- -// ImportDirectiveElement Nodes - +// All import-directive-element node (own definition for solparser) types implement the ImportDirectiveElement interface. type ImportDirectiveElement interface { Node importDirectiveElementNode() } +// All contract-body-element node types implement the ContractBodyElement interface. +type ContractBodyElement interface { + Node + contractBodyElementNode() +} + +// All type-name node types implement the TypeName interface. +type TypeName interface { + Node + typeNameNode() +} + +// All expression node types implement the Expression interface. +type Expression interface { + Node + expressionNode() +} + +// All literal node types implement the Literal interface. +type Literal interface { + Node + literalNode() + expressionNode() +} + +// All statement node types implement the Statement interface. +type Statement interface { + Node + statementNode() +} + +// ---------------------------------------------------------------------------- +// ImportDirectiveElement Nodes + type ImportDirectivePathElement struct { Path Path As *token.Pos @@ -60,6 +93,9 @@ type ImportDirective struct { Semicolon token.Pos } +func (i ImportDirective) Pos() token.Pos { return i.Import } +func (i ImportDirective) End() token.Pos { return i.Semicolon } + type Path token.Token func (p Path) Pos() token.Pos { return p.Position } @@ -76,6 +112,14 @@ type SymbolAlias struct { Alias *Identifier } +func (s SymbolAlias) Pos() token.Pos { return s.Symbol.Pos() } +func (s SymbolAlias) End() token.Pos { + if s.Alias != nil { + return s.Alias.End() + } + return s.Symbol.End() +} + type SymbolAliases struct { LBrace token.Pos Aliases []*SymbolAlias @@ -92,6 +136,9 @@ type PragmaDirective struct { Semicolon token.Pos } +func (p PragmaDirective) Pos() token.Pos { return p.Pragma } +func (p PragmaDirective) End() token.Pos { return p.Semicolon } + type ModifierList struct { Visibility *Visibility StateMutability *StateMutability @@ -118,6 +165,14 @@ type FunctionDefinitionReturns struct { RParen token.Pos } +type InheritanceSpecifier struct { + IdentifierPath string + CallArgumentList CallArgumentList +} + +// ---------------------------------------------------------------------------- +// ContractBodyElement Nodes + type FunctionDefinition struct { From token.Pos FunctionDescriptor FunctionDescriptor @@ -128,26 +183,13 @@ type FunctionDefinition struct { Block *Block } -func (f FunctionDefinition) Pos() token.Pos { - return f.From -} - -func (f FunctionDefinition) End() token.Pos { - return f.Block.End() -} - -type InheritanceSpecifier struct { - IdentifierPath string - CallArgumentList CallArgumentList -} - -type ContractBodyElement interface { - Node - contractBodyElementNode() -} +func (f FunctionDefinition) Pos() token.Pos { return f.From } +func (f FunctionDefinition) End() token.Pos { return f.Block.End() } func (f *FunctionDefinition) contractBodyElementNode() {} +// ---------------------------------------------------------------------------- + type CallArgumentList struct{} type ContractDefinition struct { @@ -170,11 +212,6 @@ type SourceUnit struct { // ---------------------------------------------------------------------------- // TypeName Nodes -type TypeName interface { - Node - typeNameNode() -} - type ElementaryTypeName []*token.Token func (e ElementaryTypeName) Pos() token.Pos { @@ -194,13 +231,6 @@ func (e ElementaryTypeName) typeNameNode() {} // ---------------------------------------------------------------------------- // Expression Nodes -type Expression interface { - Node - expressionNode() -} - -// ---------------------------------------------------------------------------- - type Identifier token.Token func (i Identifier) Pos() token.Pos { return i.Position } @@ -211,15 +241,11 @@ func (i Identifier) End() token.Pos { } } +func (i *Identifier) expressionNode() {} + // ---------------------------------------------------------------------------- // Literal Nodes -type Literal interface { - Node - literalNode() - expressionNode() -} - type BooleanLiteral struct { Token token.Token } @@ -387,22 +413,13 @@ func (b Block) End() token.Pos { // ---------------------------------------------------------------------------- // Statement Nodes -type Statement interface { - Node - statementNode() -} - type ReturnStatement struct { From token.Pos SemiPos token.Pos Expression Expression } -func (r *ReturnStatement) Pos() token.Pos { - return r.From -} -func (r *ReturnStatement) End() token.Pos { - return r.SemiPos -} +func (r ReturnStatement) Pos() token.Pos { return r.From } +func (r ReturnStatement) End() token.Pos { return r.SemiPos } func (s *ReturnStatement) statementNode() {} diff --git a/ast/ast_test.go b/ast/ast_test.go index ce3d381..6241b3d 100644 --- a/ast/ast_test.go +++ b/ast/ast_test.go @@ -8,6 +8,24 @@ import ( "github.com/uji/solparser/token" ) +var ( + _ ast.ImportDirectiveElement = &ast.ImportDirectivePathElement{} + _ ast.ImportDirectiveElement = &ast.ImportDirectiveSymbolAliasesElement{} + _ ast.ImportDirectiveElement = &ast.ImportDirectiveMulElement{} + _ ast.Node = &ast.ImportDirective{} + _ ast.Node = &ast.Path{} + _ ast.Node = &ast.SymbolAlias{} + _ ast.Node = &ast.SymbolAliases{} + _ ast.Node = &ast.PragmaDirective{} + _ ast.ContractBodyElement = &ast.FunctionDefinition{} + _ ast.TypeName = ast.ElementaryTypeName{} + _ ast.Expression = &ast.Identifier{} + _ ast.Literal = &ast.BooleanLiteral{} + _ ast.Literal = &ast.StringLiteral{} + _ ast.Node = &ast.Block{} + _ ast.Statement = &ast.ReturnStatement{} +) + func TestNode_End(t *testing.T) { tests := []struct { name string From 258d2326566d98c7b18874b41ec4f4fdce7dde06 Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 1 Mar 2023 23:29:08 +0900 Subject: [PATCH 130/138] lexer: Scan unicode-string-literal --- lexer/helper_test.go | 56 ++++++++++++++++++++++++++++++++++++++++++++ lexer/lexer.go | 53 ++++++++++++++++++++++++++++++++++------- lexer/lexer_test.go | 25 ++++++++++++++++++++ 3 files changed, 126 insertions(+), 8 deletions(-) create mode 100644 lexer/helper_test.go diff --git a/lexer/helper_test.go b/lexer/helper_test.go new file mode 100644 index 0000000..f02d7fd --- /dev/null +++ b/lexer/helper_test.go @@ -0,0 +1,56 @@ +package lexer + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser/token" +) + +func pos(c, l int) token.Pos { + return token.Pos{Column: c, Line: l} +} + +func tkn(tp token.TokenType, text string, pos token.Pos) token.Token { + return token.Token{ + Type: tp, + Value: text, + Position: pos, + } +} + +func perr(pos token.Pos, msg string) *token.PosError { + return &token.PosError{ + Pos: pos, + Msg: msg, + } +} + +type TestData[T any] []struct { + input string + want T + err error +} + +func (ts TestData[T]) Test(t *testing.T, target func(l *Lexer) (T, error)) { + for _, tt := range ts { + tt := tt + t.Run(tt.input, func(t *testing.T) { + l := New(strings.NewReader(tt.input)) + got, err := target(l) + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} diff --git a/lexer/lexer.go b/lexer/lexer.go index 459e503..d35c141 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -71,29 +71,66 @@ func (l *Lexer) Peek() (token.Token, error) { // ScanStringLiteral parse NonEmptyStringLiteral or EmptyStringLiteral then return StringLiteral token. func (l *Lexer) ScanStringLiteral() (token.Token, error) { - firstPos, str, err := l.scanner.Scan() + start, v, err := l.scanner.Scan() if err != nil { return token.Token{}, err } - if str != `"` && str != `\'` { - return token.Token{}, token.NewPosError(firstPos, `not found " or \'`) + if v != `"` && v != `\'` { + return token.Token{}, token.NewPosError(start, `not found " or \'`) } - quote, txt := str, str + quote, txt := v, v tokenType := token.EmptyStringLiteral for { - _, str, err := l.scanner.Scan() + _, v, err := l.scanner.Scan() if err != nil { return token.Token{}, err } - txt = txt + str - if str == quote { + txt = txt + v + if v == quote { return token.Token{ Type: tokenType, Value: txt, - Position: firstPos, + Position: start, }, nil } tokenType = token.NonEmptyStringLiteral } } + +func (l *Lexer) ScanUnicodeStringLiteral() (token.Token, error) { + start, v, err := l.scanner.Scan() + if err != nil { + return token.Token{}, err + } + if v != `unicode` { + return token.Token{}, token.NewPosError(start, `not found unicode prefix.`) + } + rslt := v + + pos, v, err := l.scanner.Scan() + if err != nil { + return token.Token{}, err + } + // 'unicode' and quote('"' or '\'') must not have spaces. + if v != `"` && v != `\'` || pos.Column != start.Column+7 { + return token.Token{}, token.NewPosError(pos, `not found " or \'`) + } + + rslt = rslt + v + quote := v + for { + _, v, err := l.scanner.Scan() + if err != nil { + return token.Token{}, err + } + rslt = rslt + v + if v == quote { + return token.Token{ + Type: token.UnicodeStringLiteral, + Value: rslt, + Position: start, + }, nil + } + } +} diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index a5beb05..0d73611 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -251,3 +251,28 @@ func TestLexer_ScanStringLiteral(t *testing.T) { } }) } + +func TestLexer_ScanUnicodeStringLiteral(t *testing.T) { + tests := TestData[token.Token]{ + { + input: `unicode"Hello 😃"`, + want: tkn(token.UnicodeStringLiteral, `unicode"Hello 😃"`, pos(1,1)), + }, + { + input: `unicode\'Hello 😃\'`, + want: tkn(token.UnicodeStringLiteral, `unicode\'Hello 😃\'`, pos(1,1) ), + }, + { + input: `unicode Hello 😃`, + err: perr(pos(8, 1), `not found " or \'`), + }, + { + input: `unicode "Hello 😃"`, + err: perr(pos(8, 1), `not found " or \'`), + }, + } + + tests.Test(t, func(l *Lexer) (token.Token, error) { + return l.ScanUnicodeStringLiteral() + }) +} From 6ec080284f94fed78c61360648e53f881fa741ff Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 11 Mar 2023 16:28:56 +0900 Subject: [PATCH 131/138] lexer: refactor unicodeStringLiteral --- lexer/lexer.go | 8 +++++--- lexer/lexer_test.go | 7 ++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index d35c141..2f689cc 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -98,13 +98,15 @@ func (l *Lexer) ScanStringLiteral() (token.Token, error) { } } +const unicodeStringLiteralQuoteOffset = len("unicode") + func (l *Lexer) ScanUnicodeStringLiteral() (token.Token, error) { start, v, err := l.scanner.Scan() if err != nil { return token.Token{}, err } - if v != `unicode` { - return token.Token{}, token.NewPosError(start, `not found unicode prefix.`) + if v != "unicode" { + return token.Token{}, token.NewPosError(start, "not found unicode prefix.") } rslt := v @@ -113,7 +115,7 @@ func (l *Lexer) ScanUnicodeStringLiteral() (token.Token, error) { return token.Token{}, err } // 'unicode' and quote('"' or '\'') must not have spaces. - if v != `"` && v != `\'` || pos.Column != start.Column+7 { + if v != `"` && v != `\'` && pos.Line == start.Line || pos.Column != start.Column+unicodeStringLiteralQuoteOffset { return token.Token{}, token.NewPosError(pos, `not found " or \'`) } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 0d73611..468f0c1 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -260,7 +260,7 @@ func TestLexer_ScanUnicodeStringLiteral(t *testing.T) { }, { input: `unicode\'Hello 😃\'`, - want: tkn(token.UnicodeStringLiteral, `unicode\'Hello 😃\'`, pos(1,1) ), + want: tkn(token.UnicodeStringLiteral, `unicode\'Hello 😃\'`, pos(1,1)), }, { input: `unicode Hello 😃`, @@ -270,6 +270,11 @@ func TestLexer_ScanUnicodeStringLiteral(t *testing.T) { input: `unicode "Hello 😃"`, err: perr(pos(8, 1), `not found " or \'`), }, + { + input: `unicode + "Hello 😃"`, + err: perr(pos(8, 1), `not found " or \'`), + }, } tests.Test(t, func(l *Lexer) (token.Token, error) { From 5565270b1726a15dfc1544fbfb1c639987466b37 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 12 Mar 2023 15:47:05 +0900 Subject: [PATCH 132/138] lexer, token: scan HexString. --- lexer/lexer.go | 68 +++++++++++++++++++++++++++++++++++++++++++++ lexer/lexer_test.go | 48 ++++++++++++++++++++++++++++++-- token/token.go | 6 ++-- 3 files changed, 115 insertions(+), 7 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 2f689cc..ac7ef53 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -3,6 +3,7 @@ package lexer import ( "errors" "io" + "unicode" "github.com/uji/solparser/scanner" "github.com/uji/solparser/token" @@ -136,3 +137,70 @@ func (l *Lexer) ScanUnicodeStringLiteral() (token.Token, error) { } } } + +func isEvenHexDigits(str string) bool { + if len(str) == 0 { + return true + } + runes := []rune(str) + if !unicode.Is(unicode.Hex_Digit, runes[0]) { + return false + } + + // If a chunk of HexString is being read, it will be true. + readingBlock := true + i := 1 + for i < len(runes) { + if !readingBlock && runes[i] == '_' { + i++ + continue + } + if !unicode.Is(unicode.Hex_Digit, runes[i]) { + return false + } + readingBlock = !readingBlock + i++ + } + return !readingBlock +} + +func (l *Lexer) ScanHexString() (token.Token, error) { + start, hex, err := l.scanner.Scan() + if err != nil { + return token.Token{}, err + } + + if hex != "hex" { + return token.Token{}, token.NewPosError(start, "not found hex prefix.") + } + + lqpos, lquote, err := l.scanner.Scan() + if err != nil { + return token.Token{}, err + } + if lquote != `"` && lquote != `\'` { + return token.Token{}, token.NewPosError(lqpos, `not found " or \'`) + } + + vpos, v, err := l.scanner.Scan() + if err != nil { + return token.Token{}, err + } + if !isEvenHexDigits(v) { + return token.Token{}, token.NewPosError(vpos, "invalid HexString format") + } + + rqpos, rquote, err := l.scanner.Scan() + if err != nil { + return token.Token{}, err + } + if rquote != lquote { + return token.Token{}, token.NewPosError(rqpos, `not found quote'`) + } + + return token.Token{ + Type: token.HexString, + Value: hex + lquote + v + rquote, + Position: start, + }, nil +} diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 468f0c1..4551bc2 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -256,11 +256,11 @@ func TestLexer_ScanUnicodeStringLiteral(t *testing.T) { tests := TestData[token.Token]{ { input: `unicode"Hello 😃"`, - want: tkn(token.UnicodeStringLiteral, `unicode"Hello 😃"`, pos(1,1)), + want: tkn(token.UnicodeStringLiteral, `unicode"Hello 😃"`, pos(1, 1)), }, { input: `unicode\'Hello 😃\'`, - want: tkn(token.UnicodeStringLiteral, `unicode\'Hello 😃\'`, pos(1,1)), + want: tkn(token.UnicodeStringLiteral, `unicode\'Hello 😃\'`, pos(1, 1)), }, { input: `unicode Hello 😃`, @@ -273,7 +273,7 @@ func TestLexer_ScanUnicodeStringLiteral(t *testing.T) { { input: `unicode "Hello 😃"`, - err: perr(pos(8, 1), `not found " or \'`), + err: perr(pos(8, 1), `not found " or \'`), }, } @@ -281,3 +281,45 @@ func TestLexer_ScanUnicodeStringLiteral(t *testing.T) { return l.ScanUnicodeStringLiteral() }) } + +func TestLexer_ScanHexStringLiteral(t *testing.T) { + tests := TestData[token.Token]{ + { + input: `hex"1B"`, + want: tkn(token.HexString, `hex"1B"`, pos(1, 1)), + }, + { + input: `hex\'2E\'`, + want: tkn(token.HexString, `hex\'2E\'`, pos(1, 1)), + }, + { + input: `hex"16_3F_B5"`, + want: tkn(token.HexString, `hex"16_3F_B5"`, pos(1, 1)), + }, + { + input: `hex "12"`, + err: perr(pos(4, 1), `not found " or \'`), + }, + { + input: `hex + "123"`, + err: perr(pos(4, 1), `not found " or \'`), + }, + { + input: `hex" 123"`, + err: perr(pos(5, 1), "invalid HexString format"), + }, + { + input: `hex"16_3"`, + err: perr(pos(5, 1), "invalid HexString format"), + }, + { + input: `hex"163_A13"`, + err: perr(pos(5, 1), "invalid HexString format"), + }, + } + + tests.Test(t, func(l *Lexer) (token.Token, error) { + return l.ScanHexString() + }) +} diff --git a/token/token.go b/token/token.go index 783aec6..9e98de3 100644 --- a/token/token.go +++ b/token/token.go @@ -130,7 +130,6 @@ const ( From Function Global - Hex If Immutable Import @@ -163,6 +162,8 @@ const ( Virtual While + HexString + // Literal TrueLiteral FalseLiteral @@ -170,7 +171,6 @@ const ( NonEmptyStringLiteral EmptyStringLiteral UnicodeStringLiteral - HexStringLiteral CommentLiteral // Not keywords or future reserved words @@ -405,8 +405,6 @@ func asKeyword(str string) TokenType { return Function case "global": return Global - case "hex": - return Hex case "if": return If case "immutable": From da5d95c4afedcd63891167bc4433a956e9fd8f23 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 24 Sep 2023 11:21:42 +0900 Subject: [PATCH 133/138] main: implement ParseIdentifierPath --- ast/ast.go | 11 ++++++++++ identifier.go | 8 +++++++ identifierPath.go | 49 ++++++++++++++++++++++++++++++++++++++++++ identifierPath_test.go | 32 +++++++++++++++++++++++++++ 4 files changed, 100 insertions(+) create mode 100644 identifierPath.go create mode 100644 identifierPath_test.go diff --git a/ast/ast.go b/ast/ast.go index e224820..d2f8b92 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -192,6 +192,17 @@ func (f *FunctionDefinition) contractBodyElementNode() {} type CallArgumentList struct{} +type IdentifierPathElement struct { + Identifier Identifier + Period *token.Pos +} +type IdentifierPath struct { + Elements []*IdentifierPathElement +} + +func (i IdentifierPath) Pos() token.Pos { return i.Elements[0].Identifier.Pos() } +func (i IdentifierPath) End() token.Pos { return i.Elements[len(i.Elements)-1].Identifier.End() } + type ContractDefinition struct { Abstract *token.Pos Contract token.Pos diff --git a/identifier.go b/identifier.go index aa39106..37f42bb 100644 --- a/identifier.go +++ b/identifier.go @@ -5,6 +5,14 @@ import ( "github.com/uji/solparser/token" ) +func isIdentifier(tkn token.Token) bool { + switch tkn.Type { + case token.Identifier, token.From, token.Error, token.Revert, token.Global: + return true + } + return false +} + func (p *Parser) ParseIdentifier() (ast.Identifier, error) { tkn, err := p.lexer.Scan() if err != nil { diff --git a/identifierPath.go b/identifierPath.go new file mode 100644 index 0000000..ab9f0fa --- /dev/null +++ b/identifierPath.go @@ -0,0 +1,49 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseIdentifierPath() (ast.IdentifierPath, error) { + id, err := p.lexer.Peek() + if err != nil { + return ast.IdentifierPath{}, err + } + if !isIdentifier(id) { + return ast.IdentifierPath{}, token.NewPosError(id.Position, "not found identifier.") + } + p.lexer.Scan() + + var elements []*ast.IdentifierPathElement + for { + prd, err := p.lexer.Peek() + if err != nil || prd.Type != token.Period { + break + } + p.lexer.Scan() + + elements = append(elements, &ast.IdentifierPathElement{ + Identifier: ast.Identifier(id), + Period: &prd.Position, + }) + + i, err := p.lexer.Peek() + if err != nil { + return ast.IdentifierPath{}, err + } + if !isIdentifier(i) { + return ast.IdentifierPath{}, token.NewPosError(i.Position, "not found identifier.") + } + p.lexer.Scan() + id = i + } + + elements = append(elements, &ast.IdentifierPathElement{ + Identifier: ast.Identifier(id), + }) + + return ast.IdentifierPath{ + Elements: elements, + }, nil +} diff --git a/identifierPath_test.go b/identifierPath_test.go new file mode 100644 index 0000000..eb9e631 --- /dev/null +++ b/identifierPath_test.go @@ -0,0 +1,32 @@ +package solparser_test + +import ( + "testing" + + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseIdentifierPath(t *testing.T) { + tests := TestData[ast.IdentifierPath]{ + { + input: "global . identifier", + want: ast.IdentifierPath{ + Elements: []*ast.IdentifierPathElement{ + { + Identifier: ast.Identifier(tkn(token.Global, "global", pos(1, 1))), + Period: posPtr(8, 1), + }, + { + Identifier: ast.Identifier(tkn(token.Identifier, "identifier", pos(10, 1))), + }, + }, + }, + }, + } + + tests.Test(t, func(p *solparser.Parser) (ast.IdentifierPath, error) { + return p.ParseIdentifierPath() + }) +} From e3beb3ce81213b0ab3d311a7776256ff4607a1b9 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 24 Sep 2023 13:28:17 +0900 Subject: [PATCH 134/138] main: ParseCallArgumentList --- ast/ast.go | 25 ++++++- callArgumentList.go | 154 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 callArgumentList.go diff --git a/ast/ast.go b/ast/ast.go index d2f8b92..118cccf 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -190,7 +190,30 @@ func (f *FunctionDefinition) contractBodyElementNode() {} // ---------------------------------------------------------------------------- -type CallArgumentList struct{} +type CallArgumentListExpretion struct { + Expression Expression + Comma *token.Pos +} + +type CallArgumentListNamedExpretion struct { + Identifier Identifier + Colon token.Pos + Expression Expression + Comma *token.Pos +} + +type CallArgumentListNamedExpretions struct { + RBrace token.Pos + NamedExpretions []*CallArgumentListNamedExpretion + LBrace token.Pos +} + +type CallArgumentList struct { + LParen token.Pos + Expressions []*CallArgumentListExpretion + NamedExpretionsa *CallArgumentListNamedExpretions + RParen token.Pos +} type IdentifierPathElement struct { Identifier Identifier diff --git a/callArgumentList.go b/callArgumentList.go new file mode 100644 index 0000000..096babc --- /dev/null +++ b/callArgumentList.go @@ -0,0 +1,154 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseCallArgumentList() (*ast.CallArgumentList, error) { + lparen, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + tkn, err := p.lexer.Peek() + if err != nil { + return nil, err + } + if tkn.Type == token.RParen { + p.lexer.Scan() + return &ast.CallArgumentList{ + LParen: lparen.Position, + RParen: tkn.Position, + }, nil + } + + var expretions []*ast.CallArgumentListExpretion + var namedExpretions *ast.CallArgumentListNamedExpretions + if tkn.Type == token.LBrace { + es, err := p.ParseCallArgumentListExpretions() + if err != nil { + return nil, err + } + expretions = es + } else { + nes, err := p.ParseCallArgumentListNamedExpretions() + if err != nil { + return nil, err + } + namedExpretions = nes + } + + rparen, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if rparen.Type != token.RParen { + return nil, token.NewPosError(rparen.Position, "not found RParen.") + } + + return &ast.CallArgumentList{ + LParen: lparen.Position, + Expressions: expretions, + NamedExpretionsa: namedExpretions, + RParen: rparen.Position, + }, nil +} + +func (p *Parser) ParseCallArgumentListExpretions() ([]*ast.CallArgumentListExpretion, error) { + ex, err := p.ParseExpression() + if err != nil { + return nil, err + } + + var exs []*ast.CallArgumentListExpretion + for { + cmm, err := p.lexer.Peek() + if err != nil || cmm.Type != token.Comma { + break + } + p.lexer.Scan() + + exs = append(exs, &ast.CallArgumentListExpretion{ + Expression: ex, + Comma: &cmm.Position, + }) + + e, err := p.ParseExpression() + ex = e + } + + exs = append(exs, &ast.CallArgumentListExpretion{ + Expression: ex, + }) + + return exs, nil +} + +func (p *Parser) ParseCallArgumentListNamedExpretions() (*ast.CallArgumentListNamedExpretions, error) { + lbrace, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + exs := make([]*ast.CallArgumentListNamedExpretion, 0, 1) + + for { + id, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + if !isIdentifier(id) { + return nil, token.NewPosError(id.Position, "not found identifier.") + } + p.lexer.Scan() + + cln, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if cln.Type != token.Colon { + return nil, token.NewPosError(cln.Position, "not found Colon.") + } + + ex, err := p.ParseExpression() + if err != nil { + return nil, err + } + + cmm, err := p.lexer.Peek() + if err != nil { + return nil, err + } + if cmm.Type != token.Comma { + exs = append(exs, &ast.CallArgumentListNamedExpretion{ + Identifier: ast.Identifier(id), + Colon: cln.Position, + Expression: ex, + }) + break + } + p.lexer.Scan() + exs = append(exs, &ast.CallArgumentListNamedExpretion{ + Identifier: ast.Identifier(id), + Colon: cln.Position, + Expression: ex, + Comma: &cmm.Position, + }) + } + + rbrace, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if rbrace.Type != token.RBrace { + return nil, token.NewPosError(rbrace.Position, "not found RBrace.") + } + + return &ast.CallArgumentListNamedExpretions{ + RBrace: rbrace.Position, + NamedExpretions: exs, + LBrace: lbrace.Position, + }, nil +} From bda85d40c2fb074d747faa272e35016d4539eb69 Mon Sep 17 00:00:00 2001 From: uji Date: Mon, 2 Oct 2023 00:38:14 +0900 Subject: [PATCH 135/138] main: refactor ParseIndentfierPath() --- identifierPath.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/identifierPath.go b/identifierPath.go index ab9f0fa..d46a506 100644 --- a/identifierPath.go +++ b/identifierPath.go @@ -6,14 +6,10 @@ import ( ) func (p *Parser) ParseIdentifierPath() (ast.IdentifierPath, error) { - id, err := p.lexer.Peek() + id, err := p.ParseIdentifier() if err != nil { return ast.IdentifierPath{}, err } - if !isIdentifier(id) { - return ast.IdentifierPath{}, token.NewPosError(id.Position, "not found identifier.") - } - p.lexer.Scan() var elements []*ast.IdentifierPathElement for { @@ -28,14 +24,10 @@ func (p *Parser) ParseIdentifierPath() (ast.IdentifierPath, error) { Period: &prd.Position, }) - i, err := p.lexer.Peek() + i, err := p.ParseIdentifier() if err != nil { return ast.IdentifierPath{}, err } - if !isIdentifier(i) { - return ast.IdentifierPath{}, token.NewPosError(i.Position, "not found identifier.") - } - p.lexer.Scan() id = i } From c51dea6cd9389c490874d68d49507b01d36ef5d4 Mon Sep 17 00:00:00 2001 From: uji Date: Mon, 2 Oct 2023 00:46:37 +0900 Subject: [PATCH 136/138] main, ast: implement ParseInheritanceSpecifire --- ast/ast.go | 10 +++++----- inheritanceSpecifier.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 inheritanceSpecifier.go diff --git a/ast/ast.go b/ast/ast.go index 118cccf..b210993 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -165,11 +165,6 @@ type FunctionDefinitionReturns struct { RParen token.Pos } -type InheritanceSpecifier struct { - IdentifierPath string - CallArgumentList CallArgumentList -} - // ---------------------------------------------------------------------------- // ContractBodyElement Nodes @@ -190,6 +185,11 @@ func (f *FunctionDefinition) contractBodyElementNode() {} // ---------------------------------------------------------------------------- +type InheritanceSpecifier struct { + IdentifierPath IdentifierPath + CallArgumentList *CallArgumentList +} + type CallArgumentListExpretion struct { Expression Expression Comma *token.Pos diff --git a/inheritanceSpecifier.go b/inheritanceSpecifier.go new file mode 100644 index 0000000..1c0cb24 --- /dev/null +++ b/inheritanceSpecifier.go @@ -0,0 +1,33 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseInheritanceSpecifire() (*ast.InheritanceSpecifier, error) { + ip, err := p.ParseIdentifierPath() + if err != nil { + return nil, err + } + + lparen, err := p.lexer.Peek() + if err != nil { + return nil, err + } + if lparen.Type != token.LParen { + return &ast.InheritanceSpecifier{ + IdentifierPath: ip, + }, nil + } + + cal, err := p.ParseCallArgumentList() + if err != nil { + return nil, err + } + + return &ast.InheritanceSpecifier{ + IdentifierPath: ip, + CallArgumentList: cal, + }, nil +} From 1a58404aac757dc935d23ecbae230c749f2358db Mon Sep 17 00:00:00 2001 From: uji Date: Mon, 2 Oct 2023 00:52:36 +0900 Subject: [PATCH 137/138] ast: move importDirectiveElement interface --- ast/ast.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index b210993..3896ebe 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -12,12 +12,6 @@ type Node interface { End() token.Pos } -// All import-directive-element node (own definition for solparser) types implement the ImportDirectiveElement interface. -type ImportDirectiveElement interface { - Node - importDirectiveElementNode() -} - // All contract-body-element node types implement the ContractBodyElement interface. type ContractBodyElement interface { Node @@ -52,6 +46,12 @@ type Statement interface { // ---------------------------------------------------------------------------- // ImportDirectiveElement Nodes +// All import-directive-element node (own definition for solparser) types implement the ImportDirectiveElement interface. +type ImportDirectiveElement interface { + Node + importDirectiveElementNode() +} + type ImportDirectivePathElement struct { Path Path As *token.Pos From 96bf7e0516b5b60a1e1ca195401d6bb35b5050fe Mon Sep 17 00:00:00 2001 From: uji Date: Mon, 2 Oct 2023 01:12:02 +0900 Subject: [PATCH 138/138] main, ast: fix callArgumentList type definition --- ast/ast.go | 48 +++++++++++++++++++++++++++++++++++++++------ callArgumentList.go | 18 ++++++++--------- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 3896ebe..a58ca8a 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -190,11 +190,34 @@ type InheritanceSpecifier struct { CallArgumentList *CallArgumentList } +// ---------------------------------------------------------------------------- +// ImportDirectiveElement Nodes + +// All call-argument-list-elements node (own definition for solparser) types implement the CallArgumentListElements interface. +type CallArgumentListElements interface { + Node + callArgumentListElementsNode() +} + type CallArgumentListExpretion struct { Expression Expression Comma *token.Pos } +func (c CallArgumentListExpretion) Pos() token.Pos { return c.Expression.Pos() } +func (c CallArgumentListExpretion) End() token.Pos { + if c.Comma != nil { + return *c.Comma + } + return c.Expression.End() +} + +type CallArgumentListExpretions []*CallArgumentListExpretion + +func (c CallArgumentListExpretions) Pos() token.Pos { return c[0].Pos() } +func (c CallArgumentListExpretions) End() token.Pos { return c[len(c)-1].End() } +func (c CallArgumentListExpretions) callArgumentListElementsNode() {} + type CallArgumentListNamedExpretion struct { Identifier Identifier Colon token.Pos @@ -202,17 +225,30 @@ type CallArgumentListNamedExpretion struct { Comma *token.Pos } +func (c CallArgumentListNamedExpretion) Pos() token.Pos { return c.Identifier.Pos() } +func (c CallArgumentListNamedExpretion) End() token.Pos { + if c.Comma != nil { + return *c.Comma + } + return c.Expression.End() +} + type CallArgumentListNamedExpretions struct { - RBrace token.Pos - NamedExpretions []*CallArgumentListNamedExpretion LBrace token.Pos + NamedExpretions []*CallArgumentListNamedExpretion + RBrace token.Pos } +func (c CallArgumentListNamedExpretions) Pos() token.Pos { return c.LBrace } +func (c CallArgumentListNamedExpretions) End() token.Pos { return c.RBrace } +func (c *CallArgumentListNamedExpretions) callArgumentListElementsNode() {} + +// ---------------------------------------------------------------------------- + type CallArgumentList struct { - LParen token.Pos - Expressions []*CallArgumentListExpretion - NamedExpretionsa *CallArgumentListNamedExpretions - RParen token.Pos + LParen token.Pos + Elements CallArgumentListElements + RParen token.Pos } type IdentifierPathElement struct { diff --git a/callArgumentList.go b/callArgumentList.go index 096babc..c538609 100644 --- a/callArgumentList.go +++ b/callArgumentList.go @@ -23,20 +23,19 @@ func (p *Parser) ParseCallArgumentList() (*ast.CallArgumentList, error) { }, nil } - var expretions []*ast.CallArgumentListExpretion - var namedExpretions *ast.CallArgumentListNamedExpretions + var elements ast.CallArgumentListElements if tkn.Type == token.LBrace { es, err := p.ParseCallArgumentListExpretions() if err != nil { return nil, err } - expretions = es + elements = es } else { nes, err := p.ParseCallArgumentListNamedExpretions() if err != nil { return nil, err } - namedExpretions = nes + elements = nes } rparen, err := p.lexer.Scan() @@ -48,20 +47,19 @@ func (p *Parser) ParseCallArgumentList() (*ast.CallArgumentList, error) { } return &ast.CallArgumentList{ - LParen: lparen.Position, - Expressions: expretions, - NamedExpretionsa: namedExpretions, - RParen: rparen.Position, + LParen: lparen.Position, + Elements: elements, + RParen: rparen.Position, }, nil } -func (p *Parser) ParseCallArgumentListExpretions() ([]*ast.CallArgumentListExpretion, error) { +func (p *Parser) ParseCallArgumentListExpretions() (ast.CallArgumentListExpretions, error) { ex, err := p.ParseExpression() if err != nil { return nil, err } - var exs []*ast.CallArgumentListExpretion + var exs ast.CallArgumentListExpretions for { cmm, err := p.lexer.Peek() if err != nil || cmm.Type != token.Comma {