From f627bdaa027edc96fb8ab9c564671eacce64a511 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 12 Nov 2022 21:41:42 +0900 Subject: [PATCH 01/58] .github/workflows: fix test options --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a5082e..66301f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,4 +25,4 @@ jobs: steps: - uses: actions/checkout@v3 - name: test - run: go test ./... -v + run: go test -race -cover -shuffle=on ./... From d86e97a7a2b9020e2a28aa0c6a1f93ec9ebcf1f7 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 12 Nov 2022 22:18:49 +0900 Subject: [PATCH 02/58] .gitignore: init --- .gitignore | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f9a726a --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Temporary file directory +/tmp/ + +# Go workspace file +go.work + +# Dependency directory +/vendor/ From fdb499e5ea0aabc13fc867a0b52699a4f9368cbc Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 12 Nov 2022 22:19:16 +0900 Subject: [PATCH 03/58] Makefile: init, define coverage command. --- Makefile | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6b5d723 --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +.PHONY: help coverage +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +coverage: ## generate test coverage profile to cover.html + go test -cover -coverprofile=cover.out ./... + go tool cover -html=cover.out -o cover.html From 98f035bb4b9598ca86057c0800344ba6abb8f9b8 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 12 Nov 2022 22:28:43 +0900 Subject: [PATCH 04/58] .gitignore: remove tmp directory --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index f9a726a..808b277 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ # Output of the go coverage tool, specifically when used with LiteIDE *.out -# Temporary file directory -/tmp/ +# Test coverage file +cover.html # Go workspace file go.work From 8ab745b8005e9b87dc98b3ad92c7ece76da57484 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 12 Nov 2022 23:07:08 +0900 Subject: [PATCH 05/58] lexer: remove result return value from Lexer.Scan() --- contractDefinition.go | 26 +++++++++++++++++++------- lexer/lexer.go | 31 ++++--------------------------- lexer/lexer_test.go | 38 ++++++++++++++++---------------------- pragmaDirective.go | 32 ++++++++++++++++++++++---------- solparser.go | 10 +++------- 5 files changed, 64 insertions(+), 73 deletions(-) diff --git a/contractDefinition.go b/contractDefinition.go index bad1e2b..af04f35 100644 --- a/contractDefinition.go +++ b/contractDefinition.go @@ -9,8 +9,10 @@ import ( func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { var abstract bool - p.lexer.Scan() - tkn := p.lexer.Token() + tkn, err := p.lexer.Scan() + if err != nil { + return nil, err + } if tkn.TokenType == token.Abstract { abstract = true p.lexer.Scan() @@ -19,17 +21,27 @@ func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { return nil, errors.New("not found contract definition") } - p.lexer.Scan() - if p.lexer.Token().TokenType != token.Unknown { + tkn, err = p.lexer.Scan() + if err != nil { + return nil, err + } + if tkn.TokenType != token.Unknown { return nil, errors.New("not found left brace") } - if p.lexer.Token().TokenType != token.LBrace { + tkn, err = p.lexer.Scan() + if err != nil { + return nil, err + } + if tkn.TokenType != token.LBrace { return nil, errors.New("not found left brace") } - p.lexer.Scan() - if p.lexer.Token().TokenType != token.RBrace { + tkn, err = p.lexer.Scan() + if err != nil { + return nil, err + } + if tkn.TokenType != token.RBrace { return nil, errors.New("not found right brace") } diff --git a/lexer/lexer.go b/lexer/lexer.go index 970a8e6..99869d8 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -10,10 +10,6 @@ import ( type Lexer struct { scanner *scanner.Scanner - // scan result - token token.Token - err error - // peek state peeked bool peekToken token.Token @@ -37,33 +33,14 @@ func (l *Lexer) scan() (result bool, tkn token.Token, err error) { return true, token.NewToken(lit, pos), nil } -func (l *Lexer) Scan() (result bool) { +func (l *Lexer) Scan() (token.Token, error) { if l.peeked { - l.token = l.peekToken - l.err = l.peekErr l.peeked = false - return true - } - - rslt, tkn, err := l.scan() - if err != nil { - l.err = err - return false + return l.peekToken, l.peekErr } - if !rslt { - return false - } - l.token = tkn - - return true -} - -func (l Lexer) Token() token.Token { - return l.token -} -func (l Lexer) Error() error { - return l.err + _, tkn, err := l.scan() + return tkn, err } func (l *Lexer) Peek() bool { diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 06a9e35..5754253 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -12,19 +12,17 @@ import ( func TestLexerScan(t *testing.T) { tests := []struct { - name string - input string - peeked bool - peekToken token.Token - peekErr error - wantResult bool - wantErr error - wantToken token.Token + name string + input string + peeked bool + peekToken token.Token + peekErr error + wantErr error + wantToken token.Token }{ { - name: "normal", - input: "pragma", - wantResult: true, + name: "normal", + input: "pragma", wantToken: token.Token{ TokenType: token.Pragma, Text: "pragma", @@ -35,11 +33,10 @@ func TestLexerScan(t *testing.T) { }, }, { - name: "when scan is done", - input: "", - wantResult: false, - wantToken: token.Token{}, - wantErr: io.EOF, + name: "when scan is done", + input: "", + wantToken: token.Token{}, + wantErr: io.EOF, }, { name: "when peeked", @@ -53,7 +50,6 @@ func TestLexerScan(t *testing.T) { Line: 6, }, }, - wantResult: true, wantToken: token.Token{ TokenType: token.BitXor, Text: "^", @@ -75,13 +71,11 @@ func TestLexerScan(t *testing.T) { peekToken: tt.peekToken, peekErr: tt.peekErr, } - if rslt := l.Scan(); rslt != tt.wantResult { - t.Errorf("result is wrong, want: %t, got: %t", tt.wantResult, rslt) - } - if err := l.Error(); err != tt.wantErr { + tkn, err := l.Scan() + if err != tt.wantErr { t.Errorf("error is wrong, want: %s, got: %s", tt.wantErr, err) } - if diff := cmp.Diff(tt.wantToken, l.Token()); diff != "" { + if diff := cmp.Diff(tt.wantToken, tkn); diff != "" { t.Errorf(diff) } }) diff --git a/pragmaDirective.go b/pragmaDirective.go index 2a9b689..4cfb897 100644 --- a/pragmaDirective.go +++ b/pragmaDirective.go @@ -6,31 +6,43 @@ import ( ) func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { - p.lexer.Scan() - if tkn := p.lexer.Token(); tkn.TokenType != token.Pragma { + tkn, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if tkn.TokenType != token.Pragma { return nil, newError(tkn.Pos, "not found pragma") } - p.lexer.Scan() - pragmaName := p.lexer.Token() + pragmaName, err := p.lexer.Scan() + if err != nil { + return nil, err + } if pragmaName.TokenType != token.Unknown { return nil, newError(pragmaName.Pos, "not found pragma name") } - p.lexer.Scan() - expression := p.lexer.Token() + expression, err := p.lexer.Scan() + if err != nil { + return nil, err + } if expression.TokenType != token.BitXor { return nil, newError(expression.Pos, "not found BitXor expression") } - p.lexer.Scan() - version := p.lexer.Token() + version, err := p.lexer.Scan() + if err != nil { + return nil, err + } if version.TokenType != token.Unknown { return nil, newError(version.Pos, "not found version") } - p.lexer.Scan() - if tkn := p.lexer.Token(); tkn.TokenType != token.Semicolon { + tkn, err = p.lexer.Scan() + if err != nil { + return nil, err + } + if tkn.TokenType != token.Semicolon { return nil, newError(tkn.Pos, "not found Semicolon") } diff --git a/solparser.go b/solparser.go index 680742b..294f844 100644 --- a/solparser.go +++ b/solparser.go @@ -62,20 +62,16 @@ func (p *Parser) Parse(input io.Reader) (*ast.SourceUnit, error) { } func (p *Parser) ParseBooleanLiteral() (*ast.BooleanLiteral, error) { - if !p.lexer.Scan() { - return nil, nil - } - - if err := p.lexer.Error(); err != nil { + tkn, err := p.lexer.Scan() + if err != nil { return nil, err } - tkn := p.lexer.Token() if tkn.TokenType != token.True && tkn.TokenType != token.False { return nil, newError(tkn.Pos, "not found keyword true or false") } return &ast.BooleanLiteral{ - Token: p.lexer.Token(), + Token: tkn, }, nil } From 1a29decafe9a9f7349dd05e43b0e0ef1f822e74c Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 12 Nov 2022 23:20:11 +0900 Subject: [PATCH 06/58] lexer: remove result return value from Lexer.Peek() --- lexer/lexer.go | 20 ++++---------------- lexer/lexer_test.go | 30 +++++++++++++----------------- solparser.go | 8 ++++---- 3 files changed, 21 insertions(+), 37 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 99869d8..8873484 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -43,26 +43,14 @@ func (l *Lexer) Scan() (token.Token, error) { return tkn, err } -func (l *Lexer) Peek() bool { +func (l *Lexer) Peek() (token.Token, error) { if l.peeked { - return true - } - - rslt, tkn, err := l.scan() - if !rslt { - return false + return l.peekToken, l.peekErr } + _, tkn, err := l.scan() l.peeked = true l.peekToken = tkn l.peekErr = err - return true -} - -func (l Lexer) PeekToken() token.Token { - return l.peekToken -} - -func (l Lexer) PeekError() error { - return l.peekErr + return l.peekToken, l.peekErr } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 5754253..a4e2a97 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -84,16 +84,14 @@ func TestLexerScan(t *testing.T) { func TestLexerPeek(t *testing.T) { tests := []struct { - name string - input string - result bool - token token.Token - err error + name string + input string + token token.Token + err error }{ { - name: "normal", - input: "pragma", - result: true, + name: "normal", + input: "pragma", token: token.Token{ TokenType: token.Pragma, Text: "pragma", @@ -104,10 +102,10 @@ func TestLexerPeek(t *testing.T) { }, }, { - name: "when scan is done", - input: "", - result: false, - token: token.Token{}, + name: "when scan is done", + input: "", + token: token.Token{}, + err: io.EOF, }, } for _, tt := range tests { @@ -118,13 +116,11 @@ func TestLexerPeek(t *testing.T) { l := Lexer{ scanner: s, } - if rslt := l.Peek(); tt.result != rslt { - t.Errorf("result is wrong, want: %t, got: %t", tt.result, rslt) - } - if err := l.PeekError(); err != tt.err { + tkn, err := l.Peek() + if err != tt.err { t.Errorf("want: %s, got: %s", tt.err, err) } - if diff := cmp.Diff(tt.token, l.PeekToken()); diff != "" { + if diff := cmp.Diff(tt.token, tkn); diff != "" { t.Errorf(diff) } }) diff --git a/solparser.go b/solparser.go index 294f844..4d51f67 100644 --- a/solparser.go +++ b/solparser.go @@ -21,11 +21,11 @@ func New(input io.Reader) *Parser { } func (p *Parser) Parse(input io.Reader) (*ast.SourceUnit, error) { - if !p.lexer.Peek() { + tkn, err := p.lexer.Peek() + if err == io.EOF { return nil, nil } - - if err := p.lexer.PeekError(); err != nil { + if err != nil { return nil, err } @@ -33,7 +33,7 @@ func (p *Parser) Parse(input io.Reader) (*ast.SourceUnit, error) { var contractDefinition *ast.ContractDefinition var functionDefinition *ast.FunctionDefinition - switch p.lexer.PeekToken().TokenType { + switch tkn.TokenType { case token.Pragma: prgm, err := p.ParsePragmaDirective() if err != nil { From c02cc7bd1bc35d0aeed3a8b3942187c4e121b60d Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 12 Nov 2022 23:21:42 +0900 Subject: [PATCH 07/58] lexer: remove result return value from Lexer.scan() --- lexer/lexer.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 8873484..a862b32 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -24,13 +24,13 @@ func New(input io.Reader) *Lexer { } } -func (l *Lexer) scan() (result bool, tkn token.Token, err error) { +func (l *Lexer) scan() (tkn token.Token, err error) { pos, lit, err := l.scanner.Scan() if err != nil { - return false, token.Token{}, err + return token.Token{}, err } - return true, token.NewToken(lit, pos), nil + return token.NewToken(lit, pos), nil } func (l *Lexer) Scan() (token.Token, error) { @@ -39,8 +39,7 @@ func (l *Lexer) Scan() (token.Token, error) { return l.peekToken, l.peekErr } - _, tkn, err := l.scan() - return tkn, err + return l.scan() } func (l *Lexer) Peek() (token.Token, error) { @@ -48,7 +47,7 @@ func (l *Lexer) Peek() (token.Token, error) { return l.peekToken, l.peekErr } - _, tkn, err := l.scan() + tkn, err := l.scan() l.peeked = true l.peekToken = tkn l.peekErr = err From 9aa28c4309037d13f868087888e4d75fb70d0cda Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 13 Nov 2022 11:50:07 +0900 Subject: [PATCH 08/58] lexer: refactor lexer test case --- lexer/lexer_test.go | 84 ++++++++++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index a4e2a97..0f3a030 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -10,14 +10,10 @@ import ( "github.com/uji/solparser/token" ) -func TestLexerScan(t *testing.T) { +func TestLexer_Scan(t *testing.T) { tests := []struct { name string input string - peeked bool - peekToken token.Token - peekErr error - wantErr error wantToken token.Token }{ { @@ -32,57 +28,65 @@ func TestLexerScan(t *testing.T) { }, }, }, - { - name: "when scan is done", - input: "", - wantToken: token.Token{}, - wantErr: io.EOF, - }, - { - name: "when peeked", - input: "", - peeked: true, - peekToken: token.Token{ - TokenType: token.BitXor, - Text: "^", - Pos: token.Pos{ - Column: 4, - Line: 6, - }, - }, - wantToken: token.Token{ - TokenType: token.BitXor, - Text: "^", - Pos: token.Pos{ - Column: 4, - Line: 6, - }, - }, - }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { s := scanner.New(strings.NewReader(tt.input)) - l := Lexer{ - scanner: s, - peeked: tt.peeked, - peekToken: tt.peekToken, - peekErr: tt.peekErr, + scanner: s, } + tkn, err := l.Scan() - if err != tt.wantErr { - t.Errorf("error is wrong, want: %s, got: %s", tt.wantErr, err) + if err != nil { + t.Errorf("error is not nil, got: %s", err) } if diff := cmp.Diff(tt.wantToken, tkn); diff != "" { t.Errorf(diff) } }) } + + t.Run("when scan is done", func(t *testing.T) { + s := scanner.New(strings.NewReader("")) + l := Lexer{ + scanner: s, + } + + _, err := l.Scan() + if err != io.EOF { + t.Errorf("error is not io.EOF, got: %s", err) + } + }) + + t.Run("when peeked", func(t *testing.T) { + peekToken := token.Token{ + TokenType: token.BitXor, + Text: "^", + Pos: token.Pos{ + Column: 4, + Line: 6, + }, + } + s := scanner.New(strings.NewReader("")) + l := Lexer{ + scanner: s, + peeked: true, + peekToken: peekToken, + peekErr: nil, + } + + tkn, err := l.Scan() + if err != nil { + t.Errorf("error is not nil, got: %s", err) + } + if diff := cmp.Diff(peekToken, tkn); diff != "" { + t.Errorf(diff) + } + }) } -func TestLexerPeek(t *testing.T) { +func TestLexer_Peek(t *testing.T) { tests := []struct { name string input string From ea0a72ccb6af330a18a9dc0a273be3be6ddb3721 Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 16 Nov 2022 22:23:15 +0900 Subject: [PATCH 09/58] token: add literals to token list --- token/token.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/token/token.go b/token/token.go index 83d1ff9..03ba866 100644 --- a/token/token.go +++ b/token/token.go @@ -159,6 +159,15 @@ const ( View Virtual While + + // Literal + TrueLiteral + FalseLiteral + Number + StringLiteral + UnicodeStringLiteral + HexStringLiteral + CommentLiteral ) func asKeyword(str string) TokenType { From 252628899d9c2772b69ab3da5349aadcb102c90d Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 16 Nov 2022 22:38:37 +0900 Subject: [PATCH 10/58] scanner: do not ignore space literal --- scanner/scanner.go | 45 ++++++++++++++++------------------------- scanner/scanner_test.go | 15 ++++++++++++++ 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/scanner/scanner.go b/scanner/scanner.go index 589e414..a32ff65 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -81,43 +81,32 @@ func (s *Scanner) readRune() (rune, error) { } func (s *Scanner) Scan() (token.Pos, string, error) { - runes := make([]rune, 1) - for { - // Retain the state before reading. - c := s.offset + 1 - l := s.lineOffset + 1 - - ch, err := s.readRune() - if err != nil { - return token.Pos{}, "", err - } - if ch == bufrr.EOF { - return token.Pos{}, "", io.EOF - } - if isSplitSymbol(ch) || ch == '\n' { - return token.Pos{ - Column: c, - Line: l, - }, string(ch), nil - } - if !isSpace(ch) { - runes[0] = ch - break - } + startPos := token.Pos{ + Column: s.offset + 1, + Line: s.lineOffset + 1, } - pos := token.Pos{ - Column: s.offset, - Line: s.lineOffset + 1, + ch, err := s.readRune() + if err != nil { + return token.Pos{}, "", err } + if ch == bufrr.EOF { + return token.Pos{}, "", io.EOF + } + if isSplitSymbol(ch) || ch == '\n' { + return startPos, string(ch), nil + } + + readingSpace := isSpace(ch) + runes := []rune{ch} for { ch, _, err := s.r.PeekRune() if err != nil { return token.Pos{}, "", err } - if isSpace(ch) || isSplitSymbol(ch) || ch == '\n' || ch == bufrr.EOF { - return pos, string(runes), nil + if isSpace(ch) != readingSpace || isSplitSymbol(ch) || ch == '\n' || ch == bufrr.EOF { + return startPos, string(runes), nil } ch, err = s.readRune() if err != nil { diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index c270afc..b093403 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -24,10 +24,18 @@ func TestScannerScan(t *testing.T) { Column: 1, Line: 1, }, + { + Column: 7, + Line: 1, + }, { Column: 8, Line: 1, }, + { + Column: 16, + Line: 1, + }, { Column: 17, Line: 1, @@ -43,7 +51,9 @@ func TestScannerScan(t *testing.T) { }, wantLits: []string{ "pragma", + " ", "solidity", + " ", "^", "0.8.13", ";", @@ -61,6 +71,10 @@ func TestScannerScan(t *testing.T) { Column: 7, Line: 1, }, + { + Column: 1, + Line: 2, + }, { Column: 2, Line: 2, @@ -69,6 +83,7 @@ func TestScannerScan(t *testing.T) { wantLits: []string{ "pragma", "\n", + " ", "solidity", }, }, From 533a93578d7b25c9846548aacc8e2a8c1a46cb87 Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 16 Nov 2022 22:43:51 +0900 Subject: [PATCH 11/58] lexer: implement ScanStringLiteral() scanning double-quoted-printable string literal --- lexer/lexer.go | 27 +++++++++++++++++++++++++++ lexer/lexer_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/lexer/lexer.go b/lexer/lexer.go index a862b32..62a2eff 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -1,6 +1,7 @@ package lexer import ( + "errors" "io" "github.com/uji/solparser/scanner" @@ -53,3 +54,29 @@ func (l *Lexer) Peek() (token.Token, error) { l.peekErr = err return l.peekToken, l.peekErr } + +func (l *Lexer) ScanStringLiteral() (token.Token, error) { + firstPos, lit, err := l.scanner.Scan() + if err != nil { + return token.Token{}, err + } + if lit != `"` { + return token.Token{}, errors.New("test") // TODO: include Pos in error + } + + txt := lit + for { + _, lit, err := l.scanner.Scan() + if err != nil { + return token.Token{}, err + } + txt = txt + lit + if lit == `"` { + return token.Token{ + TokenType: token.StringLiteral, + Text: txt, + Pos: firstPos, + }, nil + } + } +} diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 0f3a030..1dd7492 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -130,3 +130,34 @@ func TestLexer_Peek(t *testing.T) { }) } } + +func TestLexer_ScanStringLiteral(t *testing.T) { + tests := []struct { + name string + input string + wantText string + }{ + { + name: "double-quoted-printable", + input: `"Hello world!!";`, + wantText: `"Hello world!!"`, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + s := scanner.New(strings.NewReader(tt.input)) + l := Lexer{ + scanner: s, + } + + tkn, err := l.ScanStringLiteral() + if err != nil { + t.Errorf("error is not nil, got: %s", err) + } + if diff := cmp.Diff(tt.wantText, tkn.Text); diff != "" { + t.Errorf(diff) + } + }) + } +} From c2f9520d6ddf593e72c816507b017d8fbdbc112c Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 16 Nov 2022 23:06:37 +0900 Subject: [PATCH 12/58] main, token: move Pos Error from main package to token package --- error.go | 26 -------------------------- pragmaDirective.go | 10 +++++----- pragmaDirective_test.go | 12 ++++++------ solparser.go | 2 +- solparser_test.go | 6 +++--- token/error.go | 22 ++++++++++++++++++++++ error_test.go => token/error_test.go | 11 +++++------ 7 files changed, 42 insertions(+), 47 deletions(-) delete mode 100644 error.go create mode 100644 token/error.go rename error_test.go => token/error_test.go (79%) diff --git a/error.go b/error.go deleted file mode 100644 index 9fa3337..0000000 --- a/error.go +++ /dev/null @@ -1,26 +0,0 @@ -package solparser - -import ( - "github.com/uji/solparser/token" -) - -type Error struct { - Pos token.Pos - Msg string -} - -var _ error = &Error{} - -func newError(pos token.Pos, msg string) *Error { - return &Error{ - Pos: pos, - Msg: msg, - } -} - -func (e *Error) Error() string { - if e.Pos.IsValid() { - return e.Pos.String() + ": " + e.Msg - } - return e.Msg -} diff --git a/pragmaDirective.go b/pragmaDirective.go index 4cfb897..7a3b76a 100644 --- a/pragmaDirective.go +++ b/pragmaDirective.go @@ -11,7 +11,7 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { return nil, err } if tkn.TokenType != token.Pragma { - return nil, newError(tkn.Pos, "not found pragma") + return nil, token.NewPosError(tkn.Pos, "not found pragma") } pragmaName, err := p.lexer.Scan() @@ -19,7 +19,7 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { return nil, err } if pragmaName.TokenType != token.Unknown { - return nil, newError(pragmaName.Pos, "not found pragma name") + return nil, token.NewPosError(pragmaName.Pos, "not found pragma name") } expression, err := p.lexer.Scan() @@ -27,7 +27,7 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { return nil, err } if expression.TokenType != token.BitXor { - return nil, newError(expression.Pos, "not found BitXor expression") + return nil, token.NewPosError(expression.Pos, "not found BitXor expression") } version, err := p.lexer.Scan() @@ -35,7 +35,7 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { return nil, err } if version.TokenType != token.Unknown { - return nil, newError(version.Pos, "not found version") + return nil, token.NewPosError(version.Pos, "not found version") } tkn, err = p.lexer.Scan() @@ -43,7 +43,7 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { return nil, err } if tkn.TokenType != token.Semicolon { - return nil, newError(tkn.Pos, "not found Semicolon") + return nil, token.NewPosError(tkn.Pos, "not found Semicolon") } return &ast.PragmaDirective{ diff --git a/pragmaDirective_test.go b/pragmaDirective_test.go index 08b38b0..a1ad8a7 100644 --- a/pragmaDirective_test.go +++ b/pragmaDirective_test.go @@ -16,7 +16,7 @@ func TestParserParsePragmaDirective(t *testing.T) { name string input string want *ast.PragmaDirective - err *solparser.Error + err *token.PosError }{ { name: "normal case", @@ -34,7 +34,7 @@ func TestParserParsePragmaDirective(t *testing.T) { name: "there is no pragma keyword", input: "solidity ^0.8.13;", want: nil, - err: &solparser.Error{ + err: &token.PosError{ Pos: token.Pos{ Column: 1, Line: 1, @@ -46,7 +46,7 @@ func TestParserParsePragmaDirective(t *testing.T) { name: "there is no pragma name", input: "pragma ^0.8.13;", want: nil, - err: &solparser.Error{ + err: &token.PosError{ Pos: token.Pos{ Column: 8, Line: 1, @@ -58,7 +58,7 @@ func TestParserParsePragmaDirective(t *testing.T) { name: "there is no BitXor expression", input: "pragma solidity 0.8.13;", want: nil, - err: &solparser.Error{ + err: &token.PosError{ Pos: token.Pos{ Column: 17, Line: 1, @@ -70,7 +70,7 @@ func TestParserParsePragmaDirective(t *testing.T) { name: "there is no Semicolon", input: "pragma solidity ^0.8.13", want: nil, - err: &solparser.Error{ + err: &token.PosError{ Pos: token.Pos{ Column: 18, Line: 1, @@ -87,7 +87,7 @@ func TestParserParsePragmaDirective(t *testing.T) { got, err := p.ParsePragmaDirective() - var sErr *solparser.Error + var sErr *token.PosError if errors.As(err, &sErr) { if diff := cmp.Diff(tt.err, sErr); diff != "" { t.Errorf("%s", diff) diff --git a/solparser.go b/solparser.go index 4d51f67..052db18 100644 --- a/solparser.go +++ b/solparser.go @@ -68,7 +68,7 @@ func (p *Parser) ParseBooleanLiteral() (*ast.BooleanLiteral, error) { } if tkn.TokenType != token.True && tkn.TokenType != token.False { - return nil, newError(tkn.Pos, "not found keyword true or false") + return nil, token.NewPosError(tkn.Pos, "not found keyword true or false") } return &ast.BooleanLiteral{ diff --git a/solparser_test.go b/solparser_test.go index ac14890..e944bd5 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -16,7 +16,7 @@ func TestParserParseBooleanLiteral(t *testing.T) { name string input string want *ast.BooleanLiteral - err *solparser.Error + err *token.PosError }{ { name: "true case", @@ -52,7 +52,7 @@ func TestParserParseBooleanLiteral(t *testing.T) { name: "not true or false", input: "solidity", want: nil, - err: &solparser.Error{ + err: &token.PosError{ Pos: token.Pos{ Column: 1, Line: 1, @@ -69,7 +69,7 @@ func TestParserParseBooleanLiteral(t *testing.T) { got, err := p.ParseBooleanLiteral() - var sErr *solparser.Error + var sErr *token.PosError if errors.As(err, &sErr) { if diff := cmp.Diff(tt.err, sErr); diff != "" { t.Errorf("%s", diff) diff --git a/token/error.go b/token/error.go new file mode 100644 index 0000000..9994b95 --- /dev/null +++ b/token/error.go @@ -0,0 +1,22 @@ +package token + +type PosError struct { + Pos Pos + Msg string +} + +var _ error = &PosError{} + +func NewPosError(pos Pos, msg string) *PosError { + return &PosError{ + Pos: pos, + Msg: msg, + } +} + +func (e *PosError) Error() string { + if e.Pos.IsValid() { + return e.Pos.String() + ": " + e.Msg + } + return e.Msg +} diff --git a/error_test.go b/token/error_test.go similarity index 79% rename from error_test.go rename to token/error_test.go index 1d31ca9..5da487d 100644 --- a/error_test.go +++ b/token/error_test.go @@ -1,21 +1,20 @@ -package solparser_test +package token_test import ( "testing" - "github.com/uji/solparser" "github.com/uji/solparser/token" ) -func TestErrorError(t *testing.T) { +func TestError_PosError(t *testing.T) { tests := []struct { name string - err solparser.Error + err token.PosError want string }{ { name: "normal case", - err: solparser.Error{ + err: token.PosError{ Pos: token.Pos{ Column: 1, Line: 1, @@ -26,7 +25,7 @@ func TestErrorError(t *testing.T) { }, { name: "invalid position case", - err: solparser.Error{ + err: token.PosError{ Pos: token.Pos{ Column: 0, Line: 1, From 1c496621362b6f3041a3b59793dfbc8f462eb417 Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 16 Nov 2022 23:08:07 +0900 Subject: [PATCH 13/58] lexer: correspond to String Literal containing a space --- lexer/lexer.go | 3 +-- lexer/lexer_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 62a2eff..386d9d2 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -1,7 +1,6 @@ package lexer import ( - "errors" "io" "github.com/uji/solparser/scanner" @@ -61,7 +60,7 @@ func (l *Lexer) ScanStringLiteral() (token.Token, error) { return token.Token{}, err } if lit != `"` { - return token.Token{}, errors.New("test") // TODO: include Pos in error + return token.Token{}, token.NewPosError(firstPos, `not found "`) } txt := lit diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 1dd7492..86d15bb 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -1,6 +1,7 @@ package lexer import ( + "errors" "io" "strings" "testing" @@ -160,4 +161,27 @@ func TestLexer_ScanStringLiteral(t *testing.T) { } }) } + + t.Run(`not found "`, func(t *testing.T) { + s := scanner.New(strings.NewReader("a")) + l := Lexer{ + scanner: s, + } + exptErr := &token.PosError{ + Pos: token.Pos{ + Column: 1, + Line: 1, + }, + Msg: `not found "`, + } + + _, err := l.ScanStringLiteral() + var pErr *token.PosError + if !errors.As(err, &pErr) { + t.Errorf("error is unexpected, got: %s", err) + } + if diff := cmp.Diff(pErr, exptErr); diff != "" { + t.Errorf(diff) + } + }) } From fd25c4060bb619079daec1504eadeafc0a42a175 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 4 Feb 2023 12:47:03 +0900 Subject: [PATCH 14/58] lexer, scanner: Ignore spaces in lexer.scan() --- lexer/lexer.go | 4 ++++ lexer/lexer_test.go | 12 ++++++++++++ scanner/scanner.go | 8 ++++---- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 386d9d2..5298762 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -30,6 +30,10 @@ func (l *Lexer) scan() (tkn token.Token, err error) { return token.Token{}, err } + if scanner.IsSpace([]rune(lit)[0]) { + return l.scan() + } + return token.NewToken(lit, pos), nil } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 86d15bb..aa11632 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -29,6 +29,18 @@ func TestLexer_Scan(t *testing.T) { }, }, }, + { + name: "There is a space at the beginning", + input: " pragma", + wantToken: token.Token{ + TokenType: token.Pragma, + Text: "pragma", + Pos: token.Pos{ + Column: 2, + Line: 1, + }, + }, + }, } for _, tt := range tests { tt := tt diff --git a/scanner/scanner.go b/scanner/scanner.go index a32ff65..4a70288 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -23,10 +23,10 @@ func New(reader io.Reader) *Scanner { } } -// isSpace reports whether the character is a Unicode white space character. +// IsSpace reports whether the character is a Unicode white space character. // We avoid dependency on the unicode package, but check validity of the implementation // in the tests. -func isSpace(r rune) bool { +func IsSpace(r rune) bool { if r <= '\u00FF' { // Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs. switch r { @@ -97,7 +97,7 @@ func (s *Scanner) Scan() (token.Pos, string, error) { return startPos, string(ch), nil } - readingSpace := isSpace(ch) + readingSpace := IsSpace(ch) runes := []rune{ch} for { @@ -105,7 +105,7 @@ func (s *Scanner) Scan() (token.Pos, string, error) { if err != nil { return token.Pos{}, "", err } - if isSpace(ch) != readingSpace || isSplitSymbol(ch) || ch == '\n' || ch == bufrr.EOF { + if IsSpace(ch) != readingSpace || isSplitSymbol(ch) || ch == '\n' || ch == bufrr.EOF { return startPos, string(runes), nil } ch, err = s.readRune() From 97cb8b290f90f7ce121a472833fd763f1f6451f2 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 4 Feb 2023 14:23:30 +0900 Subject: [PATCH 15/58] all: move IsSpace() from scanner package to token package --- lexer/lexer.go | 2 +- scanner/scanner.go | 29 ++--------------------------- token/token.go | 25 +++++++++++++++++++++++++ 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 5298762..c210d90 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -30,7 +30,7 @@ func (l *Lexer) scan() (tkn token.Token, err error) { return token.Token{}, err } - if scanner.IsSpace([]rune(lit)[0]) { + if token.IsSpace([]rune(lit)[0]) { return l.scan() } diff --git a/scanner/scanner.go b/scanner/scanner.go index 4a70288..a35fc15 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -23,31 +23,6 @@ func New(reader io.Reader) *Scanner { } } -// IsSpace reports whether the character is a Unicode white space character. -// We avoid dependency on the unicode package, but check validity of the implementation -// in the tests. -func IsSpace(r rune) bool { - if r <= '\u00FF' { - // Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs. - switch r { - case ' ', '\t', '\v', '\f', '\r': - return true - case '\u0085', '\u00A0': - return true - } - return false - } - // High-valued ones. - if '\u2000' <= r && r <= '\u200a' { - return true - } - switch r { - case '\u1680', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000': - return true - } - return false -} - func isMultiLengthOperatorSymbol(r rune) bool { switch r { case '=', '|', '^', '&', '+', '-', '*', '/', '%', '<', '>', '!': @@ -97,7 +72,7 @@ func (s *Scanner) Scan() (token.Pos, string, error) { return startPos, string(ch), nil } - readingSpace := IsSpace(ch) + readingSpace := token.IsSpace(ch) runes := []rune{ch} for { @@ -105,7 +80,7 @@ func (s *Scanner) Scan() (token.Pos, string, error) { if err != nil { return token.Pos{}, "", err } - if IsSpace(ch) != readingSpace || isSplitSymbol(ch) || ch == '\n' || ch == bufrr.EOF { + if token.IsSpace(ch) != readingSpace || isSplitSymbol(ch) || ch == bufrr.EOF { return startPos, string(runes), nil } ch, err = s.readRune() diff --git a/token/token.go b/token/token.go index 03ba866..fb32cae 100644 --- a/token/token.go +++ b/token/token.go @@ -497,3 +497,28 @@ func (p Pos) String() string { return fmt.Sprintf("%d:%d", p.Line, p.Column) } + +// IsSpace reports whether the character is a Unicode white space character. +// We avoid dependency on the unicode package, but check validity of the implementation +// in the tests. +func IsSpace(r rune) bool { + if r <= '\u00FF' { + // Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs. + switch r { + case ' ', '\n', '\t', '\v', '\f', '\r': + return true + case '\u0085', '\u00A0': + return true + } + return false + } + // High-valued ones. + if '\u2000' <= r && r <= '\u200a' { + return true + } + switch r { + case '\u1680', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000': + return true + } + return false +} From 523ffbffc0b9a4da5a9358a79761f591aa0900dd Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 4 Feb 2023 22:43:30 +0900 Subject: [PATCH 16/58] lexer: fix string literal scan description --- lexer/lexer.go | 11 +++++++---- lexer/lexer_test.go | 8 +++++++- scanner/scanner.go | 2 +- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index c210d90..e085623 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -1,6 +1,7 @@ package lexer import ( + "fmt" "io" "github.com/uji/solparser/scanner" @@ -58,23 +59,25 @@ func (l *Lexer) Peek() (token.Token, error) { return l.peekToken, l.peekErr } +// ScanStringLiteral parse NonEmptyStringLiteral or EmptyStringLiteral then return StringLiteral token. func (l *Lexer) ScanStringLiteral() (token.Token, error) { firstPos, lit, err := l.scanner.Scan() if err != nil { return token.Token{}, err } - if lit != `"` { - return token.Token{}, token.NewPosError(firstPos, `not found "`) + fmt.Println(firstPos, lit) + if lit != `"` && lit != `\'` { + return token.Token{}, token.NewPosError(firstPos, `not found " or \'`) } - txt := lit + quote, txt := lit, lit for { _, lit, err := l.scanner.Scan() if err != nil { return token.Token{}, err } txt = txt + lit - if lit == `"` { + if lit == quote { return token.Token{ TokenType: token.StringLiteral, Text: txt, diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index aa11632..3e22e75 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -155,6 +155,12 @@ func TestLexer_ScanStringLiteral(t *testing.T) { input: `"Hello world!!";`, wantText: `"Hello world!!"`, }, + // TODO + // { + // name: "single-quoted-printable", + // input: `\'Hello world!!\';`, + // wantText: `\'Hello world!!\'`, + // }, } for _, tt := range tests { tt := tt @@ -184,7 +190,7 @@ func TestLexer_ScanStringLiteral(t *testing.T) { Column: 1, Line: 1, }, - Msg: `not found "`, + Msg: `not found " or \'`, } _, err := l.ScanStringLiteral() diff --git a/scanner/scanner.go b/scanner/scanner.go index a35fc15..c31fcf2 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -25,7 +25,7 @@ func New(reader io.Reader) *Scanner { func isMultiLengthOperatorSymbol(r rune) bool { switch r { - case '=', '|', '^', '&', '+', '-', '*', '/', '%', '<', '>', '!': + case '=', '|', '^', '&', '+', '-', '*', '/', '\\', '%', '<', '>', '!': return true } return false From 8bf6c037a75c84e747aa41510fc8be517fe25668 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 5 Feb 2023 15:19:18 +0900 Subject: [PATCH 17/58] scanner: scan multi length operator --- lexer/lexer_test.go | 11 +++--- scanner/scanner.go | 80 ++++++++++++++++++++++++++++++++++++++-- scanner/scanner_test.go | 81 +++++++++++++++++------------------------ token/token.go | 2 +- 4 files changed, 117 insertions(+), 57 deletions(-) diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 3e22e75..72290d6 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -155,12 +155,11 @@ func TestLexer_ScanStringLiteral(t *testing.T) { input: `"Hello world!!";`, wantText: `"Hello world!!"`, }, - // TODO - // { - // name: "single-quoted-printable", - // input: `\'Hello world!!\';`, - // wantText: `\'Hello world!!\'`, - // }, + { + name: "single-quoted-printable", + input: `\'Hello world!!\';`, + wantText: `\'Hello world!!\'`, + }, } for _, tt := range tests { tt := tt diff --git a/scanner/scanner.go b/scanner/scanner.go index c31fcf2..364121d 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -1,6 +1,7 @@ package scanner import ( + "errors" "io" "github.com/SteelSeries/bufrr" @@ -55,23 +56,96 @@ func (s *Scanner) readRune() (rune, error) { return r, nil } +func (s *Scanner) scanOperator() (string, error) { + ch, _, err := s.r.PeekRune() + if err != nil { + return "", err + } + if !isSplitSymbol(ch) { + return "", errors.New("Not operator") + } + s.readRune() + if !isMultiLengthOperatorSymbol(ch) { + return string(ch), nil + } + ch2, _, err := s.r.PeekRune() + if err != nil { + return string(ch), nil + } + oprt := string([]rune{ch, ch2}) + switch oprt { + case "=>", "->", "|=", "^=", "&=", "+=", "-=", "*=", "/=", "%=", "==", "||", "&&", "**", "!=", "<-", ">-", "++", "--", `\'`: + s.readRune() + return oprt, nil + case "<<": + s.readRune() + ch, _, err := s.r.PeekRune() + if err != nil { + return oprt, nil + } + if ch == '=' { + return "<<=", nil + } + return oprt, nil + case ">>": + s.readRune() + ch3, _, err := s.r.PeekRune() + if err != nil { + return oprt, nil + } + if ch3 == '=' { + return ">>=", nil + } + if ch3 == '>' { + s.readRune() + oprt = ">>>" + ch4, _, err := s.r.PeekRune() + if err != nil { + return oprt, nil + } + if ch4 == '=' { + s.readRune() + return ">>>=", nil + } + return oprt, nil + } + return oprt, nil + } + return string(ch), nil +} + +// Scan divides a string into the smallest units from the bufrr.Reader and returns them one by one. +// +// - If first rune is an error or EOF, return an error and exit. +// - If first rune is operator character, scan operator. +// - If first rune is a space character, scan until the end of the blank character. +// - Else scan to next space or operator string. func (s *Scanner) Scan() (token.Pos, string, error) { startPos := token.Pos{ Column: s.offset + 1, Line: s.lineOffset + 1, } - ch, err := s.readRune() + ch, _, err := s.r.PeekRune() if err != nil { + s.readRune() return token.Pos{}, "", err } if ch == bufrr.EOF { + s.readRune() return token.Pos{}, "", io.EOF } - if isSplitSymbol(ch) || ch == '\n' { - return startPos, string(ch), nil + if isSplitSymbol(ch) { + // scan operator. + oprt, err := s.scanOperator() + if err != nil { + return token.Pos{}, "", err + } + return startPos, oprt, nil } + s.readRune() + readingSpace := token.IsSpace(ch) runes := []rune{ch} diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index b093403..2cc9ca8 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -20,34 +20,13 @@ func TestScannerScan(t *testing.T) { name: "normal", input: "pragma solidity ^0.8.13;", wantPoss: []token.Pos{ - { - Column: 1, - Line: 1, - }, - { - Column: 7, - Line: 1, - }, - { - Column: 8, - Line: 1, - }, - { - Column: 16, - Line: 1, - }, - { - Column: 17, - Line: 1, - }, - { - Column: 18, - Line: 1, - }, - { - Column: 24, - Line: 1, - }, + {Column: 1, Line: 1}, + {Column: 7, Line: 1}, + {Column: 8, Line: 1}, + {Column: 16, Line: 1}, + {Column: 17, Line: 1}, + {Column: 18, Line: 1}, + {Column: 24, Line: 1}, }, wantLits: []string{ "pragma", @@ -60,33 +39,41 @@ func TestScannerScan(t *testing.T) { }, }, { - name: "normal", + name: "there is newline character", input: "pragma\n solidity", wantPoss: []token.Pos{ - { - Column: 1, - Line: 1, - }, - { - Column: 7, - Line: 1, - }, - { - Column: 1, - Line: 2, - }, - { - Column: 2, - Line: 2, - }, + {Column: 1, Line: 1}, + {Column: 7, Line: 1}, + {Column: 2, Line: 2}, }, wantLits: []string{ "pragma", - "\n", - " ", + "\n ", "solidity", }, }, + { + name: "there are multi length operators", + input: `a >> \'test\'`, + wantPoss: []token.Pos{ + {Column: 1, Line: 1}, + {Column: 2, Line: 1}, + {Column: 3, Line: 1}, + {Column: 5, Line: 1}, + {Column: 6, Line: 1}, + {Column: 8, Line: 1}, + {Column: 12, Line: 1}, + }, + wantLits: []string{ + "a", + " ", + ">>", + " ", + `\'`, + "test", + `\'`, + }, + }, } for _, tt := range tests { diff --git a/token/token.go b/token/token.go index fb32cae..898c941 100644 --- a/token/token.go +++ b/token/token.go @@ -61,7 +61,7 @@ const ( Inc // ++ Dec // -- DoubleQuote // " - SingleQuote // ' + SingleQuote // ' or \' // Reserved Keyword After From fb305f5f025ae058466a202975c9abd49d0d8486 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 5 Feb 2023 20:59:04 +0900 Subject: [PATCH 18/58] scanner: refactor Scan() --- scanner/scanner.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/scanner/scanner.go b/scanner/scanner.go index 364121d..a160547 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -128,11 +128,9 @@ func (s *Scanner) Scan() (token.Pos, string, error) { ch, _, err := s.r.PeekRune() if err != nil { - s.readRune() return token.Pos{}, "", err } if ch == bufrr.EOF { - s.readRune() return token.Pos{}, "", io.EOF } if isSplitSymbol(ch) { @@ -144,10 +142,12 @@ func (s *Scanner) Scan() (token.Pos, string, error) { return startPos, oprt, nil } - s.readRune() + if _, err := s.readRune(); err != nil { + return token.Pos{}, "", err + } readingSpace := token.IsSpace(ch) - runes := []rune{ch} + rslt := []rune{ch} for { ch, _, err := s.r.PeekRune() @@ -155,13 +155,12 @@ func (s *Scanner) Scan() (token.Pos, string, error) { return token.Pos{}, "", err } if token.IsSpace(ch) != readingSpace || isSplitSymbol(ch) || ch == bufrr.EOF { - return startPos, string(runes), nil + return startPos, string(rslt), nil } - ch, err = s.readRune() - if err != nil { + if _, err = s.readRune(); err != nil { return token.Pos{}, "", err } - runes = append(runes, ch) + rslt = append(rslt, ch) } } From 7ce29b72d07c181b69571d10e67f28d14e2aafa6 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 5 Feb 2023 21:28:09 +0900 Subject: [PATCH 19/58] scanner: add scanOperator test and fix logic. --- scanner/scanner.go | 65 ++++++++++++++++++++++++----------------- scanner/scanner_test.go | 40 ++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 28 deletions(-) diff --git a/scanner/scanner.go b/scanner/scanner.go index a160547..edeb008 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -24,15 +24,7 @@ func New(reader io.Reader) *Scanner { } } -func isMultiLengthOperatorSymbol(r rune) bool { - switch r { - case '=', '|', '^', '&', '+', '-', '*', '/', '\\', '%', '<', '>', '!': - return true - } - return false -} - -func isSplitSymbol(r rune) bool { +func isOperatorRune(r rune) bool { switch r { case '(', ')', '[', ']', '{', '}', ':', ';', '?', '=', '|', '^', '&', '<', '>', '+', '-', '*', '/', '%', ',', '!', '~', '"', '\'', '\\': return true @@ -56,62 +48,81 @@ func (s *Scanner) readRune() (rune, error) { return r, nil } +var ( + errNotOperator = errors.New("Not operator.") +) + func (s *Scanner) scanOperator() (string, error) { - ch, _, err := s.r.PeekRune() + ch1, _, err := s.r.PeekRune() if err != nil { return "", err } - if !isSplitSymbol(ch) { - return "", errors.New("Not operator") + if !isOperatorRune(ch1) { + return "", errNotOperator } - s.readRune() - if !isMultiLengthOperatorSymbol(ch) { - return string(ch), nil + if _, err := s.readRune(); err != nil { + return "", err } ch2, _, err := s.r.PeekRune() if err != nil { - return string(ch), nil + return string(ch1), nil } - oprt := string([]rune{ch, ch2}) + oprt := string([]rune{ch1, ch2}) switch oprt { case "=>", "->", "|=", "^=", "&=", "+=", "-=", "*=", "/=", "%=", "==", "||", "&&", "**", "!=", "<-", ">-", "++", "--", `\'`: - s.readRune() + if _, err := s.readRune(); err != nil { + return "", err + } return oprt, nil case "<<": - s.readRune() - ch, _, err := s.r.PeekRune() + if _, err := s.readRune(); err != nil { + return "", err + } + ch3, _, err := s.r.PeekRune() if err != nil { return oprt, nil } - if ch == '=' { + if ch3 == '=' { + if _, err := s.readRune(); err != nil { + return "", err + } return "<<=", nil } return oprt, nil case ">>": - s.readRune() + if _, err := s.readRune(); err != nil { + return "", err + } ch3, _, err := s.r.PeekRune() if err != nil { return oprt, nil } if ch3 == '=' { + if _, err := s.readRune(); err != nil { + return "", err + } return ">>=", nil } if ch3 == '>' { - s.readRune() + if _, err := s.readRune(); err != nil { + return "", err + } oprt = ">>>" ch4, _, err := s.r.PeekRune() if err != nil { return oprt, nil } if ch4 == '=' { - s.readRune() + if _, err := s.readRune(); err != nil { + return "", err + } return ">>>=", nil } return oprt, nil } return oprt, nil } - return string(ch), nil + return string(ch1), nil } // Scan divides a string into the smallest units from the bufrr.Reader and returns them one by one. @@ -133,7 +144,7 @@ func (s *Scanner) Scan() (token.Pos, string, error) { if ch == bufrr.EOF { return token.Pos{}, "", io.EOF } - if isSplitSymbol(ch) { + if isOperatorRune(ch) { // scan operator. oprt, err := s.scanOperator() if err != nil { @@ -154,7 +165,7 @@ func (s *Scanner) Scan() (token.Pos, string, error) { if err != nil { return token.Pos{}, "", err } - if token.IsSpace(ch) != readingSpace || isSplitSymbol(ch) || ch == bufrr.EOF { + if token.IsSpace(ch) != readingSpace || isOperatorRune(ch) || ch == bufrr.EOF { return startPos, string(rslt), nil } if _, err = s.readRune(); err != nil { diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index 2cc9ca8..1c86d46 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -1,6 +1,7 @@ package scanner import ( + "errors" "io" "strings" "testing" @@ -53,7 +54,7 @@ func TestScannerScan(t *testing.T) { }, }, { - name: "there are multi length operators", + name: "there are operators", input: `a >> \'test\'`, wantPoss: []token.Pos{ {Column: 1, Line: 1}, @@ -105,3 +106,40 @@ func TestScannerScan(t *testing.T) { }) } } + +func TestScanner_scanOperator(t *testing.T) { + tests := []struct { + input string + want string + wantErr error + }{ + {input: "^0.8.13", want: "^"}, + {input: "=>>", want: "=>"}, + {input: "<< ", want: "<<"}, + {input: "<<=a", want: "<<="}, + {input: ">>1", want: ">>"}, + {input: ">>=x", want: ">>="}, + {input: ">>>>", want: ">>>"}, + {input: ">>>==", want: ">>>="}, + {input: "pragma", wantErr: errNotOperator}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.input, func(t *testing.T) { + s := New(strings.NewReader(tt.input)) + + got, err := s.scanOperator() + if !errors.Is(err, tt.wantErr) { + t.Error(err) + } + if got != tt.want { + t.Errorf("got: %s, want: %s", got, tt.want) + } + expectedOffset := len(tt.want) + if s.offset != expectedOffset { + t.Errorf("offset is %d, expected is %d", s.offset, expectedOffset) + } + }) + } +} From f19d4b89015067eb028fb7b248abb722668c0e47 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 11 Feb 2023 22:58:32 +0900 Subject: [PATCH 20/58] scanner: imeplement Peek --- scanner/scanner.go | 36 +++++++++++-- scanner/scanner_test.go | 114 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 137 insertions(+), 13 deletions(-) diff --git a/scanner/scanner.go b/scanner/scanner.go index edeb008..92b402d 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -15,6 +15,12 @@ type Scanner struct { // position state offset int lineOffset int + + // peek state + peeked bool + peekStr string + peekPos token.Pos + peekErr error } func New(reader io.Reader) *Scanner { @@ -125,13 +131,13 @@ func (s *Scanner) scanOperator() (string, error) { return string(ch1), nil } -// Scan divides a string into the smallest units from the bufrr.Reader and returns them one by one. +// scan flow // // - If first rune is an error or EOF, return an error and exit. // - If first rune is operator character, scan operator. // - If first rune is a space character, scan until the end of the blank character. // - Else scan to next space or operator string. -func (s *Scanner) Scan() (token.Pos, string, error) { +func (s *Scanner) scan() (token.Pos, string, error) { startPos := token.Pos{ Column: s.offset + 1, Line: s.lineOffset + 1, @@ -175,6 +181,28 @@ func (s *Scanner) Scan() (token.Pos, string, error) { } } -func (s *Scanner) Peek() (pos token.Pos, lit string, err error) { - return token.Pos{}, "", nil +// Scan divides a string into the smallest units from the bufrr.Reader and returns them one by one. +func (s *Scanner) Scan() (token.Pos, string, error) { + if s.peeked { + s.peeked = false + return s.peekPos, s.peekStr, s.peekErr + } + + return s.scan() +} + +// Peek reads ahead and returns the result of Scan. +// The offset is unchanged. +func (s *Scanner) Peek() (pos token.Pos, str string, err error) { + if s.peeked { + return s.peekPos, s.peekStr, s.peekErr + } + + s.peeked = true + + pos, str, err = s.scan() + s.peekPos = pos + s.peekStr = str + s.peekErr = err + return } diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index 1c86d46..1aad218 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -6,16 +6,17 @@ import ( "strings" "testing" + "github.com/SteelSeries/bufrr" "github.com/google/go-cmp/cmp" "github.com/uji/solparser/token" ) -func TestScannerScan(t *testing.T) { +func TestScanner_Scan(t *testing.T) { tests := []struct { name string input string wantPoss []token.Pos - wantLits []string + wantStrs []string }{ { name: "normal", @@ -29,7 +30,7 @@ func TestScannerScan(t *testing.T) { {Column: 18, Line: 1}, {Column: 24, Line: 1}, }, - wantLits: []string{ + wantStrs: []string{ "pragma", " ", "solidity", @@ -47,7 +48,7 @@ func TestScannerScan(t *testing.T) { {Column: 7, Line: 1}, {Column: 2, Line: 2}, }, - wantLits: []string{ + wantStrs: []string{ "pragma", "\n ", "solidity", @@ -65,7 +66,7 @@ func TestScannerScan(t *testing.T) { {Column: 8, Line: 1}, {Column: 12, Line: 1}, }, - wantLits: []string{ + wantStrs: []string{ "a", " ", ">>", @@ -83,10 +84,10 @@ func TestScannerScan(t *testing.T) { s := New(strings.NewReader(tt.input)) poss := make([]token.Pos, 0, len(tt.wantPoss)) - lits := make([]string, 0, len(tt.wantLits)) + strs := make([]string, 0, len(tt.wantStrs)) for { - pos, lit, err := s.Scan() + pos, str, err := s.Scan() if err == io.EOF { break } @@ -94,17 +95,112 @@ func TestScannerScan(t *testing.T) { t.Fatalf("got unexpected error: %s", err) } poss = append(poss, pos) - lits = append(lits, lit) + strs = append(strs, str) } if diff := cmp.Diff(tt.wantPoss, poss); diff != "" { t.Errorf(diff) } - if diff := cmp.Diff(tt.wantLits, lits); diff != "" { + if diff := cmp.Diff(tt.wantStrs, strs); diff != "" { t.Errorf(diff) } }) } + + t.Run("when peeked", func(t *testing.T) { + wantStr := "str" + wantPos := token.Pos{ + Column: 3, + Line: 4, + } + wantErr := errors.New(t.Name()) + s := &Scanner{ + r: bufrr.NewReader(strings.NewReader("input will not scand")), + peeked: true, + peekStr: wantStr, + peekPos: wantPos, + peekErr: wantErr, + } + + pos, str, err := s.Scan() + if !errors.Is(err, wantErr) { + t.Fatalf("got unexpected error: %s", err) + } + if diff := cmp.Diff(wantPos, pos); diff != "" { + t.Errorf(diff) + } + if str != wantStr { + t.Errorf("got %s, want %s", str, wantStr) + } + if s.peeked != false { + t.Error("peeked status was not cleared") + } + }) +} + +func TestScanner_Peek(t *testing.T) { + terr := errors.New(t.Name()) + tests := []struct { + name string + input string + peeked bool + peekedPos token.Pos + peekedStr string + peekedErr error + wantPos token.Pos + wantStr string + wantErr error + }{ + { + name: "not peeked", + input: "pragma solidity ^0.8.13;", + peeked: false, + wantPos: token.Pos{ + Column: 1, + Line: 1, + }, + wantStr: "pragma", + }, + { + name: "peeked", + peeked: true, + peekedStr: "peekedStr", + peekedPos: token.Pos{ + Column: 2, + Line: 2, + }, + wantPos: token.Pos{ + Column: 2, + Line: 2, + }, + wantErr: terr, + wantStr: "peekedStr", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + s := &Scanner{ + r: bufrr.NewReader(strings.NewReader(tt.input)), + peeked: tt.peeked, + peekStr: tt.peekedStr, + peekPos: tt.peekedPos, + peekErr: tt.peekedErr, + } + + pos, str, err := s.Peek() + if !errors.Is(err, tt.peekedErr) { + t.Fatalf("got unexpected error: %s", err) + } + if diff := cmp.Diff(tt.wantPos, pos); diff != "" { + t.Errorf(diff) + } + if str != tt.wantStr { + t.Errorf("got %s, want %s", str, tt.wantStr) + } + }) + } } func TestScanner_scanOperator(t *testing.T) { From 4bc910f314c2de50baf79d9ea4928adfe45729c8 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 11 Feb 2023 23:07:45 +0900 Subject: [PATCH 21/58] lexer: implement Scan StringLiteral into scan. --- lexer/lexer.go | 31 ++++++++++++++++++++----------- lexer/lexer_test.go | 12 ++++++++++++ 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index e085623..90ae859 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -1,7 +1,7 @@ package lexer import ( - "fmt" + "errors" "io" "github.com/uji/solparser/scanner" @@ -26,16 +26,26 @@ func New(input io.Reader) *Lexer { } func (l *Lexer) scan() (tkn token.Token, err error) { - pos, lit, err := l.scanner.Scan() + pos, str, err := l.scanner.Peek() if err != nil { return token.Token{}, err } + if str == "" { + return token.Token{}, errors.New("Empty character scanned.") + } + + if str == `"` || str == `\"` { + return l.ScanStringLiteral() + } + + l.scanner.Scan() - if token.IsSpace([]rune(lit)[0]) { + // If space, scan for the next token + if token.IsSpace([]rune(str)[0]) { return l.scan() } - return token.NewToken(lit, pos), nil + return token.NewToken(str, pos), nil } func (l *Lexer) Scan() (token.Token, error) { @@ -61,23 +71,22 @@ func (l *Lexer) Peek() (token.Token, error) { // ScanStringLiteral parse NonEmptyStringLiteral or EmptyStringLiteral then return StringLiteral token. func (l *Lexer) ScanStringLiteral() (token.Token, error) { - firstPos, lit, err := l.scanner.Scan() + firstPos, str, err := l.scanner.Scan() if err != nil { return token.Token{}, err } - fmt.Println(firstPos, lit) - if lit != `"` && lit != `\'` { + if str != `"` && str != `\'` { return token.Token{}, token.NewPosError(firstPos, `not found " or \'`) } - quote, txt := lit, lit + quote, txt := str, str for { - _, lit, err := l.scanner.Scan() + _, str, err := l.scanner.Scan() if err != nil { return token.Token{}, err } - txt = txt + lit - if lit == quote { + txt = txt + str + if str == quote { return token.Token{ TokenType: token.StringLiteral, Text: txt, diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 72290d6..e07a7ef 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -41,6 +41,18 @@ func TestLexer_Scan(t *testing.T) { }, }, }, + { + name: "There is a StringLiteral", + input: `"pragma"`, + wantToken: token.Token{ + TokenType: token.StringLiteral, + Text: `"pragma"`, + Pos: token.Pos{ + Column: 1, + Line: 1, + }, + }, + }, } for _, tt := range tests { tt := tt From 03dd73e58fe1058610698c0b704da971b1d49f91 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 12 Feb 2023 15:09:01 +0900 Subject: [PATCH 22/58] token: remove token.True, token.False and use XxxLiteral --- solparser.go | 4 ++-- solparser_test.go | 6 +++--- token/token.go | 6 ++---- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/solparser.go b/solparser.go index 052db18..778fad8 100644 --- a/solparser.go +++ b/solparser.go @@ -20,7 +20,7 @@ func New(input io.Reader) *Parser { } } -func (p *Parser) Parse(input io.Reader) (*ast.SourceUnit, error) { +func (p *Parser) Parse() (*ast.SourceUnit, error) { tkn, err := p.lexer.Peek() if err == io.EOF { return nil, nil @@ -67,7 +67,7 @@ func (p *Parser) ParseBooleanLiteral() (*ast.BooleanLiteral, error) { return nil, err } - if tkn.TokenType != token.True && tkn.TokenType != token.False { + if tkn.TokenType != token.TrueLiteral && tkn.TokenType != token.FalseLiteral { return nil, token.NewPosError(tkn.Pos, "not found keyword true or false") } diff --git a/solparser_test.go b/solparser_test.go index e944bd5..d127fc3 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -11,7 +11,7 @@ import ( "github.com/uji/solparser/token" ) -func TestParserParseBooleanLiteral(t *testing.T) { +func TestParser_ParseBooleanLiteral(t *testing.T) { tests := []struct { name string input string @@ -23,7 +23,7 @@ func TestParserParseBooleanLiteral(t *testing.T) { input: "true", want: &ast.BooleanLiteral{ Token: token.Token{ - TokenType: token.True, + TokenType: token.TrueLiteral, Text: "true", Pos: token.Pos{ Column: 1, @@ -38,7 +38,7 @@ func TestParserParseBooleanLiteral(t *testing.T) { input: "false", want: &ast.BooleanLiteral{ Token: token.Token{ - TokenType: token.False, + TokenType: token.FalseLiteral, Text: "false", Pos: token.Pos{ Column: 1, diff --git a/token/token.go b/token/token.go index 898c941..83df08a 100644 --- a/token/token.go +++ b/token/token.go @@ -120,7 +120,6 @@ const ( Event External Fallback - False Fixed For From @@ -152,7 +151,6 @@ const ( Storage String Struct - True Try Type Using @@ -383,7 +381,7 @@ func asKeyword(str string) TokenType { case "fallback": return Fallback case "false": - return False + return FalseLiteral case "fixed": return Fixed case "for": @@ -447,7 +445,7 @@ func asKeyword(str string) TokenType { case "struct": return Struct case "true": - return True + return TrueLiteral case "try": return Try case "type": From 2fbb4aee0d66cb20ddbf580972b2e1a1d49b9134 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 12 Feb 2023 16:52:53 +0900 Subject: [PATCH 23/58] solparser: parse StringLiteral as Literal --- ast/ast.go | 13 +++++++++- literal.go | 26 +++++++++++++++++++ literal_test.go | 69 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 literal.go create mode 100644 literal_test.go diff --git a/ast/ast.go b/ast/ast.go index a869c6a..02c2bdc 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -106,8 +106,19 @@ func (b *BooleanLiteral) End() token.Pos { } } +// EmptyStringLiteral | NonEmptyStringLiteral type StringLiteral struct { - List []Node // EmptyStringLiteral | NonEmptyStringLiteral + Str string + Position token.Pos + EndPos token.Pos +} + +func (s StringLiteral) Pos() token.Pos { + return s.Position +} + +func (s StringLiteral) End() token.Pos { + return s.EndPos } type HexStringLiteral []*HexString diff --git a/literal.go b/literal.go new file mode 100644 index 0000000..60d43d2 --- /dev/null +++ b/literal.go @@ -0,0 +1,26 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseLiteral() (ast.Literal, error) { + tkn, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + if tkn.TokenType != token.StringLiteral { + return nil, token.NewPosError(tkn.Pos, "not found string literal quote") + } + + return &ast.StringLiteral{ + Str: tkn.Text, + Position: tkn.Pos, + EndPos: token.Pos{ + Column: tkn.Pos.Column + len(tkn.Text) - 1, + Line: tkn.Pos.Line, // TODO: support new line code + }, + }, nil +} diff --git a/literal_test.go b/literal_test.go new file mode 100644 index 0000000..c7f9e62 --- /dev/null +++ b/literal_test.go @@ -0,0 +1,69 @@ +package solparser_test + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseLiteral(t *testing.T) { + tests := []struct { + name string + input string + want ast.Literal + err *token.PosError + }{ + { + name: "StringLiteral", + input: `"Hello World!!";`, + want: &ast.StringLiteral{ + Str: `"Hello World!!"`, + Position: token.Pos{ + Column: 1, + Line: 1, + }, + EndPos: token.Pos{ + Column: 15, + Line: 1, + }, + }, + err: nil, + }, + { + name: "Not Literal", + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{ + Column: 1, + Line: 1, + }, + Msg: "not found string literal quote", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseLiteral() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} From 4fa9750ee451df9bed7d6fe50fdb7ef64941cc85 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 18 Feb 2023 15:01:27 +0900 Subject: [PATCH 24/58] all: parse EOF as tokentype `EOS` --- lexer/lexer_test.go | 28 ++++++++++++++++++++++------ pragmaDirective_test.go | 2 +- scanner/scanner.go | 3 +-- scanner/scanner_test.go | 7 +++---- token/token.go | 12 +++++++++++- 5 files changed, 38 insertions(+), 14 deletions(-) diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index e07a7ef..b69f5a5 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -2,7 +2,6 @@ package lexer import ( "errors" - "io" "strings" "testing" @@ -78,9 +77,20 @@ func TestLexer_Scan(t *testing.T) { scanner: s, } - _, err := l.Scan() - if err != io.EOF { - t.Errorf("error is not io.EOF, got: %s", err) + got, err := l.Scan() + if err != nil { + t.Errorf("got error: %s", err) + } + want := token.Token{ + TokenType: token.EOS, + Text: token.EOSString, + Pos: token.Pos{ + Column: 1, + Line: 1, + }, + } + if diff := cmp.Diff(want, got); diff != "" { + t.Error(diff) } }) @@ -133,8 +143,14 @@ func TestLexer_Peek(t *testing.T) { { name: "when scan is done", input: "", - token: token.Token{}, - err: io.EOF, + token: token.Token{ + TokenType: token.EOS, + Text: token.EOSString, + Pos: token.Pos{ + Column: 1, + Line: 1, + }, + }, }, } for _, tt := range tests { diff --git a/pragmaDirective_test.go b/pragmaDirective_test.go index a1ad8a7..f2a65da 100644 --- a/pragmaDirective_test.go +++ b/pragmaDirective_test.go @@ -72,7 +72,7 @@ func TestParserParsePragmaDirective(t *testing.T) { want: nil, err: &token.PosError{ Pos: token.Pos{ - Column: 18, + Column: 24, Line: 1, }, Msg: "not found Semicolon", diff --git a/scanner/scanner.go b/scanner/scanner.go index 92b402d..ee68301 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -133,7 +133,6 @@ func (s *Scanner) scanOperator() (string, error) { // scan flow // -// - If first rune is an error or EOF, return an error and exit. // - If first rune is operator character, scan operator. // - If first rune is a space character, scan until the end of the blank character. // - Else scan to next space or operator string. @@ -148,7 +147,7 @@ func (s *Scanner) scan() (token.Pos, string, error) { return token.Pos{}, "", err } if ch == bufrr.EOF { - return token.Pos{}, "", io.EOF + return startPos, token.EOSString, nil } if isOperatorRune(ch) { // scan operator. diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index 1aad218..85bedb2 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -2,7 +2,6 @@ package scanner import ( "errors" - "io" "strings" "testing" @@ -88,12 +87,12 @@ func TestScanner_Scan(t *testing.T) { for { pos, str, err := s.Scan() - if err == io.EOF { - break - } if err != nil { t.Fatalf("got unexpected error: %s", err) } + if str == token.EOSString { + break + } poss = append(poss, pos) strs = append(strs, str) } diff --git a/token/token.go b/token/token.go index 83df08a..16d4f1f 100644 --- a/token/token.go +++ b/token/token.go @@ -1,6 +1,10 @@ package token -import "fmt" +import ( + "fmt" + + "github.com/SteelSeries/bufrr" +) type TokenType int @@ -8,6 +12,8 @@ const ( Invalid TokenType = iota Unknown + EOS // End of source indicator. + LParen // ( RParen // ) LBrack // [ @@ -168,8 +174,12 @@ const ( CommentLiteral ) +var EOSString string = string([]rune{bufrr.EOF}) + func asKeyword(str string) TokenType { switch str { + case EOSString: + return EOS case "(": return LParen case ")": From 57b84f05461560aa62fa2a1c8999ea20c92f4d65 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 18 Feb 2023 23:12:06 +0900 Subject: [PATCH 25/58] main: parse literal --- literal.go | 10 ++++++++-- literal_test.go | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/literal.go b/literal.go index 60d43d2..d7c2a21 100644 --- a/literal.go +++ b/literal.go @@ -15,12 +15,18 @@ func (p *Parser) ParseLiteral() (ast.Literal, error) { return nil, token.NewPosError(tkn.Pos, "not found string literal quote") } + // peek to find literal end position. + peek, err := p.lexer.Peek() + if err != nil { + return nil, err + } + return &ast.StringLiteral{ Str: tkn.Text, Position: tkn.Pos, EndPos: token.Pos{ - Column: tkn.Pos.Column + len(tkn.Text) - 1, - Line: tkn.Pos.Line, // TODO: support new line code + Column: peek.Pos.Column - 1, + Line: peek.Pos.Line, }, }, nil } diff --git a/literal_test.go b/literal_test.go index c7f9e62..02dda01 100644 --- a/literal_test.go +++ b/literal_test.go @@ -19,7 +19,7 @@ func TestParser_ParseLiteral(t *testing.T) { err *token.PosError }{ { - name: "StringLiteral", + name: "normal", input: `"Hello World!!";`, want: &ast.StringLiteral{ Str: `"Hello World!!"`, @@ -34,6 +34,39 @@ func TestParser_ParseLiteral(t *testing.T) { }, err: nil, }, + { + name: `Including \n`, + input: "\"Hello \nWorld!!\";", + want: &ast.StringLiteral{ + Str: "\"Hello \nWorld!!\"", + Position: token.Pos{ + Column: 1, + Line: 1, + }, + EndPos: token.Pos{ + Column: 8, + Line: 2, + }, + }, + err: nil, + }, + // TODO + // { + // name: `Next token is \n`, + // input: "\"Hello World!!\"\n", + // want: &ast.StringLiteral{ + // Str: `"Hello World!!"`, + // Position: token.Pos{ + // Column: 1, + // Line: 1, + // }, + // EndPos: token.Pos{ + // Column: 15, + // Line: 1, + // }, + // }, + // err: nil, + // }, { name: "Not Literal", input: "pragma", From 9ae80170a9cb624725ed539384bbc9dce3c74225 Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 18 Feb 2023 23:33:54 +0900 Subject: [PATCH 26/58] main: parse expression (support literal only) --- expression.go | 19 +++++++++++++ expression_test.go | 69 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 expression.go create mode 100644 expression_test.go diff --git a/expression.go b/expression.go new file mode 100644 index 0000000..8a8b11b --- /dev/null +++ b/expression.go @@ -0,0 +1,19 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseExpression() (ast.Expression, error) { + tkn, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + if tkn.TokenType == token.StringLiteral { + return p.ParseLiteral() + } + + return nil, token.NewPosError(tkn.Pos, "not found expression.") +} diff --git a/expression_test.go b/expression_test.go new file mode 100644 index 0000000..72bdb1e --- /dev/null +++ b/expression_test.go @@ -0,0 +1,69 @@ +package solparser_test + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseExpression(t *testing.T) { + tests := []struct { + name string + input string + want ast.Expression + err *token.PosError + }{ + { + name: "Literal", + input: `"Hello World!!";`, + want: &ast.StringLiteral{ + Str: `"Hello World!!"`, + Position: token.Pos{ + Column: 1, + Line: 1, + }, + EndPos: token.Pos{ + Column: 15, + Line: 1, + }, + }, + err: nil, + }, + { + name: "Not expression", + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{ + Column: 1, + Line: 1, + }, + Msg: "not found expression.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseExpression() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} From f5f3f6d2bbb2994dea6a236544c4594f0999512a Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 19 Feb 2023 12:15:48 +0900 Subject: [PATCH 27/58] main, ast: rename StringLiteral fields --- ast/ast.go | 9 ++++----- expression_test.go | 6 +++--- literal.go | 6 +++--- literal_test.go | 12 ++++++------ 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 02c2bdc..c2d2bd8 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -108,17 +108,16 @@ func (b *BooleanLiteral) End() token.Pos { // EmptyStringLiteral | NonEmptyStringLiteral type StringLiteral struct { - Str string - Position token.Pos - EndPos token.Pos + From, To token.Pos + Value string } func (s StringLiteral) Pos() token.Pos { - return s.Position + return s.From } func (s StringLiteral) End() token.Pos { - return s.EndPos + return s.To } type HexStringLiteral []*HexString diff --git a/expression_test.go b/expression_test.go index 72bdb1e..0fee217 100644 --- a/expression_test.go +++ b/expression_test.go @@ -22,12 +22,12 @@ func TestParser_ParseExpression(t *testing.T) { name: "Literal", input: `"Hello World!!";`, want: &ast.StringLiteral{ - Str: `"Hello World!!"`, - Position: token.Pos{ + Value: `"Hello World!!"`, + From: token.Pos{ Column: 1, Line: 1, }, - EndPos: token.Pos{ + To: token.Pos{ Column: 15, Line: 1, }, diff --git a/literal.go b/literal.go index d7c2a21..9cdf99b 100644 --- a/literal.go +++ b/literal.go @@ -22,9 +22,9 @@ func (p *Parser) ParseLiteral() (ast.Literal, error) { } return &ast.StringLiteral{ - Str: tkn.Text, - Position: tkn.Pos, - EndPos: token.Pos{ + Value: tkn.Text, + From: tkn.Pos, + To: token.Pos{ Column: peek.Pos.Column - 1, Line: peek.Pos.Line, }, diff --git a/literal_test.go b/literal_test.go index 02dda01..1f74cf0 100644 --- a/literal_test.go +++ b/literal_test.go @@ -22,12 +22,12 @@ func TestParser_ParseLiteral(t *testing.T) { name: "normal", input: `"Hello World!!";`, want: &ast.StringLiteral{ - Str: `"Hello World!!"`, - Position: token.Pos{ + Value: `"Hello World!!"`, + From: token.Pos{ Column: 1, Line: 1, }, - EndPos: token.Pos{ + To: token.Pos{ Column: 15, Line: 1, }, @@ -38,12 +38,12 @@ func TestParser_ParseLiteral(t *testing.T) { name: `Including \n`, input: "\"Hello \nWorld!!\";", want: &ast.StringLiteral{ - Str: "\"Hello \nWorld!!\"", - Position: token.Pos{ + Value: "\"Hello \nWorld!!\"", + From: token.Pos{ Column: 1, Line: 1, }, - EndPos: token.Pos{ + To: token.Pos{ Column: 8, Line: 2, }, From 35b8fde80c21099f0b74ba0c536cd33629c0b9ea Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 19 Feb 2023 12:55:16 +0900 Subject: [PATCH 28/58] main, ast: parse statement (only returnStatement) --- ast/ast.go | 20 +++++-- statement.go | 50 +++++++++++++++++ statement_test.go | 140 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 204 insertions(+), 6 deletions(-) create mode 100644 statement.go create mode 100644 statement_test.go diff --git a/ast/ast.go b/ast/ast.go index c2d2bd8..2fa3f5d 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -252,17 +252,25 @@ type ElementaryTypeName struct { Kind ElementaryTypeNameKind } +// ---------------------------------------------------------------------------- +// Statement Nodes + +type Statement interface { + Node + statementNode() +} + type ReturnStatement struct { - Return token.Pos + From token.Pos + SemiPos token.Pos Expression Expression } func (r *ReturnStatement) Pos() token.Pos { - return r.Return + return r.From } func (r *ReturnStatement) End() token.Pos { - return token.Pos{ - Column: r.Expression.End().Column + 1, - Line: r.Expression.End().Line, - } + return r.SemiPos } + +func (s *ReturnStatement) statementNode() {} diff --git a/statement.go b/statement.go new file mode 100644 index 0000000..b9905a7 --- /dev/null +++ b/statement.go @@ -0,0 +1,50 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseStatement() (ast.Statement, error) { + tkn, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + if tkn.TokenType == token.Return { + return p.ParseReturnStatement() + } + + return nil, token.NewPosError(tkn.Pos, "not found statement.") +} + +func (p *Parser) ParseReturnStatement() (ast.Statement, error) { + rtn, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + if rtn.TokenType != token.Return { + return nil, token.NewPosError(rtn.Pos, "not found return keyword.") + } + + exp, err := p.ParseExpression() + if err != nil { + return nil, err + } + + semi, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + if semi.TokenType != token.Semicolon { + return nil, token.NewPosError(semi.Pos, "not found semicolon.") + } + + return &ast.ReturnStatement{ + From: rtn.Pos, + SemiPos: semi.Pos, + Expression: exp, + }, nil +} diff --git a/statement_test.go b/statement_test.go new file mode 100644 index 0000000..a63c1ec --- /dev/null +++ b/statement_test.go @@ -0,0 +1,140 @@ +package solparser_test + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseStatement(t *testing.T) { + tests := []struct { + name string + input string + want ast.Statement + err *token.PosError + }{ + { + name: "ReturnStatement", + input: `return "Hello World!!";`, + want: &ast.ReturnStatement{ + From: token.Pos{Column: 1, Line: 1}, + SemiPos: token.Pos{Column: 23, Line: 1}, + Expression: &ast.StringLiteral{ + From: token.Pos{Column: 8, Line: 1}, + To: token.Pos{Column: 22, Line: 1}, + Value: `"Hello World!!"`, + }, + }, + err: nil, + }, + { + name: "Not Statement", + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{Column: 1, Line: 1}, + Msg: "not found statement.", + }, + }, + { + name: "Found broken return statement", + input: "return pragma", + err: &token.PosError{ + Pos: token.Pos{Column: 8, Line: 1}, + Msg: "not found expression.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseStatement() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} + +func TestParser_ParseReturnStatement(t *testing.T) { + tests := []struct { + name string + input string + want ast.Statement + err *token.PosError + }{ + { + name: "normal", + input: `return "Hello World!!";`, + want: &ast.ReturnStatement{ + From: token.Pos{Column: 1, Line: 1}, + SemiPos: token.Pos{Column: 23, Line: 1}, + Expression: &ast.StringLiteral{ + From: token.Pos{Column: 8, Line: 1}, + To: token.Pos{Column: 22, Line: 1}, + Value: `"Hello World!!"`, + }, + }, + err: nil, + }, + { + name: "Not found return keyword", + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{Column: 1, Line: 1}, + Msg: "not found return keyword.", + }, + }, + { + name: "Not found expression", + input: "return pragma", + err: &token.PosError{ + Pos: token.Pos{Column: 8, Line: 1}, + Msg: "not found expression.", + }, + }, + { + name: "Not found semicolon.", + input: `return "test" pragma`, + err: &token.PosError{ + Pos: token.Pos{Column: 15, Line: 1}, + Msg: "not found semicolon.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseReturnStatement() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} From 8ce442a44077fbed0bf44dbb8b440c0ffbb00cb1 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 19 Feb 2023 13:15:31 +0900 Subject: [PATCH 29/58] main, ast: parse block (supported only one statement) --- ast/ast.go | 16 +++++++++++ block.go | 37 ++++++++++++++++++++++++ block_test.go | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 block.go create mode 100644 block_test.go diff --git a/ast/ast.go b/ast/ast.go index 2fa3f5d..48b8a37 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -252,6 +252,22 @@ type ElementaryTypeName struct { Kind ElementaryTypeNameKind } +// ---------------------------------------------------------------------------- + +type Block struct { + LBracePos token.Pos + RBracePos token.Pos + Nodes []Node // statement | unblocked-block +} + +func (b Block) Pos() token.Pos { + return b.LBracePos +} + +func (b Block) End() token.Pos { + return b.RBracePos +} + // ---------------------------------------------------------------------------- // Statement Nodes diff --git a/block.go b/block.go new file mode 100644 index 0000000..e1399e3 --- /dev/null +++ b/block.go @@ -0,0 +1,37 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseBlock() (*ast.Block, error) { + lblace, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + if lblace.TokenType != token.LBrace { + return nil, token.NewPosError(lblace.Pos, "not found LBrace.") + } + + stmt, err := p.ParseStatement() + if err != nil { + return nil, err + } + + rblace, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + if rblace.TokenType != token.RBrace { + return nil, token.NewPosError(rblace.Pos, "not found RBrace.") + } + + return &ast.Block{ + LBracePos: lblace.Pos, + RBracePos: rblace.Pos, + Nodes: []ast.Node{stmt}, + }, nil +} diff --git a/block_test.go b/block_test.go new file mode 100644 index 0000000..fa82134 --- /dev/null +++ b/block_test.go @@ -0,0 +1,78 @@ +package solparser_test + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseBlock(t *testing.T) { + tests := []struct { + name string + input string + want *ast.Block + err *token.PosError + }{ + { + name: "One Statement", + input: "{ \nreturn \"Hello World!!\";\n }", + want: &ast.Block{ + LBracePos: token.Pos{Column: 1, Line: 1}, + RBracePos: token.Pos{Column: 2, Line: 3}, + Nodes: []ast.Node{ + &ast.ReturnStatement{ + From: token.Pos{Column: 1, Line: 2}, + SemiPos: token.Pos{Column: 23, Line: 2}, + Expression: &ast.StringLiteral{ + From: token.Pos{Column: 8, Line: 2}, + To: token.Pos{Column: 22, Line: 2}, + Value: "\"Hello World!!\"", + }, + }, + }, + }, + err: nil, + }, + { + name: "Not found LBrace", + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{Column: 1, Line: 1}, + Msg: "not found LBrace.", + }, + }, + { + name: "Not found RBrace", + input: "{ \nreturn \"Hello World!!\";", + err: &token.PosError{ + Pos: token.Pos{Column: 24, Line: 2}, + Msg: "not found RBrace.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseBlock() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} From 2b8589e2804195e607452a575935faecf4d97ce6 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 19 Feb 2023 15:30:04 +0900 Subject: [PATCH 30/58] main, ast: parse typeName (only elementaryTypeNode) --- ast/ast.go | 48 ++++++++--------- typeName.go | 46 ++++++++++++++++ typeName_test.go | 135 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 205 insertions(+), 24 deletions(-) create mode 100644 typeName.go create mode 100644 typeName_test.go diff --git a/ast/ast.go b/ast/ast.go index 48b8a37..ef85e2b 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -29,10 +29,6 @@ type ModifierList struct { StateMutability *StateMutability } -type TypeName struct { - ElementalyTypeName string -} - type EventParameter struct { TypeName *TypeName } @@ -74,6 +70,30 @@ type SourceUnit struct { FunctionDefinition *FunctionDefinition } +// ---------------------------------------------------------------------------- +// TypeName Nodes + +type TypeName interface { + Node + typeNameNode() +} + +type ElementaryTypeName []token.Token + +func (e ElementaryTypeName) Pos() token.Pos { + return e[0].Pos +} + +func (e ElementaryTypeName) End() token.Pos { + // TODO + return token.Pos{ + Column: 1, + Line: 1, + } +} + +func (e ElementaryTypeName) typeNameNode() {} + // ---------------------------------------------------------------------------- // Expression Nodes @@ -232,26 +252,6 @@ type HexString struct { String string } -type ElementaryTypeNameKind int - -const ( - ElementaryTypeName_Address ElementaryTypeNameKind = iota + 1 - ElementaryTypeName_AddressPayable - ElementaryTypeName_Bool - ElementaryTypeName_String - ElementaryTypeName_Bytes - ElementaryTypeName_SignedIntegerType - ElementaryTypeName_UnsignedIntegerType - ElementaryTypeName_FixedBytes - ElementaryTypeName_Fixed - ElementaryTypeName_Ufixed -) - -type ElementaryTypeName struct { - Pos token.Pos - Kind ElementaryTypeNameKind -} - // ---------------------------------------------------------------------------- type Block struct { diff --git a/typeName.go b/typeName.go new file mode 100644 index 0000000..96d5ea2 --- /dev/null +++ b/typeName.go @@ -0,0 +1,46 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseTypeName() (ast.TypeName, error) { + tkn, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + switch tkn.TokenType { + case token.Address, token.String, token.Bytes, token.Fixed, token.Bool: + return p.ParseElementaryTypeName() + } + + return nil, token.NewPosError(tkn.Pos, "not found type-name.") +} + +func (p *Parser) ParseElementaryTypeName() (ast.TypeName, error) { + tkn, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + if tkn.TokenType == token.Address { + tkn2, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + if tkn2.TokenType == token.Payable { + p.lexer.Scan() + return ast.ElementaryTypeName{tkn, tkn2}, nil + } + } + + switch tkn.TokenType { + case token.Address, token.String, token.Bytes, token.Fixed, token.Bool: + return ast.ElementaryTypeName{tkn}, nil + } + + return nil, token.NewPosError(tkn.Pos, "not found elementary type name keyword.") +} diff --git a/typeName_test.go b/typeName_test.go new file mode 100644 index 0000000..b0f5730 --- /dev/null +++ b/typeName_test.go @@ -0,0 +1,135 @@ +package solparser_test + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseTypeName(t *testing.T) { + tests := []struct { + name string + input string + want ast.TypeName + err *token.PosError + }{ + { + name: "ElementaryTypeName", + input: "string)", + want: ast.ElementaryTypeName{ + { + TokenType: token.String, + Text: "string", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + }, + { + name: "Not TypeName", + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{Column: 1, Line: 1}, + Msg: "not found type-name.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseTypeName() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} + +func TestParser_ParseElementaryTypeName(t *testing.T) { + tests := []struct { + input string + want ast.TypeName + err *token.PosError + }{ + { + input: "address", + want: ast.ElementaryTypeName{ + { + TokenType: token.Address, + Text: "address", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + }, + { + input: "bool", + want: ast.ElementaryTypeName{ + { + TokenType: token.Bool, + Text: "bool", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + }, + { + input: "string", + want: ast.ElementaryTypeName{ + { + TokenType: token.String, + Text: "string", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + }, + { + input: "address payable", + want: ast.ElementaryTypeName{ + { + TokenType: token.Address, + Text: "address", + Pos: token.Pos{Column: 1, Line: 1}, + }, + { + TokenType: token.Payable, + Text: "payable", + Pos: token.Pos{Column: 9, Line: 1}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.input, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseElementaryTypeName() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} From dc95af2165bb776b90a19d35c22cf981f3b358f3 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 19 Feb 2023 16:07:56 +0900 Subject: [PATCH 31/58] main, ast: parse parameter-list --- ast/ast.go | 9 ++- parameterList.go | 40 ++++++++++++ parameterList_test.go | 149 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 193 insertions(+), 5 deletions(-) create mode 100644 parameterList.go create mode 100644 parameterList_test.go diff --git a/ast/ast.go b/ast/ast.go index ef85e2b..d67e7f4 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -29,13 +29,12 @@ type ModifierList struct { StateMutability *StateMutability } -type EventParameter struct { - TypeName *TypeName +// Parameter is type of ParameterList elements +type Parameter struct { + TypeName TypeName } -type ParameterList struct { - EventParameter *EventParameter -} +type ParameterList []*Parameter type ReturnParameters struct { ParameterList *ParameterList diff --git a/parameterList.go b/parameterList.go new file mode 100644 index 0000000..a1dd76e --- /dev/null +++ b/parameterList.go @@ -0,0 +1,40 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseParameter() (*ast.Parameter, error) { + tn, err := p.ParseTypeName() + if err != nil { + return nil, err + } + + return &ast.Parameter{ + TypeName: tn, + }, nil +} + +func (p *Parser) ParseParameterList() (ast.ParameterList, error) { + prms := make(ast.ParameterList, 0, 1) + for { + prm, err := p.ParseParameter() + if err != nil { + return nil, err + } + + prms = append(prms, prm) + + comma, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + if comma.TokenType != token.Comma { + return prms, nil + } + + p.lexer.Scan() + } +} diff --git a/parameterList_test.go b/parameterList_test.go new file mode 100644 index 0000000..366fdc3 --- /dev/null +++ b/parameterList_test.go @@ -0,0 +1,149 @@ +package solparser_test + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseParameter(t *testing.T) { + tests := []struct { + name string + input string + want *ast.Parameter + err *token.PosError + }{ + { + name: "TypeName only", + input: "string", + want: &ast.Parameter{ + TypeName: ast.ElementaryTypeName{ + { + TokenType: token.String, + Text: "string", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + }, + err: nil, + }, + { + name: "Not ParameterList", + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{ + Column: 1, + Line: 1, + }, + Msg: "not found type-name.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseParameter() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} + +func TestParser_ParseParameterList(t *testing.T) { + tests := []struct { + name string + input string + want ast.ParameterList + err *token.PosError + }{ + { + name: "One Parameter", + input: "string", + want: ast.ParameterList{ + { + TypeName: ast.ElementaryTypeName{ + { + TokenType: token.String, + Text: "string", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + }, + }, + err: nil, + }, + { + name: "Some Parameter", + input: "string, bool", + want: ast.ParameterList{ + { + TypeName: ast.ElementaryTypeName{ + { + TokenType: token.String, + Text: "string", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + }, + { + TypeName: ast.ElementaryTypeName{ + { + TokenType: token.Bool, + Text: "bool", + Pos: token.Pos{Column: 9, Line: 1}, + }, + }, + }, + }, + err: nil, + }, + { + name: "Not ParameterList", + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{ + Column: 1, + Line: 1, + }, + Msg: "not found type-name.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseParameterList() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} From dfd3d35671c54522bc0c97e9b98a5d4d3aae0537 Mon Sep 17 00:00:00 2001 From: uji Date: Thu, 23 Feb 2023 15:13:58 +0900 Subject: [PATCH 32/58] main, ast: parse visibility and state-mutability. --- ast/ast.go | 8 +-- functionDefinition.go | 35 ++++++++++--- functionDefinition_test.go | 100 +++++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 12 deletions(-) create mode 100644 functionDefinition_test.go diff --git a/ast/ast.go b/ast/ast.go index d67e7f4..390ffd5 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -21,14 +21,12 @@ type FunctionDescriptor struct { Name string } -type StateMutability struct { - Pure bool -} - type ModifierList struct { StateMutability *StateMutability } +type Visibility = token.Token + // Parameter is type of ParameterList elements type Parameter struct { TypeName TypeName @@ -36,6 +34,8 @@ type Parameter struct { type ParameterList []*Parameter +type StateMutability = token.Token + type ReturnParameters struct { ParameterList *ParameterList } diff --git a/functionDefinition.go b/functionDefinition.go index 4268ca6..bfddd4e 100644 --- a/functionDefinition.go +++ b/functionDefinition.go @@ -1,15 +1,34 @@ package solparser -import "github.com/uji/solparser/ast" +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) -func (p *Parser) ParseFunctionDefinition() (*ast.FunctionDefinition, error) { - return nil, nil -} +func (p *Parser) ParseVisibility() (ast.Visibility, error) { + tkn, err := p.lexer.Scan() + if err != nil { + return token.Token{}, err + } + + switch tkn.TokenType { + case token.Internal, token.External, token.Public, token.Private: + return tkn, nil + } -func (p *Parser) ParseModirierList() (*ast.ModifierList, error) { - return nil, nil + return token.Token{}, token.NewPosError(tkn.Pos, "not found visibility keyword.") } -func (p *Parser) ParseReturnParameters() (*ast.ReturnParameters, error) { - return nil, nil +func (p *Parser) ParseStateMutability() (ast.StateMutability, error) { + tkn, err := p.lexer.Scan() + if err != nil { + return token.Token{}, err + } + + switch tkn.TokenType { + case token.Pure, token.View, token.Payable: + return tkn, nil + } + + return token.Token{}, token.NewPosError(tkn.Pos, "not found state-mutability keyword.") } diff --git a/functionDefinition_test.go b/functionDefinition_test.go new file mode 100644 index 0000000..a258263 --- /dev/null +++ b/functionDefinition_test.go @@ -0,0 +1,100 @@ +package solparser_test + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseVisibility(t *testing.T) { + tests := []struct { + input string + want ast.Visibility + err *token.PosError + }{ + { + input: "internal", + want: ast.Visibility{ + TokenType: token.Internal, + Text: "internal", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + { + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{Column: 1, Line: 1}, + Msg: "not found visibility keyword.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.input, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseVisibility() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} + +func TestParser_ParseStateMutability(t *testing.T) { + tests := []struct { + input string + want ast.StateMutability + err *token.PosError + }{ + { + input: "pure", + want: ast.StateMutability{ + TokenType: token.Pure, + Text: "pure", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + { + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{Column: 1, Line: 1}, + Msg: "not found state-mutability keyword.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.input, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseStateMutability() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} From 4e51f4ddaa6194e5ccb30f44e2e2d8da0cbefcc3 Mon Sep 17 00:00:00 2001 From: uji Date: Thu, 23 Feb 2023 15:27:42 +0900 Subject: [PATCH 33/58] main, ast, token: implement identifier --- ast/ast.go | 4 +++ contractDefinition.go | 2 +- identifier.go | 20 +++++++++++ identifier_test.go | 80 +++++++++++++++++++++++++++++++++++++++++ pragmaDirective.go | 4 +-- pragmaDirective_test.go | 4 +-- token/token.go | 6 ++-- 7 files changed, 113 insertions(+), 7 deletions(-) create mode 100644 identifier.go create mode 100644 identifier_test.go diff --git a/ast/ast.go b/ast/ast.go index 390ffd5..0db342a 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -101,6 +101,10 @@ type Expression interface { expressionNode() } +// ---------------------------------------------------------------------------- + +type Identifier = token.Token + // ---------------------------------------------------------------------------- // Literal Nodes diff --git a/contractDefinition.go b/contractDefinition.go index af04f35..edbe354 100644 --- a/contractDefinition.go +++ b/contractDefinition.go @@ -25,7 +25,7 @@ func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { if err != nil { return nil, err } - if tkn.TokenType != token.Unknown { + if tkn.TokenType != token.Identifier { return nil, errors.New("not found left brace") } diff --git a/identifier.go b/identifier.go new file mode 100644 index 0000000..f908f3d --- /dev/null +++ b/identifier.go @@ -0,0 +1,20 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseIdentifier() (ast.Identifier, error) { + tkn, err := p.lexer.Scan() + if err != nil { + return token.Token{}, err + } + + switch tkn.TokenType { + case token.Identifier, token.From, token.Error, token.Revert, token.Global: + return tkn, nil + } + + return token.Token{}, token.NewPosError(tkn.Pos, "keyword is not available as identifier.") +} diff --git a/identifier_test.go b/identifier_test.go new file mode 100644 index 0000000..b6f658a --- /dev/null +++ b/identifier_test.go @@ -0,0 +1,80 @@ +package solparser_test + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseIdentifier(t *testing.T) { + tests := []struct { + input string + want ast.Identifier + err *token.PosError + }{ + { + input: "identifier", + want: token.Token{ + TokenType: token.Identifier, + Text: "identifier", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + { + input: "from", + want: token.Token{ + TokenType: token.From, + Text: "from", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + { + input: "error", + want: token.Token{ + TokenType: token.Error, + Text: "error", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + { + input: "global", + want: token.Token{ + TokenType: token.Global, + Text: "global", + Pos: token.Pos{Column: 1, Line: 1}, + }, + }, + { + input: "pragma", + err: &token.PosError{ + Pos: token.Pos{Column: 1, Line: 1}, + Msg: "keyword is not available as identifier.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.input, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseIdentifier() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} diff --git a/pragmaDirective.go b/pragmaDirective.go index 7a3b76a..8d3b457 100644 --- a/pragmaDirective.go +++ b/pragmaDirective.go @@ -18,7 +18,7 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { if err != nil { return nil, err } - if pragmaName.TokenType != token.Unknown { + if pragmaName.TokenType == token.Semicolon { return nil, token.NewPosError(pragmaName.Pos, "not found pragma name") } @@ -34,7 +34,7 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { if err != nil { return nil, err } - if version.TokenType != token.Unknown { + if version.TokenType != token.Identifier { return nil, token.NewPosError(version.Pos, "not found version") } diff --git a/pragmaDirective_test.go b/pragmaDirective_test.go index f2a65da..6c482d6 100644 --- a/pragmaDirective_test.go +++ b/pragmaDirective_test.go @@ -48,10 +48,10 @@ func TestParserParsePragmaDirective(t *testing.T) { want: nil, err: &token.PosError{ Pos: token.Pos{ - Column: 8, + Column: 9, Line: 1, }, - Msg: "not found pragma name", + Msg: "not found BitXor expression", }, }, { diff --git a/token/token.go b/token/token.go index 16d4f1f..2c82937 100644 --- a/token/token.go +++ b/token/token.go @@ -10,7 +10,6 @@ type TokenType int const ( Invalid TokenType = iota - Unknown EOS // End of source indicator. @@ -172,6 +171,9 @@ const ( UnicodeStringLiteral HexStringLiteral CommentLiteral + + // Not keywords or future reserved words + Identifier ) var EOSString string = string([]rune{bufrr.EOF}) @@ -470,7 +472,7 @@ func asKeyword(str string) TokenType { return While } - return Unknown + return Identifier } func asToken(str string) TokenType { From 9d2d5f0f52aabfaabb211f9398818af72194f7fa Mon Sep 17 00:00:00 2001 From: uji Date: Thu, 23 Feb 2023 17:28:31 +0900 Subject: [PATCH 34/58] main, ast: parse functionDefinition --- ast/ast.go | 23 +++-- functionDefinition.go | 129 ++++++++++++++++++++++++++ functionDefinition_test.go | 185 +++++++++++++++++++++++++++++++++++++ 3 files changed, 329 insertions(+), 8 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 0db342a..8c8a7ec 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -17,11 +17,8 @@ type PragmaDirective struct { PragmaValue PragmaValue } -type FunctionDescriptor struct { - Name string -} - type ModifierList struct { + Visibility *Visibility StateMutability *StateMutability } @@ -36,14 +33,24 @@ type ParameterList []*Parameter type StateMutability = token.Token -type ReturnParameters struct { - ParameterList *ParameterList +// identifier | fallback | recevie +type FunctionDescriptor = token.Token + +type FunctionDefinitionReturns struct { + From token.Pos + LParen token.Pos + ParameterList ParameterList + RParen token.Pos } type FunctionDefinition struct { - FunctionDescriptor *FunctionDescriptor + From token.Pos + FunctionDescriptor FunctionDescriptor + LParen token.Pos + RParen token.Pos ModifierList *ModifierList - ReturnParameters *ReturnParameters + Returns *FunctionDefinitionReturns + Block *Block } type InheritanceSpecifier struct { diff --git a/functionDefinition.go b/functionDefinition.go index bfddd4e..7c4b906 100644 --- a/functionDefinition.go +++ b/functionDefinition.go @@ -32,3 +32,132 @@ func (p *Parser) ParseStateMutability() (ast.StateMutability, error) { return token.Token{}, token.NewPosError(tkn.Pos, "not found state-mutability keyword.") } + +func (p *Parser) ParseFunctionDefinitionReturns() (*ast.FunctionDefinitionReturns, error) { + tkn, err := p.lexer.Peek() + if err != nil { + return nil, err + } + if tkn.TokenType != token.Returns { + return nil, nil + } + if _, err := p.lexer.Scan(); err != nil { + return nil, err + } + + lparen, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if lparen.TokenType != token.LParen { + return nil, token.NewPosError(lparen.Pos, "not found arguments LParen.") + } + + pl, err := p.ParseParameterList() + if err != nil { + return nil, err + } + + rparen, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if rparen.TokenType != token.RParen { + return nil, token.NewPosError(rparen.Pos, "not found arguments RParen.") + } + + return &ast.FunctionDefinitionReturns{ + From: tkn.Pos, + LParen: lparen.Pos, + ParameterList: pl, + RParen: rparen.Pos, + }, nil +} + +func (p *Parser) ParseFunctionDefinition() (*ast.FunctionDefinition, error) { + from, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + if from.TokenType != token.Function { + return nil, token.NewPosError(from.Pos, "not found function keyword.") + } + + dsc, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + switch dsc.TokenType { + case token.Identifier, token.From, token.Error, token.Revert, token.Global, token.Fallback, token.Receive: + default: + return nil, token.NewPosError(dsc.Pos, "not found function description.") + } + + lparen, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if lparen.TokenType != token.LParen { + return nil, token.NewPosError(lparen.Pos, "not found arguments LParen.") + } + + rparen, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if rparen.TokenType != token.RParen { + return nil, token.NewPosError(rparen.Pos, "not found arguments RParen.") + } + + modifierList := &ast.ModifierList{ + Visibility: nil, + StateMutability: nil, + } + + for { + tkn, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + switch tkn.TokenType { + case token.Internal, token.External, token.Public, token.Private: + vs, err := p.ParseVisibility() + if err != nil { + return nil, err + } + modifierList.Visibility = &vs + continue + case token.Pure, token.View, token.Payable: + sm, err := p.ParseStateMutability() + if err != nil { + return nil, err + } + modifierList.StateMutability = &sm + continue + } + break + } + + r, err := p.ParseFunctionDefinitionReturns() + if err != nil { + return nil, err + } + + b, err := p.ParseBlock() + if err != nil { + return nil, err + } + + return &ast.FunctionDefinition{ + From: from.Pos, + FunctionDescriptor: dsc, + LParen: lparen.Pos, + RParen: rparen.Pos, + ModifierList: modifierList, + Returns: r, + Block: b, + }, nil +} diff --git a/functionDefinition_test.go b/functionDefinition_test.go index a258263..6fed4a6 100644 --- a/functionDefinition_test.go +++ b/functionDefinition_test.go @@ -98,3 +98,188 @@ func TestParser_ParseStateMutability(t *testing.T) { }) } } + +func TestParser_ParseFunctionDefinitionReturns(t *testing.T) { + tests := []struct { + input string + want *ast.FunctionDefinitionReturns + err *token.PosError + }{ + { + input: "returns (string)", + want: &ast.FunctionDefinitionReturns{ + From: token.Pos{Column: 1, Line: 1}, + LParen: token.Pos{Column: 9, Line: 1}, + ParameterList: []*ast.Parameter{ + { + TypeName: ast.ElementaryTypeName{ + { + TokenType: token.String, + Text: "string", + Pos: token.Pos{Column: 10, Line: 1}, + }, + }, + }, + }, + RParen: token.Pos{Column: 16, Line: 1}, + }, + }, + { + input: ";", + want: nil, + }, + { + input: "returns string)", + err: &token.PosError{ + Pos: token.Pos{Column: 9, Line: 1}, + Msg: "not found arguments LParen.", + }, + }, + { + input: "returns (string {", + err: &token.PosError{ + Pos: token.Pos{Column: 17, Line: 1}, + Msg: "not found arguments RParen.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.input, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseFunctionDefinitionReturns() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} + +func TestParser_ParseFunctionDefinition(t *testing.T) { + tests := []struct { + name string + input string + want *ast.FunctionDefinition + err *token.PosError + }{ + { + name: "normal case", + input: `function hello() public pure returns (string) { + return "Hello World!!"; + }`, + want: &ast.FunctionDefinition{ + From: token.Pos{Column: 1, Line: 1}, + FunctionDescriptor: token.Token{ + TokenType: token.Identifier, + Text: "hello", + Pos: token.Pos{Column: 10, Line: 1}, + }, + LParen: token.Pos{Column: 15, Line: 1}, + RParen: token.Pos{Column: 16, Line: 1}, + ModifierList: &ast.ModifierList{ + Visibility: &token.Token{ + TokenType: token.Public, + Text: "public", + Pos: token.Pos{Column: 18, Line: 1}, + }, + StateMutability: &token.Token{ + TokenType: token.Pure, + Text: "pure", + Pos: token.Pos{Column: 25, Line: 1}, + }, + }, + Returns: &ast.FunctionDefinitionReturns{ + From: token.Pos{Column: 30, Line: 1}, + LParen: token.Pos{Column: 38, Line: 1}, + ParameterList: []*ast.Parameter{ + { + TypeName: ast.ElementaryTypeName{ + { + TokenType: token.String, + Text: "string", + Pos: token.Pos{Column: 39, Line: 1}, + }, + }, + }, + }, + RParen: token.Pos{Column: 45, Line: 1}, + }, + Block: &ast.Block{ + LBracePos: token.Pos{Column: 47, Line: 1}, + RBracePos: token.Pos{Column: 5, Line: 3}, + Nodes: []ast.Node{ + &ast.ReturnStatement{ + From: token.Pos{Column: 9, Line: 2}, + SemiPos: token.Pos{Column: 31, Line: 2}, + Expression: &ast.StringLiteral{ + From: token.Pos{Column: 16, Line: 2}, + To: token.Pos{Column: 30, Line: 2}, + Value: "\"Hello World!!\"", + }, + }, + }, + }, + }, + }, + { + name: "function description is invalid", + input: `function pragma() public pure returns (string) { + return "Hello World!!"; + }`, + err: &token.PosError{ + Pos: token.Pos{Column: 10, Line: 1}, + Msg: "not found function description.", + }, + }, + { + name: "not found lparen", + input: `function hello) public pure returns (string) { + return "Hello World!!"; + }`, + err: &token.PosError{ + Pos: token.Pos{Column: 15, Line: 1}, + Msg: "not found arguments LParen.", + }, + }, + { + name: "not found rparen", + input: `function hello( public pure returns (string) { + return "Hello World!!"; + }`, + err: &token.PosError{ + Pos: token.Pos{Column: 17, Line: 1}, + Msg: "not found arguments RParen.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseFunctionDefinition() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} From 68454fff5d282ab1a42328d530141aedb6efc8ec Mon Sep 17 00:00:00 2001 From: uji Date: Thu, 23 Feb 2023 21:30:29 +0900 Subject: [PATCH 35/58] main, ast: parse contractDefinition. --- ast/ast.go | 27 ++++- contractDefinition.go | 49 ++++---- contractDefinition_test.go | 240 +++++++++++++++++++++++++++++++++++++ 3 files changed, 290 insertions(+), 26 deletions(-) create mode 100644 contractDefinition_test.go diff --git a/ast/ast.go b/ast/ast.go index 8c8a7ec..9ad3766 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -53,20 +53,35 @@ type FunctionDefinition struct { Block *Block } +func (f FunctionDefinition) Pos() token.Pos { + return f.From +} + +func (f FunctionDefinition) End() token.Pos { + return f.Block.End() +} + type InheritanceSpecifier struct { IdentifierPath string CallArgumentList CallArgumentList } -type CallArgumentList struct{} +type ContractBodyElement interface { + Node + contractBodyElementNode() +} -type ContractBodyElement Node +func (f *FunctionDefinition) contractBodyElementNode() {} + +type CallArgumentList struct{} type ContractDefinition struct { - Position token.Pos - Abstract bool - InheritanceSpecifiers []*InheritanceSpecifier - ContractBodyElements []*ContractBodyElement + Abstract *token.Pos + Contract token.Pos + Identifier Identifier + LBrace token.Pos + ContractBodyElements []ContractBodyElement + RBrace token.Pos } // A File node represents a Solidity source file. diff --git a/contractDefinition.go b/contractDefinition.go index edbe354..503ba63 100644 --- a/contractDefinition.go +++ b/contractDefinition.go @@ -1,51 +1,60 @@ package solparser import ( - "errors" - "github.com/uji/solparser/ast" "github.com/uji/solparser/token" ) func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { - var abstract bool - tkn, err := p.lexer.Scan() + cntr, err := p.lexer.Scan() if err != nil { return nil, err } - if tkn.TokenType == token.Abstract { - abstract = true - p.lexer.Scan() + var abstractPos *token.Pos + if cntr.TokenType == token.Abstract { + pos := cntr.Pos + abstractPos = &pos + cntr, err = p.lexer.Scan() + if err != nil { + return nil, err + } } - if tkn.TokenType != token.Contract { - return nil, errors.New("not found contract definition") + if cntr.TokenType != token.Contract { + return nil, token.NewPosError(cntr.Pos, "not found contract keyword.") } - tkn, err = p.lexer.Scan() + i, err := p.ParseIdentifier() if err != nil { return nil, err } - if tkn.TokenType != token.Identifier { - return nil, errors.New("not found left brace") - } - tkn, err = p.lexer.Scan() + lbrace, err := p.lexer.Scan() if err != nil { return nil, err } - if tkn.TokenType != token.LBrace { - return nil, errors.New("not found left brace") + if lbrace.TokenType != token.LBrace { + return nil, token.NewPosError(lbrace.Pos, "not found left brace.") + } + + fn, err := p.ParseFunctionDefinition() + if err != nil { + return nil, err } - tkn, err = p.lexer.Scan() + rbrace, err := p.lexer.Scan() if err != nil { return nil, err } - if tkn.TokenType != token.RBrace { - return nil, errors.New("not found right brace") + if rbrace.TokenType != token.RBrace { + return nil, token.NewPosError(rbrace.Pos, "not found right brace.") } return &ast.ContractDefinition{ - Abstract: abstract, + Abstract: abstractPos, + Contract: cntr.Pos, + Identifier: i, + LBrace: lbrace.Pos, + ContractBodyElements: []ast.ContractBodyElement{fn}, + RBrace: rbrace.Pos, }, nil } diff --git a/contractDefinition_test.go b/contractDefinition_test.go new file mode 100644 index 0000000..15d7378 --- /dev/null +++ b/contractDefinition_test.go @@ -0,0 +1,240 @@ +package solparser_test + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseContractDefinition(t *testing.T) { + tests := []struct { + name string + input string + want *ast.ContractDefinition + err *token.PosError + }{ + { + name: "normal case", + input: `contract HelloWorld { + function hello() public pure returns (string) { + return "Hello World!!"; + } +}`, + want: &ast.ContractDefinition{ + Contract: token.Pos{Column: 1, Line: 1}, + Identifier: token.Token{ + TokenType: token.Identifier, + Text: "HelloWorld", + Pos: token.Pos{Column: 10, Line: 1}, + }, + LBrace: token.Pos{Column: 21, Line: 1}, + ContractBodyElements: []ast.ContractBodyElement{ + &ast.FunctionDefinition{ + From: token.Pos{Column: 5, Line: 2}, + FunctionDescriptor: token.Token{ + TokenType: token.Identifier, + Text: "hello", + Pos: token.Pos{Column: 14, Line: 2}, + }, + LParen: token.Pos{Column: 19, Line: 2}, + RParen: token.Pos{Column: 20, Line: 2}, + ModifierList: &ast.ModifierList{ + Visibility: &token.Token{ + TokenType: token.Public, + Text: "public", + Pos: token.Pos{Column: 22, Line: 2}, + }, + StateMutability: &token.Token{ + TokenType: token.Pure, + Text: "pure", + Pos: token.Pos{Column: 29, Line: 2}, + }, + }, + Returns: &ast.FunctionDefinitionReturns{ + From: token.Pos{Column: 34, Line: 2}, + LParen: token.Pos{Column: 42, Line: 2}, + ParameterList: []*ast.Parameter{ + { + TypeName: ast.ElementaryTypeName{ + { + TokenType: token.String, + Text: "string", + Pos: token.Pos{Column: 43, Line: 2}, + }, + }, + }, + }, + RParen: token.Pos{Column: 49, Line: 2}, + }, + Block: &ast.Block{ + LBracePos: token.Pos{Column: 51, Line: 2}, + RBracePos: token.Pos{Column: 5, Line: 4}, + Nodes: []ast.Node{ + &ast.ReturnStatement{ + From: token.Pos{Column: 9, Line: 3}, + SemiPos: token.Pos{Column: 31, Line: 3}, + Expression: &ast.StringLiteral{ + From: token.Pos{Column: 16, Line: 3}, + To: token.Pos{Column: 30, Line: 3}, + Value: "\"Hello World!!\"", + }, + }, + }, + }, + }, + }, + RBrace: token.Pos{Column: 1, Line: 5}, + }, + }, + { + name: "with abstract", + input: `abstract contract HelloWorld { + function hello() public pure returns (string) { + return "Hello World!!"; + } +}`, + want: &ast.ContractDefinition{ + Abstract: &token.Pos{Column: 1, Line: 1}, + Contract: token.Pos{Column: 10, Line: 1}, + Identifier: token.Token{ + TokenType: token.Identifier, + Text: "HelloWorld", + Pos: token.Pos{Column: 19, Line: 1}, + }, + LBrace: token.Pos{Column: 30, Line: 1}, + ContractBodyElements: []ast.ContractBodyElement{ + &ast.FunctionDefinition{ + From: token.Pos{Column: 5, Line: 2}, + FunctionDescriptor: token.Token{ + TokenType: token.Identifier, + Text: "hello", + Pos: token.Pos{Column: 14, Line: 2}, + }, + LParen: token.Pos{Column: 19, Line: 2}, + RParen: token.Pos{Column: 20, Line: 2}, + ModifierList: &ast.ModifierList{ + Visibility: &token.Token{ + TokenType: token.Public, + Text: "public", + Pos: token.Pos{Column: 22, Line: 2}, + }, + StateMutability: &token.Token{ + TokenType: token.Pure, + Text: "pure", + Pos: token.Pos{Column: 29, Line: 2}, + }, + }, + Returns: &ast.FunctionDefinitionReturns{ + From: token.Pos{Column: 34, Line: 2}, + LParen: token.Pos{Column: 42, Line: 2}, + ParameterList: []*ast.Parameter{ + { + TypeName: ast.ElementaryTypeName{ + { + TokenType: token.String, + Text: "string", + Pos: token.Pos{Column: 43, Line: 2}, + }, + }, + }, + }, + RParen: token.Pos{Column: 49, Line: 2}, + }, + Block: &ast.Block{ + LBracePos: token.Pos{Column: 51, Line: 2}, + RBracePos: token.Pos{Column: 5, Line: 4}, + Nodes: []ast.Node{ + &ast.ReturnStatement{ + From: token.Pos{Column: 9, Line: 3}, + SemiPos: token.Pos{Column: 31, Line: 3}, + Expression: &ast.StringLiteral{ + From: token.Pos{Column: 16, Line: 3}, + To: token.Pos{Column: 30, Line: 3}, + Value: "\"Hello World!!\"", + }, + }, + }, + }, + }, + }, + RBrace: token.Pos{Column: 1, Line: 5}, + }, + }, + { + name: "not found contract keyword", + input: `HelloWorld { + function hello() public pure returns (string) { + return "Hello World!!"; + } +}`, + err: &token.PosError{ + Pos: token.Pos{Column: 1, Line: 1}, + Msg: "not found contract keyword.", + }, + }, + { + name: "not found identifier", + input: `contract { + function hello() public pure returns (string) { + return "Hello World!!"; + } +}`, + err: &token.PosError{ + Pos: token.Pos{Column: 10, Line: 1}, + Msg: "keyword is not available as identifier.", + }, + }, + { + name: "not found LBrace", + input: "contract HelloWorld function", + err: &token.PosError{ + Pos: token.Pos{Column: 21, Line: 1}, + Msg: "not found left brace.", + }, + }, + { + name: "not found FunctionDefinition", + input: "contract HelloWorld { hello() }", + err: &token.PosError{ + Pos: token.Pos{Column: 23, Line: 1}, + Msg: "not found function keyword.", + }, + }, + { + name: "not found RBrace", + input: `contract HelloWorld { + function hello() public pure returns (string) { + return "Hello World!!"; + }`, + err: &token.PosError{ + Pos: token.Pos{Column: 6, Line: 4}, + Msg: "not found right brace.", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseContractDefinition() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} From f41607ed7d38d61f380953c2902b5c3279501d45 Mon Sep 17 00:00:00 2001 From: uji Date: Fri, 24 Feb 2023 23:19:36 +0900 Subject: [PATCH 36/58] main: Parse test --- solparser.go | 60 ++++++++++++------------ solparser_test.go | 114 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 30 deletions(-) diff --git a/solparser.go b/solparser.go index 778fad8..452195d 100644 --- a/solparser.go +++ b/solparser.go @@ -21,44 +21,44 @@ func New(input io.Reader) *Parser { } func (p *Parser) Parse() (*ast.SourceUnit, error) { - tkn, err := p.lexer.Peek() - if err == io.EOF { - return nil, nil - } - if err != nil { - return nil, err - } - var pragmaDirective *ast.PragmaDirective var contractDefinition *ast.ContractDefinition var functionDefinition *ast.FunctionDefinition - - switch tkn.TokenType { - case token.Pragma: - prgm, err := p.ParsePragmaDirective() - if err != nil { - return nil, err - } - pragmaDirective = prgm - case token.Abstract, token.Contract: - cntrct, err := p.ParseContractDefinition() + for { + tkn, err := p.lexer.Peek() if err != nil { return nil, err } - contractDefinition = cntrct - case token.Function: - fnc, err := p.ParseFunctionDefinition() - if err != nil { - return nil, err + + switch tkn.TokenType { + case token.Pragma: + prgm, err := p.ParsePragmaDirective() + if err != nil { + return nil, err + } + pragmaDirective = prgm + case token.Abstract, token.Contract: + cntrct, err := p.ParseContractDefinition() + if err != nil { + return nil, err + } + contractDefinition = cntrct + case token.Function: + fnc, err := p.ParseFunctionDefinition() + if err != nil { + return nil, err + } + functionDefinition = fnc + case token.EOS: + return &ast.SourceUnit{ + PragmaDirective: pragmaDirective, + ContractDefinition: contractDefinition, + FunctionDefinition: functionDefinition, + }, nil + default: + return nil, token.NewPosError(tkn.Pos, "invalid") } - functionDefinition = fnc } - - return &ast.SourceUnit{ - PragmaDirective: pragmaDirective, - ContractDefinition: contractDefinition, - FunctionDefinition: functionDefinition, - }, nil } func (p *Parser) ParseBooleanLiteral() (*ast.BooleanLiteral, error) { diff --git a/solparser_test.go b/solparser_test.go index d127fc3..78cd1e8 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -11,6 +11,120 @@ import ( "github.com/uji/solparser/token" ) +func TestParser_Parse(t *testing.T) { + tests := []struct { + name string + input string + want *ast.SourceUnit + err *token.PosError + }{ + { + name: "normal", + input: `pragma solidity ^0.8.13; + +contract HelloWorld { + function hello() public pure returns (string) { + return "Hello World!!"; + } +}`, + want: &ast.SourceUnit{ + PragmaDirective: &ast.PragmaDirective{ + PragmaName: "solidity", + PragmaValue: ast.PragmaValue{ + Version: "0.8.13", + Expression: "^", + }, + }, + ContractDefinition: &ast.ContractDefinition{ + Contract: token.Pos{Column: 1, Line: 3}, + Identifier: token.Token{ + TokenType: token.Identifier, + Text: "HelloWorld", + Pos: token.Pos{Column: 10, Line: 3}, + }, + LBrace: token.Pos{Column: 21, Line: 3}, + ContractBodyElements: []ast.ContractBodyElement{ + &ast.FunctionDefinition{ + From: token.Pos{Column: 5, Line: 4}, + FunctionDescriptor: token.Token{ + TokenType: token.Identifier, + Text: "hello", + Pos: token.Pos{Column: 14, Line: 4}, + }, + LParen: token.Pos{Column: 19, Line: 4}, + RParen: token.Pos{Column: 20, Line: 4}, + ModifierList: &ast.ModifierList{ + Visibility: &token.Token{ + TokenType: token.Public, + Text: "public", + Pos: token.Pos{Column: 22, Line: 4}, + }, + StateMutability: &token.Token{ + TokenType: token.Pure, + Text: "pure", + Pos: token.Pos{Column: 29, Line: 4}, + }, + }, + Returns: &ast.FunctionDefinitionReturns{ + From: token.Pos{Column: 34, Line: 4}, + LParen: token.Pos{Column: 42, Line: 4}, + ParameterList: []*ast.Parameter{ + { + TypeName: ast.ElementaryTypeName{ + { + TokenType: token.String, + Text: "string", + Pos: token.Pos{Column: 43, Line: 4}, + }, + }, + }, + }, + RParen: token.Pos{Column: 49, Line: 4}, + }, + Block: &ast.Block{ + LBracePos: token.Pos{Column: 51, Line: 4}, + RBracePos: token.Pos{Column: 5, Line: 6}, + Nodes: []ast.Node{ + &ast.ReturnStatement{ + From: token.Pos{Column: 9, Line: 5}, + SemiPos: token.Pos{Column: 31, Line: 5}, + Expression: &ast.StringLiteral{ + From: token.Pos{Column: 16, Line: 5}, + To: token.Pos{Column: 30, Line: 5}, + Value: "\"Hello World!!\"", + }, + }, + }, + }, + }, + }, + RBrace: token.Pos{Column: 1, Line: 7}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.Parse() + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} + func TestParser_ParseBooleanLiteral(t *testing.T) { tests := []struct { name string From 5de6da14dffde7caa4db084295c3cbfd6f6c4d74 Mon Sep 17 00:00:00 2001 From: uji Date: Fri, 24 Feb 2023 23:35:44 +0900 Subject: [PATCH 37/58] main, ast: fix pragmaDirective ast --- ast/ast.go | 10 ++----- pragmaDirective.go | 64 +++++++++++++++-------------------------- pragmaDirective_test.go | 47 +++++++++++++++--------------- solparser_test.go | 22 +++++++++++--- 4 files changed, 68 insertions(+), 75 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 9ad3766..8238d3f 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -7,14 +7,10 @@ type Node interface { End() token.Pos } -type PragmaValue struct { - Version string - Expression string -} - type PragmaDirective struct { - PragmaName string - PragmaValue PragmaValue + Pragma token.Pos + PragmaTokens []*token.Token + Semicolon token.Pos } type ModifierList struct { diff --git a/pragmaDirective.go b/pragmaDirective.go index 8d3b457..d234ed8 100644 --- a/pragmaDirective.go +++ b/pragmaDirective.go @@ -6,51 +6,33 @@ import ( ) func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { - tkn, err := p.lexer.Scan() + prgm, err := p.lexer.Scan() if err != nil { return nil, err } - if tkn.TokenType != token.Pragma { - return nil, token.NewPosError(tkn.Pos, "not found pragma") + if prgm.TokenType != token.Pragma { + return nil, token.NewPosError(prgm.Pos, "not found pragma") } - pragmaName, err := p.lexer.Scan() - if err != nil { - return nil, err - } - if pragmaName.TokenType == token.Semicolon { - return nil, token.NewPosError(pragmaName.Pos, "not found pragma name") - } - - expression, err := p.lexer.Scan() - if err != nil { - return nil, err - } - if expression.TokenType != token.BitXor { - return nil, token.NewPosError(expression.Pos, "not found BitXor expression") + tkns := make([]*token.Token, 0, 1) + for { + tkn, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if tkn.TokenType == token.EOS { + return nil, token.NewPosError(tkn.Pos, "not found Semicolon.") + } + if tkn.TokenType == token.Semicolon { + if len(tkns) == 0 { + return nil, token.NewPosError(tkn.Pos, "not found pragma tokens.") + } + return &ast.PragmaDirective{ + Pragma: prgm.Pos, + PragmaTokens: tkns, + Semicolon: tkn.Pos, + }, nil + } + tkns = append(tkns, &tkn) } - - version, err := p.lexer.Scan() - if err != nil { - return nil, err - } - if version.TokenType != token.Identifier { - return nil, token.NewPosError(version.Pos, "not found version") - } - - tkn, err = p.lexer.Scan() - if err != nil { - return nil, err - } - if tkn.TokenType != token.Semicolon { - return nil, token.NewPosError(tkn.Pos, "not found Semicolon") - } - - return &ast.PragmaDirective{ - PragmaName: pragmaName.Text, - PragmaValue: ast.PragmaValue{ - Version: version.Text, - Expression: expression.Text, - }, - }, nil } diff --git a/pragmaDirective_test.go b/pragmaDirective_test.go index 6c482d6..8daa561 100644 --- a/pragmaDirective_test.go +++ b/pragmaDirective_test.go @@ -11,7 +11,7 @@ import ( "github.com/uji/solparser/token" ) -func TestParserParsePragmaDirective(t *testing.T) { +func TestParser_ParsePragmaDirective(t *testing.T) { tests := []struct { name string input string @@ -22,13 +22,26 @@ func TestParserParsePragmaDirective(t *testing.T) { name: "normal case", input: "pragma solidity ^0.8.13;", want: &ast.PragmaDirective{ - PragmaName: "solidity", - PragmaValue: ast.PragmaValue{ - Version: "0.8.13", - Expression: "^", + Pragma: token.Pos{Column: 1, Line: 1}, + PragmaTokens: []*token.Token{ + { + TokenType: token.Identifier, + Text: "solidity", + Pos: token.Pos{Column: 8, Line: 1}, + }, + { + TokenType: token.BitXor, + Text: "^", + Pos: token.Pos{Column: 17, Line: 1}, + }, + { + TokenType: token.Identifier, + Text: "0.8.13", + Pos: token.Pos{Column: 18, Line: 1}, + }, }, + Semicolon: token.Pos{Column: 24, Line: 1}, }, - err: nil, }, { name: "there is no pragma keyword", @@ -43,27 +56,15 @@ func TestParserParsePragmaDirective(t *testing.T) { }, }, { - name: "there is no pragma name", - input: "pragma ^0.8.13;", + name: "there is no pragma tokens", + input: "pragma ;", want: nil, err: &token.PosError{ Pos: token.Pos{ - Column: 9, + Column: 8, Line: 1, }, - Msg: "not found BitXor expression", - }, - }, - { - name: "there is no BitXor expression", - input: "pragma solidity 0.8.13;", - want: nil, - err: &token.PosError{ - Pos: token.Pos{ - Column: 17, - Line: 1, - }, - Msg: "not found BitXor expression", + Msg: "not found pragma tokens.", }, }, { @@ -75,7 +76,7 @@ func TestParserParsePragmaDirective(t *testing.T) { Column: 24, Line: 1, }, - Msg: "not found Semicolon", + Msg: "not found Semicolon.", }, }, } diff --git a/solparser_test.go b/solparser_test.go index 78cd1e8..8bbb09d 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -29,11 +29,25 @@ contract HelloWorld { }`, want: &ast.SourceUnit{ PragmaDirective: &ast.PragmaDirective{ - PragmaName: "solidity", - PragmaValue: ast.PragmaValue{ - Version: "0.8.13", - Expression: "^", + Pragma: token.Pos{Column: 1, Line: 1}, + PragmaTokens: []*token.Token{ + { + TokenType: token.Identifier, + Text: "solidity", + Pos: token.Pos{Column: 8, Line: 1}, + }, + { + TokenType: token.BitXor, + Text: "^", + Pos: token.Pos{Column: 17, Line: 1}, + }, + { + TokenType: token.Identifier, + Text: "0.8.13", + Pos: token.Pos{Column: 18, Line: 1}, + }, }, + Semicolon: token.Pos{Column: 24, Line: 1}, }, ContractDefinition: &ast.ContractDefinition{ Contract: token.Pos{Column: 1, Line: 3}, From b7c229562a463376e438a72d9c5e6132d36d9ddb Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 25 Feb 2023 12:15:31 +0900 Subject: [PATCH 38/58] main: add test helper functions --- helper_test.go | 42 ++++++++++++++++++++++ pragmaDirective.go | 2 +- pragmaDirective_test.go | 79 ++++++----------------------------------- 3 files changed, 53 insertions(+), 70 deletions(-) create mode 100644 helper_test.go diff --git a/helper_test.go b/helper_test.go new file mode 100644 index 0000000..de6cd65 --- /dev/null +++ b/helper_test.go @@ -0,0 +1,42 @@ +package solparser_test + +import ( + "errors" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser/token" +) + +func pos(c, l int) token.Pos { + return token.Pos{Column: c, Line: l} +} + +func tkn(tp token.TokenType, text string, pos token.Pos) token.Token { + return token.Token{ + TokenType: tp, + Text: text, + Pos: pos, + } +} + +func perr(pos token.Pos, msg string) *token.PosError { + return &token.PosError{ + Pos: pos, + Msg: msg, + } +} + +func assert(t *testing.T, got, want interface{}, gotErr, wantErr error) { + t.Helper() + var sErr *token.PosError + if errors.As(gotErr, &sErr) { + if diff := cmp.Diff(wantErr, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("%s", diff) + } +} diff --git a/pragmaDirective.go b/pragmaDirective.go index d234ed8..aed0f54 100644 --- a/pragmaDirective.go +++ b/pragmaDirective.go @@ -11,7 +11,7 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { return nil, err } if prgm.TokenType != token.Pragma { - return nil, token.NewPosError(prgm.Pos, "not found pragma") + return nil, token.NewPosError(prgm.Pos, "not found pragma.") } tkns := make([]*token.Token, 0, 1) diff --git a/pragmaDirective_test.go b/pragmaDirective_test.go index 8daa561..dbd37e9 100644 --- a/pragmaDirective_test.go +++ b/pragmaDirective_test.go @@ -1,11 +1,9 @@ package solparser_test import ( - "errors" "strings" "testing" - "github.com/google/go-cmp/cmp" "github.com/uji/solparser" "github.com/uji/solparser/ast" "github.com/uji/solparser/token" @@ -13,91 +11,34 @@ import ( func TestParser_ParsePragmaDirective(t *testing.T) { tests := []struct { - name string input string want *ast.PragmaDirective err *token.PosError }{ { - name: "normal case", input: "pragma solidity ^0.8.13;", want: &ast.PragmaDirective{ - Pragma: token.Pos{Column: 1, Line: 1}, + Pragma: pos(1, 1), PragmaTokens: []*token.Token{ - { - TokenType: token.Identifier, - Text: "solidity", - Pos: token.Pos{Column: 8, Line: 1}, - }, - { - TokenType: token.BitXor, - Text: "^", - Pos: token.Pos{Column: 17, Line: 1}, - }, - { - TokenType: token.Identifier, - Text: "0.8.13", - Pos: token.Pos{Column: 18, Line: 1}, - }, + {TokenType: token.Identifier, Text: "solidity", Pos: pos(8, 1)}, + {TokenType: token.BitXor, Text: "^", Pos: pos(17, 1)}, + {TokenType: token.Identifier, Text: "0.8.13", Pos: pos(18, 1)}, }, - Semicolon: token.Pos{Column: 24, Line: 1}, - }, - }, - { - name: "there is no pragma keyword", - input: "solidity ^0.8.13;", - want: nil, - err: &token.PosError{ - Pos: token.Pos{ - Column: 1, - Line: 1, - }, - Msg: "not found pragma", - }, - }, - { - name: "there is no pragma tokens", - input: "pragma ;", - want: nil, - err: &token.PosError{ - Pos: token.Pos{ - Column: 8, - Line: 1, - }, - Msg: "not found pragma tokens.", - }, - }, - { - name: "there is no Semicolon", - input: "pragma solidity ^0.8.13", - want: nil, - err: &token.PosError{ - Pos: token.Pos{ - Column: 24, - Line: 1, - }, - Msg: "not found Semicolon.", + Semicolon: pos(24, 1), }, }, + {input: "solidity ^0.8.13;", err: perr(pos(1, 1), "not found pragma.")}, + {input: "pragma ;", err: perr(pos(8, 1), "not found pragma tokens.")}, + {input: "pragma solidity ^0.8.13", err: perr(pos(24, 1), "not found Semicolon.")}, } for _, tt := range tests { - t.Run(tt.name, func(*testing.T) { + t.Run(tt.input, func(*testing.T) { r := strings.NewReader(tt.input) p := solparser.New(r) got, err := p.ParsePragmaDirective() - - var sErr *token.PosError - if errors.As(err, &sErr) { - if diff := cmp.Diff(tt.err, sErr); diff != "" { - t.Errorf("%s", diff) - } - } - - if diff := cmp.Diff(tt.want, got); diff != "" { - t.Errorf("%s", diff) - } + assert(t, got, tt.want, err, tt.err) }) } } From 8e5ebf0f14f4ede9e8a92781359b1e62393c26f3 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 26 Feb 2023 15:57:56 +0900 Subject: [PATCH 39/58] main, ast: refactor elementaryTypeName --- ast/ast.go | 8 ++++---- typeName.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 8238d3f..f52c9fb 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -95,17 +95,17 @@ type TypeName interface { typeNameNode() } -type ElementaryTypeName []token.Token +type ElementaryTypeName []*token.Token func (e ElementaryTypeName) Pos() token.Pos { return e[0].Pos } func (e ElementaryTypeName) End() token.Pos { - // TODO + eTkn := e[len(e)-1] return token.Pos{ - Column: 1, - Line: 1, + Column: eTkn.Pos.Column + len(eTkn.Text), + Line: eTkn.Pos.Line, } } diff --git a/typeName.go b/typeName.go index 96d5ea2..fbf983a 100644 --- a/typeName.go +++ b/typeName.go @@ -33,13 +33,13 @@ func (p *Parser) ParseElementaryTypeName() (ast.TypeName, error) { if tkn2.TokenType == token.Payable { p.lexer.Scan() - return ast.ElementaryTypeName{tkn, tkn2}, nil + return ast.ElementaryTypeName{&tkn, &tkn2}, nil } } switch tkn.TokenType { case token.Address, token.String, token.Bytes, token.Fixed, token.Bool: - return ast.ElementaryTypeName{tkn}, nil + return ast.ElementaryTypeName{&tkn}, nil } return nil, token.NewPosError(tkn.Pos, "not found elementary type name keyword.") From 1934c4a74fadc9bc75042aac8acf71a9cd41c219 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 26 Feb 2023 16:22:46 +0900 Subject: [PATCH 40/58] all: rename token field --- ast/ast.go | 12 ++++---- block.go | 12 ++++---- contractDefinition.go | 22 +++++++------- contractDefinition_test.go | 60 +++++++++++++++++++------------------- expression.go | 4 +-- functionDefinition.go | 48 +++++++++++++++--------------- functionDefinition_test.go | 42 +++++++++++++------------- helper_test.go | 6 ++-- identifier.go | 4 +-- identifier_test.go | 24 +++++++-------- lexer/lexer.go | 6 ++-- lexer/lexer_test.go | 44 ++++++++++++++-------------- literal.go | 12 ++++---- literal_test.go | 7 ++--- parameterList.go | 2 +- parameterList_test.go | 24 +++++++-------- pragmaDirective.go | 16 +++++----- pragmaDirective_test.go | 6 ++-- solparser.go | 8 ++--- solparser_test.go | 60 +++++++++++++++++++------------------- statement.go | 16 +++++----- token/token.go | 12 ++++---- typeName.go | 12 ++++---- typeName_test.go | 36 +++++++++++------------ 24 files changed, 247 insertions(+), 248 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index f52c9fb..3776ef2 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -98,14 +98,14 @@ type TypeName interface { type ElementaryTypeName []*token.Token func (e ElementaryTypeName) Pos() token.Pos { - return e[0].Pos + return e[0].Position } func (e ElementaryTypeName) End() token.Pos { eTkn := e[len(e)-1] return token.Pos{ - Column: eTkn.Pos.Column + len(eTkn.Text), - Line: eTkn.Pos.Line, + Column: eTkn.Position.Column + len(eTkn.Value), + Line: eTkn.Position.Line, } } @@ -137,13 +137,13 @@ type BooleanLiteral struct { } func (b *BooleanLiteral) Pos() token.Pos { - return b.Token.Pos + return b.Token.Position } func (b *BooleanLiteral) End() token.Pos { return token.Pos{ - Column: b.Token.Pos.Column + len(b.Token.Text), - Line: b.Token.Pos.Line, + Column: b.Token.Position.Column + len(b.Token.Value), + Line: b.Token.Position.Line, } } diff --git a/block.go b/block.go index e1399e3..2166456 100644 --- a/block.go +++ b/block.go @@ -11,8 +11,8 @@ func (p *Parser) ParseBlock() (*ast.Block, error) { return nil, err } - if lblace.TokenType != token.LBrace { - return nil, token.NewPosError(lblace.Pos, "not found LBrace.") + if lblace.Type != token.LBrace { + return nil, token.NewPosError(lblace.Position, "not found LBrace.") } stmt, err := p.ParseStatement() @@ -25,13 +25,13 @@ func (p *Parser) ParseBlock() (*ast.Block, error) { return nil, err } - if rblace.TokenType != token.RBrace { - return nil, token.NewPosError(rblace.Pos, "not found RBrace.") + if rblace.Type != token.RBrace { + return nil, token.NewPosError(rblace.Position, "not found RBrace.") } return &ast.Block{ - LBracePos: lblace.Pos, - RBracePos: rblace.Pos, + LBracePos: lblace.Position, + RBracePos: rblace.Position, Nodes: []ast.Node{stmt}, }, nil } diff --git a/contractDefinition.go b/contractDefinition.go index 503ba63..47db899 100644 --- a/contractDefinition.go +++ b/contractDefinition.go @@ -11,16 +11,16 @@ func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { return nil, err } var abstractPos *token.Pos - if cntr.TokenType == token.Abstract { - pos := cntr.Pos + if cntr.Type == token.Abstract { + pos := cntr.Position abstractPos = &pos cntr, err = p.lexer.Scan() if err != nil { return nil, err } } - if cntr.TokenType != token.Contract { - return nil, token.NewPosError(cntr.Pos, "not found contract keyword.") + if cntr.Type != token.Contract { + return nil, token.NewPosError(cntr.Position, "not found contract keyword.") } i, err := p.ParseIdentifier() @@ -32,8 +32,8 @@ func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { if err != nil { return nil, err } - if lbrace.TokenType != token.LBrace { - return nil, token.NewPosError(lbrace.Pos, "not found left brace.") + if lbrace.Type != token.LBrace { + return nil, token.NewPosError(lbrace.Position, "not found left brace.") } fn, err := p.ParseFunctionDefinition() @@ -45,16 +45,16 @@ func (p *Parser) ParseContractDefinition() (*ast.ContractDefinition, error) { if err != nil { return nil, err } - if rbrace.TokenType != token.RBrace { - return nil, token.NewPosError(rbrace.Pos, "not found right brace.") + if rbrace.Type != token.RBrace { + return nil, token.NewPosError(rbrace.Position, "not found right brace.") } return &ast.ContractDefinition{ Abstract: abstractPos, - Contract: cntr.Pos, + Contract: cntr.Position, Identifier: i, - LBrace: lbrace.Pos, + LBrace: lbrace.Position, ContractBodyElements: []ast.ContractBodyElement{fn}, - RBrace: rbrace.Pos, + RBrace: rbrace.Position, }, nil } diff --git a/contractDefinition_test.go b/contractDefinition_test.go index 15d7378..5abd512 100644 --- a/contractDefinition_test.go +++ b/contractDefinition_test.go @@ -28,31 +28,31 @@ func TestParser_ParseContractDefinition(t *testing.T) { want: &ast.ContractDefinition{ Contract: token.Pos{Column: 1, Line: 1}, Identifier: token.Token{ - TokenType: token.Identifier, - Text: "HelloWorld", - Pos: token.Pos{Column: 10, Line: 1}, + Type: token.Identifier, + Value: "HelloWorld", + Position: token.Pos{Column: 10, Line: 1}, }, LBrace: token.Pos{Column: 21, Line: 1}, ContractBodyElements: []ast.ContractBodyElement{ &ast.FunctionDefinition{ From: token.Pos{Column: 5, Line: 2}, FunctionDescriptor: token.Token{ - TokenType: token.Identifier, - Text: "hello", - Pos: token.Pos{Column: 14, Line: 2}, + Type: token.Identifier, + Value: "hello", + Position: token.Pos{Column: 14, Line: 2}, }, LParen: token.Pos{Column: 19, Line: 2}, RParen: token.Pos{Column: 20, Line: 2}, ModifierList: &ast.ModifierList{ Visibility: &token.Token{ - TokenType: token.Public, - Text: "public", - Pos: token.Pos{Column: 22, Line: 2}, + Type: token.Public, + Value: "public", + Position: token.Pos{Column: 22, Line: 2}, }, StateMutability: &token.Token{ - TokenType: token.Pure, - Text: "pure", - Pos: token.Pos{Column: 29, Line: 2}, + Type: token.Pure, + Value: "pure", + Position: token.Pos{Column: 29, Line: 2}, }, }, Returns: &ast.FunctionDefinitionReturns{ @@ -62,9 +62,9 @@ func TestParser_ParseContractDefinition(t *testing.T) { { TypeName: ast.ElementaryTypeName{ { - TokenType: token.String, - Text: "string", - Pos: token.Pos{Column: 43, Line: 2}, + Type: token.String, + Value: "string", + Position: token.Pos{Column: 43, Line: 2}, }, }, }, @@ -102,31 +102,31 @@ func TestParser_ParseContractDefinition(t *testing.T) { Abstract: &token.Pos{Column: 1, Line: 1}, Contract: token.Pos{Column: 10, Line: 1}, Identifier: token.Token{ - TokenType: token.Identifier, - Text: "HelloWorld", - Pos: token.Pos{Column: 19, Line: 1}, + Type: token.Identifier, + Value: "HelloWorld", + Position: token.Pos{Column: 19, Line: 1}, }, LBrace: token.Pos{Column: 30, Line: 1}, ContractBodyElements: []ast.ContractBodyElement{ &ast.FunctionDefinition{ From: token.Pos{Column: 5, Line: 2}, FunctionDescriptor: token.Token{ - TokenType: token.Identifier, - Text: "hello", - Pos: token.Pos{Column: 14, Line: 2}, + Type: token.Identifier, + Value: "hello", + Position: token.Pos{Column: 14, Line: 2}, }, LParen: token.Pos{Column: 19, Line: 2}, RParen: token.Pos{Column: 20, Line: 2}, ModifierList: &ast.ModifierList{ Visibility: &token.Token{ - TokenType: token.Public, - Text: "public", - Pos: token.Pos{Column: 22, Line: 2}, + Type: token.Public, + Value: "public", + Position: token.Pos{Column: 22, Line: 2}, }, StateMutability: &token.Token{ - TokenType: token.Pure, - Text: "pure", - Pos: token.Pos{Column: 29, Line: 2}, + Type: token.Pure, + Value: "pure", + Position: token.Pos{Column: 29, Line: 2}, }, }, Returns: &ast.FunctionDefinitionReturns{ @@ -136,9 +136,9 @@ func TestParser_ParseContractDefinition(t *testing.T) { { TypeName: ast.ElementaryTypeName{ { - TokenType: token.String, - Text: "string", - Pos: token.Pos{Column: 43, Line: 2}, + Type: token.String, + Value: "string", + Position: token.Pos{Column: 43, Line: 2}, }, }, }, diff --git a/expression.go b/expression.go index 8a8b11b..f9a8d40 100644 --- a/expression.go +++ b/expression.go @@ -11,9 +11,9 @@ func (p *Parser) ParseExpression() (ast.Expression, error) { return nil, err } - if tkn.TokenType == token.StringLiteral { + if tkn.Type == token.StringLiteral { return p.ParseLiteral() } - return nil, token.NewPosError(tkn.Pos, "not found expression.") + return nil, token.NewPosError(tkn.Position, "not found expression.") } diff --git a/functionDefinition.go b/functionDefinition.go index 7c4b906..4c9ca81 100644 --- a/functionDefinition.go +++ b/functionDefinition.go @@ -11,12 +11,12 @@ func (p *Parser) ParseVisibility() (ast.Visibility, error) { return token.Token{}, err } - switch tkn.TokenType { + switch tkn.Type { case token.Internal, token.External, token.Public, token.Private: return tkn, nil } - return token.Token{}, token.NewPosError(tkn.Pos, "not found visibility keyword.") + return token.Token{}, token.NewPosError(tkn.Position, "not found visibility keyword.") } func (p *Parser) ParseStateMutability() (ast.StateMutability, error) { @@ -25,12 +25,12 @@ func (p *Parser) ParseStateMutability() (ast.StateMutability, error) { return token.Token{}, err } - switch tkn.TokenType { + switch tkn.Type { case token.Pure, token.View, token.Payable: return tkn, nil } - return token.Token{}, token.NewPosError(tkn.Pos, "not found state-mutability keyword.") + return token.Token{}, token.NewPosError(tkn.Position, "not found state-mutability keyword.") } func (p *Parser) ParseFunctionDefinitionReturns() (*ast.FunctionDefinitionReturns, error) { @@ -38,7 +38,7 @@ func (p *Parser) ParseFunctionDefinitionReturns() (*ast.FunctionDefinitionReturn if err != nil { return nil, err } - if tkn.TokenType != token.Returns { + if tkn.Type != token.Returns { return nil, nil } if _, err := p.lexer.Scan(); err != nil { @@ -49,8 +49,8 @@ func (p *Parser) ParseFunctionDefinitionReturns() (*ast.FunctionDefinitionReturn if err != nil { return nil, err } - if lparen.TokenType != token.LParen { - return nil, token.NewPosError(lparen.Pos, "not found arguments LParen.") + if lparen.Type != token.LParen { + return nil, token.NewPosError(lparen.Position, "not found arguments LParen.") } pl, err := p.ParseParameterList() @@ -62,15 +62,15 @@ func (p *Parser) ParseFunctionDefinitionReturns() (*ast.FunctionDefinitionReturn if err != nil { return nil, err } - if rparen.TokenType != token.RParen { - return nil, token.NewPosError(rparen.Pos, "not found arguments RParen.") + if rparen.Type != token.RParen { + return nil, token.NewPosError(rparen.Position, "not found arguments RParen.") } return &ast.FunctionDefinitionReturns{ - From: tkn.Pos, - LParen: lparen.Pos, + From: tkn.Position, + LParen: lparen.Position, ParameterList: pl, - RParen: rparen.Pos, + RParen: rparen.Position, }, nil } @@ -80,8 +80,8 @@ func (p *Parser) ParseFunctionDefinition() (*ast.FunctionDefinition, error) { return nil, err } - if from.TokenType != token.Function { - return nil, token.NewPosError(from.Pos, "not found function keyword.") + if from.Type != token.Function { + return nil, token.NewPosError(from.Position, "not found function keyword.") } dsc, err := p.lexer.Scan() @@ -89,26 +89,26 @@ func (p *Parser) ParseFunctionDefinition() (*ast.FunctionDefinition, error) { return nil, err } - switch dsc.TokenType { + switch dsc.Type { case token.Identifier, token.From, token.Error, token.Revert, token.Global, token.Fallback, token.Receive: default: - return nil, token.NewPosError(dsc.Pos, "not found function description.") + return nil, token.NewPosError(dsc.Position, "not found function description.") } lparen, err := p.lexer.Scan() if err != nil { return nil, err } - if lparen.TokenType != token.LParen { - return nil, token.NewPosError(lparen.Pos, "not found arguments LParen.") + if lparen.Type != token.LParen { + return nil, token.NewPosError(lparen.Position, "not found arguments LParen.") } rparen, err := p.lexer.Scan() if err != nil { return nil, err } - if rparen.TokenType != token.RParen { - return nil, token.NewPosError(rparen.Pos, "not found arguments RParen.") + if rparen.Type != token.RParen { + return nil, token.NewPosError(rparen.Position, "not found arguments RParen.") } modifierList := &ast.ModifierList{ @@ -122,7 +122,7 @@ func (p *Parser) ParseFunctionDefinition() (*ast.FunctionDefinition, error) { return nil, err } - switch tkn.TokenType { + switch tkn.Type { case token.Internal, token.External, token.Public, token.Private: vs, err := p.ParseVisibility() if err != nil { @@ -152,10 +152,10 @@ func (p *Parser) ParseFunctionDefinition() (*ast.FunctionDefinition, error) { } return &ast.FunctionDefinition{ - From: from.Pos, + From: from.Position, FunctionDescriptor: dsc, - LParen: lparen.Pos, - RParen: rparen.Pos, + LParen: lparen.Position, + RParen: rparen.Position, ModifierList: modifierList, Returns: r, Block: b, diff --git a/functionDefinition_test.go b/functionDefinition_test.go index 6fed4a6..308341e 100644 --- a/functionDefinition_test.go +++ b/functionDefinition_test.go @@ -20,9 +20,9 @@ func TestParser_ParseVisibility(t *testing.T) { { input: "internal", want: ast.Visibility{ - TokenType: token.Internal, - Text: "internal", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.Internal, + Value: "internal", + Position: token.Pos{Column: 1, Line: 1}, }, }, { @@ -64,9 +64,9 @@ func TestParser_ParseStateMutability(t *testing.T) { { input: "pure", want: ast.StateMutability{ - TokenType: token.Pure, - Text: "pure", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.Pure, + Value: "pure", + Position: token.Pos{Column: 1, Line: 1}, }, }, { @@ -114,9 +114,9 @@ func TestParser_ParseFunctionDefinitionReturns(t *testing.T) { { TypeName: ast.ElementaryTypeName{ { - TokenType: token.String, - Text: "string", - Pos: token.Pos{Column: 10, Line: 1}, + Type: token.String, + Value: "string", + Position: token.Pos{Column: 10, Line: 1}, }, }, }, @@ -180,22 +180,22 @@ func TestParser_ParseFunctionDefinition(t *testing.T) { want: &ast.FunctionDefinition{ From: token.Pos{Column: 1, Line: 1}, FunctionDescriptor: token.Token{ - TokenType: token.Identifier, - Text: "hello", - Pos: token.Pos{Column: 10, Line: 1}, + Type: token.Identifier, + Value: "hello", + Position: token.Pos{Column: 10, Line: 1}, }, LParen: token.Pos{Column: 15, Line: 1}, RParen: token.Pos{Column: 16, Line: 1}, ModifierList: &ast.ModifierList{ Visibility: &token.Token{ - TokenType: token.Public, - Text: "public", - Pos: token.Pos{Column: 18, Line: 1}, + Type: token.Public, + Value: "public", + Position: token.Pos{Column: 18, Line: 1}, }, StateMutability: &token.Token{ - TokenType: token.Pure, - Text: "pure", - Pos: token.Pos{Column: 25, Line: 1}, + Type: token.Pure, + Value: "pure", + Position: token.Pos{Column: 25, Line: 1}, }, }, Returns: &ast.FunctionDefinitionReturns{ @@ -205,9 +205,9 @@ func TestParser_ParseFunctionDefinition(t *testing.T) { { TypeName: ast.ElementaryTypeName{ { - TokenType: token.String, - Text: "string", - Pos: token.Pos{Column: 39, Line: 1}, + Type: token.String, + Value: "string", + Position: token.Pos{Column: 39, Line: 1}, }, }, }, diff --git a/helper_test.go b/helper_test.go index de6cd65..d80c961 100644 --- a/helper_test.go +++ b/helper_test.go @@ -14,9 +14,9 @@ func pos(c, l int) token.Pos { func tkn(tp token.TokenType, text string, pos token.Pos) token.Token { return token.Token{ - TokenType: tp, - Text: text, - Pos: pos, + Type: tp, + Value: text, + Position: pos, } } diff --git a/identifier.go b/identifier.go index f908f3d..335996d 100644 --- a/identifier.go +++ b/identifier.go @@ -11,10 +11,10 @@ func (p *Parser) ParseIdentifier() (ast.Identifier, error) { return token.Token{}, err } - switch tkn.TokenType { + switch tkn.Type { case token.Identifier, token.From, token.Error, token.Revert, token.Global: return tkn, nil } - return token.Token{}, token.NewPosError(tkn.Pos, "keyword is not available as identifier.") + return token.Token{}, token.NewPosError(tkn.Position, "keyword is not available as identifier.") } diff --git a/identifier_test.go b/identifier_test.go index b6f658a..793116e 100644 --- a/identifier_test.go +++ b/identifier_test.go @@ -20,33 +20,33 @@ func TestParser_ParseIdentifier(t *testing.T) { { input: "identifier", want: token.Token{ - TokenType: token.Identifier, - Text: "identifier", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.Identifier, + Value: "identifier", + Position: token.Pos{Column: 1, Line: 1}, }, }, { input: "from", want: token.Token{ - TokenType: token.From, - Text: "from", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.From, + Value: "from", + Position: token.Pos{Column: 1, Line: 1}, }, }, { input: "error", want: token.Token{ - TokenType: token.Error, - Text: "error", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.Error, + Value: "error", + Position: token.Pos{Column: 1, Line: 1}, }, }, { input: "global", want: token.Token{ - TokenType: token.Global, - Text: "global", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.Global, + Value: "global", + Position: token.Pos{Column: 1, Line: 1}, }, }, { diff --git a/lexer/lexer.go b/lexer/lexer.go index 90ae859..14eea5b 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -88,9 +88,9 @@ func (l *Lexer) ScanStringLiteral() (token.Token, error) { txt = txt + str if str == quote { return token.Token{ - TokenType: token.StringLiteral, - Text: txt, - Pos: firstPos, + Type: token.StringLiteral, + Value: txt, + Position: firstPos, }, nil } } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index b69f5a5..66f735a 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -20,9 +20,9 @@ func TestLexer_Scan(t *testing.T) { name: "normal", input: "pragma", wantToken: token.Token{ - TokenType: token.Pragma, - Text: "pragma", - Pos: token.Pos{ + Type: token.Pragma, + Value: "pragma", + Position: token.Pos{ Column: 1, Line: 1, }, @@ -32,9 +32,9 @@ func TestLexer_Scan(t *testing.T) { name: "There is a space at the beginning", input: " pragma", wantToken: token.Token{ - TokenType: token.Pragma, - Text: "pragma", - Pos: token.Pos{ + Type: token.Pragma, + Value: "pragma", + Position: token.Pos{ Column: 2, Line: 1, }, @@ -44,9 +44,9 @@ func TestLexer_Scan(t *testing.T) { name: "There is a StringLiteral", input: `"pragma"`, wantToken: token.Token{ - TokenType: token.StringLiteral, - Text: `"pragma"`, - Pos: token.Pos{ + Type: token.StringLiteral, + Value: `"pragma"`, + Position: token.Pos{ Column: 1, Line: 1, }, @@ -82,9 +82,9 @@ func TestLexer_Scan(t *testing.T) { t.Errorf("got error: %s", err) } want := token.Token{ - TokenType: token.EOS, - Text: token.EOSString, - Pos: token.Pos{ + Type: token.EOS, + Value: token.EOSString, + Position: token.Pos{ Column: 1, Line: 1, }, @@ -96,9 +96,9 @@ func TestLexer_Scan(t *testing.T) { t.Run("when peeked", func(t *testing.T) { peekToken := token.Token{ - TokenType: token.BitXor, - Text: "^", - Pos: token.Pos{ + Type: token.BitXor, + Value: "^", + Position: token.Pos{ Column: 4, Line: 6, }, @@ -132,9 +132,9 @@ func TestLexer_Peek(t *testing.T) { name: "normal", input: "pragma", token: token.Token{ - TokenType: token.Pragma, - Text: "pragma", - Pos: token.Pos{ + Type: token.Pragma, + Value: "pragma", + Position: token.Pos{ Column: 1, Line: 1, }, @@ -144,9 +144,9 @@ func TestLexer_Peek(t *testing.T) { name: "when scan is done", input: "", token: token.Token{ - TokenType: token.EOS, - Text: token.EOSString, - Pos: token.Pos{ + Type: token.EOS, + Value: token.EOSString, + Position: token.Pos{ Column: 1, Line: 1, }, @@ -201,7 +201,7 @@ func TestLexer_ScanStringLiteral(t *testing.T) { if err != nil { t.Errorf("error is not nil, got: %s", err) } - if diff := cmp.Diff(tt.wantText, tkn.Text); diff != "" { + if diff := cmp.Diff(tt.wantText, tkn.Value); diff != "" { t.Errorf(diff) } }) diff --git a/literal.go b/literal.go index 9cdf99b..681c2f4 100644 --- a/literal.go +++ b/literal.go @@ -11,8 +11,8 @@ func (p *Parser) ParseLiteral() (ast.Literal, error) { return nil, err } - if tkn.TokenType != token.StringLiteral { - return nil, token.NewPosError(tkn.Pos, "not found string literal quote") + if tkn.Type != token.StringLiteral { + return nil, token.NewPosError(tkn.Position, "not found string literal quote") } // peek to find literal end position. @@ -22,11 +22,11 @@ func (p *Parser) ParseLiteral() (ast.Literal, error) { } return &ast.StringLiteral{ - Value: tkn.Text, - From: tkn.Pos, + Value: tkn.Value, + From: tkn.Position, To: token.Pos{ - Column: peek.Pos.Column - 1, - Line: peek.Pos.Line, + Column: peek.Position.Column - 1, + Line: peek.Position.Line, }, }, nil } diff --git a/literal_test.go b/literal_test.go index 1f74cf0..1483795 100644 --- a/literal_test.go +++ b/literal_test.go @@ -50,17 +50,16 @@ func TestParser_ParseLiteral(t *testing.T) { }, err: nil, }, - // TODO // { // name: `Next token is \n`, // input: "\"Hello World!!\"\n", // want: &ast.StringLiteral{ - // Str: `"Hello World!!"`, - // Position: token.Pos{ + // Value: `"Hello World!!"`, + // From: token.Pos{ // Column: 1, // Line: 1, // }, - // EndPos: token.Pos{ + // To: token.Pos{ // Column: 15, // Line: 1, // }, diff --git a/parameterList.go b/parameterList.go index a1dd76e..b97bd8d 100644 --- a/parameterList.go +++ b/parameterList.go @@ -31,7 +31,7 @@ func (p *Parser) ParseParameterList() (ast.ParameterList, error) { return nil, err } - if comma.TokenType != token.Comma { + if comma.Type != token.Comma { return prms, nil } diff --git a/parameterList_test.go b/parameterList_test.go index 366fdc3..809bc45 100644 --- a/parameterList_test.go +++ b/parameterList_test.go @@ -24,9 +24,9 @@ func TestParser_ParseParameter(t *testing.T) { want: &ast.Parameter{ TypeName: ast.ElementaryTypeName{ { - TokenType: token.String, - Text: "string", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.String, + Value: "string", + Position: token.Pos{Column: 1, Line: 1}, }, }, }, @@ -80,9 +80,9 @@ func TestParser_ParseParameterList(t *testing.T) { { TypeName: ast.ElementaryTypeName{ { - TokenType: token.String, - Text: "string", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.String, + Value: "string", + Position: token.Pos{Column: 1, Line: 1}, }, }, }, @@ -96,18 +96,18 @@ func TestParser_ParseParameterList(t *testing.T) { { TypeName: ast.ElementaryTypeName{ { - TokenType: token.String, - Text: "string", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.String, + Value: "string", + Position: token.Pos{Column: 1, Line: 1}, }, }, }, { TypeName: ast.ElementaryTypeName{ { - TokenType: token.Bool, - Text: "bool", - Pos: token.Pos{Column: 9, Line: 1}, + Type: token.Bool, + Value: "bool", + Position: token.Pos{Column: 9, Line: 1}, }, }, }, diff --git a/pragmaDirective.go b/pragmaDirective.go index aed0f54..43cda5f 100644 --- a/pragmaDirective.go +++ b/pragmaDirective.go @@ -10,8 +10,8 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { if err != nil { return nil, err } - if prgm.TokenType != token.Pragma { - return nil, token.NewPosError(prgm.Pos, "not found pragma.") + if prgm.Type != token.Pragma { + return nil, token.NewPosError(prgm.Position, "not found pragma.") } tkns := make([]*token.Token, 0, 1) @@ -20,17 +20,17 @@ func (p *Parser) ParsePragmaDirective() (*ast.PragmaDirective, error) { if err != nil { return nil, err } - if tkn.TokenType == token.EOS { - return nil, token.NewPosError(tkn.Pos, "not found Semicolon.") + if tkn.Type == token.EOS { + return nil, token.NewPosError(tkn.Position, "not found Semicolon.") } - if tkn.TokenType == token.Semicolon { + if tkn.Type == token.Semicolon { if len(tkns) == 0 { - return nil, token.NewPosError(tkn.Pos, "not found pragma tokens.") + return nil, token.NewPosError(tkn.Position, "not found pragma tokens.") } return &ast.PragmaDirective{ - Pragma: prgm.Pos, + Pragma: prgm.Position, PragmaTokens: tkns, - Semicolon: tkn.Pos, + Semicolon: tkn.Position, }, nil } tkns = append(tkns, &tkn) diff --git a/pragmaDirective_test.go b/pragmaDirective_test.go index dbd37e9..192f270 100644 --- a/pragmaDirective_test.go +++ b/pragmaDirective_test.go @@ -20,9 +20,9 @@ func TestParser_ParsePragmaDirective(t *testing.T) { want: &ast.PragmaDirective{ Pragma: pos(1, 1), PragmaTokens: []*token.Token{ - {TokenType: token.Identifier, Text: "solidity", Pos: pos(8, 1)}, - {TokenType: token.BitXor, Text: "^", Pos: pos(17, 1)}, - {TokenType: token.Identifier, Text: "0.8.13", Pos: pos(18, 1)}, + {Type: token.Identifier, Value: "solidity", Position: pos(8, 1)}, + {Type: token.BitXor, Value: "^", Position: pos(17, 1)}, + {Type: token.Identifier, Value: "0.8.13", Position: pos(18, 1)}, }, Semicolon: pos(24, 1), }, diff --git a/solparser.go b/solparser.go index 452195d..4bdb977 100644 --- a/solparser.go +++ b/solparser.go @@ -30,7 +30,7 @@ func (p *Parser) Parse() (*ast.SourceUnit, error) { return nil, err } - switch tkn.TokenType { + switch tkn.Type { case token.Pragma: prgm, err := p.ParsePragmaDirective() if err != nil { @@ -56,7 +56,7 @@ func (p *Parser) Parse() (*ast.SourceUnit, error) { FunctionDefinition: functionDefinition, }, nil default: - return nil, token.NewPosError(tkn.Pos, "invalid") + return nil, token.NewPosError(tkn.Position, "invalid") } } } @@ -67,8 +67,8 @@ func (p *Parser) ParseBooleanLiteral() (*ast.BooleanLiteral, error) { return nil, err } - if tkn.TokenType != token.TrueLiteral && tkn.TokenType != token.FalseLiteral { - return nil, token.NewPosError(tkn.Pos, "not found keyword true or false") + if tkn.Type != token.TrueLiteral && tkn.Type != token.FalseLiteral { + return nil, token.NewPosError(tkn.Position, "not found keyword true or false") } return &ast.BooleanLiteral{ diff --git a/solparser_test.go b/solparser_test.go index 8bbb09d..4bf968e 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -32,19 +32,19 @@ contract HelloWorld { Pragma: token.Pos{Column: 1, Line: 1}, PragmaTokens: []*token.Token{ { - TokenType: token.Identifier, - Text: "solidity", - Pos: token.Pos{Column: 8, Line: 1}, + Type: token.Identifier, + Value: "solidity", + Position: token.Pos{Column: 8, Line: 1}, }, { - TokenType: token.BitXor, - Text: "^", - Pos: token.Pos{Column: 17, Line: 1}, + Type: token.BitXor, + Value: "^", + Position: token.Pos{Column: 17, Line: 1}, }, { - TokenType: token.Identifier, - Text: "0.8.13", - Pos: token.Pos{Column: 18, Line: 1}, + Type: token.Identifier, + Value: "0.8.13", + Position: token.Pos{Column: 18, Line: 1}, }, }, Semicolon: token.Pos{Column: 24, Line: 1}, @@ -52,31 +52,31 @@ contract HelloWorld { ContractDefinition: &ast.ContractDefinition{ Contract: token.Pos{Column: 1, Line: 3}, Identifier: token.Token{ - TokenType: token.Identifier, - Text: "HelloWorld", - Pos: token.Pos{Column: 10, Line: 3}, + Type: token.Identifier, + Value: "HelloWorld", + Position: token.Pos{Column: 10, Line: 3}, }, LBrace: token.Pos{Column: 21, Line: 3}, ContractBodyElements: []ast.ContractBodyElement{ &ast.FunctionDefinition{ From: token.Pos{Column: 5, Line: 4}, FunctionDescriptor: token.Token{ - TokenType: token.Identifier, - Text: "hello", - Pos: token.Pos{Column: 14, Line: 4}, + Type: token.Identifier, + Value: "hello", + Position: token.Pos{Column: 14, Line: 4}, }, LParen: token.Pos{Column: 19, Line: 4}, RParen: token.Pos{Column: 20, Line: 4}, ModifierList: &ast.ModifierList{ Visibility: &token.Token{ - TokenType: token.Public, - Text: "public", - Pos: token.Pos{Column: 22, Line: 4}, + Type: token.Public, + Value: "public", + Position: token.Pos{Column: 22, Line: 4}, }, StateMutability: &token.Token{ - TokenType: token.Pure, - Text: "pure", - Pos: token.Pos{Column: 29, Line: 4}, + Type: token.Pure, + Value: "pure", + Position: token.Pos{Column: 29, Line: 4}, }, }, Returns: &ast.FunctionDefinitionReturns{ @@ -86,9 +86,9 @@ contract HelloWorld { { TypeName: ast.ElementaryTypeName{ { - TokenType: token.String, - Text: "string", - Pos: token.Pos{Column: 43, Line: 4}, + Type: token.String, + Value: "string", + Position: token.Pos{Column: 43, Line: 4}, }, }, }, @@ -151,9 +151,9 @@ func TestParser_ParseBooleanLiteral(t *testing.T) { input: "true", want: &ast.BooleanLiteral{ Token: token.Token{ - TokenType: token.TrueLiteral, - Text: "true", - Pos: token.Pos{ + Type: token.TrueLiteral, + Value: "true", + Position: token.Pos{ Column: 1, Line: 1, }, @@ -166,9 +166,9 @@ func TestParser_ParseBooleanLiteral(t *testing.T) { input: "false", want: &ast.BooleanLiteral{ Token: token.Token{ - TokenType: token.FalseLiteral, - Text: "false", - Pos: token.Pos{ + Type: token.FalseLiteral, + Value: "false", + Position: token.Pos{ Column: 1, Line: 1, }, diff --git a/statement.go b/statement.go index b9905a7..0184548 100644 --- a/statement.go +++ b/statement.go @@ -11,11 +11,11 @@ func (p *Parser) ParseStatement() (ast.Statement, error) { return nil, err } - if tkn.TokenType == token.Return { + if tkn.Type == token.Return { return p.ParseReturnStatement() } - return nil, token.NewPosError(tkn.Pos, "not found statement.") + return nil, token.NewPosError(tkn.Position, "not found statement.") } func (p *Parser) ParseReturnStatement() (ast.Statement, error) { @@ -24,8 +24,8 @@ func (p *Parser) ParseReturnStatement() (ast.Statement, error) { return nil, err } - if rtn.TokenType != token.Return { - return nil, token.NewPosError(rtn.Pos, "not found return keyword.") + if rtn.Type != token.Return { + return nil, token.NewPosError(rtn.Position, "not found return keyword.") } exp, err := p.ParseExpression() @@ -38,13 +38,13 @@ func (p *Parser) ParseReturnStatement() (ast.Statement, error) { return nil, err } - if semi.TokenType != token.Semicolon { - return nil, token.NewPosError(semi.Pos, "not found semicolon.") + if semi.Type != token.Semicolon { + return nil, token.NewPosError(semi.Position, "not found semicolon.") } return &ast.ReturnStatement{ - From: rtn.Pos, - SemiPos: semi.Pos, + From: rtn.Position, + SemiPos: semi.Position, Expression: exp, }, nil } diff --git a/token/token.go b/token/token.go index 2c82937..8b3952f 100644 --- a/token/token.go +++ b/token/token.go @@ -480,16 +480,16 @@ func asToken(str string) TokenType { } type Token struct { - TokenType TokenType - Text string - Pos Pos + Type TokenType + Value string + Position Pos } func NewToken(ch string, pos Pos) Token { return Token{ - TokenType: asToken(ch), - Text: ch, - Pos: pos, + Type: asToken(ch), + Value: ch, + Position: pos, } } diff --git a/typeName.go b/typeName.go index fbf983a..704017c 100644 --- a/typeName.go +++ b/typeName.go @@ -11,12 +11,12 @@ func (p *Parser) ParseTypeName() (ast.TypeName, error) { return nil, err } - switch tkn.TokenType { + switch tkn.Type { case token.Address, token.String, token.Bytes, token.Fixed, token.Bool: return p.ParseElementaryTypeName() } - return nil, token.NewPosError(tkn.Pos, "not found type-name.") + return nil, token.NewPosError(tkn.Position, "not found type-name.") } func (p *Parser) ParseElementaryTypeName() (ast.TypeName, error) { @@ -25,22 +25,22 @@ func (p *Parser) ParseElementaryTypeName() (ast.TypeName, error) { return nil, err } - if tkn.TokenType == token.Address { + if tkn.Type == token.Address { tkn2, err := p.lexer.Peek() if err != nil { return nil, err } - if tkn2.TokenType == token.Payable { + if tkn2.Type == token.Payable { p.lexer.Scan() return ast.ElementaryTypeName{&tkn, &tkn2}, nil } } - switch tkn.TokenType { + switch tkn.Type { case token.Address, token.String, token.Bytes, token.Fixed, token.Bool: return ast.ElementaryTypeName{&tkn}, nil } - return nil, token.NewPosError(tkn.Pos, "not found elementary type name keyword.") + return nil, token.NewPosError(tkn.Position, "not found elementary type name keyword.") } diff --git a/typeName_test.go b/typeName_test.go index b0f5730..20d4e3f 100644 --- a/typeName_test.go +++ b/typeName_test.go @@ -23,9 +23,9 @@ func TestParser_ParseTypeName(t *testing.T) { input: "string)", want: ast.ElementaryTypeName{ { - TokenType: token.String, - Text: "string", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.String, + Value: "string", + Position: token.Pos{Column: 1, Line: 1}, }, }, }, @@ -70,9 +70,9 @@ func TestParser_ParseElementaryTypeName(t *testing.T) { input: "address", want: ast.ElementaryTypeName{ { - TokenType: token.Address, - Text: "address", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.Address, + Value: "address", + Position: token.Pos{Column: 1, Line: 1}, }, }, }, @@ -80,9 +80,9 @@ func TestParser_ParseElementaryTypeName(t *testing.T) { input: "bool", want: ast.ElementaryTypeName{ { - TokenType: token.Bool, - Text: "bool", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.Bool, + Value: "bool", + Position: token.Pos{Column: 1, Line: 1}, }, }, }, @@ -90,9 +90,9 @@ func TestParser_ParseElementaryTypeName(t *testing.T) { input: "string", want: ast.ElementaryTypeName{ { - TokenType: token.String, - Text: "string", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.String, + Value: "string", + Position: token.Pos{Column: 1, Line: 1}, }, }, }, @@ -100,14 +100,14 @@ func TestParser_ParseElementaryTypeName(t *testing.T) { input: "address payable", want: ast.ElementaryTypeName{ { - TokenType: token.Address, - Text: "address", - Pos: token.Pos{Column: 1, Line: 1}, + Type: token.Address, + Value: "address", + Position: token.Pos{Column: 1, Line: 1}, }, { - TokenType: token.Payable, - Text: "payable", - Pos: token.Pos{Column: 9, Line: 1}, + Type: token.Payable, + Value: "payable", + Position: token.Pos{Column: 9, Line: 1}, }, }, }, From 1a2ec4edd7528940240dc65416d9a9c1fd7bf4c1 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 26 Feb 2023 17:08:53 +0900 Subject: [PATCH 41/58] ast: refactor StringLiteral --- ast/ast.go | 27 +++++++++++++++++++-------- ast/ast_test.go | 30 ++++++++++++++++++++++++++++++ block_test.go | 6 +++--- contractDefinition_test.go | 12 ++++++------ expression_test.go | 12 +++--------- functionDefinition_test.go | 6 +++--- literal.go | 16 ++-------------- literal_test.go | 26 ++++++-------------------- solparser_test.go | 6 +++--- statement_test.go | 12 ++++++------ 10 files changed, 81 insertions(+), 72 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 3776ef2..86af812 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -1,6 +1,10 @@ package ast -import "github.com/uji/solparser/token" +import ( + "strings" + + "github.com/uji/solparser/token" +) type Node interface { Pos() token.Pos @@ -148,17 +152,24 @@ func (b *BooleanLiteral) End() token.Pos { } // EmptyStringLiteral | NonEmptyStringLiteral -type StringLiteral struct { - From, To token.Pos - Value string -} +type StringLiteral token.Token func (s StringLiteral) Pos() token.Pos { - return s.From + return s.Position } -func (s StringLiteral) End() token.Pos { - return s.To +func (s StringLiteral) End() (pos token.Pos) { + // Calculate Line and Offset by referring to the number of new line codes + nc := strings.Count(s.Value, "\n") + + pos.Line = s.Position.Line + nc + if nc == 0 { + pos.Column = s.Position.Column + len(s.Value) - 1 + return + } + + pos.Column = len(s.Value) - strings.LastIndexByte(s.Value, '\n') - 1 + return } type HexStringLiteral []*HexString diff --git a/ast/ast_test.go b/ast/ast_test.go index 1deb8e0..dbc6252 100644 --- a/ast/ast_test.go +++ b/ast/ast_test.go @@ -56,6 +56,36 @@ func TestNode_End(t *testing.T) { Line: 3, }, }, + { + name: "StringLiteral", + node: &ast.StringLiteral{ + Type: token.StringLiteral, + Value: "\"Hello world!!\"", + Position: token.Pos{ + Column: 4, + Line: 3, + }, + }, + exptEnd: token.Pos{ + Column: 18, + Line: 3, + }, + }, + { + name: "StringLiteral with new line codes", + node: &ast.StringLiteral{ + Type: token.StringLiteral, + Value: "\"Hello \n new \nworld!!\"", + Position: token.Pos{ + Column: 4, + Line: 3, + }, + }, + exptEnd: token.Pos{ + Column: 8, + Line: 5, + }, + }, } for _, tt := range tests { diff --git a/block_test.go b/block_test.go index fa82134..4d4e0bd 100644 --- a/block_test.go +++ b/block_test.go @@ -29,9 +29,9 @@ func TestParser_ParseBlock(t *testing.T) { From: token.Pos{Column: 1, Line: 2}, SemiPos: token.Pos{Column: 23, Line: 2}, Expression: &ast.StringLiteral{ - From: token.Pos{Column: 8, Line: 2}, - To: token.Pos{Column: 22, Line: 2}, - Value: "\"Hello World!!\"", + Type: token.StringLiteral, + Position: token.Pos{Column: 8, Line: 2}, + Value: "\"Hello World!!\"", }, }, }, diff --git a/contractDefinition_test.go b/contractDefinition_test.go index 5abd512..41cacbd 100644 --- a/contractDefinition_test.go +++ b/contractDefinition_test.go @@ -79,9 +79,9 @@ func TestParser_ParseContractDefinition(t *testing.T) { From: token.Pos{Column: 9, Line: 3}, SemiPos: token.Pos{Column: 31, Line: 3}, Expression: &ast.StringLiteral{ - From: token.Pos{Column: 16, Line: 3}, - To: token.Pos{Column: 30, Line: 3}, - Value: "\"Hello World!!\"", + Type: token.StringLiteral, + Position: token.Pos{Column: 16, Line: 3}, + Value: "\"Hello World!!\"", }, }, }, @@ -153,9 +153,9 @@ func TestParser_ParseContractDefinition(t *testing.T) { From: token.Pos{Column: 9, Line: 3}, SemiPos: token.Pos{Column: 31, Line: 3}, Expression: &ast.StringLiteral{ - From: token.Pos{Column: 16, Line: 3}, - To: token.Pos{Column: 30, Line: 3}, - Value: "\"Hello World!!\"", + Type: token.StringLiteral, + Position: token.Pos{Column: 16, Line: 3}, + Value: "\"Hello World!!\"", }, }, }, diff --git a/expression_test.go b/expression_test.go index 0fee217..64fd674 100644 --- a/expression_test.go +++ b/expression_test.go @@ -22,15 +22,9 @@ func TestParser_ParseExpression(t *testing.T) { name: "Literal", input: `"Hello World!!";`, want: &ast.StringLiteral{ - Value: `"Hello World!!"`, - From: token.Pos{ - Column: 1, - Line: 1, - }, - To: token.Pos{ - Column: 15, - Line: 1, - }, + Type: token.StringLiteral, + Value: `"Hello World!!"`, + Position: pos(1, 1), }, err: nil, }, diff --git a/functionDefinition_test.go b/functionDefinition_test.go index 308341e..cdfda0d 100644 --- a/functionDefinition_test.go +++ b/functionDefinition_test.go @@ -222,9 +222,9 @@ func TestParser_ParseFunctionDefinition(t *testing.T) { From: token.Pos{Column: 9, Line: 2}, SemiPos: token.Pos{Column: 31, Line: 2}, Expression: &ast.StringLiteral{ - From: token.Pos{Column: 16, Line: 2}, - To: token.Pos{Column: 30, Line: 2}, - Value: "\"Hello World!!\"", + Type: token.StringLiteral, + Position: token.Pos{Column: 16, Line: 2}, + Value: "\"Hello World!!\"", }, }, }, diff --git a/literal.go b/literal.go index 681c2f4..0e814e9 100644 --- a/literal.go +++ b/literal.go @@ -15,18 +15,6 @@ func (p *Parser) ParseLiteral() (ast.Literal, error) { return nil, token.NewPosError(tkn.Position, "not found string literal quote") } - // peek to find literal end position. - peek, err := p.lexer.Peek() - if err != nil { - return nil, err - } - - return &ast.StringLiteral{ - Value: tkn.Value, - From: tkn.Position, - To: token.Pos{ - Column: peek.Position.Column - 1, - Line: peek.Position.Line, - }, - }, nil + lit := ast.StringLiteral(tkn) + return &lit, nil } diff --git a/literal_test.go b/literal_test.go index 1483795..3509bf6 100644 --- a/literal_test.go +++ b/literal_test.go @@ -22,33 +22,19 @@ func TestParser_ParseLiteral(t *testing.T) { name: "normal", input: `"Hello World!!";`, want: &ast.StringLiteral{ - Value: `"Hello World!!"`, - From: token.Pos{ - Column: 1, - Line: 1, - }, - To: token.Pos{ - Column: 15, - Line: 1, - }, + Type: token.StringLiteral, + Value: `"Hello World!!"`, + Position: pos(1, 1), }, - err: nil, }, { name: `Including \n`, input: "\"Hello \nWorld!!\";", want: &ast.StringLiteral{ - Value: "\"Hello \nWorld!!\"", - From: token.Pos{ - Column: 1, - Line: 1, - }, - To: token.Pos{ - Column: 8, - Line: 2, - }, + Type: token.StringLiteral, + Value: "\"Hello \nWorld!!\"", + Position: pos(1, 1), }, - err: nil, }, // { // name: `Next token is \n`, diff --git a/solparser_test.go b/solparser_test.go index 4bf968e..3d5e43f 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -103,9 +103,9 @@ contract HelloWorld { From: token.Pos{Column: 9, Line: 5}, SemiPos: token.Pos{Column: 31, Line: 5}, Expression: &ast.StringLiteral{ - From: token.Pos{Column: 16, Line: 5}, - To: token.Pos{Column: 30, Line: 5}, - Value: "\"Hello World!!\"", + Type: token.StringLiteral, + Position: token.Pos{Column: 16, Line: 5}, + Value: "\"Hello World!!\"", }, }, }, diff --git a/statement_test.go b/statement_test.go index a63c1ec..a85023f 100644 --- a/statement_test.go +++ b/statement_test.go @@ -25,9 +25,9 @@ func TestParser_ParseStatement(t *testing.T) { From: token.Pos{Column: 1, Line: 1}, SemiPos: token.Pos{Column: 23, Line: 1}, Expression: &ast.StringLiteral{ - From: token.Pos{Column: 8, Line: 1}, - To: token.Pos{Column: 22, Line: 1}, - Value: `"Hello World!!"`, + Type: token.StringLiteral, + Position: token.Pos{Column: 8, Line: 1}, + Value: `"Hello World!!"`, }, }, err: nil, @@ -85,9 +85,9 @@ func TestParser_ParseReturnStatement(t *testing.T) { From: token.Pos{Column: 1, Line: 1}, SemiPos: token.Pos{Column: 23, Line: 1}, Expression: &ast.StringLiteral{ - From: token.Pos{Column: 8, Line: 1}, - To: token.Pos{Column: 22, Line: 1}, - Value: `"Hello World!!"`, + Type: token.StringLiteral, + Position: token.Pos{Column: 8, Line: 1}, + Value: `"Hello World!!"`, }, }, err: nil, From bb73c42bf9ae8acf6b115607b2eaf5bd789f6610 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 26 Feb 2023 18:04:05 +0900 Subject: [PATCH 42/58] main: parser example --- example_test.go | 35 +++++++++++++++++++++++++++++++++++ solparser.go | 1 + 2 files changed, 36 insertions(+) create mode 100644 example_test.go diff --git a/example_test.go b/example_test.go new file mode 100644 index 0000000..22bdb95 --- /dev/null +++ b/example_test.go @@ -0,0 +1,35 @@ +package solparser_test + +import ( + "fmt" + "strings" + + "github.com/uji/solparser" + "github.com/uji/solparser/ast" +) + +func Example() { + input := + `pragma solidity ^0.8.13; + +contract HelloWorld { + function hello() public pure returns (string) { + return "Hello World!!"; + } +}` + + parser := solparser.New(strings.NewReader(input)) + + got, err := parser.Parse() + if err != nil { + fmt.Println(err) + return + } + + fmt.Println(got.ContractDefinition.Identifier.Value) + fmt.Println(got.ContractDefinition.ContractBodyElements[0].(*ast.FunctionDefinition).FunctionDescriptor.Value) + + // Output: + // HelloWorld + // hello +} diff --git a/solparser.go b/solparser.go index 4bdb977..2c11763 100644 --- a/solparser.go +++ b/solparser.go @@ -8,6 +8,7 @@ import ( "github.com/uji/solparser/token" ) +// Parser parses "Solidity" code and outputs ASTs. type Parser struct { input io.Reader lexer *lexer.Lexer From 89618c37b1a3db593787c46f66ab40fcdb37fb20 Mon Sep 17 00:00:00 2001 From: uji Date: Mon, 27 Feb 2023 00:11:09 +0900 Subject: [PATCH 43/58] main, ast: parse symbolAliases --- ast/ast.go | 13 +++++ helper_test.go | 12 ++++ symbolAliases.go | 112 +++++++++++++++++++++++++++++++++++ symbolAliases_test.go | 132 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 269 insertions(+) create mode 100644 symbolAliases.go create mode 100644 symbolAliases_test.go diff --git a/ast/ast.go b/ast/ast.go index 86af812..5613761 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -11,6 +11,19 @@ type Node interface { End() token.Pos } +type SymbolAlias struct { + Symbol Identifier + As *token.Pos + Alias *Identifier +} + +type SymbolAliases struct { + LBrace token.Pos + Aliases []*SymbolAlias + Commas []*token.Pos + RBrace token.Pos +} + type PragmaDirective struct { Pragma token.Pos PragmaTokens []*token.Token diff --git a/helper_test.go b/helper_test.go index d80c961..de20174 100644 --- a/helper_test.go +++ b/helper_test.go @@ -12,6 +12,10 @@ func pos(c, l int) token.Pos { return token.Pos{Column: c, Line: l} } +func posPtr(c, l int) *token.Pos { + return &token.Pos{Column: c, Line: l} +} + func tkn(tp token.TokenType, text string, pos token.Pos) token.Token { return token.Token{ Type: tp, @@ -20,6 +24,14 @@ func tkn(tp token.TokenType, text string, pos token.Pos) token.Token { } } +func tknPtr(tp token.TokenType, text string, pos token.Pos) *token.Token { + return &token.Token{ + Type: tp, + Value: text, + Position: pos, + } +} + func perr(pos token.Pos, msg string) *token.PosError { return &token.PosError{ Pos: pos, diff --git a/symbolAliases.go b/symbolAliases.go new file mode 100644 index 0000000..f71c71d --- /dev/null +++ b/symbolAliases.go @@ -0,0 +1,112 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseSymbolAlias() (*ast.SymbolAlias, error) { + smbl, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + if smbl.Type != token.Identifier { + return nil, token.NewPosError(smbl.Position, "not found identifier.") + } + + if _, err := p.lexer.Scan(); err != nil { + return nil, err + } + + as, err := p.lexer.Peek() + if err != nil { + return nil, err + } + if as.Type != token.As { + return &ast.SymbolAlias{Symbol: smbl}, nil + } + if _, err := p.lexer.Scan(); err != nil { + return nil, err + } + + alias, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + if alias.Type != token.Identifier { + return nil, token.NewPosError(alias.Position, "not found identifier.") + } + + return &ast.SymbolAlias{ + Symbol: smbl, + As: &as.Position, + Alias: &alias, + }, nil +} + +func (p *Parser) ParseSymbolAliases() (*ast.SymbolAliases, error) { + lbrace, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + if lbrace.Type != token.LBrace { + return nil, token.NewPosError(lbrace.Position, "not found LBrace.") + } + + if _, err := p.lexer.Scan(); err != nil { + return nil, err + } + + alias, err := p.ParseSymbolAlias() + if err != nil { + return nil, err + } + + aliases := []*ast.SymbolAlias{alias} + commas := make([]*token.Pos, 0) + + for { + comma, err := p.lexer.Peek() + if err != nil { + return nil, err + } + if comma.Type != token.Comma { + break + } + + if _, err := p.lexer.Scan(); err != nil { + return nil, err + } + + alias, err := p.ParseSymbolAlias() + if err != nil { + return nil, err + } + + aliases = append(aliases, alias) + commas = append(commas, &comma.Position) + } + + rbrace, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + if rbrace.Type != token.RBrace { + return nil, token.NewPosError(rbrace.Position, "not found RBrace.") + } + + if _, err := p.lexer.Scan(); err != nil { + return nil, err + } + + return &ast.SymbolAliases{ + LBrace: lbrace.Position, + Aliases: aliases, + Commas: commas, + RBrace: rbrace.Position, + }, nil +} diff --git a/symbolAliases_test.go b/symbolAliases_test.go new file mode 100644 index 0000000..fc37cc3 --- /dev/null +++ b/symbolAliases_test.go @@ -0,0 +1,132 @@ +package solparser_test + +import ( + "strings" + "testing" + + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseSymbolAlias(t *testing.T) { + tests := []struct { + input string + want *ast.SymbolAlias + err *token.PosError + }{ + { + input: "symbol as alias1}", + want: &ast.SymbolAlias{ + Symbol: tkn(token.Identifier, "symbol", pos(1, 1)), + As: posPtr(8, 1), + Alias: tknPtr(token.Identifier, "alias1", pos(11, 1)), + }, + }, + { + input: "symbol}", + want: &ast.SymbolAlias{ + Symbol: tkn(token.Identifier, "symbol", pos(1, 1)), + As: nil, + Alias: nil, + }, + }, + { + input: "pragma", + err: perr(pos(1, 1), "not found identifier."), + }, + { + input: "symbol as pragma", + err: perr(pos(11, 1), "not found identifier."), + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.input, func(t *testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseSymbolAlias() + assert(t, got, tt.want, err, tt.err) + }) + } +} + +func TestParser_ParseSymbolAliases(t *testing.T) { + tests := []struct { + input string + want *ast.SymbolAliases + err *token.PosError + }{ + { + input: "{symbol1}", + want: &ast.SymbolAliases{ + LBrace: pos(1, 1), + Aliases: []*ast.SymbolAlias{ + { + Symbol: tkn(token.Identifier, "symbol1", pos(2, 1)), + }, + }, + Commas: []*token.Pos{}, + RBrace: pos(9, 1), + }, + }, + { + input: "{symbol2 as alias1}", + want: &ast.SymbolAliases{ + LBrace: pos(1, 1), + Aliases: []*ast.SymbolAlias{ + { + Symbol: tkn(token.Identifier, "symbol2", pos(2, 1)), + As: posPtr(10, 1), + Alias: tknPtr(token.Identifier, "alias1", pos(13, 1)), + }, + }, + Commas: []*token.Pos{}, + RBrace: pos(19, 1), + }, + }, + { + input: "{symbol1, symbol2 as alias1}", + want: &ast.SymbolAliases{ + LBrace: pos(1, 1), + Aliases: []*ast.SymbolAlias{ + { + Symbol: tkn(token.Identifier, "symbol1", pos(2, 1)), + }, + { + Symbol: tkn(token.Identifier, "symbol2", pos(11, 1)), + As: posPtr(19, 1), + Alias: tknPtr(token.Identifier, "alias1", pos(22, 1)), + }, + }, + Commas: []*token.Pos{posPtr(9, 1)}, + RBrace: pos(28, 1), + }, + }, + { + input: "symbol1, symbol2 as alian1}", + err: perr(pos(1, 1), "not found LBrace."), + }, + { + input: "{symbol1, as alian1", + err: perr(pos(11, 1), "not found identifier."), + }, + { + input: "{symbol1, symbol2 as alian1", + err: perr(pos(28, 1), "not found RBrace."), + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.input, func(t *testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseSymbolAliases() + assert(t, got, tt.want, err, tt.err) + }) + } +} From 202f304aa62563561f91a97645defa6392e9adf4 Mon Sep 17 00:00:00 2001 From: uji Date: Tue, 28 Feb 2023 22:39:09 +0900 Subject: [PATCH 44/58] main: refactor ParseSymbolAlias --- symbolAliases.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/symbolAliases.go b/symbolAliases.go index f71c71d..7dd4849 100644 --- a/symbolAliases.go +++ b/symbolAliases.go @@ -90,7 +90,7 @@ func (p *Parser) ParseSymbolAliases() (*ast.SymbolAliases, error) { commas = append(commas, &comma.Position) } - rbrace, err := p.lexer.Peek() + rbrace, err := p.lexer.Scan() if err != nil { return nil, err } @@ -99,10 +99,6 @@ func (p *Parser) ParseSymbolAliases() (*ast.SymbolAliases, error) { return nil, token.NewPosError(rbrace.Position, "not found RBrace.") } - if _, err := p.lexer.Scan(); err != nil { - return nil, err - } - return &ast.SymbolAliases{ LBrace: lbrace.Position, Aliases: aliases, From eea44cec43389852a27357353820460753aa7ea3 Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 1 Mar 2023 11:54:55 +0900 Subject: [PATCH 45/58] all: split token.StringLiteral to Empty and NoEmpty --- ast/ast_test.go | 4 ++-- block_test.go | 2 +- contractDefinition_test.go | 4 ++-- expression.go | 2 +- expression_test.go | 2 +- functionDefinition_test.go | 2 +- lexer/lexer.go | 4 +++- lexer/lexer_test.go | 45 ++++++++++++++++++++++++++++---------- literal.go | 2 +- literal_test.go | 4 ++-- solparser_test.go | 2 +- statement_test.go | 4 ++-- token/token.go | 3 ++- 13 files changed, 52 insertions(+), 28 deletions(-) diff --git a/ast/ast_test.go b/ast/ast_test.go index dbc6252..ce3d381 100644 --- a/ast/ast_test.go +++ b/ast/ast_test.go @@ -59,7 +59,7 @@ func TestNode_End(t *testing.T) { { name: "StringLiteral", node: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Value: "\"Hello world!!\"", Position: token.Pos{ Column: 4, @@ -74,7 +74,7 @@ func TestNode_End(t *testing.T) { { name: "StringLiteral with new line codes", node: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Value: "\"Hello \n new \nworld!!\"", Position: token.Pos{ Column: 4, diff --git a/block_test.go b/block_test.go index 4d4e0bd..403991d 100644 --- a/block_test.go +++ b/block_test.go @@ -29,7 +29,7 @@ func TestParser_ParseBlock(t *testing.T) { From: token.Pos{Column: 1, Line: 2}, SemiPos: token.Pos{Column: 23, Line: 2}, Expression: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Position: token.Pos{Column: 8, Line: 2}, Value: "\"Hello World!!\"", }, diff --git a/contractDefinition_test.go b/contractDefinition_test.go index 41cacbd..b1928c2 100644 --- a/contractDefinition_test.go +++ b/contractDefinition_test.go @@ -79,7 +79,7 @@ func TestParser_ParseContractDefinition(t *testing.T) { From: token.Pos{Column: 9, Line: 3}, SemiPos: token.Pos{Column: 31, Line: 3}, Expression: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Position: token.Pos{Column: 16, Line: 3}, Value: "\"Hello World!!\"", }, @@ -153,7 +153,7 @@ func TestParser_ParseContractDefinition(t *testing.T) { From: token.Pos{Column: 9, Line: 3}, SemiPos: token.Pos{Column: 31, Line: 3}, Expression: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Position: token.Pos{Column: 16, Line: 3}, Value: "\"Hello World!!\"", }, diff --git a/expression.go b/expression.go index f9a8d40..5ad23a8 100644 --- a/expression.go +++ b/expression.go @@ -11,7 +11,7 @@ func (p *Parser) ParseExpression() (ast.Expression, error) { return nil, err } - if tkn.Type == token.StringLiteral { + if tkn.Type == token.NonEmptyStringLiteral { return p.ParseLiteral() } diff --git a/expression_test.go b/expression_test.go index 64fd674..a9683f5 100644 --- a/expression_test.go +++ b/expression_test.go @@ -22,7 +22,7 @@ func TestParser_ParseExpression(t *testing.T) { name: "Literal", input: `"Hello World!!";`, want: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Value: `"Hello World!!"`, Position: pos(1, 1), }, diff --git a/functionDefinition_test.go b/functionDefinition_test.go index cdfda0d..34aa549 100644 --- a/functionDefinition_test.go +++ b/functionDefinition_test.go @@ -222,7 +222,7 @@ func TestParser_ParseFunctionDefinition(t *testing.T) { From: token.Pos{Column: 9, Line: 2}, SemiPos: token.Pos{Column: 31, Line: 2}, Expression: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Position: token.Pos{Column: 16, Line: 2}, Value: "\"Hello World!!\"", }, diff --git a/lexer/lexer.go b/lexer/lexer.go index 14eea5b..459e503 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -80,6 +80,7 @@ func (l *Lexer) ScanStringLiteral() (token.Token, error) { } quote, txt := str, str + tokenType := token.EmptyStringLiteral for { _, str, err := l.scanner.Scan() if err != nil { @@ -88,10 +89,11 @@ func (l *Lexer) ScanStringLiteral() (token.Token, error) { txt = txt + str if str == quote { return token.Token{ - Type: token.StringLiteral, + Type: tokenType, Value: txt, Position: firstPos, }, nil } + tokenType = token.NonEmptyStringLiteral } } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 66f735a..a5beb05 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -44,7 +44,7 @@ func TestLexer_Scan(t *testing.T) { name: "There is a StringLiteral", input: `"pragma"`, wantToken: token.Token{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Value: `"pragma"`, Position: token.Pos{ Column: 1, @@ -174,24 +174,45 @@ func TestLexer_Peek(t *testing.T) { func TestLexer_ScanStringLiteral(t *testing.T) { tests := []struct { - name string - input string - wantText string + input string + want token.Token }{ { - name: "double-quoted-printable", - input: `"Hello world!!";`, - wantText: `"Hello world!!"`, + input: `"Hello world!!";`, + want: token.Token{ + Type: token.NonEmptyStringLiteral, + Value: `"Hello world!!"`, + Position: token.Pos{Column: 1, Line: 1}, + }, + }, + { + input: `\'Hello world!!\';`, + want: token.Token{ + Type: token.NonEmptyStringLiteral, + Value: `\'Hello world!!\'`, + Position: token.Pos{Column: 1, Line: 1}, + }, + }, + { + input: `"";`, + want: token.Token{ + Type: token.EmptyStringLiteral, + Value: `""`, + Position: token.Pos{Column: 1, Line: 1}, + }, }, { - name: "single-quoted-printable", - input: `\'Hello world!!\';`, - wantText: `\'Hello world!!\'`, + input: `\'\';`, + want: token.Token{ + Type: token.EmptyStringLiteral, + Value: `\'\'`, + Position: token.Pos{Column: 1, Line: 1}, + }, }, } for _, tt := range tests { tt := tt - t.Run(tt.name, func(t *testing.T) { + t.Run(tt.input, func(t *testing.T) { s := scanner.New(strings.NewReader(tt.input)) l := Lexer{ scanner: s, @@ -201,7 +222,7 @@ func TestLexer_ScanStringLiteral(t *testing.T) { if err != nil { t.Errorf("error is not nil, got: %s", err) } - if diff := cmp.Diff(tt.wantText, tkn.Value); diff != "" { + if diff := cmp.Diff(tt.want, tkn); diff != "" { t.Errorf(diff) } }) diff --git a/literal.go b/literal.go index 0e814e9..b2117d9 100644 --- a/literal.go +++ b/literal.go @@ -11,7 +11,7 @@ func (p *Parser) ParseLiteral() (ast.Literal, error) { return nil, err } - if tkn.Type != token.StringLiteral { + if tkn.Type != token.NonEmptyStringLiteral { return nil, token.NewPosError(tkn.Position, "not found string literal quote") } diff --git a/literal_test.go b/literal_test.go index 3509bf6..0082803 100644 --- a/literal_test.go +++ b/literal_test.go @@ -22,7 +22,7 @@ func TestParser_ParseLiteral(t *testing.T) { name: "normal", input: `"Hello World!!";`, want: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Value: `"Hello World!!"`, Position: pos(1, 1), }, @@ -31,7 +31,7 @@ func TestParser_ParseLiteral(t *testing.T) { name: `Including \n`, input: "\"Hello \nWorld!!\";", want: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Value: "\"Hello \nWorld!!\"", Position: pos(1, 1), }, diff --git a/solparser_test.go b/solparser_test.go index 3d5e43f..ef1060e 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -103,7 +103,7 @@ contract HelloWorld { From: token.Pos{Column: 9, Line: 5}, SemiPos: token.Pos{Column: 31, Line: 5}, Expression: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Position: token.Pos{Column: 16, Line: 5}, Value: "\"Hello World!!\"", }, diff --git a/statement_test.go b/statement_test.go index a85023f..ff0cb66 100644 --- a/statement_test.go +++ b/statement_test.go @@ -25,7 +25,7 @@ func TestParser_ParseStatement(t *testing.T) { From: token.Pos{Column: 1, Line: 1}, SemiPos: token.Pos{Column: 23, Line: 1}, Expression: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Position: token.Pos{Column: 8, Line: 1}, Value: `"Hello World!!"`, }, @@ -85,7 +85,7 @@ func TestParser_ParseReturnStatement(t *testing.T) { From: token.Pos{Column: 1, Line: 1}, SemiPos: token.Pos{Column: 23, Line: 1}, Expression: &ast.StringLiteral{ - Type: token.StringLiteral, + Type: token.NonEmptyStringLiteral, Position: token.Pos{Column: 8, Line: 1}, Value: `"Hello World!!"`, }, diff --git a/token/token.go b/token/token.go index 8b3952f..783aec6 100644 --- a/token/token.go +++ b/token/token.go @@ -167,7 +167,8 @@ const ( TrueLiteral FalseLiteral Number - StringLiteral + NonEmptyStringLiteral + EmptyStringLiteral UnicodeStringLiteral HexStringLiteral CommentLiteral From 3ba67460850e3bf59d98cab0a9d91fb642711d42 Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 1 Mar 2023 13:06:40 +0900 Subject: [PATCH 46/58] main, ast: refactor identifier type --- ast/ast.go | 10 +++++++++- contractDefinition_test.go | 4 ++-- identifier.go | 6 +++--- identifier_test.go | 8 ++++---- solparser_test.go | 2 +- symbolAliases.go | 16 ++-------------- symbolAliases_test.go | 36 ++++++++++++++++++++++++------------ 7 files changed, 45 insertions(+), 37 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 5613761..34a3039 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -138,7 +138,15 @@ type Expression interface { // ---------------------------------------------------------------------------- -type Identifier = token.Token +type Identifier token.Token + +func (i Identifier) Pos() token.Pos { return i.Position } +func (i Identifier) End() token.Pos { + return token.Pos{ + Column: i.Position.Column + len(i.Value), + Line: i.Position.Line, + } +} // ---------------------------------------------------------------------------- // Literal Nodes diff --git a/contractDefinition_test.go b/contractDefinition_test.go index b1928c2..3fd6a1c 100644 --- a/contractDefinition_test.go +++ b/contractDefinition_test.go @@ -27,7 +27,7 @@ func TestParser_ParseContractDefinition(t *testing.T) { }`, want: &ast.ContractDefinition{ Contract: token.Pos{Column: 1, Line: 1}, - Identifier: token.Token{ + Identifier: ast.Identifier{ Type: token.Identifier, Value: "HelloWorld", Position: token.Pos{Column: 10, Line: 1}, @@ -101,7 +101,7 @@ func TestParser_ParseContractDefinition(t *testing.T) { want: &ast.ContractDefinition{ Abstract: &token.Pos{Column: 1, Line: 1}, Contract: token.Pos{Column: 10, Line: 1}, - Identifier: token.Token{ + Identifier: ast.Identifier{ Type: token.Identifier, Value: "HelloWorld", Position: token.Pos{Column: 19, Line: 1}, diff --git a/identifier.go b/identifier.go index 335996d..aa39106 100644 --- a/identifier.go +++ b/identifier.go @@ -8,13 +8,13 @@ import ( func (p *Parser) ParseIdentifier() (ast.Identifier, error) { tkn, err := p.lexer.Scan() if err != nil { - return token.Token{}, err + return ast.Identifier{}, err } switch tkn.Type { case token.Identifier, token.From, token.Error, token.Revert, token.Global: - return tkn, nil + return ast.Identifier(tkn), nil } - return token.Token{}, token.NewPosError(tkn.Position, "keyword is not available as identifier.") + return ast.Identifier{}, token.NewPosError(tkn.Position, "keyword is not available as identifier.") } diff --git a/identifier_test.go b/identifier_test.go index 793116e..826c2ed 100644 --- a/identifier_test.go +++ b/identifier_test.go @@ -19,7 +19,7 @@ func TestParser_ParseIdentifier(t *testing.T) { }{ { input: "identifier", - want: token.Token{ + want: ast.Identifier{ Type: token.Identifier, Value: "identifier", Position: token.Pos{Column: 1, Line: 1}, @@ -27,7 +27,7 @@ func TestParser_ParseIdentifier(t *testing.T) { }, { input: "from", - want: token.Token{ + want: ast.Identifier{ Type: token.From, Value: "from", Position: token.Pos{Column: 1, Line: 1}, @@ -35,7 +35,7 @@ func TestParser_ParseIdentifier(t *testing.T) { }, { input: "error", - want: token.Token{ + want: ast.Identifier{ Type: token.Error, Value: "error", Position: token.Pos{Column: 1, Line: 1}, @@ -43,7 +43,7 @@ func TestParser_ParseIdentifier(t *testing.T) { }, { input: "global", - want: token.Token{ + want: ast.Identifier{ Type: token.Global, Value: "global", Position: token.Pos{Column: 1, Line: 1}, diff --git a/solparser_test.go b/solparser_test.go index ef1060e..b2f12ea 100644 --- a/solparser_test.go +++ b/solparser_test.go @@ -51,7 +51,7 @@ contract HelloWorld { }, ContractDefinition: &ast.ContractDefinition{ Contract: token.Pos{Column: 1, Line: 3}, - Identifier: token.Token{ + Identifier: ast.Identifier{ Type: token.Identifier, Value: "HelloWorld", Position: token.Pos{Column: 10, Line: 3}, diff --git a/symbolAliases.go b/symbolAliases.go index 7dd4849..d9a96ad 100644 --- a/symbolAliases.go +++ b/symbolAliases.go @@ -6,19 +6,11 @@ import ( ) func (p *Parser) ParseSymbolAlias() (*ast.SymbolAlias, error) { - smbl, err := p.lexer.Peek() + smbl, err := p.ParseIdentifier() if err != nil { return nil, err } - if smbl.Type != token.Identifier { - return nil, token.NewPosError(smbl.Position, "not found identifier.") - } - - if _, err := p.lexer.Scan(); err != nil { - return nil, err - } - as, err := p.lexer.Peek() if err != nil { return nil, err @@ -30,15 +22,11 @@ func (p *Parser) ParseSymbolAlias() (*ast.SymbolAlias, error) { return nil, err } - alias, err := p.lexer.Scan() + alias, err := p.ParseIdentifier() if err != nil { return nil, err } - if alias.Type != token.Identifier { - return nil, token.NewPosError(alias.Position, "not found identifier.") - } - return &ast.SymbolAlias{ Symbol: smbl, As: &as.Position, diff --git a/symbolAliases_test.go b/symbolAliases_test.go index fc37cc3..67a9d71 100644 --- a/symbolAliases_test.go +++ b/symbolAliases_test.go @@ -18,26 +18,30 @@ func TestParser_ParseSymbolAlias(t *testing.T) { { input: "symbol as alias1}", want: &ast.SymbolAlias{ - Symbol: tkn(token.Identifier, "symbol", pos(1, 1)), + Symbol: ast.Identifier(tkn(token.Identifier, "symbol", pos(1, 1))), As: posPtr(8, 1), - Alias: tknPtr(token.Identifier, "alias1", pos(11, 1)), + Alias: &ast.Identifier{ + Type: token.Identifier, + Value: "alias1", + Position: pos(11, 1), + }, }, }, { input: "symbol}", want: &ast.SymbolAlias{ - Symbol: tkn(token.Identifier, "symbol", pos(1, 1)), + Symbol: ast.Identifier(tkn(token.Identifier, "symbol", pos(1, 1))), As: nil, Alias: nil, }, }, { input: "pragma", - err: perr(pos(1, 1), "not found identifier."), + err: perr(pos(1, 1), "keyword is not available as identifier."), }, { input: "symbol as pragma", - err: perr(pos(11, 1), "not found identifier."), + err: perr(pos(11, 1), "keyword is not available as identifier."), }, } @@ -65,7 +69,7 @@ func TestParser_ParseSymbolAliases(t *testing.T) { LBrace: pos(1, 1), Aliases: []*ast.SymbolAlias{ { - Symbol: tkn(token.Identifier, "symbol1", pos(2, 1)), + Symbol: ast.Identifier(tkn(token.Identifier, "symbol1", pos(2, 1))), }, }, Commas: []*token.Pos{}, @@ -78,9 +82,13 @@ func TestParser_ParseSymbolAliases(t *testing.T) { LBrace: pos(1, 1), Aliases: []*ast.SymbolAlias{ { - Symbol: tkn(token.Identifier, "symbol2", pos(2, 1)), + Symbol: ast.Identifier(tkn(token.Identifier, "symbol2", pos(2, 1))), As: posPtr(10, 1), - Alias: tknPtr(token.Identifier, "alias1", pos(13, 1)), + Alias: &ast.Identifier{ + Type: token.Identifier, + Value: "alias1", + Position: pos(13, 1), + }, }, }, Commas: []*token.Pos{}, @@ -93,12 +101,16 @@ func TestParser_ParseSymbolAliases(t *testing.T) { LBrace: pos(1, 1), Aliases: []*ast.SymbolAlias{ { - Symbol: tkn(token.Identifier, "symbol1", pos(2, 1)), + Symbol: ast.Identifier(tkn(token.Identifier, "symbol1", pos(2, 1))), }, { - Symbol: tkn(token.Identifier, "symbol2", pos(11, 1)), + Symbol: ast.Identifier(tkn(token.Identifier, "symbol2", pos(11, 1))), As: posPtr(19, 1), - Alias: tknPtr(token.Identifier, "alias1", pos(22, 1)), + Alias: &ast.Identifier{ + Type: token.Identifier, + Value: "alias1", + Position: pos(22, 1), + }, }, }, Commas: []*token.Pos{posPtr(9, 1)}, @@ -111,7 +123,7 @@ func TestParser_ParseSymbolAliases(t *testing.T) { }, { input: "{symbol1, as alian1", - err: perr(pos(11, 1), "not found identifier."), + err: perr(pos(11, 1), "keyword is not available as identifier."), }, { input: "{symbol1, symbol2 as alian1", From 163d45c9a78be24b20935f7c4b7e5ebbd9402f0e Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 1 Mar 2023 14:41:40 +0900 Subject: [PATCH 47/58] main, ast: implement import-directive --- ast/ast.go | 63 ++++++++++ importDirective.go | 169 +++++++++++++++++++++++++++ importDirective_test.go | 248 ++++++++++++++++++++++++++++++++++++++++ solparser.go | 8 ++ 4 files changed, 488 insertions(+) create mode 100644 importDirective.go create mode 100644 importDirective_test.go diff --git a/ast/ast.go b/ast/ast.go index 34a3039..4889e4f 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -11,6 +11,65 @@ type Node interface { End() token.Pos } +// ---------------------------------------------------------------------------- +// ImportDirectiveElement Nodes + +type ImportDirectiveElement interface { + Node + importDirectiveElementNode() +} + +type ImportDirectivePathElement struct { + Path Path + As *token.Pos + Identifier *Identifier +} + +func (i ImportDirectivePathElement) Pos() token.Pos { return i.Path.Pos() } +func (i ImportDirectivePathElement) End() token.Pos { return i.Identifier.End() } + +type ImportDirectiveSymbolAliasesElement struct { + SymbolAliases *SymbolAliases + From token.Pos + Path Path +} + +func (i ImportDirectiveSymbolAliasesElement) Pos() token.Pos { return i.SymbolAliases.Pos() } +func (i ImportDirectiveSymbolAliasesElement) End() token.Pos { return i.Path.End() } + +type ImportDirectiveMulElement struct { + Mul token.Pos + As token.Pos + Identifier Identifier + From token.Pos + Path Path +} + +func (i ImportDirectiveMulElement) Pos() token.Pos { return i.Mul } +func (i ImportDirectiveMulElement) End() token.Pos { return i.Path.End() } + +func (i *ImportDirectivePathElement) importDirectiveElementNode() {} +func (i *ImportDirectiveSymbolAliasesElement) importDirectiveElementNode() {} +func (i *ImportDirectiveMulElement) importDirectiveElementNode() {} + +// ---------------------------------------------------------------------------- + +type ImportDirective struct { + Import token.Pos + Element ImportDirectiveElement + Semicolon token.Pos +} + +type Path token.Token + +func (p Path) Pos() token.Pos { return p.Position } +func (p Path) End() token.Pos { + return token.Pos{ + Column: p.Position.Column + len(p.Value), // Path does not include newline code. + Line: p.Position.Line, + } +} + type SymbolAlias struct { Symbol Identifier As *token.Pos @@ -24,6 +83,9 @@ type SymbolAliases struct { RBrace token.Pos } +func (s SymbolAliases) Pos() token.Pos { return s.LBrace } +func (s SymbolAliases) End() token.Pos { return s.RBrace } + type PragmaDirective struct { Pragma token.Pos PragmaTokens []*token.Token @@ -100,6 +162,7 @@ type ContractDefinition struct { // A File node represents a Solidity source file. type SourceUnit struct { PragmaDirective *PragmaDirective + ImportDirective *ImportDirective ContractDefinition *ContractDefinition FunctionDefinition *FunctionDefinition } diff --git a/importDirective.go b/importDirective.go new file mode 100644 index 0000000..2fff313 --- /dev/null +++ b/importDirective.go @@ -0,0 +1,169 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParsePath() (ast.Path, error) { + lit, err := p.lexer.Peek() + if err != nil { + return ast.Path{}, err + } + if lit.Type != token.NonEmptyStringLiteral { + return ast.Path{}, token.NewPosError(lit.Position, "not found non-empty-string-literal.") + } + if _, err := p.lexer.Scan(); err != nil { + return ast.Path{}, err + } + return ast.Path(lit), nil +} + +func (p *Parser) ParseImportDirectivePathElement() (*ast.ImportDirectivePathElement, error) { + path, err := p.ParsePath() + if err != nil { + return nil, err + } + + as, err := p.lexer.Peek() + if err != nil { + return nil, err + } + if as.Type != token.As { + return &ast.ImportDirectivePathElement{Path: path}, nil + } + if _, err := p.lexer.Scan(); err != nil { + return nil, err + } + + id, err := p.ParseIdentifier() + if err != nil { + return nil, err + } + + return &ast.ImportDirectivePathElement{ + Path: path, + As: &as.Position, + Identifier: &id, + }, nil +} + +func (p *Parser) ParseImportDirectiveSymbolAliasesElement() (*ast.ImportDirectiveSymbolAliasesElement, error) { + aliases, err := p.ParseSymbolAliases() + if err != nil { + return nil, err + } + + from, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if from.Type != token.From { + return nil, token.NewPosError(from.Position, "not found from keyword.") + } + + path, err := p.ParsePath() + if err != nil { + return nil, err + } + + return &ast.ImportDirectiveSymbolAliasesElement{ + SymbolAliases: aliases, + From: from.Position, + Path: path, + }, nil +} + +func (p *Parser) ParseImportDirectiveMulElement() (*ast.ImportDirectiveMulElement, error) { + mul, err := p.lexer.Peek() + if err != nil { + return nil, err + } + if mul.Type != token.Mul { + return nil, token.NewPosError(mul.Position, "not found mul keyword.") + } + if _, err := p.lexer.Scan(); err != nil { + return nil, err + } + + as, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if as.Type != token.As { + return nil, token.NewPosError(as.Position, "not found as keyword.") + } + + id, err := p.ParseIdentifier() + if err != nil { + return nil, err + } + + from, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if from.Type != token.From { + return nil, token.NewPosError(from.Position, "not found from keyword.") + } + + path, err := p.ParsePath() + if err != nil { + return nil, err + } + + return &ast.ImportDirectiveMulElement{ + Mul: mul.Position, + As: as.Position, + Identifier: id, + From: from.Position, + Path: path, + }, nil +} + +func (p *Parser) ParseImportDirective() (*ast.ImportDirective, error) { + impt, err := p.lexer.Peek() + if err != nil { + return nil, err + } + if impt.Type != token.Import { + return nil, token.NewPosError(impt.Position, "not found import keyword.") + } + if _, err := p.lexer.Scan(); err != nil { + return nil, err + } + + tkn, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + var el ast.ImportDirectiveElement + switch tkn.Type { + case token.NonEmptyStringLiteral: + el, err = p.ParseImportDirectivePathElement() + case token.LBrace: + el, err = p.ParseImportDirectiveSymbolAliasesElement() + case token.Mul: + el, err = p.ParseImportDirectiveMulElement() + default: + return nil, token.NewPosError(tkn.Position, "not found import-directive element.") + } + if err != nil { + return nil, err + } + + semi, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if semi.Type != token.Semicolon { + return nil, token.NewPosError(semi.Position, "not found semicolon.") + } + + return &ast.ImportDirective{ + Import: impt.Position, + Element: el, + Semicolon: semi.Position, + }, nil +} diff --git a/importDirective_test.go b/importDirective_test.go new file mode 100644 index 0000000..dea4854 --- /dev/null +++ b/importDirective_test.go @@ -0,0 +1,248 @@ +package solparser_test + +import ( + "strings" + "testing" + + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParsePath(t *testing.T) { + tests := []struct { + input string + want ast.Path + err *token.PosError + }{ + { + input: `"test.sol"`, + want: ast.Path(tkn(token.NonEmptyStringLiteral, `"test.sol"`, pos(1, 1))), + }, + { + input: "pragma", + err: perr(pos(1, 1), "not found non-empty-string-literal."), + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.input, func(t *testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParsePath() + assert(t, got, tt.want, err, tt.err) + }) + } +} + +func TestParser_ParseImportDirectivePathElement(t *testing.T) { + tests := []struct { + input string + want *ast.ImportDirectivePathElement + err *token.PosError + }{ + { + input: `"test.sol";`, + want: &ast.ImportDirectivePathElement{ + Path: ast.Path(tkn(token.NonEmptyStringLiteral, `"test.sol"`, pos(1, 1))), + }, + }, + { + input: `"test.sol" as alias1 ;`, + want: &ast.ImportDirectivePathElement{ + Path: ast.Path(tkn(token.NonEmptyStringLiteral, `"test.sol"`, pos(1, 1))), + As: posPtr(12, 1), + Identifier: &ast.Identifier{ + Type: token.Identifier, + Value: "alias1", + Position: pos(15, 1), + }, + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.input, func(t *testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseImportDirectivePathElement() + assert(t, got, tt.want, err, tt.err) + }) + } +} + +func TestParser_ParseImportDirectiveSymbolAliasesElement(t *testing.T) { + tests := []struct { + input string + want *ast.ImportDirectiveSymbolAliasesElement + err *token.PosError + }{ + { + input: `{symbol1} from "test.sol"`, + want: &ast.ImportDirectiveSymbolAliasesElement{ + SymbolAliases: &ast.SymbolAliases{ + LBrace: pos(1, 1), + Aliases: []*ast.SymbolAlias{ + {Symbol: ast.Identifier{ + Type: token.Identifier, + Value: "symbol1", + Position: pos(2, 1), + }}, + }, + Commas: []*token.Pos{}, + RBrace: pos(9, 1), + }, + From: pos(11, 1), + Path: ast.Path(tkn(token.NonEmptyStringLiteral, `"test.sol"`, pos(16, 1))), + }, + }, + { + input: `{symbol1} "test.sol"`, + err: perr(pos(11, 1), "not found from keyword."), + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.input, func(t *testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseImportDirectiveSymbolAliasesElement() + assert(t, got, tt.want, err, tt.err) + }) + } +} + +func TestParser_ParseImportDirectiveMulElement(t *testing.T) { + tests := []struct { + input string + want *ast.ImportDirectiveMulElement + err *token.PosError + }{ + { + input: `* as alias1 from "test.sol"`, + want: &ast.ImportDirectiveMulElement{ + Mul: pos(1, 1), + As: pos(3, 1), + Identifier: ast.Identifier{ + Type: token.Identifier, + Value: "alias1", + Position: pos(6, 1), + }, + From: pos(13, 1), + Path: ast.Path(tkn(token.NonEmptyStringLiteral, `"test.sol"`, pos(18, 1))), + }, + }, + { + input: `{symbol1} "test.sol"`, + err: perr(pos(1, 1), "not found mul keyword."), + }, + { + input: `* alias1 from "test.sol"`, + err: perr(pos(3, 1), "not found as keyword."), + }, + { + input: `* as alias1 "test.sol"`, + err: perr(pos(13, 1), "not found from keyword."), + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.input, func(t *testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseImportDirectiveMulElement() + assert(t, got, tt.want, err, tt.err) + }) + } +} + +func TestParser_ParseImportDirective(t *testing.T) { + tests := []struct { + input string + want *ast.ImportDirective + err *token.PosError + }{ + { + input: `import "test.sol";`, + want: &ast.ImportDirective{ + Import: pos(1, 1), + Element: &ast.ImportDirectivePathElement{ + Path: ast.Path(tkn(token.NonEmptyStringLiteral, `"test.sol"`, pos(8, 1))), + }, + Semicolon: pos(18, 1), + }, + }, + { + input: `import {symbol1} from "test.sol";`, + want: &ast.ImportDirective{ + Import: pos(1, 1), + Element: &ast.ImportDirectiveSymbolAliasesElement{ + SymbolAliases: &ast.SymbolAliases{ + LBrace: pos(8, 1), + Aliases: []*ast.SymbolAlias{ + {Symbol: ast.Identifier{ + Type: token.Identifier, + Value: "symbol1", + Position: pos(9, 1), + }}, + }, + Commas: []*token.Pos{}, + RBrace: pos(16, 1), + }, + From: pos(18, 1), + Path: ast.Path(tkn(token.NonEmptyStringLiteral, `"test.sol"`, pos(23, 1))), + }, + Semicolon: pos(33, 1), + }, + }, + { + input: `import * as alias1 from "test.sol";`, + want: &ast.ImportDirective{ + Import: pos(1, 1), + Element: &ast.ImportDirectiveMulElement{ + Mul: pos(8, 1), + As: pos(10, 1), + Identifier: ast.Identifier{ + Type: token.Identifier, + Value: "alias1", + Position: pos(13, 1), + }, + From: pos(20, 1), + Path: ast.Path(tkn(token.NonEmptyStringLiteral, `"test.sol"`, pos(25, 1))), + }, + Semicolon: pos(35, 1), + }, + }, + { + input: "symbol as pragma", + err: perr(pos(1, 1), "not found import keyword."), + }, + { + input: `import alias1 from "test.sol";`, + err: perr(pos(8, 1), "not found import-directive element."), + }, + { + input: `import * as alias1 from "test.sol"`, + err: perr(pos(35, 1), "not found semicolon."), + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.input, func(t *testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + + got, err := p.ParseImportDirective() + assert(t, got, tt.want, err, tt.err) + }) + } +} diff --git a/solparser.go b/solparser.go index 2c11763..a2a02fd 100644 --- a/solparser.go +++ b/solparser.go @@ -23,6 +23,7 @@ func New(input io.Reader) *Parser { func (p *Parser) Parse() (*ast.SourceUnit, error) { var pragmaDirective *ast.PragmaDirective + var importDirective *ast.ImportDirective var contractDefinition *ast.ContractDefinition var functionDefinition *ast.FunctionDefinition for { @@ -38,6 +39,12 @@ func (p *Parser) Parse() (*ast.SourceUnit, error) { return nil, err } pragmaDirective = prgm + case token.Import: + imp, err := p.ParseImportDirective() + if err != nil { + return nil, err + } + importDirective = imp case token.Abstract, token.Contract: cntrct, err := p.ParseContractDefinition() if err != nil { @@ -53,6 +60,7 @@ func (p *Parser) Parse() (*ast.SourceUnit, error) { case token.EOS: return &ast.SourceUnit{ PragmaDirective: pragmaDirective, + ImportDirective: importDirective, ContractDefinition: contractDefinition, FunctionDefinition: functionDefinition, }, nil From 567a618fbb1959eb224f8e5cfc0f5cb622e7abc7 Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 1 Mar 2023 15:15:37 +0900 Subject: [PATCH 48/58] all: add generics test helper struct --- go.mod | 2 +- helper_test.go | 20 +++++++++ importDirective_test.go | 96 +++++++++-------------------------------- 3 files changed, 41 insertions(+), 77 deletions(-) diff --git a/go.mod b/go.mod index 5096641..c51c2e3 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/uji/solparser -go 1.16 +go 1.18 require ( github.com/SteelSeries/bufrr v0.0.0-20161129220322-72103137aa3c diff --git a/helper_test.go b/helper_test.go index de20174..43008db 100644 --- a/helper_test.go +++ b/helper_test.go @@ -2,9 +2,11 @@ package solparser_test import ( "errors" + "strings" "testing" "github.com/google/go-cmp/cmp" + "github.com/uji/solparser" "github.com/uji/solparser/token" ) @@ -52,3 +54,21 @@ func assert(t *testing.T, got, want interface{}, gotErr, wantErr error) { t.Errorf("%s", diff) } } + +type TestData[T any] []struct { + input string + want T + err error +} + +func (ts TestData[T]) Test(t *testing.T, target func(p *solparser.Parser) (T, error)) { + for _, tt := range ts { + tt := tt + t.Run(tt.input, func(t *testing.T) { + r := strings.NewReader(tt.input) + p := solparser.New(r) + got, err := target(p) + assert(t, got, tt.want, err, tt.err) + }) + } +} diff --git a/importDirective_test.go b/importDirective_test.go index dea4854..aa91d4d 100644 --- a/importDirective_test.go +++ b/importDirective_test.go @@ -1,7 +1,6 @@ package solparser_test import ( - "strings" "testing" "github.com/uji/solparser" @@ -10,11 +9,7 @@ import ( ) func TestParser_ParsePath(t *testing.T) { - tests := []struct { - input string - want ast.Path - err *token.PosError - }{ + tests := TestData[ast.Path]{ { input: `"test.sol"`, want: ast.Path(tkn(token.NonEmptyStringLiteral, `"test.sol"`, pos(1, 1))), @@ -25,24 +20,13 @@ func TestParser_ParsePath(t *testing.T) { }, } - for _, tt := range tests { - tt := tt - t.Run(tt.input, func(t *testing.T) { - r := strings.NewReader(tt.input) - p := solparser.New(r) - - got, err := p.ParsePath() - assert(t, got, tt.want, err, tt.err) - }) - } + tests.Test(t, func(p *solparser.Parser) (ast.Path, error) { + return p.ParsePath() + }) } func TestParser_ParseImportDirectivePathElement(t *testing.T) { - tests := []struct { - input string - want *ast.ImportDirectivePathElement - err *token.PosError - }{ + tests := TestData[*ast.ImportDirectivePathElement]{ { input: `"test.sol";`, want: &ast.ImportDirectivePathElement{ @@ -63,24 +47,13 @@ func TestParser_ParseImportDirectivePathElement(t *testing.T) { }, } - for _, tt := range tests { - tt := tt - t.Run(tt.input, func(t *testing.T) { - r := strings.NewReader(tt.input) - p := solparser.New(r) - - got, err := p.ParseImportDirectivePathElement() - assert(t, got, tt.want, err, tt.err) - }) - } + tests.Test(t, func(p *solparser.Parser) (*ast.ImportDirectivePathElement, error) { + return p.ParseImportDirectivePathElement() + }) } func TestParser_ParseImportDirectiveSymbolAliasesElement(t *testing.T) { - tests := []struct { - input string - want *ast.ImportDirectiveSymbolAliasesElement - err *token.PosError - }{ + tests := TestData[*ast.ImportDirectiveSymbolAliasesElement]{ { input: `{symbol1} from "test.sol"`, want: &ast.ImportDirectiveSymbolAliasesElement{ @@ -106,24 +79,13 @@ func TestParser_ParseImportDirectiveSymbolAliasesElement(t *testing.T) { }, } - for _, tt := range tests { - tt := tt - t.Run(tt.input, func(t *testing.T) { - r := strings.NewReader(tt.input) - p := solparser.New(r) - - got, err := p.ParseImportDirectiveSymbolAliasesElement() - assert(t, got, tt.want, err, tt.err) - }) - } + tests.Test(t, func(p *solparser.Parser) (*ast.ImportDirectiveSymbolAliasesElement, error) { + return p.ParseImportDirectiveSymbolAliasesElement() + }) } func TestParser_ParseImportDirectiveMulElement(t *testing.T) { - tests := []struct { - input string - want *ast.ImportDirectiveMulElement - err *token.PosError - }{ + tests := TestData[*ast.ImportDirectiveMulElement]{ { input: `* as alias1 from "test.sol"`, want: &ast.ImportDirectiveMulElement{ @@ -152,24 +114,13 @@ func TestParser_ParseImportDirectiveMulElement(t *testing.T) { }, } - for _, tt := range tests { - tt := tt - t.Run(tt.input, func(t *testing.T) { - r := strings.NewReader(tt.input) - p := solparser.New(r) - - got, err := p.ParseImportDirectiveMulElement() - assert(t, got, tt.want, err, tt.err) - }) - } + tests.Test(t, func(p *solparser.Parser) (*ast.ImportDirectiveMulElement, error) { + return p.ParseImportDirectiveMulElement() + }) } func TestParser_ParseImportDirective(t *testing.T) { - tests := []struct { - input string - want *ast.ImportDirective - err *token.PosError - }{ + tests := TestData[*ast.ImportDirective]{ { input: `import "test.sol";`, want: &ast.ImportDirective{ @@ -235,14 +186,7 @@ func TestParser_ParseImportDirective(t *testing.T) { }, } - for _, tt := range tests { - tt := tt - t.Run(tt.input, func(t *testing.T) { - r := strings.NewReader(tt.input) - p := solparser.New(r) - - got, err := p.ParseImportDirective() - assert(t, got, tt.want, err, tt.err) - }) - } + tests.Test(t, func(p *solparser.Parser) (*ast.ImportDirective, error) { + return p.ParseImportDirective() + }) } From 83d92cad83fcb36f4fbd3c8bf36e597e75c5128d Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 1 Mar 2023 20:32:15 +0900 Subject: [PATCH 49/58] ast: refactor ast Nodes --- ast/ast.go | 115 +++++++++++++++++++++++++++--------------------- ast/ast_test.go | 18 ++++++++ 2 files changed, 84 insertions(+), 49 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 4889e4f..e224820 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -6,19 +6,52 @@ import ( "github.com/uji/solparser/token" ) +// All node types implement the Node interface. type Node interface { Pos() token.Pos End() token.Pos } -// ---------------------------------------------------------------------------- -// ImportDirectiveElement Nodes - +// All import-directive-element node (own definition for solparser) types implement the ImportDirectiveElement interface. type ImportDirectiveElement interface { Node importDirectiveElementNode() } +// All contract-body-element node types implement the ContractBodyElement interface. +type ContractBodyElement interface { + Node + contractBodyElementNode() +} + +// All type-name node types implement the TypeName interface. +type TypeName interface { + Node + typeNameNode() +} + +// All expression node types implement the Expression interface. +type Expression interface { + Node + expressionNode() +} + +// All literal node types implement the Literal interface. +type Literal interface { + Node + literalNode() + expressionNode() +} + +// All statement node types implement the Statement interface. +type Statement interface { + Node + statementNode() +} + +// ---------------------------------------------------------------------------- +// ImportDirectiveElement Nodes + type ImportDirectivePathElement struct { Path Path As *token.Pos @@ -60,6 +93,9 @@ type ImportDirective struct { Semicolon token.Pos } +func (i ImportDirective) Pos() token.Pos { return i.Import } +func (i ImportDirective) End() token.Pos { return i.Semicolon } + type Path token.Token func (p Path) Pos() token.Pos { return p.Position } @@ -76,6 +112,14 @@ type SymbolAlias struct { Alias *Identifier } +func (s SymbolAlias) Pos() token.Pos { return s.Symbol.Pos() } +func (s SymbolAlias) End() token.Pos { + if s.Alias != nil { + return s.Alias.End() + } + return s.Symbol.End() +} + type SymbolAliases struct { LBrace token.Pos Aliases []*SymbolAlias @@ -92,6 +136,9 @@ type PragmaDirective struct { Semicolon token.Pos } +func (p PragmaDirective) Pos() token.Pos { return p.Pragma } +func (p PragmaDirective) End() token.Pos { return p.Semicolon } + type ModifierList struct { Visibility *Visibility StateMutability *StateMutability @@ -118,6 +165,14 @@ type FunctionDefinitionReturns struct { RParen token.Pos } +type InheritanceSpecifier struct { + IdentifierPath string + CallArgumentList CallArgumentList +} + +// ---------------------------------------------------------------------------- +// ContractBodyElement Nodes + type FunctionDefinition struct { From token.Pos FunctionDescriptor FunctionDescriptor @@ -128,26 +183,13 @@ type FunctionDefinition struct { Block *Block } -func (f FunctionDefinition) Pos() token.Pos { - return f.From -} - -func (f FunctionDefinition) End() token.Pos { - return f.Block.End() -} - -type InheritanceSpecifier struct { - IdentifierPath string - CallArgumentList CallArgumentList -} - -type ContractBodyElement interface { - Node - contractBodyElementNode() -} +func (f FunctionDefinition) Pos() token.Pos { return f.From } +func (f FunctionDefinition) End() token.Pos { return f.Block.End() } func (f *FunctionDefinition) contractBodyElementNode() {} +// ---------------------------------------------------------------------------- + type CallArgumentList struct{} type ContractDefinition struct { @@ -170,11 +212,6 @@ type SourceUnit struct { // ---------------------------------------------------------------------------- // TypeName Nodes -type TypeName interface { - Node - typeNameNode() -} - type ElementaryTypeName []*token.Token func (e ElementaryTypeName) Pos() token.Pos { @@ -194,13 +231,6 @@ func (e ElementaryTypeName) typeNameNode() {} // ---------------------------------------------------------------------------- // Expression Nodes -type Expression interface { - Node - expressionNode() -} - -// ---------------------------------------------------------------------------- - type Identifier token.Token func (i Identifier) Pos() token.Pos { return i.Position } @@ -211,15 +241,11 @@ func (i Identifier) End() token.Pos { } } +func (i *Identifier) expressionNode() {} + // ---------------------------------------------------------------------------- // Literal Nodes -type Literal interface { - Node - literalNode() - expressionNode() -} - type BooleanLiteral struct { Token token.Token } @@ -387,22 +413,13 @@ func (b Block) End() token.Pos { // ---------------------------------------------------------------------------- // Statement Nodes -type Statement interface { - Node - statementNode() -} - type ReturnStatement struct { From token.Pos SemiPos token.Pos Expression Expression } -func (r *ReturnStatement) Pos() token.Pos { - return r.From -} -func (r *ReturnStatement) End() token.Pos { - return r.SemiPos -} +func (r ReturnStatement) Pos() token.Pos { return r.From } +func (r ReturnStatement) End() token.Pos { return r.SemiPos } func (s *ReturnStatement) statementNode() {} diff --git a/ast/ast_test.go b/ast/ast_test.go index ce3d381..6241b3d 100644 --- a/ast/ast_test.go +++ b/ast/ast_test.go @@ -8,6 +8,24 @@ import ( "github.com/uji/solparser/token" ) +var ( + _ ast.ImportDirectiveElement = &ast.ImportDirectivePathElement{} + _ ast.ImportDirectiveElement = &ast.ImportDirectiveSymbolAliasesElement{} + _ ast.ImportDirectiveElement = &ast.ImportDirectiveMulElement{} + _ ast.Node = &ast.ImportDirective{} + _ ast.Node = &ast.Path{} + _ ast.Node = &ast.SymbolAlias{} + _ ast.Node = &ast.SymbolAliases{} + _ ast.Node = &ast.PragmaDirective{} + _ ast.ContractBodyElement = &ast.FunctionDefinition{} + _ ast.TypeName = ast.ElementaryTypeName{} + _ ast.Expression = &ast.Identifier{} + _ ast.Literal = &ast.BooleanLiteral{} + _ ast.Literal = &ast.StringLiteral{} + _ ast.Node = &ast.Block{} + _ ast.Statement = &ast.ReturnStatement{} +) + func TestNode_End(t *testing.T) { tests := []struct { name string From 258d2326566d98c7b18874b41ec4f4fdce7dde06 Mon Sep 17 00:00:00 2001 From: uji Date: Wed, 1 Mar 2023 23:29:08 +0900 Subject: [PATCH 50/58] lexer: Scan unicode-string-literal --- lexer/helper_test.go | 56 ++++++++++++++++++++++++++++++++++++++++++++ lexer/lexer.go | 53 ++++++++++++++++++++++++++++++++++------- lexer/lexer_test.go | 25 ++++++++++++++++++++ 3 files changed, 126 insertions(+), 8 deletions(-) create mode 100644 lexer/helper_test.go diff --git a/lexer/helper_test.go b/lexer/helper_test.go new file mode 100644 index 0000000..f02d7fd --- /dev/null +++ b/lexer/helper_test.go @@ -0,0 +1,56 @@ +package lexer + +import ( + "errors" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/uji/solparser/token" +) + +func pos(c, l int) token.Pos { + return token.Pos{Column: c, Line: l} +} + +func tkn(tp token.TokenType, text string, pos token.Pos) token.Token { + return token.Token{ + Type: tp, + Value: text, + Position: pos, + } +} + +func perr(pos token.Pos, msg string) *token.PosError { + return &token.PosError{ + Pos: pos, + Msg: msg, + } +} + +type TestData[T any] []struct { + input string + want T + err error +} + +func (ts TestData[T]) Test(t *testing.T, target func(l *Lexer) (T, error)) { + for _, tt := range ts { + tt := tt + t.Run(tt.input, func(t *testing.T) { + l := New(strings.NewReader(tt.input)) + got, err := target(l) + + var sErr *token.PosError + if errors.As(err, &sErr) { + if diff := cmp.Diff(tt.err, sErr); diff != "" { + t.Errorf("%s", diff) + } + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("%s", diff) + } + }) + } +} diff --git a/lexer/lexer.go b/lexer/lexer.go index 459e503..d35c141 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -71,29 +71,66 @@ func (l *Lexer) Peek() (token.Token, error) { // ScanStringLiteral parse NonEmptyStringLiteral or EmptyStringLiteral then return StringLiteral token. func (l *Lexer) ScanStringLiteral() (token.Token, error) { - firstPos, str, err := l.scanner.Scan() + start, v, err := l.scanner.Scan() if err != nil { return token.Token{}, err } - if str != `"` && str != `\'` { - return token.Token{}, token.NewPosError(firstPos, `not found " or \'`) + if v != `"` && v != `\'` { + return token.Token{}, token.NewPosError(start, `not found " or \'`) } - quote, txt := str, str + quote, txt := v, v tokenType := token.EmptyStringLiteral for { - _, str, err := l.scanner.Scan() + _, v, err := l.scanner.Scan() if err != nil { return token.Token{}, err } - txt = txt + str - if str == quote { + txt = txt + v + if v == quote { return token.Token{ Type: tokenType, Value: txt, - Position: firstPos, + Position: start, }, nil } tokenType = token.NonEmptyStringLiteral } } + +func (l *Lexer) ScanUnicodeStringLiteral() (token.Token, error) { + start, v, err := l.scanner.Scan() + if err != nil { + return token.Token{}, err + } + if v != `unicode` { + return token.Token{}, token.NewPosError(start, `not found unicode prefix.`) + } + rslt := v + + pos, v, err := l.scanner.Scan() + if err != nil { + return token.Token{}, err + } + // 'unicode' and quote('"' or '\'') must not have spaces. + if v != `"` && v != `\'` || pos.Column != start.Column+7 { + return token.Token{}, token.NewPosError(pos, `not found " or \'`) + } + + rslt = rslt + v + quote := v + for { + _, v, err := l.scanner.Scan() + if err != nil { + return token.Token{}, err + } + rslt = rslt + v + if v == quote { + return token.Token{ + Type: token.UnicodeStringLiteral, + Value: rslt, + Position: start, + }, nil + } + } +} diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index a5beb05..0d73611 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -251,3 +251,28 @@ func TestLexer_ScanStringLiteral(t *testing.T) { } }) } + +func TestLexer_ScanUnicodeStringLiteral(t *testing.T) { + tests := TestData[token.Token]{ + { + input: `unicode"Hello 😃"`, + want: tkn(token.UnicodeStringLiteral, `unicode"Hello 😃"`, pos(1,1)), + }, + { + input: `unicode\'Hello 😃\'`, + want: tkn(token.UnicodeStringLiteral, `unicode\'Hello 😃\'`, pos(1,1) ), + }, + { + input: `unicode Hello 😃`, + err: perr(pos(8, 1), `not found " or \'`), + }, + { + input: `unicode "Hello 😃"`, + err: perr(pos(8, 1), `not found " or \'`), + }, + } + + tests.Test(t, func(l *Lexer) (token.Token, error) { + return l.ScanUnicodeStringLiteral() + }) +} From 6ec080284f94fed78c61360648e53f881fa741ff Mon Sep 17 00:00:00 2001 From: uji Date: Sat, 11 Mar 2023 16:28:56 +0900 Subject: [PATCH 51/58] lexer: refactor unicodeStringLiteral --- lexer/lexer.go | 8 +++++--- lexer/lexer_test.go | 7 ++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index d35c141..2f689cc 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -98,13 +98,15 @@ func (l *Lexer) ScanStringLiteral() (token.Token, error) { } } +const unicodeStringLiteralQuoteOffset = len("unicode") + func (l *Lexer) ScanUnicodeStringLiteral() (token.Token, error) { start, v, err := l.scanner.Scan() if err != nil { return token.Token{}, err } - if v != `unicode` { - return token.Token{}, token.NewPosError(start, `not found unicode prefix.`) + if v != "unicode" { + return token.Token{}, token.NewPosError(start, "not found unicode prefix.") } rslt := v @@ -113,7 +115,7 @@ func (l *Lexer) ScanUnicodeStringLiteral() (token.Token, error) { return token.Token{}, err } // 'unicode' and quote('"' or '\'') must not have spaces. - if v != `"` && v != `\'` || pos.Column != start.Column+7 { + if v != `"` && v != `\'` && pos.Line == start.Line || pos.Column != start.Column+unicodeStringLiteralQuoteOffset { return token.Token{}, token.NewPosError(pos, `not found " or \'`) } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 0d73611..468f0c1 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -260,7 +260,7 @@ func TestLexer_ScanUnicodeStringLiteral(t *testing.T) { }, { input: `unicode\'Hello 😃\'`, - want: tkn(token.UnicodeStringLiteral, `unicode\'Hello 😃\'`, pos(1,1) ), + want: tkn(token.UnicodeStringLiteral, `unicode\'Hello 😃\'`, pos(1,1)), }, { input: `unicode Hello 😃`, @@ -270,6 +270,11 @@ func TestLexer_ScanUnicodeStringLiteral(t *testing.T) { input: `unicode "Hello 😃"`, err: perr(pos(8, 1), `not found " or \'`), }, + { + input: `unicode + "Hello 😃"`, + err: perr(pos(8, 1), `not found " or \'`), + }, } tests.Test(t, func(l *Lexer) (token.Token, error) { From 5565270b1726a15dfc1544fbfb1c639987466b37 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 12 Mar 2023 15:47:05 +0900 Subject: [PATCH 52/58] lexer, token: scan HexString. --- lexer/lexer.go | 68 +++++++++++++++++++++++++++++++++++++++++++++ lexer/lexer_test.go | 48 ++++++++++++++++++++++++++++++-- token/token.go | 6 ++-- 3 files changed, 115 insertions(+), 7 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 2f689cc..ac7ef53 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -3,6 +3,7 @@ package lexer import ( "errors" "io" + "unicode" "github.com/uji/solparser/scanner" "github.com/uji/solparser/token" @@ -136,3 +137,70 @@ func (l *Lexer) ScanUnicodeStringLiteral() (token.Token, error) { } } } + +func isEvenHexDigits(str string) bool { + if len(str) == 0 { + return true + } + runes := []rune(str) + if !unicode.Is(unicode.Hex_Digit, runes[0]) { + return false + } + + // If a chunk of HexString is being read, it will be true. + readingBlock := true + i := 1 + for i < len(runes) { + if !readingBlock && runes[i] == '_' { + i++ + continue + } + if !unicode.Is(unicode.Hex_Digit, runes[i]) { + return false + } + readingBlock = !readingBlock + i++ + } + return !readingBlock +} + +func (l *Lexer) ScanHexString() (token.Token, error) { + start, hex, err := l.scanner.Scan() + if err != nil { + return token.Token{}, err + } + + if hex != "hex" { + return token.Token{}, token.NewPosError(start, "not found hex prefix.") + } + + lqpos, lquote, err := l.scanner.Scan() + if err != nil { + return token.Token{}, err + } + if lquote != `"` && lquote != `\'` { + return token.Token{}, token.NewPosError(lqpos, `not found " or \'`) + } + + vpos, v, err := l.scanner.Scan() + if err != nil { + return token.Token{}, err + } + if !isEvenHexDigits(v) { + return token.Token{}, token.NewPosError(vpos, "invalid HexString format") + } + + rqpos, rquote, err := l.scanner.Scan() + if err != nil { + return token.Token{}, err + } + if rquote != lquote { + return token.Token{}, token.NewPosError(rqpos, `not found quote'`) + } + + return token.Token{ + Type: token.HexString, + Value: hex + lquote + v + rquote, + Position: start, + }, nil +} diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 468f0c1..4551bc2 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -256,11 +256,11 @@ func TestLexer_ScanUnicodeStringLiteral(t *testing.T) { tests := TestData[token.Token]{ { input: `unicode"Hello 😃"`, - want: tkn(token.UnicodeStringLiteral, `unicode"Hello 😃"`, pos(1,1)), + want: tkn(token.UnicodeStringLiteral, `unicode"Hello 😃"`, pos(1, 1)), }, { input: `unicode\'Hello 😃\'`, - want: tkn(token.UnicodeStringLiteral, `unicode\'Hello 😃\'`, pos(1,1)), + want: tkn(token.UnicodeStringLiteral, `unicode\'Hello 😃\'`, pos(1, 1)), }, { input: `unicode Hello 😃`, @@ -273,7 +273,7 @@ func TestLexer_ScanUnicodeStringLiteral(t *testing.T) { { input: `unicode "Hello 😃"`, - err: perr(pos(8, 1), `not found " or \'`), + err: perr(pos(8, 1), `not found " or \'`), }, } @@ -281,3 +281,45 @@ func TestLexer_ScanUnicodeStringLiteral(t *testing.T) { return l.ScanUnicodeStringLiteral() }) } + +func TestLexer_ScanHexStringLiteral(t *testing.T) { + tests := TestData[token.Token]{ + { + input: `hex"1B"`, + want: tkn(token.HexString, `hex"1B"`, pos(1, 1)), + }, + { + input: `hex\'2E\'`, + want: tkn(token.HexString, `hex\'2E\'`, pos(1, 1)), + }, + { + input: `hex"16_3F_B5"`, + want: tkn(token.HexString, `hex"16_3F_B5"`, pos(1, 1)), + }, + { + input: `hex "12"`, + err: perr(pos(4, 1), `not found " or \'`), + }, + { + input: `hex + "123"`, + err: perr(pos(4, 1), `not found " or \'`), + }, + { + input: `hex" 123"`, + err: perr(pos(5, 1), "invalid HexString format"), + }, + { + input: `hex"16_3"`, + err: perr(pos(5, 1), "invalid HexString format"), + }, + { + input: `hex"163_A13"`, + err: perr(pos(5, 1), "invalid HexString format"), + }, + } + + tests.Test(t, func(l *Lexer) (token.Token, error) { + return l.ScanHexString() + }) +} diff --git a/token/token.go b/token/token.go index 783aec6..9e98de3 100644 --- a/token/token.go +++ b/token/token.go @@ -130,7 +130,6 @@ const ( From Function Global - Hex If Immutable Import @@ -163,6 +162,8 @@ const ( Virtual While + HexString + // Literal TrueLiteral FalseLiteral @@ -170,7 +171,6 @@ const ( NonEmptyStringLiteral EmptyStringLiteral UnicodeStringLiteral - HexStringLiteral CommentLiteral // Not keywords or future reserved words @@ -405,8 +405,6 @@ func asKeyword(str string) TokenType { return Function case "global": return Global - case "hex": - return Hex case "if": return If case "immutable": From da5d95c4afedcd63891167bc4433a956e9fd8f23 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 24 Sep 2023 11:21:42 +0900 Subject: [PATCH 53/58] main: implement ParseIdentifierPath --- ast/ast.go | 11 ++++++++++ identifier.go | 8 +++++++ identifierPath.go | 49 ++++++++++++++++++++++++++++++++++++++++++ identifierPath_test.go | 32 +++++++++++++++++++++++++++ 4 files changed, 100 insertions(+) create mode 100644 identifierPath.go create mode 100644 identifierPath_test.go diff --git a/ast/ast.go b/ast/ast.go index e224820..d2f8b92 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -192,6 +192,17 @@ func (f *FunctionDefinition) contractBodyElementNode() {} type CallArgumentList struct{} +type IdentifierPathElement struct { + Identifier Identifier + Period *token.Pos +} +type IdentifierPath struct { + Elements []*IdentifierPathElement +} + +func (i IdentifierPath) Pos() token.Pos { return i.Elements[0].Identifier.Pos() } +func (i IdentifierPath) End() token.Pos { return i.Elements[len(i.Elements)-1].Identifier.End() } + type ContractDefinition struct { Abstract *token.Pos Contract token.Pos diff --git a/identifier.go b/identifier.go index aa39106..37f42bb 100644 --- a/identifier.go +++ b/identifier.go @@ -5,6 +5,14 @@ import ( "github.com/uji/solparser/token" ) +func isIdentifier(tkn token.Token) bool { + switch tkn.Type { + case token.Identifier, token.From, token.Error, token.Revert, token.Global: + return true + } + return false +} + func (p *Parser) ParseIdentifier() (ast.Identifier, error) { tkn, err := p.lexer.Scan() if err != nil { diff --git a/identifierPath.go b/identifierPath.go new file mode 100644 index 0000000..ab9f0fa --- /dev/null +++ b/identifierPath.go @@ -0,0 +1,49 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseIdentifierPath() (ast.IdentifierPath, error) { + id, err := p.lexer.Peek() + if err != nil { + return ast.IdentifierPath{}, err + } + if !isIdentifier(id) { + return ast.IdentifierPath{}, token.NewPosError(id.Position, "not found identifier.") + } + p.lexer.Scan() + + var elements []*ast.IdentifierPathElement + for { + prd, err := p.lexer.Peek() + if err != nil || prd.Type != token.Period { + break + } + p.lexer.Scan() + + elements = append(elements, &ast.IdentifierPathElement{ + Identifier: ast.Identifier(id), + Period: &prd.Position, + }) + + i, err := p.lexer.Peek() + if err != nil { + return ast.IdentifierPath{}, err + } + if !isIdentifier(i) { + return ast.IdentifierPath{}, token.NewPosError(i.Position, "not found identifier.") + } + p.lexer.Scan() + id = i + } + + elements = append(elements, &ast.IdentifierPathElement{ + Identifier: ast.Identifier(id), + }) + + return ast.IdentifierPath{ + Elements: elements, + }, nil +} diff --git a/identifierPath_test.go b/identifierPath_test.go new file mode 100644 index 0000000..eb9e631 --- /dev/null +++ b/identifierPath_test.go @@ -0,0 +1,32 @@ +package solparser_test + +import ( + "testing" + + "github.com/uji/solparser" + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func TestParser_ParseIdentifierPath(t *testing.T) { + tests := TestData[ast.IdentifierPath]{ + { + input: "global . identifier", + want: ast.IdentifierPath{ + Elements: []*ast.IdentifierPathElement{ + { + Identifier: ast.Identifier(tkn(token.Global, "global", pos(1, 1))), + Period: posPtr(8, 1), + }, + { + Identifier: ast.Identifier(tkn(token.Identifier, "identifier", pos(10, 1))), + }, + }, + }, + }, + } + + tests.Test(t, func(p *solparser.Parser) (ast.IdentifierPath, error) { + return p.ParseIdentifierPath() + }) +} From e3beb3ce81213b0ab3d311a7776256ff4607a1b9 Mon Sep 17 00:00:00 2001 From: uji Date: Sun, 24 Sep 2023 13:28:17 +0900 Subject: [PATCH 54/58] main: ParseCallArgumentList --- ast/ast.go | 25 ++++++- callArgumentList.go | 154 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 callArgumentList.go diff --git a/ast/ast.go b/ast/ast.go index d2f8b92..118cccf 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -190,7 +190,30 @@ func (f *FunctionDefinition) contractBodyElementNode() {} // ---------------------------------------------------------------------------- -type CallArgumentList struct{} +type CallArgumentListExpretion struct { + Expression Expression + Comma *token.Pos +} + +type CallArgumentListNamedExpretion struct { + Identifier Identifier + Colon token.Pos + Expression Expression + Comma *token.Pos +} + +type CallArgumentListNamedExpretions struct { + RBrace token.Pos + NamedExpretions []*CallArgumentListNamedExpretion + LBrace token.Pos +} + +type CallArgumentList struct { + LParen token.Pos + Expressions []*CallArgumentListExpretion + NamedExpretionsa *CallArgumentListNamedExpretions + RParen token.Pos +} type IdentifierPathElement struct { Identifier Identifier diff --git a/callArgumentList.go b/callArgumentList.go new file mode 100644 index 0000000..096babc --- /dev/null +++ b/callArgumentList.go @@ -0,0 +1,154 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseCallArgumentList() (*ast.CallArgumentList, error) { + lparen, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + tkn, err := p.lexer.Peek() + if err != nil { + return nil, err + } + if tkn.Type == token.RParen { + p.lexer.Scan() + return &ast.CallArgumentList{ + LParen: lparen.Position, + RParen: tkn.Position, + }, nil + } + + var expretions []*ast.CallArgumentListExpretion + var namedExpretions *ast.CallArgumentListNamedExpretions + if tkn.Type == token.LBrace { + es, err := p.ParseCallArgumentListExpretions() + if err != nil { + return nil, err + } + expretions = es + } else { + nes, err := p.ParseCallArgumentListNamedExpretions() + if err != nil { + return nil, err + } + namedExpretions = nes + } + + rparen, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if rparen.Type != token.RParen { + return nil, token.NewPosError(rparen.Position, "not found RParen.") + } + + return &ast.CallArgumentList{ + LParen: lparen.Position, + Expressions: expretions, + NamedExpretionsa: namedExpretions, + RParen: rparen.Position, + }, nil +} + +func (p *Parser) ParseCallArgumentListExpretions() ([]*ast.CallArgumentListExpretion, error) { + ex, err := p.ParseExpression() + if err != nil { + return nil, err + } + + var exs []*ast.CallArgumentListExpretion + for { + cmm, err := p.lexer.Peek() + if err != nil || cmm.Type != token.Comma { + break + } + p.lexer.Scan() + + exs = append(exs, &ast.CallArgumentListExpretion{ + Expression: ex, + Comma: &cmm.Position, + }) + + e, err := p.ParseExpression() + ex = e + } + + exs = append(exs, &ast.CallArgumentListExpretion{ + Expression: ex, + }) + + return exs, nil +} + +func (p *Parser) ParseCallArgumentListNamedExpretions() (*ast.CallArgumentListNamedExpretions, error) { + lbrace, err := p.lexer.Scan() + if err != nil { + return nil, err + } + + exs := make([]*ast.CallArgumentListNamedExpretion, 0, 1) + + for { + id, err := p.lexer.Peek() + if err != nil { + return nil, err + } + + if !isIdentifier(id) { + return nil, token.NewPosError(id.Position, "not found identifier.") + } + p.lexer.Scan() + + cln, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if cln.Type != token.Colon { + return nil, token.NewPosError(cln.Position, "not found Colon.") + } + + ex, err := p.ParseExpression() + if err != nil { + return nil, err + } + + cmm, err := p.lexer.Peek() + if err != nil { + return nil, err + } + if cmm.Type != token.Comma { + exs = append(exs, &ast.CallArgumentListNamedExpretion{ + Identifier: ast.Identifier(id), + Colon: cln.Position, + Expression: ex, + }) + break + } + p.lexer.Scan() + exs = append(exs, &ast.CallArgumentListNamedExpretion{ + Identifier: ast.Identifier(id), + Colon: cln.Position, + Expression: ex, + Comma: &cmm.Position, + }) + } + + rbrace, err := p.lexer.Scan() + if err != nil { + return nil, err + } + if rbrace.Type != token.RBrace { + return nil, token.NewPosError(rbrace.Position, "not found RBrace.") + } + + return &ast.CallArgumentListNamedExpretions{ + RBrace: rbrace.Position, + NamedExpretions: exs, + LBrace: lbrace.Position, + }, nil +} From bda85d40c2fb074d747faa272e35016d4539eb69 Mon Sep 17 00:00:00 2001 From: uji Date: Mon, 2 Oct 2023 00:38:14 +0900 Subject: [PATCH 55/58] main: refactor ParseIndentfierPath() --- identifierPath.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/identifierPath.go b/identifierPath.go index ab9f0fa..d46a506 100644 --- a/identifierPath.go +++ b/identifierPath.go @@ -6,14 +6,10 @@ import ( ) func (p *Parser) ParseIdentifierPath() (ast.IdentifierPath, error) { - id, err := p.lexer.Peek() + id, err := p.ParseIdentifier() if err != nil { return ast.IdentifierPath{}, err } - if !isIdentifier(id) { - return ast.IdentifierPath{}, token.NewPosError(id.Position, "not found identifier.") - } - p.lexer.Scan() var elements []*ast.IdentifierPathElement for { @@ -28,14 +24,10 @@ func (p *Parser) ParseIdentifierPath() (ast.IdentifierPath, error) { Period: &prd.Position, }) - i, err := p.lexer.Peek() + i, err := p.ParseIdentifier() if err != nil { return ast.IdentifierPath{}, err } - if !isIdentifier(i) { - return ast.IdentifierPath{}, token.NewPosError(i.Position, "not found identifier.") - } - p.lexer.Scan() id = i } From c51dea6cd9389c490874d68d49507b01d36ef5d4 Mon Sep 17 00:00:00 2001 From: uji Date: Mon, 2 Oct 2023 00:46:37 +0900 Subject: [PATCH 56/58] main, ast: implement ParseInheritanceSpecifire --- ast/ast.go | 10 +++++----- inheritanceSpecifier.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 inheritanceSpecifier.go diff --git a/ast/ast.go b/ast/ast.go index 118cccf..b210993 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -165,11 +165,6 @@ type FunctionDefinitionReturns struct { RParen token.Pos } -type InheritanceSpecifier struct { - IdentifierPath string - CallArgumentList CallArgumentList -} - // ---------------------------------------------------------------------------- // ContractBodyElement Nodes @@ -190,6 +185,11 @@ func (f *FunctionDefinition) contractBodyElementNode() {} // ---------------------------------------------------------------------------- +type InheritanceSpecifier struct { + IdentifierPath IdentifierPath + CallArgumentList *CallArgumentList +} + type CallArgumentListExpretion struct { Expression Expression Comma *token.Pos diff --git a/inheritanceSpecifier.go b/inheritanceSpecifier.go new file mode 100644 index 0000000..1c0cb24 --- /dev/null +++ b/inheritanceSpecifier.go @@ -0,0 +1,33 @@ +package solparser + +import ( + "github.com/uji/solparser/ast" + "github.com/uji/solparser/token" +) + +func (p *Parser) ParseInheritanceSpecifire() (*ast.InheritanceSpecifier, error) { + ip, err := p.ParseIdentifierPath() + if err != nil { + return nil, err + } + + lparen, err := p.lexer.Peek() + if err != nil { + return nil, err + } + if lparen.Type != token.LParen { + return &ast.InheritanceSpecifier{ + IdentifierPath: ip, + }, nil + } + + cal, err := p.ParseCallArgumentList() + if err != nil { + return nil, err + } + + return &ast.InheritanceSpecifier{ + IdentifierPath: ip, + CallArgumentList: cal, + }, nil +} From 1a58404aac757dc935d23ecbae230c749f2358db Mon Sep 17 00:00:00 2001 From: uji Date: Mon, 2 Oct 2023 00:52:36 +0900 Subject: [PATCH 57/58] ast: move importDirectiveElement interface --- ast/ast.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index b210993..3896ebe 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -12,12 +12,6 @@ type Node interface { End() token.Pos } -// All import-directive-element node (own definition for solparser) types implement the ImportDirectiveElement interface. -type ImportDirectiveElement interface { - Node - importDirectiveElementNode() -} - // All contract-body-element node types implement the ContractBodyElement interface. type ContractBodyElement interface { Node @@ -52,6 +46,12 @@ type Statement interface { // ---------------------------------------------------------------------------- // ImportDirectiveElement Nodes +// All import-directive-element node (own definition for solparser) types implement the ImportDirectiveElement interface. +type ImportDirectiveElement interface { + Node + importDirectiveElementNode() +} + type ImportDirectivePathElement struct { Path Path As *token.Pos From 96bf7e0516b5b60a1e1ca195401d6bb35b5050fe Mon Sep 17 00:00:00 2001 From: uji Date: Mon, 2 Oct 2023 01:12:02 +0900 Subject: [PATCH 58/58] main, ast: fix callArgumentList type definition --- ast/ast.go | 48 +++++++++++++++++++++++++++++++++++++++------ callArgumentList.go | 18 ++++++++--------- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 3896ebe..a58ca8a 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -190,11 +190,34 @@ type InheritanceSpecifier struct { CallArgumentList *CallArgumentList } +// ---------------------------------------------------------------------------- +// ImportDirectiveElement Nodes + +// All call-argument-list-elements node (own definition for solparser) types implement the CallArgumentListElements interface. +type CallArgumentListElements interface { + Node + callArgumentListElementsNode() +} + type CallArgumentListExpretion struct { Expression Expression Comma *token.Pos } +func (c CallArgumentListExpretion) Pos() token.Pos { return c.Expression.Pos() } +func (c CallArgumentListExpretion) End() token.Pos { + if c.Comma != nil { + return *c.Comma + } + return c.Expression.End() +} + +type CallArgumentListExpretions []*CallArgumentListExpretion + +func (c CallArgumentListExpretions) Pos() token.Pos { return c[0].Pos() } +func (c CallArgumentListExpretions) End() token.Pos { return c[len(c)-1].End() } +func (c CallArgumentListExpretions) callArgumentListElementsNode() {} + type CallArgumentListNamedExpretion struct { Identifier Identifier Colon token.Pos @@ -202,17 +225,30 @@ type CallArgumentListNamedExpretion struct { Comma *token.Pos } +func (c CallArgumentListNamedExpretion) Pos() token.Pos { return c.Identifier.Pos() } +func (c CallArgumentListNamedExpretion) End() token.Pos { + if c.Comma != nil { + return *c.Comma + } + return c.Expression.End() +} + type CallArgumentListNamedExpretions struct { - RBrace token.Pos - NamedExpretions []*CallArgumentListNamedExpretion LBrace token.Pos + NamedExpretions []*CallArgumentListNamedExpretion + RBrace token.Pos } +func (c CallArgumentListNamedExpretions) Pos() token.Pos { return c.LBrace } +func (c CallArgumentListNamedExpretions) End() token.Pos { return c.RBrace } +func (c *CallArgumentListNamedExpretions) callArgumentListElementsNode() {} + +// ---------------------------------------------------------------------------- + type CallArgumentList struct { - LParen token.Pos - Expressions []*CallArgumentListExpretion - NamedExpretionsa *CallArgumentListNamedExpretions - RParen token.Pos + LParen token.Pos + Elements CallArgumentListElements + RParen token.Pos } type IdentifierPathElement struct { diff --git a/callArgumentList.go b/callArgumentList.go index 096babc..c538609 100644 --- a/callArgumentList.go +++ b/callArgumentList.go @@ -23,20 +23,19 @@ func (p *Parser) ParseCallArgumentList() (*ast.CallArgumentList, error) { }, nil } - var expretions []*ast.CallArgumentListExpretion - var namedExpretions *ast.CallArgumentListNamedExpretions + var elements ast.CallArgumentListElements if tkn.Type == token.LBrace { es, err := p.ParseCallArgumentListExpretions() if err != nil { return nil, err } - expretions = es + elements = es } else { nes, err := p.ParseCallArgumentListNamedExpretions() if err != nil { return nil, err } - namedExpretions = nes + elements = nes } rparen, err := p.lexer.Scan() @@ -48,20 +47,19 @@ func (p *Parser) ParseCallArgumentList() (*ast.CallArgumentList, error) { } return &ast.CallArgumentList{ - LParen: lparen.Position, - Expressions: expretions, - NamedExpretionsa: namedExpretions, - RParen: rparen.Position, + LParen: lparen.Position, + Elements: elements, + RParen: rparen.Position, }, nil } -func (p *Parser) ParseCallArgumentListExpretions() ([]*ast.CallArgumentListExpretion, error) { +func (p *Parser) ParseCallArgumentListExpretions() (ast.CallArgumentListExpretions, error) { ex, err := p.ParseExpression() if err != nil { return nil, err } - var exs []*ast.CallArgumentListExpretion + var exs ast.CallArgumentListExpretions for { cmm, err := p.lexer.Peek() if err != nil || cmm.Type != token.Comma {