diff --git a/ast/ast.go b/ast/ast.go index e06808d..dae99eb 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -9,12 +9,12 @@ import ( ) // Parse parses a full script, returning the root node of the AST. -func Parse(r io.Reader) (Node, error) { - return parse(r, tokenStack{pgen.NTerm("script")}, pgen.Table) +func Parse(r io.Reader, macros scanner.MacroMap) (Node, error) { + return parse(r, tokenStack{pgen.NTerm("script")}, pgen.Table, macros) } -func parse(r io.Reader, g tokenStack, table map[pgen.Lookup]pgen.Rule) (ast Node, err error) { - s := scanner.New(r) +func parse(r io.Reader, g tokenStack, table map[pgen.Lookup]pgen.Rule, macros scanner.MacroMap) (ast Node, err error) { + s := scanner.New(r, macros) more := s.Scan() var cur *NTerm diff --git a/go.mod b/go.mod index 23bd535..2f2a167 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,6 @@ module github.com/DeedleFake/wdte require ( github.com/mattn/go-runewidth v0.0.4 // indirect github.com/peterh/liner v1.1.0 - golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc - golang.org/x/sys v0.0.0-20190108104531-7fbe1cd0fcc2 // indirect + golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 + golang.org/x/sys v0.0.0-20190204203706-41f3e6584952 // indirect ) diff --git a/go.sum b/go.sum index 5870154..a75cfe9 100644 --- a/go.sum +++ b/go.sum @@ -4,7 +4,7 @@ github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/ github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/peterh/liner v1.1.0 h1:f+aAedNJA6uk7+6rXsYBnhdo4Xux7ESLe+kcuVUF5os= github.com/peterh/liner v1.1.0/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= -golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc h1:F5tKCVGp+MUAHhKp5MZtGqAlGX3+oCsiL1Q629FL90M= -golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/sys v0.0.0-20190108104531-7fbe1cd0fcc2 h1:ku9Kvp2ZBWAz3GyvuUH3UV1bZCd7RxH0Qf1epWfIDKc= -golang.org/x/sys v0.0.0-20190108104531-7fbe1cd0fcc2/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 h1:MQ/ZZiDsUapFFiMS+vzwXkCTeEKaum+Do5rINYJDmxc= +golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/sys v0.0.0-20190204203706-41f3e6584952 h1:FDfvYgoVsA7TTZSbgiqjAbfPbK47CNHdWl3h/PJtii0= +golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/scanner/scanner.go b/scanner/scanner.go index 21dacfb..9788749 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -8,6 +8,8 @@ import ( "unicode" ) +type MacroMap map[string]func(string) ([]Token, error) + // A Scanner tokenizes runes from an io.Reader. type Scanner struct { r io.RuneReader @@ -20,10 +22,14 @@ type Scanner struct { tbuf bytes.Buffer quote rune + macro string + + macroMap MacroMap + macroBuf []Token } // New returns a new Scanner that reads from r. -func New(r io.Reader) *Scanner { +func New(r io.Reader, macros MacroMap) *Scanner { var rr io.RuneReader switch r := r.(type) { case io.RuneReader: @@ -35,6 +41,8 @@ func New(r io.Reader) *Scanner { return &Scanner{ r: rr, line: 1, + + macroMap: macros, } } @@ -46,6 +54,13 @@ func (s *Scanner) Scan() bool { return false } + if len(s.macroBuf) > 0 { + tok := s.macroBuf[len(s.macroBuf)-1] + s.macroBuf = s.macroBuf[:len(s.macroBuf)-1] + s.setTok(tok.Type, tok.Val) + return s.err == nil + } + s.tbuf.Reset() var eof bool @@ -129,7 +144,8 @@ func (s *Scanner) unread(r rune) { } func (s *Scanner) setTok(t TokenType, v interface{}) { - if t == Keyword { + switch t { + case Keyword: switch v { case ")", "]", "}": if (s.tok.Type != Keyword) || (s.tok.Val != ";") { @@ -137,6 +153,24 @@ func (s *Scanner) setTok(t TokenType, v interface{}) { v = ";" } } + + case Macro: + v := v.([2]string) + + macro := s.macroMap[v[0]] + if macro == nil { + break + } + + toks, err := macro(v[1]) + if len(toks) > 0 { + for i := len(toks) - 1; i >= 1; i-- { + s.macroBuf = append(s.macroBuf, toks[i]) + } + s.tok = toks[0] + } + s.err = err + return } s.tok = Token{ @@ -164,6 +198,11 @@ func (s *Scanner) whitespace(r rune) stateFunc { return s.maybeNumber } + if r == '@' { + s.tline, s.tcol = s.line, s.col + return s.macroName + } + if unicode.IsDigit(r) { s.tline, s.tcol = s.line, s.col s.unread(r) @@ -293,3 +332,25 @@ func (s *Scanner) id(r rune) stateFunc { s.unread(r) return nil } + +func (s *Scanner) macroName(r rune) stateFunc { + if unicode.IsDigit(r) || unicode.IsLetter(r) { + s.tbuf.WriteRune(r) + return s.macroName + } + + s.quote = endQuote(r) + s.macro = s.tbuf.String() + s.tbuf.Reset() + return s.macroInput +} + +func (s *Scanner) macroInput(r rune) stateFunc { + if r == s.quote { + s.setTok(Macro, [2]string{s.macro, s.tbuf.String()}) + return nil + } + + s.tbuf.WriteRune(r) + return s.macroInput +} diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index af43810..49aff7b 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -151,11 +151,20 @@ o => print "double\n" 'single\\';`, {Type: scanner.EOF}, }, }, + { + name: "Macro", + in: `@fmt[{q}, 'greetings'];`, + out: []scanner.Token{ + {Type: scanner.Macro, Val: [2]string{"fmt", "{q}, 'greetings'"}}, + {Type: scanner.Keyword, Val: ";"}, + {Type: scanner.EOF}, + }, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - s := scanner.New(strings.NewReader(test.in)) + s := scanner.New(strings.NewReader(test.in), nil) for i := 0; s.Scan(); i++ { if i >= len(test.out) { t.Fatalf("Extra token: %#v", s.Tok()) @@ -194,7 +203,7 @@ var ( ) func ExampleScanner() { - s := scanner.New(r) + s := scanner.New(r, nil) for s.Scan() { /* Do something with s.Tok(). */ } diff --git a/scanner/token.go b/scanner/token.go index 29f1e4a..7647f13 100644 --- a/scanner/token.go +++ b/scanner/token.go @@ -20,6 +20,7 @@ const ( String ID Keyword + Macro EOF ) @@ -35,6 +36,8 @@ func (t TokenType) String() string { return "id" case Keyword: return "keyword" + case Macro: + return "macro" case EOF: return "EOF" } diff --git a/scanner/util.go b/scanner/util.go index 64f2ee8..ebf5bd2 100644 --- a/scanner/util.go +++ b/scanner/util.go @@ -57,3 +57,23 @@ func symbolicSuffix(str string) string { func isQuote(r rune) bool { return (r == '\'') || (r == '"') } + +func endQuote(r rune) rune { + switch r { + case '(': + return ')' + case ')': + return '(' + case '[': + return ']' + case ']': + return '[' + case '{': + return '}' + case '}': + return '{' + + default: + return r + } +} diff --git a/wdte.go b/wdte.go index c016653..ae77ab0 100644 --- a/wdte.go +++ b/wdte.go @@ -9,13 +9,14 @@ import ( "strings" "github.com/DeedleFake/wdte/ast" + "github.com/DeedleFake/wdte/scanner" ) // Parse parses an AST from r and then translates it into a top-level // compound. im is used to handle import statements. If im is nil, a // no-op importer is used. In most cases, std.Import is a good default. -func Parse(r io.Reader, im Importer) (Compound, error) { - root, err := ast.Parse(r) +func Parse(r io.Reader, im Importer, macros scanner.MacroMap) (Compound, error) { + root, err := ast.Parse(r, macros) if err != nil { return nil, err } diff --git a/wdte_test.go b/wdte_test.go index 9b17125..f244a71 100644 --- a/wdte_test.go +++ b/wdte_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/DeedleFake/wdte" + "github.com/DeedleFake/wdte/scanner" "github.com/DeedleFake/wdte/std" _ "github.com/DeedleFake/wdte/std/arrays" wdteio "github.com/DeedleFake/wdte/std/io" @@ -24,6 +25,7 @@ type test struct { script string im wdte.Importer + macros scanner.MacroMap args []wdte.Func ret wdte.Func @@ -68,7 +70,7 @@ func runTests(t *testing.T, tests []test) { }) } - m, err := wdte.Parse(strings.NewReader(test.script), im) + m, err := wdte.Parse(strings.NewReader(test.script), im, test.macros) if err != nil { t.Fatalf("Failed to parse script: %v", err) } @@ -332,6 +334,29 @@ func TestBasics(t *testing.T) { return std.Import(im) }), }, + { + name: "Macro/ROT13", + script: `@rot13[test];`, + ret: wdte.String("grfg"), + macros: scanner.MacroMap{ + "rot13": func(input string) ([]scanner.Token, error) { + r := make([]rune, 0, len(input)) + for _, c := range input { + switch { + case (c >= 'a') && (c <= 'z'): + r = append(r, (c-'a'+13)%26+'a') + case (c >= 'A') && (c <= 'Z'): + r = append(r, (c-'A'+13)%26+'A') + default: + r = append(r, c) + } + } + return []scanner.Token{ + {Type: scanner.String, Val: string(r)}, + }, nil + }, + }, + }, }) }