multiple: Add Macros (#179)

* scanner: Add Macro token type.

* wdte: Update some dependencies.

* scanner: Add support for specifying a macro map.

* ast: Add support for specifying a macro map for the scanner.

* wdte: Add support for specifying a macro map for the scanner.

ast/ast.go

@@ -9,12 +9,12 @@ import (
 )
 
 // Parse parses a full script, returning the root node of the AST.
-func Parse(r io.Reader) (Node, error) {
-	return parse(r, tokenStack{pgen.NTerm("script")}, pgen.Table)
+func Parse(r io.Reader, macros scanner.MacroMap) (Node, error) {
+	return parse(r, tokenStack{pgen.NTerm("script")}, pgen.Table, macros)
 }
 
-func parse(r io.Reader, g tokenStack, table map[pgen.Lookup]pgen.Rule) (ast Node, err error) {
-	s := scanner.New(r)
+func parse(r io.Reader, g tokenStack, table map[pgen.Lookup]pgen.Rule, macros scanner.MacroMap) (ast Node, err error) {
+	s := scanner.New(r, macros)
 
 	more := s.Scan()
 	var cur *NTerm

go.mod

@@ -3,6 +3,6 @@ module github.com/DeedleFake/wdte
 require (
 	github.com/mattn/go-runewidth v0.0.4 // indirect
 	github.com/peterh/liner v1.1.0
-	golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc
-	golang.org/x/sys v0.0.0-20190108104531-7fbe1cd0fcc2 // indirect
+	golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613
+	golang.org/x/sys v0.0.0-20190204203706-41f3e6584952 // indirect
 )

go.sum

@@ -4,7 +4,7 @@ github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/peterh/liner v1.1.0 h1:f+aAedNJA6uk7+6rXsYBnhdo4Xux7ESLe+kcuVUF5os=
 github.com/peterh/liner v1.1.0/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0=
-golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc h1:F5tKCVGp+MUAHhKp5MZtGqAlGX3+oCsiL1Q629FL90M=
-golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/sys v0.0.0-20190108104531-7fbe1cd0fcc2 h1:ku9Kvp2ZBWAz3GyvuUH3UV1bZCd7RxH0Qf1epWfIDKc=
-golang.org/x/sys v0.0.0-20190108104531-7fbe1cd0fcc2/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 h1:MQ/ZZiDsUapFFiMS+vzwXkCTeEKaum+Do5rINYJDmxc=
+golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/sys v0.0.0-20190204203706-41f3e6584952 h1:FDfvYgoVsA7TTZSbgiqjAbfPbK47CNHdWl3h/PJtii0=
+golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

scanner/scanner.go

@@ -8,6 +8,8 @@ import (
 	"unicode"
 )
 
+type MacroMap map[string]func(string) ([]Token, error)
+
 // A Scanner tokenizes runes from an io.Reader.
 type Scanner struct {
 	r               io.RuneReader
@@ -20,10 +22,14 @@ type Scanner struct {
 
 	tbuf  bytes.Buffer
 	quote rune
+	macro string
+
+	macroMap MacroMap
+	macroBuf []Token
 }
 
 // New returns a new Scanner that reads from r.
-func New(r io.Reader) *Scanner {
+func New(r io.Reader, macros MacroMap) *Scanner {
 	var rr io.RuneReader
 	switch r := r.(type) {
 	case io.RuneReader:
@@ -35,6 +41,8 @@ func New(r io.Reader) *Scanner {
 	return &Scanner{
 		r:    rr,
 		line: 1,
+
+		macroMap: macros,
 	}
 }
 
@@ -46,6 +54,13 @@ func (s *Scanner) Scan() bool {
 		return false
 	}
 
+	if len(s.macroBuf) > 0 {
+		tok := s.macroBuf[len(s.macroBuf)-1]
+		s.macroBuf = s.macroBuf[:len(s.macroBuf)-1]
+		s.setTok(tok.Type, tok.Val)
+		return s.err == nil
+	}
+
 	s.tbuf.Reset()
 
 	var eof bool
@@ -129,14 +144,33 @@ func (s *Scanner) unread(r rune) {
 }
 
 func (s *Scanner) setTok(t TokenType, v interface{}) {
-	if t == Keyword {
+	switch t {
+	case Keyword:
 		switch v {
 		case ")", "]", "}":
 			if (s.tok.Type != Keyword) || (s.tok.Val != ";") {
 				s.unread(rune(v.(string)[0]))
 				v = ";"
 			}
 		}
+
+	case Macro:
+		v := v.([2]string)
+
+		macro := s.macroMap[v[0]]
+		if macro == nil {
+			break
+		}
+
+		toks, err := macro(v[1])
+		if len(toks) > 0 {
+			for i := len(toks) - 1; i >= 1; i-- {
+				s.macroBuf = append(s.macroBuf, toks[i])
+			}
+			s.tok = toks[0]
+		}
+		s.err = err
+		return
 	}
 
 	s.tok = Token{
@@ -164,6 +198,11 @@ func (s *Scanner) whitespace(r rune) stateFunc {
 		return s.maybeNumber
 	}
 
+	if r == '@' {
+		s.tline, s.tcol = s.line, s.col
+		return s.macroName
+	}
+
 	if unicode.IsDigit(r) {
 		s.tline, s.tcol = s.line, s.col
 		s.unread(r)
@@ -293,3 +332,25 @@ func (s *Scanner) id(r rune) stateFunc {
 	s.unread(r)
 	return nil
 }
+
+func (s *Scanner) macroName(r rune) stateFunc {
+	if unicode.IsDigit(r) || unicode.IsLetter(r) {
+		s.tbuf.WriteRune(r)
+		return s.macroName
+	}
+
+	s.quote = endQuote(r)
+	s.macro = s.tbuf.String()
+	s.tbuf.Reset()
+	return s.macroInput
+}
+
+func (s *Scanner) macroInput(r rune) stateFunc {
+	if r == s.quote {
+		s.setTok(Macro, [2]string{s.macro, s.tbuf.String()})
+		return nil
+	}
+
+	s.tbuf.WriteRune(r)
+	return s.macroInput
+}

scanner/scanner_test.go

@@ -151,11 +151,20 @@ o => print "double\n" 'single\\';`,
 				{Type: scanner.EOF},
 			},
 		},
+		{
+			name: "Macro",
+			in:   `@fmt[{q}, 'greetings'];`,
+			out: []scanner.Token{
+				{Type: scanner.Macro, Val: [2]string{"fmt", "{q}, 'greetings'"}},
+				{Type: scanner.Keyword, Val: ";"},
+				{Type: scanner.EOF},
+			},
+		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			s := scanner.New(strings.NewReader(test.in))
+			s := scanner.New(strings.NewReader(test.in), nil)
 			for i := 0; s.Scan(); i++ {
 				if i >= len(test.out) {
 					t.Fatalf("Extra token: %#v", s.Tok())
@@ -194,7 +203,7 @@ var (
 )
 
 func ExampleScanner() {
-	s := scanner.New(r)
+	s := scanner.New(r, nil)
 	for s.Scan() {
 		/* Do something with s.Tok(). */
 	}

scanner/token.go

@@ -20,6 +20,7 @@ const (
 	String
 	ID
 	Keyword
+	Macro
 	EOF
 )
 
@@ -35,6 +36,8 @@ func (t TokenType) String() string {
 		return "id"
 	case Keyword:
 		return "keyword"
+	case Macro:
+		return "macro"
 	case EOF:
 		return "EOF"
 	}

scanner/util.go

@@ -57,3 +57,23 @@ func symbolicSuffix(str string) string {
 func isQuote(r rune) bool {
 	return (r == '\'') || (r == '"')
 }
+
+func endQuote(r rune) rune {
+	switch r {
+	case '(':
+		return ')'
+	case ')':
+		return '('
+	case '[':
+		return ']'
+	case ']':
+		return '['
+	case '{':
+		return '}'
+	case '}':
+		return '{'
+
+	default:
+		return r
+	}
+}

wdte.go

@@ -9,13 +9,14 @@ import (
 	"strings"
 
 	"github.com/DeedleFake/wdte/ast"
+	"github.com/DeedleFake/wdte/scanner"
 )
 
 // Parse parses an AST from r and then translates it into a top-level
 // compound. im is used to handle import statements. If im is nil, a
 // no-op importer is used. In most cases, std.Import is a good default.
-func Parse(r io.Reader, im Importer) (Compound, error) {
-	root, err := ast.Parse(r)
+func Parse(r io.Reader, im Importer, macros scanner.MacroMap) (Compound, error) {
+	root, err := ast.Parse(r, macros)
 	if err != nil {
 		return nil, err
 	}

wdte_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 
 	"github.com/DeedleFake/wdte"
+	"github.com/DeedleFake/wdte/scanner"
 	"github.com/DeedleFake/wdte/std"
 	_ "github.com/DeedleFake/wdte/std/arrays"
 	wdteio "github.com/DeedleFake/wdte/std/io"
@@ -24,6 +25,7 @@ type test struct {
 
 	script string
 	im     wdte.Importer
+	macros scanner.MacroMap
 
 	args []wdte.Func
 	ret  wdte.Func
@@ -68,7 +70,7 @@ func runTests(t *testing.T, tests []test) {
 				})
 			}
 
-			m, err := wdte.Parse(strings.NewReader(test.script), im)
+			m, err := wdte.Parse(strings.NewReader(test.script), im, test.macros)
 			if err != nil {
 				t.Fatalf("Failed to parse script: %v", err)
 			}
@@ -332,6 +334,29 @@ func TestBasics(t *testing.T) {
 				return std.Import(im)
 			}),
 		},
+		{
+			name:   "Macro/ROT13",
+			script: `@rot13[test];`,
+			ret:    wdte.String("grfg"),
+			macros: scanner.MacroMap{
+				"rot13": func(input string) ([]scanner.Token, error) {
+					r := make([]rune, 0, len(input))
+					for _, c := range input {
+						switch {
+						case (c >= 'a') && (c <= 'z'):
+							r = append(r, (c-'a'+13)%26+'a')
+						case (c >= 'A') && (c <= 'Z'):
+							r = append(r, (c-'A'+13)%26+'A')
+						default:
+							r = append(r, c)
+						}
+					}
+					return []scanner.Token{
+						{Type: scanner.String, Val: string(r)},
+					}, nil
+				},
+			},
+		},
 	})
 }