-
Notifications
You must be signed in to change notification settings - Fork 0
/
lexer.js
103 lines (91 loc) · 3.26 KB
/
lexer.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
lexer_t.prototype.valid_tokens = new Set(['HEY', 'ROBOT_NICK', 'DIRECTION', 'ACTION', 'DISTANCE', 'UNIT']);
function lexer_t(text, opts){
_this = this;
this.preprocessor = new preprocessor_t();
_this.text = this.preprocessor.preprocess(text);
_this.text_idx = 0;
_this.keywords = {};
_this.tokens = [];
_this.add_keywords(['hey'], 'HEY');
_this.add_keywords([opts.robot_nick || 'robot'], 'ROBOT_NICK');
_this.add_keywords(['left', 'right', 'forward', 'backwards'], 'DIRECTION');
_this.add_keywords(['drive', 'turn'], 'ACTION');
_this.add_keywords(['then'], 'THEN');
_this.add_keywords(['and'], 'THEN');
_this.add_keywords(['pi'], 'PI', function(lexeme){lexeme.text = Math.PI; return lexeme});
_this.add_keywords(['tau'], 'TAU', function(lexeme){lexeme.text = Math.PI*2; return lexeme;});
_this.add_keywords(['radians', 'degrees'], 'ANGLE_UNIT');
_this.add_keywords(['over'], 'DIVIDE');
_this.add_token([/[0-9]+(?:.[0-9]+)?/], 'NUMBER');
_this.add_token([/(?:centi)?meters?/, /inch(?:es)?/, 'feet', 'foot'], "DISTANCE_UNIT", this.clean_distance_unit);
}
lexer_t.prototype.clean_distance_unit = function(lex){
//remove the plural from the unit, just makes code more readable later
//also, turn inches into inch
if(lex.text == 'inches'){
lex.text = 'inch';
}
if(lex.text == 'feet'){
lex.text = 'foot';
}
if(lex.text.slice(-1) == 's'){
lex.text = lex.text.slice(0,-1);
}
return lex;
}
lexer_t.prototype[Symbol.iterator] = function*(){
while(!this.finished()){
yield this.get_next();
}
}
lexer_t.prototype.add_keywords = function(list, ret, callback){
for(kw of list){
this.keywords[kw] = {token_return:ret, callback:callback};
}
}
lexer_t.prototype.add_token = function(list, ret, callback){
for(var t of this.tokens){
if(t.token_return == ret){
for(i of list)
t.expected_texts.push(i);
return;
}
}
this.tokens.push({expected_texts: list, token_return: ret, callback: callback});
}
lexer_t.prototype.get_text = function(){
old_idx = this.text_idx;
while(this.text[this.text_idx] != ' ' && this.text_idx < this.text.length) this.text_idx++;
return this.text.slice(old_idx, this.text_idx);
}
lexer_t.prototype.get_next = function(){
lexeme = {}
lexeme.text = this.get_text();
lexeme = this.get_token(lexeme);
this.text_idx++;
console.log(lexeme);
return lexeme;
}
lexer_t.prototype.get_token = function(lexeme){
if(lexeme.text == '') return 'NULL';
if(lexeme.text in this.keywords){
lexeme.token = this.keywords[lexeme.text].token_return;
if(this.keywords[lexeme.text].callback){ lexeme = this.keywords[lexeme.text].callback(lexeme)}
return lexeme;
}
for(t of this.tokens){
for(et of t.expected_texts){
var res = lexeme.text.match(et);
if(res && res[0].length == res.input.length){
if(t.callback) lexeme = t.callback(lexeme);
lexeme.token = t.token_return;
return lexeme;
}
}
}
lexeme.token = 'ID';
return lexeme;
}
lexer_t.prototype.finished = function(){
return (this.text_idx >= this.text.length);
}