1 include "token-table.h";
3 constant TOKEN__NUMBER 1;
4 constant TOKEN__STRING 2;
6 constant TOKEN__SPACE 4;
7 constant TOKEN__PLUS 5;
8 constant TOKEN__MINUS 6;
9 constant TOKEN__STAR 7;
10 constant TOKEN__SLASH 8;
11 constant TOKEN__COLON 9;
12 constant TOKEN__EQUALS 10;
13 constant TOKEN__COMMA 11;
14 constant TOKEN__LPAREN 12;
15 constant TOKEN__RPAREN 13;
16 constant TOKEN__LARROW 14;
17 constant TOKEN__RARROW 15;
18 constant TOKEN__SEMICOLON 16;
19 constant TOKEN__GEQUAL 17;
20 constant TOKEN__LEQUAL 18;
21 constant TOKEN__NEQUAL 19;
23 ! Is this an invalid alphabetical token character?
25 [ token_invalidchar c;
26 return ((c < 'a') || (c > 'z'));
29 ! Is this a valid variable name character?
31 [ token_validvarnamechar c;
32 return (((c >= 'a') && (c <= 'z')) ||
33 ((c >= 'A') && (c <= 'Z')) ||
34 ((c >= '0') && (c <= '9')) ||
35 (c == '_') || (c == '%') || (c == '$'));
41 return ((c >= '0') && (c <= '9'));
47 return ((c == 32) || (c == 9));
50 ! Tokenise an input stream.
52 ! The input and output pointers must point to different regions of memory.
54 [ tokenise_stream in out incount outcount i j k;
61 i = token_encode(in+incount, out+outcount);
64 incount = incount + i;
65 outcount = outcount + 1;
69 ! Not a recognised token. We test against all the
70 ! other things we recognise. Note the order! This
79 while (token_isspace(in->(incount)))
81 out->(outcount++) = TOKEN__SPACE;
87 if (token_isnumber(i))
89 out->(outcount++) = TOKEN__NUMBER;
92 i = i*10 + (in->incount - '0');
94 } until (token_isnumber(in->incount) == 0);
95 (out+outcount)-->0 = i;
96 outcount = outcount + 2;
104 ! Work out the size of the string.
110 } until ((k == '"') || (k == 0));
115 out->(outcount++) = TOKEN__STRING;
116 out->(outcount++) = j;
118 ! And now emit the string itself.
120 memcpy(out+outcount, in+i, j);
121 outcount = outcount + j;
123 ! Remember to skip over the close quote
134 '+': out->(outcount++) = TOKEN__PLUS;
138 '-': out->(outcount++) = TOKEN__MINUS;
142 '*': out->(outcount++) = TOKEN__STAR;
146 '/': out->(outcount++) = TOKEN__SLASH;
150 ':': out->(outcount++) = TOKEN__COLON;
154 '=': out->(outcount++) = TOKEN__EQUALS;
158 ',': out->(outcount++) = TOKEN__COMMA;
162 '(': out->(outcount++) = TOKEN__LPAREN;
166 ')': out->(outcount++) = TOKEN__RPAREN;
170 '<': switch (in->(++incount))
172 '>': out->(outcount++) = TOKEN__NEQUAL;
176 '=': out->(outcount++) = TOKEN__LEQUAL;
180 default: out->(outcount++) = TOKEN__LARROW;
185 '>': switch (in->(++incount))
187 '=': out->(outcount++) = TOKEN__GEQUAL;
191 default: out->(outcount++) = TOKEN__RARROW;
196 ';': out->(outcount++) = TOKEN__SEMICOLON;
201 ! Is it a variable name?
203 if (token_validvarnamechar(i))
205 out->(outcount++) = TOKEN__VAR;
207 out->(outcount++) = in->(incount++);
208 } until (token_validvarnamechar(in->incount) == 0);
209 out->(outcount++) = 0;
217 ! Patch up the line length.
219 out->outcount = TOKEN__EOL;
220 out->0 = outcount + 1;
225 ! Detokenise a stream.
227 [ detokenise_stream in i;
238 while (i = (in++)->0)
255 print (char) (in++)->0;
319 print (string) token_decode(i);