+long token_to_packed(const char token[6])
+{
+ size_t t_len = strlen(token);
+ long packed = 0;
+ for (size_t i = 0; i < t_len; ++i)
+ {
+ char mapped = new_ascii_to_advent[(int) token[i]];
+ packed |= (mapped << (6 * i));
+ }
+ return(packed);
+}
+
+void tokenize(char* raw, long tokens[4])
+{
+ // set each token to 0
+ for (int i = 0; i < 4; ++i)
+ tokens[i] = 0;
+
+ // grab the first two words
+ char* words[2];
+ words[0] = (char*) xmalloc(strlen(raw));
+ words[1] = (char*) xmalloc(strlen(raw));
+ int word_count = sscanf(raw, "%s%s", words[0], words[1]);
+
+ // make space for substrings and zero it out
+ char chunk_data[][6] = {
+ {"\0\0\0\0\0"},
+ {"\0\0\0\0\0"},
+ {"\0\0\0\0\0"},
+ {"\0\0\0\0\0"},
+ };
+
+ // break the words into up to 4 5-char substrings
+ sscanf(words[0], "%5s%5s", chunk_data[0], chunk_data[1]);
+ if (word_count == 2)
+ sscanf(words[1], "%5s%5s", chunk_data[2], chunk_data[3]);
+ free(words[0]);
+ free(words[1]);
+
+ // uppercase all the substrings
+ for (int i = 0; i < 4; ++i)
+ for (unsigned int j = 0; j < strlen(chunk_data[i]); ++j)
+ chunk_data[i][j] = (char) toupper(chunk_data[i][j]);
+
+ // pack the substrings
+ for (int i = 0; i < 4; ++i)
+ tokens[i] = token_to_packed(chunk_data[i]);
+}
+
+/* Hide the fact that wods are corrently packed longs */
+
+bool wordeq(token_t a, token_t b)
+{
+ return a == b;
+}
+
+bool wordempty(token_t a)
+{
+ return a == 0;
+}
+
+void wordclear(token_t *v)
+{
+ *v = 0;
+}
+
+/* I/O routines (speak, pspeak, rspeak, GETIN, YES) */