From 4b51b06b7327966f6e7ba893430dad7e2b09c7cd Mon Sep 17 00:00:00 2001
From: "Eric S. Raymond" <esr@thyrsus.com>
Date: Mon, 3 Jul 2017 17:13:28 -0400
Subject: [PATCH] Simplify tokenization code.

---
 advent.h |  2 +-
 misc.c   | 28 ++++++----------------------
 2 files changed, 7 insertions(+), 23 deletions(-)

diff --git a/advent.h b/advent.h
index 61d3e4e..c0e320e 100644
--- a/advent.h
+++ b/advent.h
@@ -192,7 +192,7 @@ extern struct settings_t settings;
 extern char* xstrdup(const char* s);
 extern void* xmalloc(size_t size);
 extern void packed_to_token(long, char token[]);
-extern long token_to_packed(const char token[TOKLEN+1]);
+extern long token_to_packed(const char token[]);
 extern void tokenize(char*, struct command_t *);
 extern void vspeak(const char*, bool, va_list);
 extern bool wordeq(token_t, token_t);
diff --git a/misc.c b/misc.c
index c8cf9f1..30b3277 100644
--- a/misc.c
+++ b/misc.c
@@ -68,7 +68,7 @@ void packed_to_token(long packed, char token[TOKLEN+1])
     }
 }
 
-long token_to_packed(const char token[TOKLEN+1])
+long token_to_packed(const char token[])
 {
     const char ascii_to_advent[] = {
         63, 63, 63, 63, 63, 63, 63, 63,
@@ -92,9 +92,11 @@ long token_to_packed(const char token[TOKLEN+1])
     };
 
     size_t t_len = strlen(token);
+    if (t_len > TOKLEN)
+	t_len = TOKLEN;
     long packed = 0;
     for (size_t i = 0; i < t_len; ++i) {
-        char mapped = ascii_to_advent[(int) token[i]];
+        char mapped = ascii_to_advent[(int) toupper(token[i])];
         packed |= (mapped << (6 * i));
     }
     return (packed);
@@ -107,27 +109,9 @@ void tokenize(char* raw, struct command_t *cmd)
     /* FIXME: put a bound prefix on the %s to prevent buffer overflow */
     int word_count = sscanf(raw, "%s%s", cmd->raw1, cmd->raw2);
 
-    // make space for substrings and zero it out
-    char chunk_data[][TOKLEN+1] = {
-        {"\0\0\0\0\0"},
-        {"\0\0\0\0\0"},
-        {"\0\0\0\0\0"},
-        {"\0\0\0\0\0"},
-    };
-
-    // break the words into up to 4 5-char substrings
-    sscanf(cmd->raw1, "%5s%5s", chunk_data[0], chunk_data[1]);
-    if (word_count == 2)
-        sscanf(cmd->raw2, "%5s%5s", chunk_data[2], chunk_data[3]);
-
-    // uppercase all the substrings
-    for (int i = 0; i < 4; ++i)
-        for (unsigned int j = 0; j < strlen(chunk_data[i]); ++j)
-            chunk_data[i][j] = (char) toupper(chunk_data[i][j]);
-
     // pack the substrings
-    cmd->wd1  = token_to_packed(chunk_data[0]);
-    cmd->wd2  = token_to_packed(chunk_data[2]);
+    cmd->wd1  = token_to_packed(cmd->raw1);
+    cmd->wd2  = token_to_packed(cmd->raw2);
 }
 
 /* Hide the fact that wods are corrently packed longs */
-- 
2.31.1