sdl_runtime_compiler git · main

Better tokenization for Numbers and stuff With Claude

Christian Immanuel · 2026-06-01 09:26 · ff74854ca9356757c36ea630b92321cfa62ff641

 src/c_token/token_color.hpp |  12 ++--
 src/c_token/token_db.cpp    | 156 ++++++++++++++++++++++++++++++++++++++++++--
 src/c_token/token_db.hpp    |   1 +
 src/text_3d/text_3d.cpp     |   2 +-
 src/wall/text_wall.cpp      |   2 +-
 5 files changed, 159 insertions(+), 14 deletions(-)

diff --git a/src/c_token/token_color.hpp b/src/c_token/token_color.hpp
index b99795f..ff24ab2 100644
--- a/src/c_token/token_color.hpp
+++ b/src/c_token/token_color.hpp
@@ -25,19 +25,15 @@ inline void
 apply(Text3d* word)
 {
     if (!word) return;
-    const Token* tok = TokenDB::find(word->text);
-    if (tok)
-        word->set_Color(get(tok->type));
+    const Token* tok = TokenDB::find_OrSynthesize(word->text);
+    if (tok) word->set_Color(get(tok->type));
 }
 
 inline SDL_FColor
 get_ByString(std::string token_text)
 {
-    const Token* tok = TokenDB::find(token_text);
-    if (tok)
-        return get(tok->type);
-    else
-        return Text3d::color_white;
+    const Token* tok = TokenDB::find_OrSynthesize(token_text);
+    return tok ? get(tok->type) : Text3d::color_white;
 }
 
 }
diff --git a/src/c_token/token_db.cpp b/src/c_token/token_db.cpp
index afdd9a1..6720443 100644
--- a/src/c_token/token_db.cpp
+++ b/src/c_token/token_db.cpp
@@ -84,7 +84,6 @@ all_tokens = {
     { ">",      TokenType::Operator, "Greater than",          "Checks whether left is greater than right"  },
     { "<=",     TokenType::Operator, "Less or equal",         "Checks whether left is less than or equal to right" },
     { ">=",     TokenType::Operator, "Greater or equal",      "Checks whether left is greater than or equal to right" },
-    { "--",     TokenType::Operator, "Decrement",             "Decreases value by 1"                       },
     { "+=",     TokenType::Operator, "Addition assignment",   "Adds and assigns the result"                },
     { "-=",     TokenType::Operator, "Subtraction assignment","Subtracts and assigns the result"           },
     { "*=",     TokenType::Operator, "Multiplication assignment", "Multiplies and assigns the result"      },
@@ -109,9 +108,7 @@ all_tokens = {
     { ".",      TokenType::Operator, "Member access",         "Accesses a member of an object"             },
     { "?",      TokenType::Operator, "Conditional operator",  "Begins a ternary conditional expression"    },
     { ":",      TokenType::Operator, "Conditional separator", "Separates true and false branches"          },
-    { ",",      TokenType::Operator, "Comma operator",        "Separates expressions or arguments"         },
     { "*",      TokenType::Operator, "Dereference",           "Accesses the object pointed to by a pointer" },
-    { "&",      TokenType::Operator, "Address-of",            "Obtains the address of an object"           },
 
     { "(",      TokenType::Delimiter,  "Open parenthesis",    "Starts a function call or condition" },
     { ")",      TokenType::Delimiter,  "Close parenthesis",   "Ends a function call or condition"   },
@@ -149,7 +146,7 @@ static const std::unordered_map<std::string, const Token*>
 token_index = build_index();
 
 
-DynArray<std::string> 
+DynArray<std::string>
 TokenDB::get_TestArrayAllKeywords()
 {
     return { "float", "return", "main", "0", "++", "(", ",", ";", "\"", "\"", "#define" };
@@ -197,3 +194,154 @@ TokenDB::known(const std::string& text)
     return token_index.find(text) != token_index.end();
 }
 
+
+// ---------------------------------------------------------------------------
+// Pattern helpers
+// ---------------------------------------------------------------------------
+
+static bool
+is_number(const std::string& s)
+{
+    if (s.empty()) return false;
+    size_t i = (s[0] == '-' || s[0] == '+') ? 1 : 0;
+    if (i == s.size()) return false;
+
+    // hex: 0x / 0X
+    if (i + 1 < s.size() && s[i] == '0' && (s[i+1] == 'x' || s[i+1] == 'X')) {
+        for (size_t j = i + 2; j < s.size(); ++j) {
+            char c = s[j];
+            if (std::isxdigit(c)) continue;
+            if (std::string("uUlL").find(c) != std::string::npos) continue;
+            return false;
+        }
+        return i + 2 < s.size();
+    }
+
+    bool has_dot = false, has_exp = false;
+    for (; i < s.size(); ++i) {
+        char c = s[i];
+        if (std::isdigit(c)) continue;
+        if (c == '.' && !has_dot && !has_exp)       { has_dot = true;  continue; }
+        if ((c == 'e' || c == 'E') && !has_exp && i > 0) {
+            has_exp = true;
+            if (i + 1 < s.size() && (s[i+1] == '+' || s[i+1] == '-')) ++i;
+            continue;
+        }
+        if (std::string("fFuUlL").find(c) != std::string::npos && i == s.size() - 1) continue;
+        return false;
+    }
+    return true;
+}
+
+static bool
+is_string_literal(const std::string& s)
+{
+    if (s.size() < 2) return false;
+    if (s.front() != '"' || s.back() != '"') return false;
+    // ensure closing " is not escaped
+    size_t backslashes = 0;
+    for (size_t i = s.size() - 2; i > 0 && s[i] == '\\'; --i)
+        ++backslashes;
+    return (backslashes % 2) == 0;
+}
+
+static bool
+is_char_literal(const std::string& s)
+{
+    // 'a'  '\n'  '\0'  '\x41'
+    if (s.size() < 3) return false;
+    if (s.front() != '\'' || s.back() != '\'') return false;
+    std::string inner = s.substr(1, s.size() - 2);
+    if (inner.empty() || inner.size() > 6) return false;
+    if (inner[0] != '\\') return inner.size() == 1;  // plain char
+    return inner.size() >= 2;                         // escape sequence
+}
+
+
+// ---------------------------------------------------------------------------
+// Token synthesizers
+// ---------------------------------------------------------------------------
+
+static const Token*
+synth_number(const std::string& s)
+{
+    bool is_hex = s.size() > 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X');
+
+    // collect suffix characters (trailing u U l L f F)
+    size_t suffix_start = s.size();
+    while (suffix_start > 0 && std::string("uUlLfF").find(s[suffix_start - 1]) != std::string::npos)
+        --suffix_start;
+    std::string suffix = s.substr(suffix_start);
+
+    // lowercase the suffix for easier comparison
+    std::string suf_low;
+    for (char c : suffix) suf_low += static_cast<char>(std::tolower(c));
+
+    bool has_f_suffix   = suf_low == "f";
+    bool has_fp_content = s.find('.') != std::string::npos ||
+                          s.find('e') != std::string::npos ||
+                          s.find('E') != std::string::npos;
+    bool is_fp          = has_fp_content || has_f_suffix;
+
+    bool is_unsigned = suf_low.find('u') != std::string::npos;
+    bool is_long     = suf_low.find('l') != std::string::npos;
+
+    thread_local Token t;
+    if (is_hex) {
+        if (is_unsigned && is_long)
+            t = { s, TokenType::Literal, "Hex literal",          "Unsigned long hexadecimal constant"   };
+        else if (is_unsigned)
+            t = { s, TokenType::Literal, "Hex literal",          "Unsigned hexadecimal constant"        };
+        else
+            t = { s, TokenType::Literal, "Hex literal",          "Hexadecimal integer constant"         };
+    } else if (is_fp) {
+        if (has_f_suffix)
+            t = { s, TokenType::Literal, "Float literal",        "Single-precision floating-point (float)"  };
+        else if (is_long)
+            t = { s, TokenType::Literal, "Long double literal",  "Extended-precision floating-point (long double)" };
+        else
+            t = { s, TokenType::Literal, "Double literal",       "Double-precision floating-point (double)" };
+    } else {
+        if (is_unsigned && is_long)
+            t = { s, TokenType::Literal, "Unsigned long literal","Unsigned long integer constant"       };
+        else if (is_unsigned)
+            t = { s, TokenType::Literal, "Unsigned literal",     "Unsigned integer constant"            };
+        else if (is_long)
+            t = { s, TokenType::Literal, "Long literal",         "Long integer constant"                };
+        else
+            t = { s, TokenType::Literal, "Integer literal",      "Integer constant"                     };
+    }
+    return &t;
+}
+
+static const Token*
+synth_string(const std::string& s)
+{
+    thread_local Token t;
+    t = { s, TokenType::Literal, "String literal", "Null-terminated character string" };
+    return &t;
+}
+
+static const Token*
+synth_char(const std::string& s)
+{
+    thread_local Token t;
+    t = { s, TokenType::Literal, "Char literal", "Single character constant" };
+    return &t;
+}
+
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+const Token*
+TokenDB::find_OrSynthesize(const std::string& text)
+{
+    const Token* t = find(text);
+    if (t)                        return t;
+    if (is_string_literal(text))  return synth_string(text);
+    if (is_char_literal(text))    return synth_char(text);
+    if (is_number(text))          return synth_number(text);
+    return nullptr;
+}
diff --git a/src/c_token/token_db.hpp b/src/c_token/token_db.hpp
index 103c72d..12b8abd 100644
--- a/src/c_token/token_db.hpp
+++ b/src/c_token/token_db.hpp
@@ -6,6 +6,7 @@
 namespace TokenDB
 {
     const Token*           find(const std::string& text);
+    const Token*           find_OrSynthesize(const std::string& text);
     DynArray<const Token*> get_AllOfType(TokenType type);
     const char*            get_TypeName(TokenType type);
 
diff --git a/src/text_3d/text_3d.cpp b/src/text_3d/text_3d.cpp
index 5b24441..0774b94 100644
--- a/src/text_3d/text_3d.cpp
+++ b/src/text_3d/text_3d.cpp
@@ -19,7 +19,7 @@ Text3d::Text3d(const std::string& txt,
        : GameObject(position, ObjectType::Text,
                     CollisionShape{}, RigidBody{PhysicsMode::Hover},
                     true, true, true),
-         text(txt), color(colr), depth(dpth), scale(scle), token_info(TokenDB::find(text))
+         text(txt), color(colr), depth(dpth), scale(scle), token_info(TokenDB::find_OrSynthesize(text))
 {
     if (token_info)
         set_Color(TokenColor::get(token_info->type));
diff --git a/src/wall/text_wall.cpp b/src/wall/text_wall.cpp
index 35fbf61..4d7bdf2 100644
--- a/src/wall/text_wall.cpp
+++ b/src/wall/text_wall.cpp
@@ -131,7 +131,7 @@ TextWall::show_TokenInfo(TextWallList& texts,
                            const std::string& token_text,
                            float world_scale)
 {
-    const Token* tok = TokenDB::find(token_text);
+    const Token* tok = TokenDB::find_OrSynthesize(token_text);
 
     // Clear previous info wall content
     texts.erase(