#pragma once #include #include #include #include #include "utils/globals.hpp" namespace Nimbin { namespace SourceTokens { inline DynArray tokenize(const std::string& src) { DynArray out; const size_t n = src.size(); size_t i = 0; while (i < n) { const char c = src[i]; if (std::isspace(static_cast(c))) { ++i; continue; } // String literal — consume through the closing quote. if (c == '"') { std::string s; s += c; ++i; while (i < n) { s += src[i]; const bool closing = (src[i] == '"' && src[i - 1] != '\\'); ++i; if (closing) break; } out.push_back(std::move(s)); continue; } // Identifier / number run. if (std::isalnum(static_cast(c)) || c == '.' || c == '_') { std::string s; while (i < n) { const char d = src[i]; if (std::isalnum(static_cast(d)) || d == '.' || d == '_') { s += d; ++i; } else { break; } } out.push_back(std::move(s)); continue; } // Two-char operators: keep them whole so C expressions survive the // round-trip (!=, ==, <=, >=, &&, ||, ++, --, +=, -=, *=, /=, ->, <<, >>). if (i + 1 < n) { const char d = src[i + 1]; const bool two = (c == '!' && d == '=') || (c == '=' && d == '=') || (c == '<' && d == '=') || (c == '>' && d == '=') || (c == '&' && d == '&') || (c == '|' && d == '|') || (c == '+' && d == '+') || (c == '-' && d == '-') || (c == '+' && d == '=') || (c == '-' && d == '=') || (c == '*' && d == '=') || (c == '/' && d == '=') || (c == '-' && d == '>') || (c == '<' && d == '<') || (c == '>' && d == '>'); if (two) { out.push_back(std::string{c, d}); i += 2; continue; } } // Any other character: single-char token. out.push_back(std::string(1, c)); ++i; } return out; } // Tokenize per source LINE, preserving line breaks. Returns one vector of // tokens per non-empty line of the file, so the wall layout can mirror the // file's line structure (one editor row per source line). inline DynArray> tokenize_Lines(const std::string& src) { DynArray> lines; std::istringstream in(src); std::string line; while (std::getline(in, line)) { DynArray toks = tokenize(line); if (!toks.empty()) lines.push_back(std::move(toks)); } return lines; } inline bool read_File(const std::string& path, std::string& out) { std::ifstream f(path, std::ios::binary); if (!f) return false; std::ostringstream ss; ss << f.rdbuf(); out = ss.str(); return true; } inline bool write_File(const std::string& path, const std::string& contents) { std::ofstream f(path, std::ios::binary | std::ios::trunc); if (!f) return false; f << contents; return static_cast(f); } inline bool copy_File(const std::string& from, const std::string& to) { std::string data; if (!read_File(from, data)) return false; return write_File(to, data); } } // namespace SourceTokens } // namespace Nimbin