sdl_runtime_compiler git · main

#pragma once
#include <cctype>
#include <fstream>
#include <sstream>
#include <string>

#include "utils/globals.hpp"

namespace
Nimbin
{
namespace SourceTokens
{

inline DynArray<std::string>
tokenize(const std::string& src)
{
    DynArray<std::string> out;
    const
    size_t n = src.size();
    size_t i = 0;
    while (i < n) {
        const char c = src[i];

        if (std::isspace(static_cast<unsigned char>(c))) { ++i; continue; }

        // String literal — consume through the closing quote.
        if (c == '"') {
            std::string s;
            s += c;
            ++i;
            while (i < n) {
                s += src[i];
                const bool closing = (src[i] == '"' && src[i - 1] != '\\');
                ++i;
                if (closing) break;
            }
            out.push_back(std::move(s));
            continue;
        }

        // Identifier / number run.
        if (std::isalnum(static_cast<unsigned char>(c)) || c == '.' || c == '_') {
            std::string s;
            while (i < n) {
                const char d = src[i];
                if (std::isalnum(static_cast<unsigned char>(d)) || d == '.' || d == '_') {
                    s += d;
                    ++i;
                }
                else {
                    break;
                }
            }
            out.push_back(std::move(s));
            continue;
        }

        // Two-char operators: keep them whole so C expressions survive the
        // round-trip (!=, ==, <=, >=, &&, ||, ++, --, +=, -=, *=, /=, ->, <<, >>).
        if (i + 1 < n) {
            const char d = src[i + 1];
            const bool two =
                (c == '!' && d == '=') ||
                (c == '=' && d == '=') ||
                (c == '<' && d == '=') ||
                (c == '>' && d == '=') ||
                (c == '&' && d == '&') ||
                (c == '|' && d == '|') ||
                (c == '+' && d == '+') ||
                (c == '-' && d == '-') ||
                (c == '+' && d == '=') ||
                (c == '-' && d == '=') ||
                (c == '*' && d == '=') ||
                (c == '/' && d == '=') ||
                (c == '-' && d == '>') ||
                (c == '<' && d == '<') ||
                (c == '>' && d == '>');
            if (two) {
                out.push_back(std::string{c, d});
                i += 2;
                continue;
            }
        }

        // Any other character: single-char token.
        out.push_back(std::string(1, c));
        ++i;
    }
    return out;
}

// Tokenize per source LINE, preserving line breaks. Returns one vector of
// tokens per non-empty line of the file, so the wall layout can mirror the
// file's line structure (one editor row per source line).
inline DynArray<DynArray<std::string>>
tokenize_Lines(const std::string& src)
{
    DynArray<DynArray<std::string>> lines;
    std::istringstream in(src);
    std::string line;
    while (std::getline(in, line)) {
        DynArray<std::string> toks = tokenize(line);
        if (!toks.empty())
            lines.push_back(std::move(toks));
    }
    return lines;
}

inline bool
read_File(const std::string& path, std::string& out)
{
    std::ifstream f(path, std::ios::binary);
    if (!f) return false;
    std::ostringstream ss;
    ss << f.rdbuf();
    out = ss.str();
    return true;
}

inline bool
write_File(const std::string& path, const std::string& contents)
{
    std::ofstream f(path, std::ios::binary | std::ios::trunc);
    if (!f) return false;
    f << contents;
    return static_cast<bool>(f);
}

inline bool
copy_File(const std::string& from, const std::string& to)
{
    std::string data;
    if (!read_File(from, data)) return false;
    return write_File(to, data);
}

}  // namespace SourceTokens

} // namespace Nimbin