// SOURCE: ARCHITECTURE/32-LANGUAGE-OFFICIAL-13-IMPLEMENTATION-5.md
// AUTO-EXTRACTED - Do not edit directly. Edit the source document.

export type Tok =
  | { tag: "LParen" }
  | { tag: "RParen" }
  | { tag: "LBracket" }   // [
  | { tag: "RBracket" }   // ]
  | { tag: "LBrace" }     // {
  | { tag: "RBrace" }     // }
  | { tag: "Quote" }
  | { tag: "Backtick" }   // ` (quasiquote)
  | { tag: "Comma" }      // , (unquote)
  | { tag: "CommaSplice" } // ,@ (unquote-splicing)
  | { tag: "Str"; s: string }
  | { tag: "Char"; c: string }  // Character literal #\x
  | { tag: "Atom"; s: string };

export function tokenize(src: string): Tok[] {
  const toks: Tok[] = [];
  let i = 0;

  const isWS = (c: string) => c === " " || c === "\t" || c === "\n" || c === "\r";

  while (i < src.length) {
    const c = src[i];

    // comments
    if (c === ";") {
      while (i < src.length && src[i] !== "\n") i++;
      continue;
    }

    if (isWS(c)) { i++; continue; }

    if (c === "(") { toks.push({ tag: "LParen" }); i++; continue; }
    if (c === ")") { toks.push({ tag: "RParen" }); i++; continue; }
    if (c === "[") { toks.push({ tag: "LBracket" }); i++; continue; }
    if (c === "]") { toks.push({ tag: "RBracket" }); i++; continue; }
    if (c === "{") { toks.push({ tag: "LBrace" }); i++; continue; }
    if (c === "}") { toks.push({ tag: "RBrace" }); i++; continue; }
    if (c === "'") { toks.push({ tag: "Quote" }); i++; continue; }
    if (c === "`") { toks.push({ tag: "Backtick" }); i++; continue; }
    if (c === ",") {
      if (src[i + 1] === "@") { toks.push({ tag: "CommaSplice" }); i += 2; continue; }
      toks.push({ tag: "Comma" }); i++; continue;
    }

    if (c === "\"") {
      i++;
      let s = "";
      while (i < src.length) {
        const d = src[i];
        if (d === "\"") { i++; break; }
        if (d === "\\") {
          const e = src[i + 1];
          if (e === "n") { s += "\n"; i += 2; continue; }
          if (e === "t") { s += "\t"; i += 2; continue; }
          s += e ?? "";
          i += 2;
          continue;
        }
        s += d;
        i++;
      }
      toks.push({ tag: "Str", s });
      continue;
    }

    // Character literals: #\x, #\newline, #\space, #\tab
    if (c === "#" && src[i + 1] === "\\") {
      i += 2; // skip #\
      // Read the character name
      let charName = "";
      while (i < src.length) {
        const d = src[i];
        if (isWS(d) || d === "(" || d === ")" || d === "[" || d === "]" || d === "{" || d === "}" || d === "'" || d === "`" || d === "," || d === ";") break;
        charName += d;
        i++;
      }
      // Convert character name to actual character
      let charVal: string;
      if (charName.length === 1) {
        charVal = charName;
      } else {
        const lower = charName.toLowerCase();
        if (lower === "newline") charVal = "\n";
        else if (lower === "space") charVal = " ";
        else if (lower === "tab") charVal = "\t";
        else if (lower === "return") charVal = "\r";
        else throw new Error(`Unknown character name: #\\${charName}`);
      }
      toks.push({ tag: "Char", c: charVal });
      continue;
    }

    // atom: read until whitespace or delimiter
    let a = "";
    while (i < src.length) {
      const d = src[i];
      if (isWS(d) || d === "(" || d === ")" || d === "[" || d === "]" || d === "{" || d === "}" || d === "'" || d === "`" || d === "," || d === ";") break;
      a += d;
      i++;
    }
    if (a.length === 0) throw new Error("tokenize: internal error");
    toks.push({ tag: "Atom", s: a });
  }

  return toks;
}