Something went wrong. Try again.
[READ-ONLY] Mirror of https://github.com/haydn/ulm. Universal Language for Mathematics
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720import Token from "./Token";import errorMessage from "./errorMessage";
const SIGNS = new Set(["+", "-"]);
const LEADING_DIGITS = new Set(["1", "2", "3", "4", "5", "6", "7", "8", "9"]);
const TRUE_DIGITS = new Set(["0", ...LEADING_DIGITS]);
const DIGITS = new Set([...TRUE_DIGITS, "_"]);
const WHITESPACE = new Set(["\u0009", "\u0020"]);
const IGNORED = new Set([",", ";"]);
const NAME = new Set([ ..."ABCDEFGHIJKLMNOPQRSTUVWXYZ", ..."abcdefghijklmnopqrstuvwxyz", ..."_-!*/\\%^+÷=≠|"]);
const format = (string?: string): string => string ? `'${string}' (${string .split("") .map( char => `U+${char .charCodeAt(0) .toString(16) .toUpperCase() .padStart(4, "0")}` ) .join(" ")})` : "NULL";
const tokenize = (input: string, file: string | null = null): Token[] => { let index = 0; let line = 1; let column = 1; let char = input[index];
let tokens: Token[] = [];
while (index < input.length) { if (char === "\r") { const nextChar = input[index + 1];
if (nextChar === "\n") { line += 1;
index += 2; column = 1; char = input[index];
continue; }
throw errorMessage( `Unexpected character ${format( nextChar )} encountered. Expected ${format("\n")}.`, { index, line, column }, file ); }
if (char === "\n") { line += 1;
index += 1; column = 1; char = input[index];
continue; }
if (WHITESPACE.has(char) || IGNORED.has(char)) { index += 1; column += 1; char = input[index];
continue; }
if (char === "(") { tokens.push({ type: "LeftParenthesis", value: char, file, location: { index, line, column } });
index += 1; column += 1; char = input[index];
continue; }
if (char === ")") { tokens.push({ type: "RightParenthesis", value: char, file, location: { index, line, column } });
index += 1; column += 1; char = input[index];
continue; }
if (char === "{") { tokens.push({ type: "LeftBracket", value: char, file, location: { index, line, column } });
index += 1; column += 1; char = input[index];
continue; }
if (char === "}") { tokens.push({ type: "RightBracket", value: char, file, location: { index, line, column } });
index += 1; column += 1; char = input[index];
continue; }
if (char === ":") { tokens.push({ type: "Colon", value: char, file, location: { index, line, column } });
index += 1; column += 1; char = input[index];
continue; }
if (char === "0") { let value = char; let location = { index, line, column };
index += 1; column += 1; char = input[index];
if (char === "w") { value += char;
tokens.push({ type: "WholeNumberLiteral", value, file, location });
index += 1; column += 1; char = input[index];
continue; }
if (char === "i") { value += char;
tokens.push({ type: "IntegerNumberLiteral", value, file, location });
index += 1; column += 1; char = input[index];
continue; }
if (char === ".") { value += char;
index += 1; column += 1; char = input[index];
if (TRUE_DIGITS.has(char)) { do { value += char;
index += 1; column += 1; char = input[index]; } while (DIGITS.has(char));
tokens.push({ type: "RationalNumberLiteral", value, file, location });
continue; }
throw errorMessage( `Unexpected character ${format( char )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, { index, line, column }, file ); }
if (char === "/") { value += char;
index += 1; column += 1; char = input[index];
if (LEADING_DIGITS.has(char)) { value += char;
index += 1; column += 1; char = input[index];
while (DIGITS.has(char)) { value += char;
index += 1; column += 1; char = input[index]; }
tokens.push({ type: "RationalNumberLiteral", value, file, location });
continue; }
throw errorMessage( `Unexpected character ${format( char )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, { index, line, column }, file ); }
tokens.push({ type: "RationalNumberLiteral", value, file, location });
continue; }
if (LEADING_DIGITS.has(char)) { let value = char; let location = { index, line, column };
index += 1; column += 1; char = input[index];
while (DIGITS.has(char)) { value += char;
index += 1; column += 1; char = input[index]; }
if (char === "n") { value += char;
tokens.push({ type: "NaturalNumberLiteral", value, file, location });
index += 1; column += 1; char = input[index];
continue; }
if (char === "w") { value += char;
tokens.push({ type: "WholeNumberLiteral", value, file, location });
index += 1; column += 1; char = input[index];
continue; }
if (char === "i") { value += char;
tokens.push({ type: "IntegerNumberLiteral", value, file, location });
index += 1; column += 1; char = input[index];
continue; }
if (char === ".") { value += char;
index += 1; column += 1; char = input[index];
if (TRUE_DIGITS.has(char)) { do { value += char;
index += 1; column += 1; char = input[index]; } while (DIGITS.has(char));
tokens.push({ type: "RationalNumberLiteral", value, file, location });
continue; }
throw errorMessage( `Unexpected character ${format( char )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, { index, line, column }, file ); }
if (char === "/") { value += char;
index += 1; column += 1; char = input[index];
if (LEADING_DIGITS.has(char)) { value += char;
index += 1; column += 1; char = input[index];
while (DIGITS.has(char)) { value += char;
index += 1; column += 1; char = input[index]; }
tokens.push({ type: "RationalNumberLiteral", value, file, location });
continue; }
throw errorMessage( `Unexpected character ${format( char )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, { index, line, column }, file ); }
tokens.push({ type: "RationalNumberLiteral", value, file, location });
continue; }
if (SIGNS.has(char) && TRUE_DIGITS.has(input[index + 1])) { let value = char; let location = { index, line, column };
index += 1; column += 1; char = input[index];
if (char === "0") { value += char;
index += 1; column += 1; char = input[index];
if (char === ".") { value += char;
index += 1; column += 1; char = input[index];
if (TRUE_DIGITS.has(char)) { do { value += char;
index += 1; column += 1; char = input[index]; } while (DIGITS.has(char));
tokens.push({ type: "RationalNumberLiteral", value, file, location });
continue; }
throw errorMessage( `Unexpected character ${format( char )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, { index, line, column }, file ); }
if (char === "/") { value += char;
index += 1; column += 1; char = input[index];
if (LEADING_DIGITS.has(char)) { value += char;
index += 1; column += 1; char = input[index];
while (DIGITS.has(char)) { value += char;
index += 1; column += 1; char = input[index]; }
tokens.push({ type: "RationalNumberLiteral", value, file, location });
continue; }
throw errorMessage( `Unexpected character ${format( char )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, { index, line, column }, file ); } }
while (DIGITS.has(char)) { value += char;
index += 1; column += 1; char = input[index]; }
if (char === "i") { value += char;
tokens.push({ type: "IntegerNumberLiteral", value, file, location });
index += 1; column += 1; char = input[index];
continue; }
if (char === ".") { value += char;
index += 1; column += 1; char = input[index];
if (TRUE_DIGITS.has(char)) { do { value += char;
index += 1; column += 1; char = input[index]; } while (DIGITS.has(char));
tokens.push({ type: "RationalNumberLiteral", value, file, location });
continue; }
throw errorMessage( `Unexpected character ${format( char )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, { index, line, column }, file ); }
if (char === "/") { value += char;
index += 1; column += 1; char = input[index];
if (LEADING_DIGITS.has(char)) { value += char;
index += 1; column += 1; char = input[index];
while (DIGITS.has(char)) { value += char;
index += 1; column += 1; char = input[index]; }
tokens.push({ type: "RationalNumberLiteral", value, file, location });
continue; }
throw errorMessage( `Unexpected character ${format( char )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, { index, line, column }, file ); }
tokens.push({ type: "RationalNumberLiteral", value, file, location });
continue; }
if (NAME.has(char)) { let value = ""; let location = { index, line, column };
while (NAME.has(char)) { value += char;
index += 1; column += 1; char = input[index]; }
tokens.push({ type: "OperatorName", value, file, location });
continue; }
// // FIXME: Is something like this needed? // if (!/[\u0009\u000A\u000D\u0020-\uFFFF]/.test(char)) { // throw `Unexpected ( ${char} ) encountered. ${line}:${column}`; // }
throw errorMessage( `Unknown character encountered: ${format(char)}`, { index, line, column }, file ); }
return tokens;};
export { Token };export default tokenize;