From 93541694e1c4ef688f42f387b53660853fcc0ba2 Mon Sep 17 00:00:00 2001 From: Miraculous Ladybugreport <3642643+PeyTy@users.noreply.github.com> Date: Wed, 11 Oct 2023 05:57:41 +0300 Subject: [PATCH] [Refactor] Use new token names elsewhere --- source/compiler/lexer.hexa | 76 ++--- source/compiler/normalizer.hexa | 72 ++--- source/compiler/parser.hexa | 496 +++++++++++++++--------------- source/compiler/preprocessor.hexa | 6 +- source/compiler/typer.hexa | 49 ++- source/data/token.hexa | 94 +++--- source/data/types.hexa | 2 +- source/targets/genC.hexa | 12 +- source/targets/genCxx.hexa | 4 +- source/targets/genNasm.hexa | 16 +- source/tests/testLexer.hexa | 114 +++---- source/tests/testParser.hexa | 61 ++-- 12 files changed, 507 insertions(+), 495 deletions(-) diff --git a/source/compiler/lexer.hexa b/source/compiler/lexer.hexa index bf7c416..6fe78c7 100644 --- a/source/compiler/lexer.hexa +++ b/source/compiler/lexer.hexa @@ -498,7 +498,7 @@ class Lexer { if (_8 < 58) { var p = position _8 = get_8(p) - var found: Token = Token.LInt + var found: Token = Token.Integer while _8 >= 48 && _8 <= 57 // 0...9 { @@ -514,7 +514,7 @@ class Lexer { { _8 = get_8(++p) } - found = Token.LFloat + found = Token.FloatingPoint } // exponent @@ -530,7 +530,7 @@ class Lexer { { _8 = get_8(++p) } - found = Token.LFloat + found = Token.FloatingPoint } var m = Meta.Default @@ -649,9 +649,9 @@ class Lexer { "try" : Token.KTry, "var" : Token.KVar, "while" : Token.KWhile, - "and" : Token.OpBoolAnd, - "or" : Token.OpBoolOr, - "not" : Token.OpNot, + "and" : Token.LogicalAnd, + "or" : Token.LogicalOr, + "not" : Token.KNot, "is" : Token.KIs ] @@ -661,31 +661,31 @@ class Lexer { "@".charCodeAt(0) : Token.At, "$".charCodeAt(0) : Token.Query, "#".charCodeAt(0) : Token.Sharp, - "!".charCodeAt(0) : Token.OpNot, // TODO - "%".charCodeAt(0) : Token.OpMod, - "&".charCodeAt(0) : Token.OpAnd, - "(".charCodeAt(0) : Token.POpen, - ")".charCodeAt(0) : Token.PClose, - "*".charCodeAt(0) : Token.OpMult, - "+".charCodeAt(0) : Token.OpAdd, + "!".charCodeAt(0) : Token.KNot, // TODO + "%".charCodeAt(0) : Token.Remainder, + "&".charCodeAt(0) : Token.BitwiseAnd, + "(".charCodeAt(0) : Token.CallOpen, + ")".charCodeAt(0) : Token.CallClose, + "*".charCodeAt(0) : Token.Multiply, + "+".charCodeAt(0) : Token.Add, ",".charCodeAt(0) : Token.Comma, - "-".charCodeAt(0) : Token.OpSub, + "-".charCodeAt(0) : Token.Subtract, ".".charCodeAt(0) : Token.Dot, - "/".charCodeAt(0) : Token.OpDiv, + "/".charCodeAt(0) : Token.Divide, ":".charCodeAt(0) : Token.Colon, ";".charCodeAt(0) : Token.Semicolon, - "<".charCodeAt(0) : Token.OpLt, - "=".charCodeAt(0) : Token.OpAssign, - ">".charCodeAt(0) : Token.OpGt, + "<".charCodeAt(0) : Token.Less, + "=".charCodeAt(0) : Token.Assign, + ">".charCodeAt(0) : Token.Greater, "?".charCodeAt(0) : Token.Question, - "[".charCodeAt(0) : Token.BkOpen, - "\\".charCodeAt(0) : Token.OpIntDiv, - "]".charCodeAt(0) : Token.BkClose, - "^".charCodeAt(0) : Token.OpXor, - "{".charCodeAt(0) : Token.BrOpen, - "|".charCodeAt(0) : Token.OpOr, - "}".charCodeAt(0) : Token.BrClose, - "~".charCodeAt(0) : Token.OpNegBits + "[".charCodeAt(0) : Token.IndexOpen, + "\\".charCodeAt(0) : Token.IntegerDivide, + "]".charCodeAt(0) : Token.IndexClose, + "^".charCodeAt(0) : Token.BitwiseXor, + "{".charCodeAt(0) : Token.BlockOpen, + "|".charCodeAt(0) : Token.BitwiseOr, + "}".charCodeAt(0) : Token.BlockClose, + "~".charCodeAt(0) : Token.BitwiseNot ] for key in ops8.keys() { @@ -697,18 +697,18 @@ class Lexer { // `Hash = charCodeAt(0) + charCodeAt(1) * 256` let ops16: [Int : Token] = [ - 11051 : Token.OpIncrement, // ++ - 11565 : Token.OpDecrement, // -- - 15420 : Token.OpShl, // << - 15649 : Token.OpNotEq, // != - 15676 : Token.OpLte, // <= - 15677 : Token.OpEq, // == - 15678 : Token.OpGte, // >= - 15934 : Token.OpShr, // >> - 31868 : Token.OpBoolOr, // || - 9766 : Token.OpBoolAnd, // && - 15933 : Token.OpArrow, // => - 11839 : Token.OpChain // ?. + 11051 : Token.Increment, // ++ + 11565 : Token.Decrement, // -- + 15420 : Token.BitwiseLeftShift, // << + 15649 : Token.Unequal, // != + 15676 : Token.LessOrEqual, // <= + 15677 : Token.Equal, // == + 15678 : Token.GreaterOrEqual, // >= + 15934 : Token.BitwiseRightShift, // >> + 31868 : Token.LogicalOr, // || + 9766 : Token.LogicalAnd, // && + 15933 : Token.RightArrow, // => + 11839 : Token.OptionalChain // ?. ] // TODO automate as macro diff --git a/source/compiler/normalizer.hexa b/source/compiler/normalizer.hexa index 3405ab3..c460ee1 100644 --- a/source/compiler/normalizer.hexa +++ b/source/compiler/normalizer.hexa @@ -493,7 +493,7 @@ class Normalizer { let eb = nodeToExpression(b) // TODO refactor `if op` to `switch op` - if op == Token.OpAdd { switch ea { + if op == Token.Add { switch ea { case String(sa): switch eb { // `"abc" + "dce" = "abcdce"` case String(sb): return Expression.String(sa + sb) @@ -519,7 +519,7 @@ class Normalizer { // TODO use nested pattern matching here // `case ea, op, eb:` - if op == Token.OpBoolOr { switch ea { + if op == Token.LogicalOr { switch ea { case True: switch eb { // `true || false = true` case False: return Expression.True @@ -534,7 +534,7 @@ class Normalizer { } }} - if op == Token.OpBoolAnd { + if op == Token.LogicalAnd { switch ea { case True: switch eb { // `true && false = false` @@ -551,7 +551,7 @@ class Normalizer { }} // `a == b` - if op == Token.OpEq { + if op == Token.Equal { switch ea { case True: switch eb { // `true == false = false` @@ -600,7 +600,7 @@ class Normalizer { }} // `a != b` - if op == Token.OpNotEq { + if op == Token.Unequal { switch ea { case True: switch eb { // `true != false = true` @@ -735,8 +735,8 @@ class Normalizer { block.push(Statement.Const(i, Expression.Int(-1), typer.typeInt)) block.push(Statement.While( Expression.Binop( - Expression.Parenthesis(Expression.Binop(Expression.Ident(i, typer.typeInt), Token.OpAdd, Expression.Int(1), typer.typeInt)), - Token.OpLt, + Expression.Parenthesis(Expression.Binop(Expression.Ident(i, typer.typeInt), Token.Add, Expression.Int(1), typer.typeInt)), + Token.Less, Expression.Ident(value, valueType), typer.typeBool), Statement.Block([ @@ -776,7 +776,7 @@ class Normalizer { let resultType = typer.types.get(nullable) return Expression.Call(Expression.Parenthesis(Expression.Function(null, Statement.Block([ Statement.Const(result, nodeToExpression(nullable), resultType), - Statement.If(Expression.Binop(Expression.Ident(result, resultType), Token.OpNotEq, Expression.Null(resultType), typer.typeBool), Statement.Return(Expression.Ident(result, resultType)), null), + Statement.If(Expression.Binop(Expression.Ident(result, resultType), Token.Unequal, Expression.Null(resultType), typer.typeBool), Statement.Return(Expression.Ident(result, resultType)), null), Statement.Return(nodeToExpression(otherwise)) ]), [], [], Type.Function([], typer.types.get(otherwise), false))), [], typer.types.get(otherwise), CallKind.Function @@ -785,7 +785,7 @@ class Normalizer { case Unop(op, postfix, e): switch op { - case OpIncrement | OpDecrement: + case Increment | Decrement: var source: Node? = typer.parents.get(e) if let source = source { switch source { @@ -797,7 +797,7 @@ class Normalizer { let expression = nodeToExpression(e) switch op { - case OpNot: + case KNot: switch expression { // `not true = false` case True: return Expression.False @@ -849,7 +849,7 @@ class Normalizer { var parens = Expression.Parenthesis(nodeToExpression(econd[0])) var i = 1 while i < econd.length { - parens = Expression.Binop(parens, Token.OpBoolAnd, Expression.Parenthesis(nodeToExpression(econd[i])), typer.typeBool) + parens = Expression.Binop(parens, Token.LogicalAnd, Expression.Parenthesis(nodeToExpression(econd[i])), typer.typeBool) i++ } return Expression.If(parens, nodeToExpression(eif), nodeToExpression(eelse)) @@ -863,7 +863,7 @@ class Normalizer { case As(e, kind, t): switch kind { case Question: console.log('`as?` is not yet supported by normalizer, only `as!`') - case OpNot: + case KNot: return Expression.UnsafeBitCast(nodeToExpression(e), typer.types.get(t)) case _: console.log('`as` is not yet supported by normalizer, only `as!`') @@ -1070,7 +1070,7 @@ class Normalizer { let values = [] for f in fields { switch f { // `ValueName = b` - case Binop(a, op, b): // op == OpAssign + case Binop(a, op, b): // op == Assign switch a { case NodeTypeValue(t): switch t { case Type(name): @@ -1122,7 +1122,7 @@ class Normalizer { case Var(name, t, expr, const): a.push(unique(name, arg)) types.push(t) - if expr != null { + if (expr != null) { defaults.push(nodeToExpression(expr)) } else { defaults.push(null) @@ -1274,7 +1274,7 @@ class Normalizer { case Var(name, t, expr, const): a.push(unique(name, arg)) types.push(t) - if (expr != null) { + if expr != null { defaults.push(nodeToExpression(expr)) } else { defaults.push(null) @@ -1504,7 +1504,7 @@ class Normalizer { Statement.Var(i, Expression.Int(0), typer.typeInt), Statement.Var(it, Expression.Int(0), typer.typeInt), Statement.While( - Expression.Binop(Expression.Ident(it, typer.typeInt), Token.OpLt, Expression.Int(s), typer.typeBool), + Expression.Binop(Expression.Ident(it, typer.typeInt), Token.Less, Expression.Int(s), typer.typeBool), Statement.Block([ Statement.Assign(Expression.Ident(i, typer.typeInt), Expression.Ident(it, typer.typeInt), null), Statement.Increment(Expression.Ident(it, typer.typeInt)), @@ -1547,7 +1547,7 @@ class Normalizer { Statement.While( Expression.Binop( Expression.Ident(it, overType), - Token.OpLt, + Token.Less, Expression.Ident(finish, overType), typer.typeBool ), @@ -1583,7 +1583,7 @@ class Normalizer { Statement.Var(it, nodeToExpression(over), typer.typeInt), Statement.Var(finish, nodeToExpression(range), typer.typeInt), Statement.While( - Expression.Binop(Expression.Ident(it, typer.typeInt), Token.OpLt, Expression.Ident(finish, typer.typeInt), typer.typeBool), + Expression.Binop(Expression.Ident(it, typer.typeInt), Token.Less, Expression.Ident(finish, typer.typeInt), typer.typeBool), Statement.Block([ Statement.Assign(Expression.Ident(i, typer.typeInt), Expression.Ident(it, typer.typeInt), null), Statement.Increment(Expression.Ident(it, typer.typeInt)), @@ -1621,7 +1621,7 @@ class Normalizer { ), Statement.While( // `while it < at` - Expression.Binop(Expression.Ident(it, typer.typeInt), Token.OpLt, Expression.Ident(finish, typer.typeInt), typer.typeBool), + Expression.Binop(Expression.Ident(it, typer.typeInt), Token.Less, Expression.Ident(finish, typer.typeInt), typer.typeBool), // `{` Statement.Block([ // `var name = over[it]` TODO `name` must be nullable @@ -1688,7 +1688,7 @@ class Normalizer { var parens = Expression.Parenthesis(nodeToExpression(econd[0])) var i = 1 while i < econd.length { - parens = Expression.Binop(parens, Token.OpBoolAnd, Expression.Parenthesis(nodeToExpression(econd[i])), typer.typeBool) + parens = Expression.Binop(parens, Token.LogicalAnd, Expression.Parenthesis(nodeToExpression(econd[i])), typer.typeBool) i++ } return Statement.If(parens, nodeToStatement(eif), nodeToStatement(eelse)) @@ -1719,9 +1719,9 @@ class Normalizer { block.push(Statement.Var(name, Expression.Null(typer.types.get(expr)), typer.types.get(expr))) block.push(Statement.If( Expression.Binop( - Expression.Binop(Expression.Ident(step, typer.typeInt), Token.OpEq, Expression.Int(current + 0), typer.typeBool), - Token.OpBoolAnd, - Expression.Binop(Expression.Ident(last, lastType), Token.OpNotEq, nullable? Expression.Null(lastType) : Expression.False, typer.typeBool), + Expression.Binop(Expression.Ident(step, typer.typeInt), Token.Equal, Expression.Int(current + 0), typer.typeBool), + Token.LogicalAnd, + Expression.Binop(Expression.Ident(last, lastType), Token.Unequal, nullable? Expression.Null(lastType) : Expression.False, typer.typeBool), typer.typeBool ), Statement.Block([ @@ -1737,9 +1737,9 @@ class Normalizer { block.push(Statement.Var(next, Expression.Null(nextType), nextType)) block.push(Statement.If( Expression.Binop( - Expression.Binop(Expression.Ident(step, typer.typeInt), Token.OpEq, Expression.Int(current + 0), typer.typeBool), - Token.OpBoolAnd, - Expression.Binop(Expression.Ident(last, lastType), Token.OpNotEq, nullable? Expression.Null(lastType) : Expression.False, typer.typeBool), + Expression.Binop(Expression.Ident(step, typer.typeInt), Token.Equal, Expression.Int(current + 0), typer.typeBool), + Token.LogicalAnd, + Expression.Binop(Expression.Ident(last, lastType), Token.Unequal, nullable? Expression.Null(lastType) : Expression.False, typer.typeBool), typer.typeBool ), Statement.Block([ @@ -1754,16 +1754,16 @@ class Normalizer { // Last step block.push(Statement.If( Expression.Binop( - Expression.Binop(Expression.Ident(step, typer.typeInt), Token.OpEq, Expression.Int(econd.length - 1), typer.typeBool), - Token.OpBoolAnd, - Expression.Binop(Expression.Ident(last, lastType), Token.OpNotEq, nullable? Expression.Null(lastType) : Expression.False, typer.typeBool), + Expression.Binop(Expression.Ident(step, typer.typeInt), Token.Equal, Expression.Int(econd.length - 1), typer.typeBool), + Token.LogicalAnd, + Expression.Binop(Expression.Ident(last, lastType), Token.Unequal, nullable? Expression.Null(lastType) : Expression.False, typer.typeBool), typer.typeBool ), Statement.Assign(Expression.Ident(step, typer.typeInt), Expression.Int(econd.length), null), null)) // if (step == steps) block.push(Statement.If( - Expression.Binop(Expression.Ident(step, typer.typeInt), Token.OpEq, Expression.Int(econd.length), typer.typeBool), + Expression.Binop(Expression.Ident(step, typer.typeInt), Token.Equal, Expression.Int(econd.length), typer.typeBool), nodeToStatement(eif), nodeToStatement(eelse))) return Statement.Block(block) @@ -1800,7 +1800,7 @@ class Normalizer { case Binop(a, op, b): switch op { - case OpAssign: + case Assign: var source: Node? = typer.parents.get(a) if let source = source { switch source { @@ -1829,7 +1829,7 @@ class Normalizer { case Unop(op, postfix, e): switch op { - case OpIncrement | OpDecrement: + case Increment | Decrement: var source: Node? = typer.parents.get(e) if let source = source { switch source { @@ -1839,8 +1839,8 @@ class Normalizer { } } switch op { - case OpIncrement: return Statement.Increment(nodeToExpression(e)) - case OpDecrement: return Statement.Decrement(nodeToExpression(e)) + case Increment: return Statement.Increment(nodeToExpression(e)) + case Decrement: return Statement.Decrement(nodeToExpression(e)) case _: // Optimize away } case Parenthesis(inner): @@ -1872,7 +1872,7 @@ class Normalizer { case Dot(expr, name): return Expression.Dot(nodeToExpression(expr), name, null) case DotUpper(expr, name): return Expression.Dot(nodeToExpression(expr), name, null) case Binop(a, op, b): switch op { - case OpOr: return Expression.Binop(toCase(a), Token.OpOr, toCase(b), null) + case BitwiseOr: return Expression.Binop(toCase(a), Token.BitwiseOr, toCase(b), null) case _: console.error('(internal warning) Unknown native case binop kind:', e, typer.parents.get(e)) } case Call(e, args, argNames): switch e { @@ -1979,7 +1979,7 @@ class Normalizer { cx.push(getTag(typer.parents.get(expr), name)) case Binop(a, op, b): switch op { - case OpOr: + case BitwiseOr: addTag(a) addTag(b) case _: console.error('(internal warning) Unknown case binop kind:', e, typer.parents.get(e)) diff --git a/source/compiler/parser.hexa b/source/compiler/parser.hexa index ba46388..2fc803d 100644 --- a/source/compiler/parser.hexa +++ b/source/compiler/parser.hexa @@ -63,7 +63,7 @@ class Parser { fun parseFields() { var fields = [] - while tok() != Token.BrClose { + while tok() != Token.BlockClose { var atts: Array = [] while tok() == Token.At { atts.push(parseDecorator()) @@ -131,15 +131,15 @@ class Parser { } else { let token = tok() if - token == Token.BkOpen or - token == Token.BrOpen or - token == Token.LUpper or - token == Token.POpen + token == Token.IndexOpen or + token == Token.BlockOpen or + token == Token.Title or + token == Token.CallOpen { t = parseType() } } - if tok() == Token.OpAssign { + if tok() == Token.Assign { i++ expr = parseExpr() } @@ -151,7 +151,7 @@ class Parser { break } }} - step(Token.PClose) + step(Token.CallClose) // TODO Decorators parsing breaks this var tmp = i while tok() == Token.At { @@ -159,7 +159,7 @@ class Parser { } if - tok() != Token.BrClose, + tok() != Token.BlockClose, tok() != Token.KFun { i = tmp @@ -183,24 +183,24 @@ class Parser { let field = Node.Function('new', expr, v, null, false, variadic: false) project.data.set(field, pos) fields.push(field) - case LLower: - let name = getgo(Token.LLower) + case Identifier: + let name = getgo(Token.Identifier) // JavaScript's `get field()` to property - if name == 'get', tok() == Token.LLower { - let field = getgo(Token.LLower) + if name == 'get', tok() == Token.Identifier { + let field = getgo(Token.Identifier) fail('Use property `var \(field) { get { return value } }` instead of `get \(field)()`') } // JavaScript's `set field(value)` to property - if name == 'set', tok() == Token.LLower { - let field = getgo(Token.LLower) - step(Token.POpen) - let value = getgo(Token.LLower) + if name == 'set', tok() == Token.Identifier { + let field = getgo(Token.Identifier) + step(Token.CallOpen) + let value = getgo(Token.Identifier) fail('Use property `var \(field) { set(\(value)) { ... } }` instead of `set \(field)(\(value))`') } // JavaScript's `method()` to `function method()` - if tok() == Token.POpen { + if tok() == Token.CallOpen { if name == 'constructor' { fail('Use `new()` syntax instead of `constructor()`') } @@ -211,10 +211,10 @@ class Parser { } } fail('Fields should start with `var ' + name + '` or `let ' + name + '`') - case LUpper: + case Title: // TODO toCamelCase fail('Fields cannot start with uppercase character `' + print() + '` -> `' + print().toLowerCase() + '`') - case BrOpen: + case BlockOpen: let pos = { line: lex.line[i], column: lex.column[i] } let field: Node? = fields.pop() var v = null @@ -262,9 +262,9 @@ class Parser { if kind == 'set' { i++ - step(Token.POpen) - setterVar = getgo(Token.LLower) - step(Token.PClose) + step(Token.CallOpen) + setterVar = getgo(Token.Identifier) + step(Token.CallClose) setterExpr = parseExpr() } @@ -272,7 +272,7 @@ class Parser { fail('Variables `var` should have both getter `get` and setter `set`') } - step(Token.BrClose) + step(Token.BlockClose) let getter: Node? = (getterExpr == null)? null : Node.Function('get_' + n, getterExpr, [], t, false, variadic: false) let setter: Node? = (setterExpr == null)? null : Node.Function('set_' + n, setterExpr, [ @@ -431,9 +431,9 @@ class Parser { } case KClass | KInterface: result = parseClass(external: true) - case LUpper: + case Title: let alias = parseType() - step(Token.OpAssign) + step(Token.Assign) let value = parseType() result = Node.TypeAlias(alias, value) case _: @@ -446,16 +446,16 @@ class Parser { switch tok() { case KFunction | KFun: e = parseFunction(false) - case LUpper: + case Title: var left = parseType() - step(Token.OpAssign) + step(Token.Assign) let leftNode = Node.NodeTypeValue(left) project.data.set(leftNode, new NodeData(nodePosition.line, nodePosition.column, lex.fileName)) let rightNode = Node.NodeTypeValue(parseType()) project.data.set(rightNode, new NodeData(nodePosition.line, nodePosition.column, lex.fileName)) - e = Node.Binop(leftNode, Token.OpAssign, rightNode) + e = Node.Binop(leftNode, Token.Assign, rightNode) case _: e = parseExpr() @@ -477,7 +477,7 @@ class Parser { switch extracted { case Binop(a, op, b): - if (op != Token.OpAssign) fail('declare =') + if (op != Token.Assign) fail('declare =') switch a { case NodeTypeValue(t): switch t { @@ -514,13 +514,13 @@ class Parser { result = Node.TDeclare(name, e) */ //switch tok() { - // case LUpper: - // var name = getgo(LUpper) - // step(OpAssign) + // case Title: + // var name = getgo(Title) + // step(Assign) // TDeclare(name, parseExpr()) - // // case LLower: - // // var name = getgo(LLower) - // // step(OpAssign) + // // case Identifier: + // // var name = getgo(Identifier) + // // step(Assign) // // TDeclare(name, parseExpr()) // case KClass: // classExternal = true @@ -540,24 +540,24 @@ class Parser { //} // {} - case BrOpen: + case BlockOpen: // TODO handle EOF let startsAt = i i++ - if tok() == Token.BrClose { // Empty block {} + if tok() == Token.BlockClose { // Empty block {} i++ result = Node.Block([]) - } else if (tok() == Token.LLower && offset(1) == Token.Colon) { // Object { k:v } + } else if (tok() == Token.Identifier && offset(1) == Token.Colon) { // Object { k:v } var names: [String] = [] var el: [Node] = [] while true { - names.push(getgo(Token.LLower)) + names.push(getgo(Token.Identifier)) step(Token.Colon) el.push(parseExpr()) if tok() == Token.Comma { // Object { k:v, ... } i++ - if tok() == Token.BrClose { // Object { k:v, } + if tok() == Token.BlockClose { // Object { k:v, } fail("Unexpected `}`, remove trailing `,` comma") } continue @@ -565,21 +565,21 @@ class Parser { break } } - if tok() != Token.BrClose { + if tok() != Token.BlockClose { fail('Expected `}` or `,` before `\(print())`') } - step(Token.BrClose) + step(Token.BlockClose) result = Node.Object(names, el) } else { // Block { expr expr expr } var el = [] - while tok() != Token.BrClose { + while tok() != Token.BlockClose { if tok() == Token.Eof { i = startsAt fail('The block `{` has got unclosed to the end of the file') } el.push(parseExpr()) } - step(Token.BrClose) + step(Token.BlockClose) result = Node.Block(el) } // if econd { eif } @@ -661,17 +661,17 @@ class Parser { // (e) just parenthesis // (e) => // () => - case POpen: + case CallOpen: let startsAt = i next() if ( // () => (tok() == Token.PClose && offset(1) == Token.OpArrow) || // (a, ...) => - (tok() == Token.LLower && offset(1) == Token.Comma) || + (tok() == Token.Identifier && offset(1) == Token.Comma) || // (a: ...) => // TODO colon-less - (tok() == Token.LLower && offset(1) == Token.Colon) || + (tok() == Token.Identifier && offset(1) == Token.Colon) || // (a T) => (tok() == Token.LLower && offset(1) == Token.LUpper) || // (a) => @@ -680,17 +680,17 @@ class Parser { var vars: [String] = [] var types = [] var values = [] - while tok() != Token.PClose { - vars.push(getgo(Token.LLower)) + while tok() != Token.CallClose { + vars.push(getgo(Token.Identifier)) if tok() == Token.Colon { // TODO colon-less i++ types.push(parseType()) } - if tok() == Token.LUpper { + if tok() == Token.Title { types.push(parseType()) } - if tok() == Token.OpAssign { + if tok() == Token.Assign { i++ values.push(parseExpr()) } @@ -698,8 +698,8 @@ class Parser { i++ } } - step(Token.PClose) - step(Token.OpArrow) + step(Token.CallClose) + step(Token.RightArrow) var v = [] for i in vars.length { v.push(Node.Var(vars[i], types[i], values[i], true, false)) @@ -719,7 +719,7 @@ class Parser { i = startsAt fail('The parenthesis `(` has got unclosed to the end of the file') } - step(Token.PClose) + step(Token.CallClose) result = Node.Parenthesis(expr) } // return e @@ -728,7 +728,7 @@ class Parser { i++ switch tok() { // TODO `throw` etc? - case BrClose: result = Node.Return(null) + case BlockClose: result = Node.Return(null) case KVar: result = Node.Return(null) case KLet: result = Node.Return(null) case KCase: result = Node.Return(null) @@ -757,7 +757,7 @@ class Parser { // !i // ~i // -i - case OpIncrement | OpDecrement | OpNot | OpNegBits | OpSub: + case Increment | Decrement | KNot | BitwiseNot | Subtract: let id = lex.token[i] as! Token i++ let value = parseExpr() @@ -775,13 +775,13 @@ class Parser { result = fix(value) - case LFloat: + case FloatingPoint: // TODO handle Meta let meta = lex.meta[i] result = Node.Float(parseFloat(getgo(Token.LFloat)), meta) - case LInt: + case Integer: let meta = lex.meta[i] - let value = getgo(Token.LInt) + let value = getgo(Token.Integer) switch meta { // TODO rename `Default` token to `Empty` case Default: @@ -831,28 +831,28 @@ class Parser { // `T` // `T?` // `T` - case LUpper: - if offset(1) == Token.OpLt { + case Title: + if offset(1) == Token.Less { let res = parseType() result = Node.NodeTypeValue(res) } else if offset(1) == Token.Question { - let name = getgo(Token.LUpper) + let name = getgo(Token.Title) i++ result = Node.NodeTypeValue(NodeType.Optional(NodeType.Type(name, null))) } else { - let name = getgo(Token.LUpper) + let name = getgo(Token.Title) result = Node.NodeTypeValue(NodeType.Type(name, null)) } // `varName` // `arrow => body` // `genericCall(args)` - case LLower: + case Identifier: let pos = lex.column[i] - let name = getgo(Token.LLower) - if tok() == Token.OpArrow { + let name = getgo(Token.Identifier) + if tok() == Token.RightArrow { next() result = Node.Arrow(parseExpr(), [Node.Var(name, null, null, true, false)], null) - } else if tok() == Token.OpLt, lex.column[i] == pos + name.length { + } else if tok() == Token.Less, lex.column[i] == pos + name.length { i++ let params: [NodeType] = [parseType()] // TODO reusable code @@ -860,7 +860,7 @@ class Parser { i++ params.push(parseType()) } - step(Token.OpGt) + step(Token.Greater) result = Node.Ident(name, params) } else { result = Node.Ident(name, null) @@ -1014,9 +1014,9 @@ class Parser { i++ extend = parseType() } - step(Token.BrOpen) + step(Token.BlockOpen) var names = [] - while tok() != Token.BrClose { + while tok() != Token.BlockClose { while tok() == Token.At { atts.push(parseDecorator()) } @@ -1025,16 +1025,16 @@ class Parser { atts = [] switch tok() { - case LUpper: + case Title: let pos = new NodeData(lex.line[i], lex.column[i], lex.fileName) - var tag = Node.NodeTypeValue(NodeType.Type(getgo(Token.LUpper), null)) - if tok() == Token.POpen { + var tag = Node.NodeTypeValue(NodeType.Type(getgo(Token.Title), null)) + if tok() == Token.CallOpen { i++ // TODO `let args [] = []` causes no error and `let args [test] = []` too let args: [Node] = [] let argNames: [String] = [] while true { - argNames.push(getgo(Token.LLower)) + argNames.push(getgo(Token.Identifier)) // TODO drop if tok() == Token.Colon { @@ -1051,13 +1051,13 @@ class Parser { break } } - step(Token.PClose) + step(Token.CallClose) project.data.set(tag, pos) tag = Node.Call(tag, args, argNames) - } else if tok() == Token.OpAssign { + } else if tok() == Token.Assign { i++ project.data.set(tag, pos) - tag = Node.Binop(tag, Token.OpAssign, parseExpr()) + tag = Node.Binop(tag, Token.Assign, parseExpr()) } project.data.set(tag, pos) @@ -1066,7 +1066,7 @@ class Parser { names.push(parseExpr()) } } - step(Token.BrClose) + step(Token.BlockClose) result = Node.Enum(t, names, valuesType, extend) project.mapDecorators.set(result, att) @@ -1086,7 +1086,7 @@ class Parser { var values = [] var isMap = false - if tok() != Token.BkClose { while true { + if tok() != Token.IndexClose { while true { if tok() == Token.Colon { // [:] isMap = true next() @@ -1101,14 +1101,14 @@ class Parser { if tok() == Token.Comma { i++ // Trailing comma [a,] - if tok() == Token.BkClose { + if tok() == Token.IndexClose { fail('Unexpected `]`, remove trailing comma `,` before `]` or add a value after `,` if required') } } else { break } }} - step(Token.BkClose) + step(Token.IndexClose) if isMap { result = Node.Map(el, values) @@ -1120,30 +1120,30 @@ class Parser { var t = parseType() var names: [String] = [] var values: [Node] = [] - if tok() == Token.BrOpen { // {} + if tok() == Token.BlockOpen { // {} i++ - if tok() == Token.BrClose { // Empty {} - step(Token.BrClose) - } else if (tok() == Token.LLower && offset(1) == Token.Colon) { // Object { k:v } - while tok() != Token.BrClose { - names.push(getgo(Token.LLower)) + if tok() == Token.BlockClose { // Empty {} + step(Token.BlockClose) + } else if (tok() == Token.Identifier && offset(1) == Token.Colon) { // Object { k:v } + while tok() != Token.BlockClose { + names.push(getgo(Token.Identifier)) step(Token.Colon) values.push(parseExpr()) if tok() == Token.Comma { i++ } } - step(Token.BrClose) + step(Token.BlockClose) } } - step(Token.POpen) + step(Token.CallOpen) var args = [] var argNames: [String] = [] - if tok() != Token.PClose { while true { + if tok() != Token.CallClose { while true { switch tok() { - case PClose: + case CallClose: fail("Unexpected `)`, remove trailing `,` comma") - case LUpper: + case Title: argNames.push(null) args.push(parseExpr()) if tok() == Token.Colon { @@ -1152,7 +1152,7 @@ class Parser { } case _: if offset(1) == Token.Colon { - argNames.push(getgo(Token.LLower)) + argNames.push(getgo(Token.Identifier)) step(Token.Colon) } else { @@ -1166,7 +1166,7 @@ class Parser { break } }} - step(Token.PClose) + step(Token.CallClose) result = Node.New([], t, args, names, values, argNames) case KSwitch: @@ -1176,13 +1176,13 @@ class Parser { i++ exprs.push(parseExpr()) } - step(Token.BrOpen) + step(Token.BlockOpen) var cases = [] var conds = [] var guards = [] - while tok() != Token.BrClose { + while tok() != Token.BlockClose { step(Token.KCase) if tok() == Token.Underscore { conds.push(Node.Underscore) @@ -1207,7 +1207,7 @@ class Parser { cases.push(Node.Block(exs)) } - step(Token.BrClose) + step(Token.BlockClose) result = Node.Switch(exprs, conds, guards, cases) case KFor: @@ -1215,7 +1215,7 @@ class Parser { if tok() == Token.KLet or tok() == Token.KVar { i++ - var name = getgo(Token.LLower) + var name = getgo(Token.Identifier) // Approximately detect `for (let/var field of/in fields)` // TODO write tests for this with proper validation of err message @@ -1225,8 +1225,8 @@ class Parser { i++ var values = print() i++ - while tok() != Token.PClose and tok() != Token.Eof { - if tok() == Token.OpSub or tok() == Token.OpAdd { + while tok() != Token.CallClose and tok() != Token.Eof { + if tok() == Token.Subtract or tok() == Token.Add { values += ' ' + print() + ' ' } else { values += print() @@ -1242,31 +1242,31 @@ class Parser { // TODO `++i` `const i` `var i` `auto i` `int i` // TODO ensure `i` name is the same at all places var classic = true - if tok() == Token.OpAssign { + if tok() == Token.Assign { i++ } else { classic = false } var specialCase = '' - if tok() == Token.LInt, lex.value[i] == '0' { + if tok() == Token.Integer, lex.value[i] == '0' { i++ } else { // TODO always consume values, and compare iwht '0' // `Blah.blah().blah - 1 + 1` // TODO cache to `tok()` + `switch` while - tok() == Token.LLower || - tok() == Token.LUpper || - tok() == Token.POpen || - tok() == Token.PClose || - tok() == Token.OpAdd || - tok() == Token.OpSub || - tok() == Token.LInt || - tok() == Token.OpMult || + tok() == Token.Identifier || + tok() == Token.Title || + tok() == Token.CallOpen || + tok() == Token.CallClose || + tok() == Token.Add || + tok() == Token.Subtract || + tok() == Token.Integer || + tok() == Token.Multiply || tok() == Token.Dot { // `a-1` -> `a - 1` - if tok() == Token.OpSub or tok() == Token.OpAdd { + if tok() == Token.Subtract or tok() == Token.Add { specialCase += ' ' + print() + ' ' } else { specialCase += print() @@ -1281,16 +1281,16 @@ class Parser { classic = false } - if tok() == Token.LLower, lex.value[i] == name { + if tok() == Token.Identifier, lex.value[i] == name { i++ } else { classic = false } var lesserOrEqual = false - if tok() == Token.OpLt { + if tok() == Token.Less { i++ - } else if tok() == Token.OpLte { + } else if tok() == Token.LessOrEqual { i++ lesserOrEqual = true } else { @@ -1302,18 +1302,18 @@ class Parser { // `Blah.blah().blah - 1 + 1` while - tok() == Token.LLower || - tok() == Token.LUpper || - tok() == Token.POpen || // TODO align || in formatter - tok() == Token.PClose || - tok() == Token.OpAdd || - tok() == Token.OpSub || - tok() == Token.LInt || - tok() == Token.OpMult || + tok() == Token.Identifier || + tok() == Token.Title || + tok() == Token.CallOpen || // TODO align || in formatter + tok() == Token.CallClose || + tok() == Token.Add || + tok() == Token.Subtract || + tok() == Token.Integer || + tok() == Token.Multiply || tok() == Token.Dot { // `a-1` -> `a - 1` - if (tok() == Token.OpSub || tok() == Token.OpAdd) { + if (tok() == Token.Subtract || tok() == Token.Add) { values += ' ' + print() + ' ' } else { values += print() @@ -1332,9 +1332,9 @@ class Parser { } // `i++` - if tok() == Token.LLower, lex.value[i] == name { + if tok() == Token.Identifier, lex.value[i] == name { i++ - if (tok() == Token.OpIncrement) { + if (tok() == Token.Increment) { i++ } else { classic = false @@ -1342,9 +1342,9 @@ class Parser { } else // `++i` - if (tok() == Token.OpIncrement) { + if (tok() == Token.Increment) { i++ - if tok() == Token.LLower, lex.value[i] == name { + if tok() == Token.Identifier, lex.value[i] == name { i++ } else { classic = false @@ -1353,7 +1353,7 @@ class Parser { classic = false } - if (tok() == Token.PClose) {} else { + if (tok() == Token.CallClose) {} else { classic = false } @@ -1368,7 +1368,7 @@ class Parser { fail("Hexa has only `for \(name) in values` syntax") } - var name = getgo(Token.LLower) + var name = getgo(Token.Identifier) step(Token.KIn) var values = parseExpr() //let range = if (tok() == Token.Interval) { TODO not works! @@ -1413,7 +1413,7 @@ class Parser { project.data.set(result, new NodeData(lex.line[i], lex.column[i], lex.fileName)) switch tok() { case Eof: done = true - case BkOpen: + case IndexOpen: i++ var index = parseExpr() if tok() == Token.Comma { @@ -1423,12 +1423,12 @@ class Parser { "To create array here, wrap it with `{` as `{[\(Node.stringify(index)), ...]}` so it doesn't index previous expression." ) } - step(Token.BkClose) + step(Token.IndexClose) result = Node.Index(result, index) case KIs: i++ switch tok() { - case LUpper: + case Title: result = Node.Is(result, parseType()) case _: fail("Cannot parse type `" + Token.stringify(tok()) + "`") @@ -1436,7 +1436,7 @@ class Parser { case KAs: i++ var kind = tok() - if (tok() == Token.OpNot) { + if (tok() == Token.KNot) { i++ } else if (tok() == Token.Question) { i++ @@ -1444,15 +1444,15 @@ class Parser { kind = Token.Default } result = Node.As(result, kind, parseType()) - case POpen: { // call(a, name: b, c) + case CallOpen: { // call(a, name: b, c) var args: [Node] = [] var argNames: [String] = [] i++ - if tok() != Token.PClose { while true { + if tok() != Token.CallClose { while true { switch tok() { - case PClose: + case CallClose: fail("Unexpected `)`, remove trailing `,` comma") - case LUpper: + case Title: argNames.push(null) args.push(parseExpr()) if (tok() == Token.Colon) { @@ -1460,8 +1460,8 @@ class Parser { parseType() } case _: - if tok() == Token.LLower, offset(1) == Token.Colon { - argNames.push(getgo(Token.LLower)) + if tok() == Token.Identifier, offset(1) == Token.Colon { + argNames.push(getgo(Token.Identifier)) step(Token.Colon) } else { argNames.push(null) @@ -1474,38 +1474,38 @@ class Parser { break } }} - step(Token.PClose) + step(Token.CallClose) result = Node.Call(result, args, argNames) } - case OpArrow: + case RightArrow: next() result = Node.Arrow(parseExpr(), [result], null) // i++ - case OpIncrement: + case Increment: switch result { case Unop(_): return result } i++ - result = Node.Unop(Token.OpIncrement, true, result) + result = Node.Unop(Token.Increment, true, result) // i-- - case OpDecrement: + case Decrement: switch result { case Unop(_): return result } i++ - result = Node.Unop(Token.OpDecrement, true, result) + result = Node.Unop(Token.Decrement, true, result) // a.b // a.B case Dot: i++ switch tok() { - case LUpper: - result = Node.DotUpper(result, getgo(Token.LUpper)) + case Title: + result = Node.DotUpper(result, getgo(Token.Title)) case _: - result = Node.Dot(result, getgo(Token.LLower)) + result = Node.Dot(result, getgo(Token.Identifier)) } // a ? b : c @@ -1513,11 +1513,11 @@ class Parser { // a ?. b case Question: i++ - // TODO Probably keep just Token.OpChain? or only this? + // TODO Probably keep just Token.OptionalChain? or only this? // This way it allows to parse `a? .xx()? .yy()` on separate lines if tok() == Token.Dot { // a ?. b - var name = getgo(Token.LLower) + var name = getgo(Token.Identifier) // TODO Node.Chain() result = Node.Dot(result, name) } else if (tok() == Token.Question) { @@ -1532,7 +1532,7 @@ class Parser { result = Node.If([result], eif, eelse, true) } // a ?. b - case OpChain: + case OptionalChain: i++ // TODO result = parseExpr() @@ -1540,7 +1540,7 @@ class Parser { case _: let failAt = i let t = tok() - if isBinop(t) && offset(1) == Token.OpAssign { + if isBinop(t) && offset(1) == Token.Assign { var op = tok() i++ i++ @@ -1562,7 +1562,7 @@ class Parser { // a + b else if isBinop(t) { i++ - //if (tok() == OpAssign) { // += + //if (tok() == Assign) { // += // i++ //} var b = parseExpr() @@ -1584,7 +1584,7 @@ class Parser { } // Ternary operator has the lowest precedence case If(econd, eif, eelse, ternary): - if ternary == true && t != Token.OpAssign { + if ternary == true && t != Token.Assign { result = Node.If([Node.Binop(result, t, econd[0])], eif, eelse, true) } else { result = Node.Binop(result, t, b) @@ -1612,21 +1612,21 @@ class Parser { fun parseBlock(): Node { // TODO handle EOF - step(Token.BrOpen) - if tok() == Token.BrClose { // Empty block {} + step(Token.BlockOpen) + if tok() == Token.BlockClose { // Empty block {} i++ return Node.Block([]) } else if (tok() == Token.LLower && offset(1) == Token.Colon) { // Object { k:v } var names: [String] = [] var el: [Node] = [] while true { - names.push(getgo(Token.LLower)) + names.push(getgo(Token.Identifier)) step(Token.Colon) el.push(parseExpr()) if tok() == Token.Comma { // Object { k:v, ... } i++ - if tok() == Token.BrClose { // Object { k:v, } + if tok() == Token.BlockClose { // Object { k:v, } fail("Unexpected `}`, remove trailing `,` comma") } continue @@ -1634,17 +1634,17 @@ class Parser { break } } - if tok() != Token.BrClose { + if tok() != Token.BlockClose { fail('Expected `}` or `,` before `\(print())`') } - step(Token.BrClose) + step(Token.BlockClose) return Node.Object(names, el) } else { // Block { expr expr expr } var el = [] - while tok() != Token.BrClose { + while tok() != Token.BlockClose { el.push(parseExpr()) } - step(Token.BrClose) + step(Token.BlockClose) return Node.Block(el) } @@ -1659,7 +1659,7 @@ class Parser { /// Disallows `var $name` if tok() == Token.Query { i++ - let name = if tok() == Token.LLower { + let name = if tok() == Token.Identifier { print() } else { 'name' @@ -1669,9 +1669,9 @@ class Parser { } fun parseSingleVar(): Node { - // TODO proper error if LUpper `var cannot start with upper case letter` + // TODO proper error if Title `var cannot start with upper case letter` // ^ same for fun args - let varName = getgo(Token.LLower) + let varName = getgo(Token.Identifier) // TODO document this and actually use, beginning underscores too? if varName.endsWith("___") { fail("Variables can't end with `___`, it is reserved.") @@ -1692,7 +1692,7 @@ class Parser { if token == Token.At { // TODO soft parsing error i++ - let name = print() // TODO `getgo(Token.LLower)` + let name = print() // TODO `getgo(Token.Identifier)` // TODO `failHintAt` but consume decorator first with `parserDec` cause may have parameters i-- // TODO should know if it `var`, `let` or function argument @@ -1716,7 +1716,7 @@ class Parser { var expr = null - if tok() == Token.OpAssign { + if tok() == Token.Assign { i++ expr = parseExpr() } @@ -1732,28 +1732,28 @@ class Parser { i++ } // Enum type - path.push(getgo(Token.LUpper)) + path.push(getgo(Token.Title)) step(Token.Dot) // Enum exact tag - path.push(getgo(Token.LUpper)) + path.push(getgo(Token.Title)) // Enum bind vars or none - if (tok() == Token.POpen && offset(1) == Token.PClose) { + if (tok() == Token.CallOpen && offset(1) == Token.CallClose) { fail("Don't use empty parenthesis for `let " + path.join('.') + '()` bindings') } var bind = [] // Variables T(var, var, var) - if (tok() == Token.POpen) { + if (tok() == Token.CallOpen) { do { i++ if (tok() == Token.Underscore) { i++ bind.push(null) } else { - bind.push(Node.Var(getgo(Token.LLower), null, null, const, false)) + bind.push(Node.Var(getgo(Token.Identifier), null, null, const, false)) } } while tok() == Token.Comma - step(Token.PClose) + step(Token.CallClose) } - step(Token.OpAssign) + step(Token.Assign) var expr = parseExpr() return Node.EnumExtract(path, bind, expr) @@ -1769,17 +1769,17 @@ class Parser { var vars: [Node] = [] switch tok() { // var a - case LLower: + case Identifier: // TODO Refactor while true { - var varname = getgo(Token.LLower) + var varname = getgo(Token.Identifier) if (varname.endsWith("___")) { fail("Variables can't end with `___`, it is reserved.") } var type = null if (tok() == Token.Colon) { i++ type = parseType() } - var expr = null if (tok() == Token.OpAssign) { i++ expr = parseExpr() } + var expr = null if (tok() == Token.Assign) { i++ expr = parseExpr() } vars.push(Node.Var(varname, type, expr, const, external)) if (tok() == Token.Comma && offset(1) == Token.LLower && (offset(2) == Token.OpAssign || offset(2) == Token.Colon)) { @@ -1789,20 +1789,20 @@ class Parser { } } // var Left - case LUpper: + case Title: // if (offset(1) != Dot) { // fail("Please use lowercase for variable") // } // // var Left.B // else { - var left = Node.Ident(getgo(Token.LUpper), null) + var left = Node.Ident(getgo(Token.Title), null) var res = left // var Left.B.C.D... while tok() == Token.Dot { - res = Node.Dot(res, getgo(Token.LUpper)) + res = Node.Dot(res, getgo(Token.Title)) } switch tok() { - case POpen: + case CallOpen: var args: [String] = [] while tok() != Token.PClose { args.push(getgo(Token.LLower)) @@ -1834,19 +1834,19 @@ class Parser { let params: [NodeType] = null // TODO must error var params: [NodeType]? = null switch tok() { - case LLower: - name = getgo(Token.LLower) - if tok() == Token.OpLt { + case Identifier: + name = getgo(Token.Identifier) + if tok() == Token.Less { i++ let param: [NodeType] = [parseType()] while tok() == Token.Comma { i++ param.push(parseType()) } - step(Token.OpGt) + step(Token.Greater) params = param } - case LUpper: + case Title: fail("Function names can't start with uppercase `\(print())` -> `\(Hint.toCamelCase(print()))`") case _: } @@ -1856,9 +1856,9 @@ class Parser { } var variadic = false - step(Token.POpen) + step(Token.CallOpen) { - if tok() != Token.PClose { + if tok() != Token.CallClose { // TODO indent properly while true { let decorators: [Decorator] = [] @@ -1874,7 +1874,7 @@ class Parser { i++ } pos.push({ line: lex.line[i], column: lex.column[i] }) - let name = getgo(Token.LLower) + let name = getgo(Token.Identifier) if tok() == Token.Colon { // TODO no colon i++ @@ -1887,7 +1887,7 @@ class Parser { // TODO well `tok()` could be just `@inline let token { get }` or something if tok() == Token.At { i++ - let decName = print() // TODO `getgo(Token.LLower)` + `failHintAt` + let decName = print() // TODO `getgo(Token.Identifier)` + `failHintAt` i-- fail("Decorators should be placed at argument's declaration like `@\(decName) \(name)` instead of their type.") } @@ -1896,18 +1896,18 @@ class Parser { if // Note: don't do it like that! Some type syntax uses lowercases - //tok() != Token.OpAssign and - //tok() != Token.PClose and + //tok() != Token.Assign and + //tok() != Token.CallClose and //tok() != Token.Comma - token == Token.BkOpen or - token == Token.BrOpen or - token == Token.LUpper or - token == Token.POpen + token == Token.IndexOpen or + token == Token.BlockOpen or + token == Token.Title or + token == Token.CallOpen { t = parseType() } - if tok() == Token.OpAssign { + if tok() == Token.Assign { i++ expr = parseExpr() } @@ -1924,7 +1924,7 @@ class Parser { break } }} - step(Token.PClose) + step(Token.CallClose) } // TODO `retType` var rettype = null @@ -1935,20 +1935,20 @@ class Parser { } else { let token = tok() if - token == Token.BkOpen or - // TODO `{` conflicts with no-retType `token == Token.BrOpen or` - token == Token.LUpper or - token == Token.POpen + token == Token.IndexOpen or + // TODO `{` conflicts with no-retType `token == Token.BlockOpen or` + token == Token.Title or + token == Token.CallOpen { rettype = parseType() } } if parseBody { switch tok() { - case KNew: if (offset(1) == Token.POpen) {} else { + case KNew: if (offset(1) == Token.CallOpen) {} else { expr = parseExpr() } - case BrClose: {} + case BlockClose: {} case KStatic: {} case KPrivate: {} case KFun: {} @@ -1961,7 +1961,7 @@ class Parser { } if ( - tok() != Token.BrClose + tok() != Token.BlockClose && tok() != Token.KStatic && tok() != Token.KPrivate && tok() != Token.KFun @@ -2040,9 +2040,9 @@ class Parser { impl.push(parseType()) } - step(Token.BrOpen) + step(Token.BlockOpen) var fields = parseFields() - step(Token.BrClose) + step(Token.BlockClose) var me = Node.Class(t, ext, impl, fields, external ?? false, isInterface) classExternal = _classExternal return me @@ -2147,11 +2147,11 @@ class Parser { fun parseDecorator() { i++ - let name = getgo(Token.LLower) + let name = getgo(Token.Identifier) let values = [] - if tok() == Token.POpen { + if tok() == Token.CallOpen { i++ - if tok() != Token.PClose { while true { + if tok() != Token.CallClose { while true { values.push(parseExpr()) if tok() == Token.Comma { i++ @@ -2159,7 +2159,7 @@ class Parser { break } }} - step(Token.PClose) + step(Token.CallClose) } return new Decorator(name, values) } @@ -2186,32 +2186,32 @@ class Parser { fun parseUndecoratedType(): NodeType { // Plain values - if tok() == Token.LLower { + if tok() == Token.Identifier { // TODO `ident` maybe? - return NodeType.Ident(getgo(Token.LLower)) + return NodeType.Ident(getgo(Token.Identifier)) } - if tok() == Token.LInt { - return NodeType.Int(parseInt(getgo(Token.LInt))) + if tok() == Token.Integer { + return NodeType.Int(parseInt(getgo(Token.Integer))) } - if tok() == Token.LFloat { - return NodeType.Float(parseFloat(getgo(Token.LFloat))) + if tok() == Token.FloatingPoint { + return NodeType.Float(parseFloat(getgo(Token.FloatingPoint))) } - if tok() == Token.LString { - return NodeType.String(getgo(Token.LString)) + if tok() == Token.QuotedString { + return NodeType.String(getgo(Token.QuotedString)) } - if (tok() == Token.LLower && offset(1) == Token.POpen) { - let name = getgo(Token.LLower) + if (tok() == Token.Identifier && offset(1) == Token.CallOpen) { + let name = getgo(Token.Identifier) i++ let argNames: [String] = [] let values: [NodeType] = [] - while tok() != Token.PClose { - if tok() == Token.LLower { - argNames.push(getgo(Token.LLower)) + while tok() != Token.CallClose { + if tok() == Token.Identifier { + argNames.push(getgo(Token.Identifier)) step(Token.Colon) } else { argNames.push(null) @@ -2219,14 +2219,14 @@ class Parser { values.push(parseType()) - if tok() == Token.PClose { + if tok() == Token.CallClose { break } else { step(Token.Comma) } } - step(Token.PClose) + step(Token.CallClose) return NodeType.Call(name, argNames, values) } @@ -2263,9 +2263,9 @@ class Parser { } switch parametricTypeNestingToken { - case OpGt: parametricTypeNesting-=1 parametricTypeNestingToken = Token.Eof i++ - case OpShr: parametricTypeNesting-=1 parametricTypeNestingToken = Token.OpGt - case OpUShr: parametricTypeNesting-=1 parametricTypeNestingToken = Token.OpUShr + case Greater: parametricTypeNesting-=1 parametricTypeNestingToken = Token.Eof i++ + case BitwiseRightShift: parametricTypeNesting-=1 parametricTypeNestingToken = Token.Greater + case UnsignedRightShift: parametricTypeNesting-=1 parametricTypeNestingToken = Token.UnsignedRightShift case _: unexpected() } if (parametricTypeNesting < 0) { @@ -2278,14 +2278,14 @@ class Parser { } // `A => B` - if tok() == Token.OpArrow { + if tok() == Token.RightArrow { i++ sresult = NodeType.Function([sresult], parseType()) } result = sresult // `[Type]` - case BkOpen: + case IndexOpen: if path != null { fail("This path syntax is incorrect") } @@ -2293,12 +2293,12 @@ class Parser { i++ var res: NodeType? = null switch tok() { - case BkClose: + case IndexClose: i++ res = NodeType.ParametricType("Array", [NodeType.Object([], [])], path) case Colon: i++ - if (tok() == Token.BkClose) { + if (tok() == Token.IndexClose) { i++ res = NodeType.ParametricType("Map", [NodeType.Object([], []), NodeType.Object([], [])], path) } @@ -2313,8 +2313,8 @@ class Parser { } else { NodeType.ParametricType("Array", [key], path) } - step(Token.BkClose) - if (tok() == Token.OpArrow) { + step(Token.IndexClose) + if (tok() == Token.RightArrow) { i++ innerRes = NodeType.Function([res], parseType()) } @@ -2322,13 +2322,13 @@ class Parser { } result = res // `{}` - case BrOpen: + case BlockOpen: if path != null { fail("This path syntax is incorrect") } i++ - var sresult = if (tok() == Token.BrClose) { // Empty + var sresult = if (tok() == Token.BlockClose) { // Empty NodeType.Object([], []) } else { var names: [String] = [] @@ -2345,23 +2345,23 @@ class Parser { } NodeType.Object(names, types) } - step(Token.BrClose) - if (tok() == Token.OpArrow) { + step(Token.BlockClose) + if (tok() == Token.RightArrow) { i++ sresult = NodeType.Function([sresult], parseType()) } result = sresult // `()` - case POpen: + case CallOpen: if path != null { fail("This path syntax is incorrect") } i++ var args = [] - while tok() != Token.PClose { - if tok() == Token.LLower { - step(Token.LLower) + while tok() != Token.CallClose { + if tok() == Token.Identifier { + step(Token.Identifier) if tok() == Token.Colon { // TODO colon-less i++ @@ -2372,13 +2372,13 @@ class Parser { i++ } } - step(Token.PClose) - step(Token.OpArrow) + step(Token.CallClose) + step(Token.RightArrow) result = NodeType.Function(args, parseType()) // var.invar // var.Invar - case LLower: + case Identifier: var res: NodeType? = null switch offset(1) { case Colon: { @@ -2407,7 +2407,7 @@ class Parser { while tok() == Token.Question { i++ } - if tok() == Token.OpArrow { + if tok() == Token.RightArrow { i++ result = NodeType.Function([result], parseType()) } @@ -2439,6 +2439,7 @@ class Parser { case OpBoolAnd: return 7 + left case OpBoolOr: return 8 + left case OpAssign: return 10 + right + //case Question: return 100 + right case _: // TODO proper message, this is internal error fail("No precedence for " + Token.stringify(op)) } @@ -2467,6 +2468,7 @@ class Parser { case OpShr: return true case OpUShr: return true case OpMod: return true + //case Question: return true case _: return false } } diff --git a/source/compiler/preprocessor.hexa b/source/compiler/preprocessor.hexa index 79c9f78..a6cc12b 100644 --- a/source/compiler/preprocessor.hexa +++ b/source/compiler/preprocessor.hexa @@ -65,7 +65,7 @@ class Preprocessor { let token: Token = tokens.token[i] as! Token switch token { - case LLower: // Ok + case Identifier: // Ok case KFalse: defName = 'false' case KTrue: defName = 'true' case _: fail(tokens, i, 'Incorrect `#if` syntax') @@ -108,7 +108,7 @@ class Preprocessor { } state = states[states.length - 1] // `#end` - } else if token == Token.LLower, tokens.value[i] == 'end' { + } else if token == Token.Identifier, tokens.value[i] == 'end' { i++ // `end` switch state { case ConsumeUntilElseOrEnd: @@ -132,7 +132,7 @@ class Preprocessor { fail(tokens, i - 1, 'Unexpected `#else`') } states[states.length - 1] = state - } else if token == Token.LLower, tokens.value[i] == 'error' { + } else if token == Token.Identifier, tokens.value[i] == 'error' { i++ // `error` let message = tokens.value[i] i++ // `"message"` diff --git a/source/compiler/typer.hexa b/source/compiler/typer.hexa index f3e8d82..373bdb3 100644 --- a/source/compiler/typer.hexa +++ b/source/compiler/typer.hexa @@ -947,7 +947,7 @@ class Typer { // `enum E { A = b }` // TODO allow BigInt as possible enum value type? case Binop(a, op, b): - if op != Token.OpAssign { + if op != Token.Assign { fail('Should use `Value = value` not `Value \(Token.stringify(op)) value` format for enum tag value', node) } if valuesType == null { @@ -2225,7 +2225,7 @@ class Typer { var parent: Node = parents.get(a) // `a = b` // TODO separate assign node? or just nested pattern... - if op == Token.OpAssign { switch parent { + if op == Token.Assign { switch parent { case null: switch a { case Ident(_): // Ok @@ -2437,8 +2437,7 @@ class Typer { switch e { case Binop(a, op, b): - // TODO rename OpEq to Equal, also confusion with OpAssign - if op == Token.OpAssign { + if op == Token.Assign { fail('Probably change `=` to `==` here. `if value` expects `value` only of type `Bool`, but got `' + Type.stringify(type) + '`', e) } } @@ -2531,7 +2530,7 @@ class Typer { } // Expressions return values - fun fillExpression(node: Node) { + fun fillExpression(node Node) { switch node { // Just `name` case Ident(name, params): @@ -3329,13 +3328,13 @@ class Typer { inferNullType(typeB, typeA, b) // Anything + String = String - if op == Token.OpAdd, typeA == typeString { + if op == Token.Add, typeA == typeString { types.set(node, typeString) } else if ( - op == Token.OpBoolAnd or op == Token.OpBoolOr + op == Token.LogicalAnd or op == Token.LogicalOr ) { - let token = op == Token.OpBoolAnd? 'and' : 'or' + let token = op == Token.LogicalAnd? 'and' : 'or' if not booleanable(typeA) { fail('Operator `a \(token) b` takes `a` of type `Bool` only, but got `\(Type.stringify(typeA))`', a) } @@ -3347,14 +3346,14 @@ class Typer { } else // Always provides boolean result if (typeA == typeB) and (typeA == typeString) and ( - (op == Token.OpGte) || - (op == Token.OpGt) || - (op == Token.OpLte) || - (op == Token.OpLt) + (op == Token.GreaterOrEqual) || + (op == Token.Greater) || + (op == Token.LessOrEqual) || + (op == Token.Less) ) { types.set(node, typeBool) } else - if op == Token.OpEq or op == Token.OpNotEq { + if op == Token.Equal or op == Token.Unequal { let booleans = (typeA == typeB) and (typeA == typeBool) let comparable = areComparable(typeA, typeB) let covariant = areCovariant(typeA, typeB) @@ -3408,12 +3407,12 @@ class Typer { types.set(node, typeBool) } else if ( - (op == Token.OpGte) || - (op == Token.OpGt) || - (op == Token.OpLte) || - (op == Token.OpLt) || - (op == Token.OpEq) || - (op == Token.OpNotEq) + (op == Token.GreaterOrEqual) || + (op == Token.Greater) || + (op == Token.LessOrEqual) || + (op == Token.Less) || + (op == Token.Equal) || + (op == Token.Unequal) ) { let comparable = areComparable(typeA, typeB) @@ -3467,7 +3466,7 @@ class Typer { fillExpression(e) // Always provides boolean result - if op == Token.OpNot, !postfix { + if op == Token.KNot, !postfix { let type = types.get(e) if not booleanable(type) { fail('Operator `not value` takes `value` of type `Bool` only, but got `\(Type.stringify(types.get(e)))`', e) @@ -3478,7 +3477,7 @@ class Typer { types.set(node, types.get(e)) - let mutates = op == Token.OpIncrement || op == Token.OpDecrement + let mutates = op == Token.Increment || op == Token.Decrement // TODO ^ must allow only integer ++/-- if not mutates { return } var parent: Node = parents.get(e) @@ -3516,7 +3515,7 @@ class Typer { // TODO maybe require `a[i] == a[i] ?? n + 1` instead? // cause also string/etc arrays... case _: - let ops = Token.OpIncrement? 'increment `++`' : 'decrement `--`' + let ops = Token.Increment? 'increment `++`' : 'decrement `--`' fail('You can \(ops) only a variable, .field or array element', node) // TODO `x={i:123} x.i++` <- by type and const factor } @@ -3609,11 +3608,11 @@ class Typer { // `a op= b` case AssignOp(a, op, b): // `===` - if op == Token.OpEq { + if op == Token.Equal { fail('`===` not supported, use `strictEqual`', node) } // `!==` - if op == Token.OpEq { + if op == Token.Equal { fail('`!==` not supported, use `strictNotEqual`', node) } @@ -4081,7 +4080,7 @@ class Typer { } } case Binop(a, op, b): - if op == Token.OpOr { + if op == Token.BitwiseOr { follow(a) follow(b) } else { diff --git a/source/data/token.hexa b/source/data/token.hexa index 6416e3f..640251e 100644 --- a/source/data/token.hexa +++ b/source/data/token.hexa @@ -58,7 +58,7 @@ enum Token : Int { KIs = 57 KAs = 58 KSuper = 59 - KNot = 89 + KNot = 89 // TODO rename to LogicalNot // Literals FloatingPoint = 60 // 0.123 @@ -95,12 +95,14 @@ enum Token : Int { BitwiseNot = 88 // ~ // Binary + // TODO Note: Op- prefix for names used cause in IR binary operators use Token as operator kind + // ^ somehow create an enum subset of them? RightArrow = 90 // => Assign = 91 // = - LogicalAnd = 92 // && - LogicalOr = 93 // || + LogicalAnd = 92 // && TODO `and`, detect && in parser and propose replacement + LSP API quickFix=span,text + LogicalOr = 93 // || TODO `or` Divide = 94 // / - Equal = 95 // == TODO // com as /// doc for all values + Equal = 95 // == TODO `// comment` as `/// doc` for all Token.Tag values Greater = 96 // > GreaterOrEqual = 97 // >= Less = 99 // < @@ -120,18 +122,20 @@ enum Token : Int { IntegerDivide = 110 // \ Add = 111 // + BitwiseAnd = 112 // & - OptionalChain = 113 // ?. + OptionalChain = 113 // ?. TODO ts allows `a? \n .b`? + // TODO `??` static fun stringify(token: Token, param: String? = null): String { // TODO should be `return switch` cause *no* final return (must be detected for non-exhaustive cases) + // TODO generate this stringify with macro switch token { case At: return "@" case Query: return "$" case Sharp: return "#" - case BkClose: return "]" - case BkOpen: return "[" - case BrClose: return "}" - case BrOpen: return "{" + case IndexClose: return "]" + case IndexOpen: return "[" + case BlockClose: return "}" + case BlockOpen: return "{" case Colon: return ":" case Comma: return "," case Default: return "" @@ -171,45 +175,45 @@ enum Token : Int { case KTry: return "try" case KVar: return "var" case KWhile: return "while" - case OpAdd: return "+" - case OpAnd: return "&" - case OpArrow: return "=>" - case OpAssign: return "=" - case OpBoolAnd: return "&&" // We keep it like this (not `and`) to simplify JavaScript generation - case OpBoolOr: return "||" - case OpDecrement: return "--" - case OpDiv: return "/" - case OpEq: return "==" - case OpGt: return ">" - case OpGte: return ">=" - case OpIncrement: return "++" - case OpIntDiv: return "\\" - case OpLt: return "<" - case OpLte: return "<=" - case OpMod: return "%" - case OpMult: return "*" - case OpNegBits: return "~" - case OpNot: return "!" - case OpNotEq: return "!=" - case OpOr: return "|" - case OpShl: return "<<" - case OpShr: return ">>" - case OpSub: return "-" - case OpUShr: return ">>>" - case OpXor: return "^" - case OpChain: return "?." - case PClose: return ")" - case POpen: return "(" + case Add: return "+" + case BitwiseAnd: return "&" + case RightArrow: return "=>" + case Assign: return "=" + case LogicalAnd: return "&&" // We keep it like this (not `and`) to simplify JavaScript generation + case LogicalOr: return "||" + case Decrement: return "--" + case Divide: return "/" + case Equal: return "==" + case Greater: return ">" + case GreaterOrEqual: return ">=" + case Increment: return "++" + case IntegerDivide: return "\\" + case Less: return "<" + case LessOrEqual: return "<=" + case Remainder: return "%" + case Multiply: return "*" + case BitwiseNot: return "~" + case KNot: return "!" + case Unequal: return "!=" + case BitwiseOr: return "|" + case BitwiseLeftShift: return "<<" + case BitwiseRightShift: return ">>" + case Subtract: return "-" + case UnsignedRightShift: return ">>>" + case BitwiseXor: return "^" + case OptionalChain: return "?." + case CallClose: return ")" + case CallOpen: return "(" case Question: return "?" case Semicolon: return ";" case Underscore: return "_" - case LUpper: return (param == null)? 'TitleCase' : param - case LString: return (param == null)? 'string' : '\'\(param)\'' - case LBacktick: return (param == null)? 'backtick' : '`\(param)`' - case LLower: return (param == null)? 'identifier' : param - case LInt: return (param == null)? 'integer' : param - case LFloat: return (param == null)? 'float' : param - case LDoc: return '///\(param)' + case Title: return (param == null)? 'TitleCase' : param + case QuotedString: return (param == null)? 'string' : '\'\(param)\'' + case Backtick: return (param == null)? 'backtick' : '`\(param)`' + case Identifier: return (param == null)? 'identifier' : param + case Integer: return (param == null)? 'integer' : param + case FloatingPoint: return (param == null)? 'float' : param + case Documentation: return '///\(param)' case KIs: return "is" } } diff --git a/source/data/types.hexa b/source/data/types.hexa index d1936ae..b52f704 100644 --- a/source/data/types.hexa +++ b/source/data/types.hexa @@ -88,7 +88,7 @@ enum Type { return DataHelper.nameOf(type.parent) case EnumInstance(type, tag): // TODO return DataHelper.nameOf(type.parent) + '.' + tag - // TODO print tag name `Token.OpNot` + // TODO print tag name `Token.KNot` let tagName = type.fieldNames[tag] return DataHelper.nameOf(type.parent) + '.' + tagName case Unknown(handle): diff --git a/source/targets/genC.hexa b/source/targets/genC.hexa index c4fd812..d6b6563 100644 --- a/source/targets/genC.hexa +++ b/source/targets/genC.hexa @@ -1293,7 +1293,7 @@ class GenCxx { case Assignop(a, op, value): // TODO rename to AssignOp - if getType(a) == project.typer.typeString, op == Token.OpAdd { + if getType(a) == project.typer.typeString, op == Token.Add { switch a { case Index(expr, index): { // Cache `index` to not re-evaluate twice @@ -2524,18 +2524,18 @@ class GenCxx { return '(' + printExpression(e) + ')' case Binop(a, op, b): - if op == Token.OpEq or op == Token.OpNotEq { + if op == Token.Equal or op == Token.Unequal { // TODO commutative if let aType = getType(a), aType == project.typer.typeNullUInt64 { - return (op == Token.OpNotEq? '!' : '') + 'equalNull$UInt64(' + printExpression(a) + ', ' + printTypeConverter(printExpression(b), getType(b), aType) + ')' + return (op == Token.Unequal? '!' : '') + 'equalNull$UInt64(' + printExpression(a) + ', ' + printTypeConverter(printExpression(b), getType(b), aType) + ')' } if let aType = getType(a), aType == project.typer.typeNullFloat32 { - return (op == Token.OpNotEq? '!' : '') + 'equalNull$Float32(' + printExpression(a) + ', ' + printTypeConverter(printExpression(b), getType(b), aType) + ')' + return (op == Token.Unequal? '!' : '') + 'equalNull$Float32(' + printExpression(a) + ', ' + printTypeConverter(printExpression(b), getType(b), aType) + ')' } } - if op == Token.OpAdd { + if op == Token.Add { if getType(a) == project.typer.typeString { return 'String_$combine_(' + printExpression(a) + ', ' + printTypeConverter(printExpression(b), getType(b), project.typer.typeString) + ')' } @@ -2545,7 +2545,7 @@ class GenCxx { } } - let token = op == Token.OpIntDiv? '/' : Token.stringify(op) + let token = op == Token.IntegerDivide? '/' : Token.stringify(op) return printExpression(a) + ' ' + token + ' ' + printExpression(b) case Index(expr, index): diff --git a/source/targets/genCxx.hexa b/source/targets/genCxx.hexa index 4ef79d0..65d960c 100644 --- a/source/targets/genCxx.hexa +++ b/source/targets/genCxx.hexa @@ -174,9 +174,9 @@ types.push('struct ' + c.name + '_ {\n\t') for v in c.vars { switch v { - case NiceStatement.Var(name, expr, type): + case Var(name, expr, type): types.push('\n\t\(stringifyType(type)) ' + name + '_;') - case NiceStatement.Const(name, expr, type): + case Const(name, expr, type): types.push('\n\t\(stringifyType(type)) ' + name + '_;') }} diff --git a/source/targets/genNasm.hexa b/source/targets/genNasm.hexa index 81b7f41..4187e36 100644 --- a/source/targets/genNasm.hexa +++ b/source/targets/genNasm.hexa @@ -1200,7 +1200,7 @@ class GenNasm { } switch op { - case OpAdd: { + case Add: { if asC { return aResult + ' + ' + bResult } @@ -1211,32 +1211,32 @@ class GenNasm { return temp } - case OpEq | OpNotEq | OpGt | OpGte | OpLt | OpLte: { + case Equal | Unequal | Greater | GreaterOrEqual | Less | LessOrEqual: { let temp = 'R10' // We do NOT modify left operand! functionNow.body.push('cmp \(aResult), ' + bResult) switch op { - case OpEq: { + case Equal: { functionNow.body.push('sete al') } - case OpNotEq: { + case Unequal: { functionNow.body.push('setne al') } // TODO `if a > b` can be optimized to `jle` - case OpGt: { + case Greater: { functionNow.body.push('setg al') } - case OpGte: { + case GreaterOrEqual: { functionNow.body.push('setge al') } - case OpLt: { + case Less: { functionNow.body.push('setl al') } - case OpLte: { + case LessOrEqual: { functionNow.body.push('setle al') } } diff --git a/source/tests/testLexer.hexa b/source/tests/testLexer.hexa index 0c1b781..3fe62f3 100644 --- a/source/tests/testLexer.hexa +++ b/source/tests/testLexer.hexa @@ -1,5 +1,5 @@ // The Hexa Compiler -// Copyright (C) 2021-2022 Oleh Petrenko +// Copyright (C) 2021-2023 Oleh Petrenko // Copyright (C) 2018 Bogdan Danylchenko // // This program is free software: you can redistribute it and/or modify @@ -22,9 +22,9 @@ class TestLexer { compare("#!/bin", [], []) compare("#!/bin\n", [], []) compare("#!/bin\n//", [], []) - compare("#!/bin\r\n123", [Token.LInt], ["123"]) // May not work on Linux because of \r if no `bin\r` file exist - compare("#!/bin\n\r123", [Token.LInt], ["123"]) // \r is completely ignored by lexer - compare("#!/bin\n123", [Token.LInt], ["123"]) + compare("#!/bin\r\n123", [Token.Integer], ["123"]) // May not work on Linux because of \r if no `bin\r` file exist + compare("#!/bin\n\r123", [Token.Integer], ["123"]) // \r is completely ignored by lexer + compare("#!/bin\n123", [Token.Integer], ["123"]) // Empty (whitespace, comments) compare("", [], []) @@ -43,76 +43,76 @@ class TestLexer { compare("//\n", [], []) // Int - compare("123", [Token.LInt], ["123"]) - compare(" 0 ", [Token.LInt], ["0"]) - compare("0", [Token.LInt], ["0"]) - compare("1 2 3", [Token.LInt, Token.LInt, Token.LInt], ["1", "2", "3"]) - compare("0x1", [Token.LInt], ["0x1"]) - compare("0x0", [Token.LInt], ["0x0"]) - compare("0xF", [Token.LInt], ["0xF"]) - compare("0xFA", [Token.LInt], ["0xFA"]) - compare("0xFABCDEF", [Token.LInt], ["0xFABCDEF"]) - compare("0x1F2A3B4C5D6E7F0", [Token.LInt], ["0x1F2A3B4C5D6E7F0"]) - compare("-123", [Token.OpSub, Token.LInt], ["-", "123"]) - compare("+123", [Token.OpAdd, Token.LInt], ["+", "123"]) + compare("123", [Token.Integer], ["123"]) + compare(" 0 ", [Token.Integer], ["0"]) + compare("0", [Token.Integer], ["0"]) + compare("1 2 3", [Token.Integer, Token.Integer, Token.Integer], ["1", "2", "3"]) + compare("0x1", [Token.Integer], ["0x1"]) + compare("0x0", [Token.Integer], ["0x0"]) + compare("0xF", [Token.Integer], ["0xF"]) + compare("0xFA", [Token.Integer], ["0xFA"]) + compare("0xFABCDEF", [Token.Integer], ["0xFABCDEF"]) + compare("0x1F2A3B4C5D6E7F0", [Token.Integer], ["0x1F2A3B4C5D6E7F0"]) + compare("-123", [Token.Subtract, Token.Integer], ["-", "123"]) + compare("+123", [Token.Add, Token.Integer], ["+", "123"]) // Float - compare("0.123", [Token.LFloat], ["0.123"]) - compare("0.0", [Token.LFloat], ["0.0"]) - compare("0.0e+1", [Token.LFloat], ["0.0e+1"]) - compare("0.0E-1", [Token.LFloat], ["0.0E-1"]) - compare("0E-123", [Token.LFloat], ["0E-123"]) - compare("123e123", [Token.LFloat], ["123e123"]) - compare("1 2.0 3", [Token.LInt, Token.LFloat, Token.LInt], ["1", "2.0", "3"]) + compare("0.123", [Token.FloatingPoint], ["0.123"]) + compare("0.0", [Token.FloatingPoint], ["0.0"]) + compare("0.0e+1", [Token.FloatingPoint], ["0.0e+1"]) + compare("0.0E-1", [Token.FloatingPoint], ["0.0E-1"]) + compare("0E-123", [Token.FloatingPoint], ["0E-123"]) + compare("123e123", [Token.FloatingPoint], ["123e123"]) + compare("1 2.0 3", [Token.Integer, Token.FloatingPoint, Token.Integer], ["1", "2.0", "3"]) // String - compare("'s'", [Token.LString], ["'s'"]) - compare('"s"', [Token.LString], ["'s'"]) // TODO Quotes `"` and `'` should be differentiated for pretty printing - compare("\"s\"", [Token.LString], ["'s'"]) // TODO All `\*` should be kept for pretty printing - compare("`s`", [Token.LBacktick], ["`s`"]) - compare("`aaa bbb``ccc` `ddd`", [Token.LBacktick, Token.LBacktick], ["`aaa bbbccc`", "`ddd`"]) - compare("````````", [Token.LBacktick], ["``"]) - compare("``", [Token.LBacktick], ["``"]) - compare("``\n", [Token.LBacktick], ["``"]) - compare("``\n``", [Token.LBacktick, Token.LBacktick], ["``", "``"]) + compare("'s'", [Token.QuotedString], ["'s'"]) + compare('"s"', [Token.QuotedString], ["'s'"]) // TODO Quotes `"` and `'` should be differentiated for pretty printing + compare("\"s\"", [Token.QuotedString], ["'s'"]) // TODO All `\*` should be kept for pretty printing + compare("`s`", [Token.Backtick], ["`s`"]) + compare("`aaa bbb``ccc` `ddd`", [Token.Backtick, Token.Backtick], ["`aaa bbbccc`", "`ddd`"]) + compare("````````", [Token.Backtick], ["``"]) + compare("``", [Token.Backtick], ["``"]) + compare("``\n", [Token.Backtick], ["``"]) + compare("``\n``", [Token.Backtick, Token.Backtick], ["``", "``"]) // String consistency - repair \r\n to \n for non-raw ones - compare("'s\n'", [Token.LString], ["'s\n'"]) // ' - compare("'s\r\n'", [Token.LString], ["'s\n'"]) // ' + compare("'s\n'", [Token.QuotedString], ["'s\n'"]) // ' + compare("'s\r\n'", [Token.QuotedString], ["'s\n'"]) // ' compare("'s - s'", [Token.LString], ["'s\n\t\t\ts'"]) // Test on actual code - compare('"s\n"', [Token.LString], ["'s\n'"]) // " - compare('"s\r\n"', [Token.LString], ["'s\n'"]) // " - compare('"\r\ns\r\n"', [Token.LString], ["'\ns\n'"]) // " - compare("`s\n`", [Token.LBacktick], ["`s\n`"]) // ` - compare("`s\r\n`", [Token.LBacktick], ["`s\n`"]) // ` - compare("`s\r`", [Token.LBacktick], ["`s\r`"]) // ` - compare("`s\\r`", [Token.LBacktick], ["`s\\r`"]) // ` + s'", [Token.QuotedString], ["'s\n\t\t\ts'"]) // Test on actual code + compare('"s\n"', [Token.QuotedString], ["'s\n'"]) // " + compare('"s\r\n"', [Token.QuotedString], ["'s\n'"]) // " + compare('"\r\ns\r\n"', [Token.QuotedString], ["'\ns\n'"]) // " + compare("`s\n`", [Token.Backtick], ["`s\n`"]) // ` + compare("`s\r\n`", [Token.Backtick], ["`s\n`"]) // ` + compare("`s\r`", [Token.Backtick], ["`s\r`"]) // ` + compare("`s\\r`", [Token.Backtick], ["`s\\r`"]) // ` // Identifiers - compare("T", [Token.LUpper], ["T"]) - compare("T val", [Token.LUpper, Token.LLower], ["T", "val"]) - compare("T val Type", [Token.LUpper, Token.LLower, Token.LUpper], ["T", "val", "Type"]) - compare("_T", [Token.LLower], ["_T"]) - compare("v", [Token.LLower], ["v"]) - compare("_v", [Token.LLower], ["_v"]) - compare("_123", [Token.LLower], ["_123"]) + compare("T", [Token.Title], ["T"]) + compare("T val", [Token.Title, Token.Identifier], ["T", "val"]) + compare("T val Type", [Token.Title, Token.Identifier, Token.Title], ["T", "val", "Type"]) + compare("_T", [Token.Identifier], ["_T"]) + compare("v", [Token.Identifier], ["v"]) + compare("_v", [Token.Identifier], ["_v"]) + compare("_123", [Token.Identifier], ["_123"]) // Multichar tokens - compare(" } ", [Token.BrClose], ["}"]) - compare("==", [Token.OpEq], ["=="]) - compare("===", [Token.OpEq, Token.OpAssign], ["==", "="]) - compare("== =", [Token.OpEq, Token.OpAssign], ["==", "="]) - compare("= ==", [Token.OpAssign, Token.OpEq], ["=", "=="]) - compare("=====", [Token.OpEq, Token.OpEq, Token.OpAssign], ["==", "==", "="]) + compare(" } ", [Token.BlockClose], ["}"]) + compare("==", [Token.Equal], ["=="]) + compare("===", [Token.Equal, Token.Assign], ["==", "="]) + compare("== =", [Token.Equal, Token.Assign], ["==", "="]) + compare("= ==", [Token.Assign, Token.Equal], ["=", "=="]) + compare("=====", [Token.Equal, Token.Equal, Token.Assign], ["==", "==", "="]) compare( "> >> >>> . .. ...", - [Token.OpGt, Token.OpShr, Token.OpUShr, Token.Dot, Token.Dot, Token.Dot, Token.Interval], + [Token.Greater, Token.BitwiseRightShift, Token.UnsignedRightShift, Token.Dot, Token.Dot, Token.Dot, Token.Interval], [">", ">>", ">>>", ".", ".", ".", "..."] ) compare( ">>>>>>.......", - [Token.OpUShr, Token.OpUShr, Token.Interval, Token.Interval, Token.Dot], + [Token.UnsignedRightShift, Token.UnsignedRightShift, Token.Interval, Token.Interval, Token.Dot], [">>>", ">>>", "...", "...", "."] ) diff --git a/source/tests/testParser.hexa b/source/tests/testParser.hexa index 0de03ce..7d9661b 100644 --- a/source/tests/testParser.hexa +++ b/source/tests/testParser.hexa @@ -14,12 +14,13 @@ // You should have received a copy of the GNU Lesser General Public License // along with this program. If not, see . +// TODO include into project only when target is debug one in hexa.json class TestParser { static fun test() { - console.log("TestParser begin") + console.log("TestParser begin") - // Basics and indentaion - shouldAllEqual([ + // Basics and indentation + shouldAllEqual([ '' : '', ' ' : '', ' ' : '', @@ -48,7 +49,7 @@ class TestParser { "'s'" : 'String(s)', "\"s\"" : 'String(s)' - ]) + ]) // shouldAllEqual([ // '1 2 3 trace("Hello!", "World!") + 5 * 6 / 3': @@ -131,12 +132,12 @@ class TestParser { shouldAllError([ 'import', - //'import ""', - 'import hehe', + // TODO 'import ""', + 'import test', 'import _ as B in "test"', 'import * in "test"', 'import * as B in "test"' - // 'import a as B in ""' + // TODO 'import a as B in ""' ]) // Types parsing in expressions @@ -180,8 +181,8 @@ class TestParser { 'class A {} var a = new A { } ()' : "Block([Class(Type(A),null,[],[],false),Var(a,null,New([],Type(A),[],[],[],[]),false,false)])", 'class A { var field: String } var a = new A { field: "Value" } ()' : "Block([Class(Type(A),null,[],[Var(field,Type(String),null,false,false)],false),Var(a,null,New([],Type(A),[],[field],[String(Value)],[]),false,false)])", - 'class A { var field: String var otherfield: Int } var a = new A { field: "Value", otherfield: 25 } ()' : - "Block([Class(Type(A),null,[],[Var(field,Type(String),null,false,false),Var(otherfield,Type(Int),null,false,false)],false),Var(a,null,New([],Type(A),[],[field,otherfield],[String(Value),Int(25)],[]),false,false)])" + 'class A { var field: String var otherField: Int } var a = new A { field: "Value", otherField: 25 } ()' : + "Block([Class(Type(A),null,[],[Var(field,Type(String),null,false,false),Var(otherField,Type(Int),null,false,false)],false),Var(a,null,New([],Type(A),[],[field,otherField],[String(Value),Int(25)],[]),false,false)])" ]) // Functions @@ -271,21 +272,21 @@ class TestParser { 'declare A = B' : 'TypeAlias(Type(A),Type(B))' ]) - shouldAllError([ + shouldAllError([ "declare var a, b, c", "declare anything" - ]) + ]) - // Array and map - shouldAllEqual([ + // Array and map + shouldAllEqual([ 'push([:])' : 'Call(Ident(push),[Map([],[])])', 'push([])' : 'Call(Ident(push),[Array([])])', '[].length' : 'Dot(Array([]),length)', '[:].length' : 'Dot(Map([],[]),length)', '[[:]]': 'Array([Map([],[])])' - ]) + ]) - console.log("TestParser done \(Math.round((passed/overall)*100))% (\(passed)/\(overall))") + console.log("TestParser done \(Math.round((passed/overall)*100))% (\(passed)/\(overall))") } static var passed = 0 @@ -294,6 +295,7 @@ class TestParser { // // Helpers // + // TODO #region Helpers + require #end_region? or implicitly start next region /// Parses `input` into AST and stringifies, checks by trimmed `test` static fun shouldEqual(input: String, test: String) { @@ -304,9 +306,9 @@ class TestParser { /// Same as `shouldEqual`, but does no trimming of `test` static fun shouldEqualWithoutTrim(input: String, test: String) { overall++ - var lexe = Lexer.tokenize(Buffer.from(input), "TEST") + var lexed = Lexer.tokenize(Buffer.from(input), "TEST") try { - var parsed = Parser.toNode(Parser.parseNodes(lexe, new Project())) + var parsed = Parser.toNode(Parser.parseNodes(lexed, new Project())) var res = stringify(parsed) if test != res { throw 'TestParser test fail: `\(input)`\n\nExpect: `\(test)`\n\nParsed: `\(res)`\n\nRaw: `\(parsed)`' @@ -334,11 +336,12 @@ class TestParser { } } + /// Input should fail to parse static fun shouldError(input: String) { overall++ try { - var lexe = Lexer.tokenize(Buffer.from(input), "TEST") - var parser = Parser.toNode(Parser.parseNodes(lexe, new Project())) + var lexed = Lexer.tokenize(Buffer.from(input), "TEST") + var parser = Parser.toNode(Parser.parseNodes(lexed, new Project())) } catch e: Any { passed++ @@ -347,6 +350,7 @@ class TestParser { throw 'TestParser test fail: `\(input)` did not throw exception.' } + /// Every input should fail to parse static fun shouldAllError(input: [String]) { for str in input { shouldError(str) @@ -355,6 +359,7 @@ class TestParser { /// Removes all internal and trailing whitespace and newlines static fun deepTrim(s: String) { + // TODO String.replaceAll return s.split('\n').join('').split('\r').join('').split('\t').join('').split(' ').join('') } @@ -386,15 +391,15 @@ class TestParser { "," + ((expr != null) ? stringify(expr) : "null") +',\(const),\(external))' //case Vars(vars): return 'Vars(' + stringifyNodeArray(vars) + ")" - case Function(name, expr, vars, rettype, external): + case Function(name, expr, vars, retType, external): return 'Function(\(name),' + (expr == null ? "null" : stringify(expr)) - + "," + stringifyNodeArray(vars) + ',' + ((rettype != null) ? stringifyType(rettype) : "null") + ",\(external))" + + "," + stringifyNodeArray(vars) + ',' + ((retType != null) ? stringifyType(retType) : "null") + ",\(external))" - case Arrow(expr, vars, rettype): + case Arrow(expr, vars, retType): return 'Arrow(' + (expr == null ? "null" : stringify(expr)) - + "," + stringifyNodeArray(vars) + ',' + ((rettype != null) ? stringifyType(rettype) : "null") + ")" + + "," + stringifyNodeArray(vars) + ',' + ((retType != null) ? stringifyType(retType) : "null") + ")" case Call(e, el, argNames): var res = 'Call(' + stringify(e) + ',[' @@ -441,15 +446,17 @@ class TestParser { case DotUpper(l, r): return "DotUpper(" + stringify(l) + "," + r + ")" case Private(expr): return 'Private(' + stringify(expr) + ')' - case If(econd, eif, eelse): - return 'If(' + stringifyNodeArray(econd) + ',' + stringify(eif) + ',' + stringify(eelse) + ')' + case If(eCond, eif, eElse): + return 'If(' + stringifyNodeArray(eCond) + ',' + stringify(eif) + ',' + stringify(eElse) + ')' case Array(el): return 'Array(' + stringifyNodeArray(el) + ')' case Unop(op, postfix, e): return 'Unop(' + Token.stringify(op) + ',' + postfix + ',' + stringify(e) + ')' + // TODO exhaustive switch case _: throw '' } } static fun stringifyArray(nodes: [Node]): String { + // TODO drop `[for in]` syntax return [for node in nodes stringify(node)].join(',') } @@ -461,12 +468,12 @@ class TestParser { return "[" + [for e in arr stringifyType(e)].join(',') + "]" } - // Creates a testable consistent string representation of type + /// Creates a testable consistent string representation of type static fun stringifyType(node: NodeType): String { switch node { case Type(s): return 'Type(\(s))' case ParametricType(name, params): return 'ParametricType(\(name),[' + [for p in params stringifyType(p)].join(',') + "])" - case Function(args, rettype): return 'Function([' + [for e in args stringifyType(e)].join(',') + "]," + stringifyType(rettype) + ')' + case Function(args, retType): return 'Function([' + [for e in args stringifyType(e)].join(',') + "]," + stringifyType(retType) + ')' case Object(names, types): return 'Object([' + names.join(',') + "],[" + [for e in types stringifyType(e)].join(',') +"])" case Optional(type): return 'Optional(' + stringifyType(type) + ')' case _: throw ''