Skip to content
Snippets Groups Projects
Commit b4d1f41e authored by Leo's avatar Leo Committed by Leo
Browse files

fix: improve lexer and parser code trace for more accurate traces + exchange... 

fix: improve lexer and parser code trace for more accurate traces + exchange dot operator for 'probes'
parent 91bcb906
No related branches found
No related tags found
No related merge requests found
Expand all Show whitespace changes
Inline Side-by-side
examples/07_objects.styve
+ 12
− 11
View file @ b4d1f41e
......@@ -22,19 +22,20 @@ Arne Arne Arne person Arne 8=
=D
=D
STANDUP("Name:", person@name) Styve
STANDUP("Age:", person@age) Styve
STANDUP("Name:", person probes name) Styve
STANDUP("Age:", person probes age) Styve
person@greet() Styve
person@print("Run print native fn") Styve
person probes greet() Styve
person probes print("Run print native fn") Styve
person@random() Styve
person probes random() Styve
STANDUP() Styve
--- Test inner object
STANDUP("Inner name:", person@innerObj@innerName) Styve
STANDUP("Inner age:", person@innerObj@innerAge) Styve
person@innerObj@innerGreet() Styve
STANDUP("Inner name:", person probes innerObj probes innerName) Styve
STANDUP("Inner age:", person probes innerObj["innerAge"]) Styve
Arne Arne Styve innerGreetFnName Arne "innerGreet" Styve
person probes innerObj["innerGreet"]() Styve
STANDUP("Smallest:", ArneAlgebra@UnderArne(1, 2), "Largest:", ArneAlgebra@OverArne(1,4,3,2)) Styve
\ No newline at end of file
STANDUP("Smallest:", ArneAlgebra probes UnderArne(1, 2), "Largest:", ArneAlgebra probes OverArne(1,4,3,2)) Styve
\ No newline at end of file
examples/objects.styve
+ 10
− 10
View file @ b4d1f41e
......@@ -8,20 +8,20 @@ Arne Arne Arne obj Arne 8=
STANDUP("Object expressions:") Styve
STANDUP("First name:", obj@firstName) Styve
STANDUP("Last name:", obj@lastName) Styve
STANDUP("Age:", obj@age) Styve
STANDUP("Is alive:", obj@isAlive) Styve
STANDUP("Is dead:", obj@isDead) Styve
STANDUP("First name:", obj probes firstName) Styve
STANDUP("Last name:", obj probes lastName) Styve
STANDUP("Age:", obj probes age) Styve
STANDUP("Is alive:", obj probes isAlive) Styve
STANDUP("Is dead:", obj probes isDead) Styve
--- Assignment
Arne Arne Styve constVal Arne obj@firstName Styve
Arne Arne Styve constVal Arne obj probes firstName Styve
STANDUP("Assigned value:", constVal) Styve
--- Test setting a value
obj@firstName = "Not Arne" Styve
STANDUP("Changed value:", obj@firstName) Styve
obj probes firstName = "Not Arne" Styve
STANDUP("Changed value:", obj probes firstName) Styve
--- Test setting a value using unary
obj@isAlive = ! obj@isAlive Styve
STANDUP("Changed value using unary:", obj@isAlive) Styve
obj probes isAlive = ! obj probes isAlive Styve
STANDUP("Changed value using unary:", obj probes isAlive) Styve
src/frontend/Constants.java
+ 3
− 1
View file @ b4d1f41e
......@@ -30,7 +30,9 @@ public class Constants {
new Keyword("=>", TokenType.Colon),
new Keyword("8=", TokenType.LBrace),
new Keyword("=D", TokenType.RBrace),
new Keyword("@", TokenType.Dot),
// new Keyword("@", TokenType.Dot),
new Keyword("probes", TokenType.Dot),
new Keyword("++", TokenType.AnyParen),
new Keyword("Arne?", TokenType.If),
new Keyword("Styve?", TokenType.Else),
new Keyword("Samesies", TokenType.TripleEquals),
......
src/frontend/lexer/Lexer.java
+ 47
− 26
View file @ b4d1f41e
......@@ -11,6 +11,9 @@ public class Lexer {
private Token getToken(String value, TokenType type, int line, int start, int end) {
return new Token(type, value, line, start, end);
}
private Token getToken(String value, TokenType type, int line, int start, int end, int leftPadding) {
return new Token(type, value, line, start, end, leftPadding);
}
// Helper method to create a token when no value is provided
private Token getToken(String value, TokenType type) {
......@@ -122,10 +125,17 @@ public class Lexer {
int tokenStartCol;
boolean lastWasNewline = false;
boolean currentLineIsEmpty = true; // Track if the current line is empty
int currWhitespace = 0;
int prevTokensLength = 0;
while (!src.isEmpty()) {
tokenStartCol = col;
// When a token was added, reset the whitespace counter
if (prevTokensLength < tokens.size()) {
prevTokensLength = tokens.size();
currWhitespace = 0;
}
String firstChar = src.getFirst();
// Handle newlines and carriage returns
......@@ -142,7 +152,7 @@ public class Lexer {
// If the last line was empty (only newlines), add an EmptyLine token
if (currentLineIsEmpty && lastWasNewline) {
tokens.add(getToken("", TokenType.EmptyLine, lineNo - 1, 0, 0));
tokens.add(getToken("", TokenType.EmptyLine, lineNo - 1, 0, 0, 0));
}
lastWasNewline = true; // Mark that the last character was a newline
......@@ -154,6 +164,7 @@ public class Lexer {
if (isWhitespace(firstChar)) {
src.removeFirst(); // Consume whitespace
col++;
currWhitespace++;
continue;
}
......@@ -216,6 +227,7 @@ public class Lexer {
currentLineIsEmpty = true;
lastWasNewline = true;
col = 1;
currWhitespace = 0;
} else {
token.setValue(token.getValue() + " " + word);
col += word.length() + 1;
......@@ -231,10 +243,13 @@ public class Lexer {
currentLineIsEmpty = true;
lastWasNewline = true;
col = 1; // Ensure column is reset for next line
currWhitespace = 0;
}
token.setLine(lineNo++);
token.setStart(tokenStartCol);
token.setEnd(tokenStartCol + token.getValue().length());
token.setLeftPadding(currWhitespace);
currWhitespace = 0;
tokens.add(token);
continue;
}
......@@ -244,6 +259,7 @@ public class Lexer {
token.setLine(lineNo);
token.setStart(tokenStartCol);
token.setEnd(col);
token.setLeftPadding(currWhitespace);
tokens.add(token);
continue;
}
......@@ -252,50 +268,50 @@ public class Lexer {
// Handle individual tokens
switch (src.getFirst()) {
case "(":
tokens.add(getToken(src.removeFirst(), TokenType.LParen, lineNo, tokenStartCol, col++));
tokens.add(getToken(src.removeFirst(), TokenType.LParen, lineNo, tokenStartCol, col++, currWhitespace));
break;
case ")":
tokens.add(getToken(src.removeFirst(), TokenType.RParen, lineNo, tokenStartCol, col++));
tokens.add(getToken(src.removeFirst(), TokenType.RParen, lineNo, tokenStartCol, col++, currWhitespace));
break;
case "{":
tokens.add(getToken(src.removeFirst(), TokenType.LBrace, lineNo, tokenStartCol, col++));
tokens.add(getToken(src.removeFirst(), TokenType.LBrace, lineNo, tokenStartCol, col++, currWhitespace));
break;
case "}":
tokens.add(getToken(src.removeFirst(), TokenType.RBrace, lineNo, tokenStartCol, col++));
tokens.add(getToken(src.removeFirst(), TokenType.RBrace, lineNo, tokenStartCol, col++, currWhitespace));
break;
case "[":
tokens.add(getToken(src.removeFirst(), TokenType.LBracket, lineNo, tokenStartCol, col++));
tokens.add(getToken(src.removeFirst(), TokenType.LBracket, lineNo, tokenStartCol, col++, currWhitespace));
break;
case "]":
tokens.add(getToken(src.removeFirst(), TokenType.RBracket, lineNo, tokenStartCol, col++));
tokens.add(getToken(src.removeFirst(), TokenType.RBracket, lineNo, tokenStartCol, col++, currWhitespace));
break;
case ",":
tokens.add(getToken(src.removeFirst(), TokenType.Comma, lineNo, tokenStartCol, col++));
break;
case ":":
tokens.add(getToken(src.removeFirst(), TokenType.Colon, lineNo, tokenStartCol, col++));
break;
case "=":
tokens.add(getToken(src.removeFirst(), TokenType.Equals, lineNo, tokenStartCol, col++));
break;
case ".":
case "@":
tokens.add(getToken(src.removeFirst(), TokenType.Dot, lineNo, tokenStartCol, col++));
tokens.add(getToken(src.removeFirst(), TokenType.Comma, lineNo, tokenStartCol, col++, currWhitespace));
break;
// case ":":
// tokens.add(getToken(src.removeFirst(), TokenType.Colon, lineNo, tokenStartCol, col++, currWhitespace));
// break;
// case "=":
// tokens.add(getToken(src.removeFirst(), TokenType.Equals, lineNo, tokenStartCol, col++, currWhitespace));
// break;
// case ".":
// case "@": // Dot
// tokens.add(getToken(src.removeFirst(), TokenType.Dot, lineNo, tokenStartCol, col++, currWhitespace));
// break;
case "!":
tokens.add(getToken(src.removeFirst(), TokenType.Not, lineNo, tokenStartCol, col++));
tokens.add(getToken(src.removeFirst(), TokenType.Not, lineNo, tokenStartCol, col++, currWhitespace));
break;
case "+":
case "-":
case "*":
case "/":
case "%":
tokens.add(getToken(src.removeFirst(), TokenType.BinaryOperator, lineNo, tokenStartCol, col++));
tokens.add(getToken(src.removeFirst(), TokenType.BinaryOperator, lineNo, tokenStartCol, col++, currWhitespace));
break;
default:
if (isNumeric(src.getFirst())) {
String nm = consumeNumber(src);
tokens.add(getToken(nm, TokenType.Number, lineNo, tokenStartCol, col += nm.length()));
tokens.add(getToken(nm, TokenType.Number, lineNo, tokenStartCol, col += nm.length(), currWhitespace));
} else if (isAlphabetic(src.getFirst())) {
String word = consumeWord(src);
Keyword hit = Constants.keywords.stream()
......@@ -304,12 +320,13 @@ public class Lexer {
.orElse(null);
if (hit != null) {
tokens.add(getToken(word, hit.getValue(), lineNo, tokenStartCol, col += word.length()));
tokens.add(getToken(word, hit.getValue(), lineNo, tokenStartCol, col += word.length(), currWhitespace));
} else {
tokens.add(getToken(word, TokenType.Identifier, lineNo, tokenStartCol, col += word.length()));
tokens.add(getToken(word, TokenType.Identifier, lineNo, tokenStartCol, col += word.length(), currWhitespace));
}
} else if (isWhitespace(src.getFirst())) {
src.removeFirst();
currWhitespace++;
} else if (src.getFirst().equals("\"")) {
src.removeFirst();
StringBuilder literal = new StringBuilder();
......@@ -317,9 +334,13 @@ public class Lexer {
literal.append(src.removeFirst());
}
src.removeFirst();
tokens.add(getToken(literal.toString(), TokenType.Literal, lineNo, tokenStartCol, col += literal.length() + 2));
tokens.add(getToken(literal.toString(), TokenType.Literal, lineNo, tokenStartCol, col += literal.length() + 2, currWhitespace));
} else {
throw new IllegalArgumentException("Unknown token: '" + src.getFirst() + "'");
// Handle unknown tokens
// String unknown = consumeAnything(src, false);
String unknown = src.removeFirst();
tokens.add(getToken(unknown, TokenType.Unknown, lineNo, tokenStartCol, col += unknown.length() - 1, currWhitespace));
// throw new IllegalArgumentException("Unknown token: '" + src.getFirst() + "'");
}
break;
}
......
src/frontend/lexer/Token.java
+ 20
− 1
View file @ b4d1f41e
......@@ -6,6 +6,7 @@ public class Token {
public int line;
public int start;
public int end;
public int leftPadding;
public Token(TokenType type, String value, int line, int start, int end) {
this.type = type;
......@@ -13,6 +14,16 @@ public class Token {
this.line = line;
this.start = start;
this.end = end;
this.leftPadding = 0;
}
public Token(TokenType type, String value, int line, int start, int end, int leftPadding) {
this.type = type;
this.value = value;
this.line = line;
this.start = start;
this.end = end;
this.leftPadding = leftPadding;
}
public TokenType getType() {
......@@ -55,8 +66,16 @@ public class Token {
this.end = end;
}
public int getLeftPadding() {
return leftPadding;
}
public void setLeftPadding(int leftPadding) {
this.leftPadding = leftPadding;
}
public String toString() {
return "Token(type=" + type + ", val='" + value + "', lineNo=" + line + ", start=" + start + ", end=" + end + ")";
return "Token(type=" + type + ", val='" + value + "', lineNo=" + line + ", start=" + start + ", end=" + end + ", leftPadding=" + leftPadding + ")";
}
public boolean is(TokenType type) {
......
src/frontend/lexer/TokenType.java
+ 2
− 0
View file @ b4d1f41e
......@@ -20,6 +20,7 @@ public enum TokenType {
Equals, // Equals
LParen, // Open parenthesis
RParen, // Close parenthesis
AnyParen, // Any parenthesis
LBrace, // Open brace
RBrace, // Close brace
LBracket, // Open bracket
......@@ -66,5 +67,6 @@ public enum TokenType {
// Other
EmptyLine, // Empty line
Unknown // Unknown token
;
}
\ No newline at end of file
src/frontend/parser/Parser.java
+ 151
− 65
View file @ b4d1f41e
This diff is collapsed.
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment