- Removed whitespace handling from the bare parser

- Tokens no longer store whitespace

Parser/src/main/java/org/openzen/zenscript/lexer/Token.java

@@ -12,6 +12,4 @@ public interface Token<TT extends TokenType>
 	TT getType();
 	
 	String getContent();
-	
-	String getWhitespaceBefore();
 }

Parser/src/main/java/org/openzen/zenscript/lexer/TokenFactory.java

@@ -10,5 +10,5 @@ package org.openzen.zenscript.lexer;
  * @author Hoofdgebruiker
  */
 public interface TokenFactory<T, TT> {
-	T create(TT type, String content, String whitespaceBefore);
+	T create(TT type, String content);
 }

Parser/src/main/java/org/openzen/zenscript/lexer/TokenParser.java

@@ -106,7 +106,6 @@ public class TokenParser<T extends Token<TT>, TT extends TokenType> implements T
 				next.position,
 				factory.create(
 						other,
-						next.token.getWhitespaceBefore(),
 						next.token.getContent()));
 	}
 
@@ -126,27 +125,8 @@ public class TokenParser<T extends Token<TT>, TT extends TokenType> implements T
     // =======================
     // === Private methods ===
     // =======================
-
-    /**
-     * Advances to the next non - whitespace token.
-     */
+	
     private void advance()
-	{
-		StringBuilder whitespace = new StringBuilder();
-        while (true) {
-            advanceToken(whitespace.toString());
-			if (next.token.getType().isWhitespace()) {
-				whitespace.append(next.token.getContent());
-			} else {
-				break;
-			}
-        }
-    }
-
-    /**
-     * Advances to the next token.
-     */
-    private void advanceToken(String whitespace)
 	{
         if (nextChar < 0) {
 			CodePosition position = new CodePosition(
@@ -156,7 +136,7 @@ public class TokenParser<T extends Token<TT>, TT extends TokenType> implements T
 					line,
 					lineOffset);
 			
-            next = new PositionedToken(position, factory.create(eof, whitespace, ""));
+            next = new PositionedToken(position, factory.create(eof, ""));
             return;
         }
 		
@@ -180,17 +160,17 @@ public class TokenParser<T extends Token<TT>, TT extends TokenType> implements T
             if (dfa.finals[state] != null) {
                 if (state == 0) {
 					value.append((char) nextChar);
-					next = new PositionedToken(position, factory.create(invalid, value.toString(), whitespace));
+					next = new PositionedToken(position, factory.create(invalid, value.toString()));
 					nextChar = reader.read();
 				}
 				
-				next = new PositionedToken(position, factory.create(dfa.finals[state], value.toString(), whitespace));
+				next = new PositionedToken(position, factory.create(dfa.finals[state], value.toString()));
             } else {
 				if (nextChar < 0 && value.length() == 0)
 					return; // happens on comments at the end of files
 				
 				value.append((char) nextChar);
-				next = new PositionedToken(position, factory.create(invalid, value.toString(), whitespace));
+				next = new PositionedToken(position, factory.create(invalid, value.toString()));
 				nextChar = reader.read();
             }
         } catch (IOException ex) {

Parser/src/main/java/org/openzen/zenscript/lexer/ZSToken.java

@@ -12,12 +12,10 @@ package org.openzen.zenscript.lexer;
 public class ZSToken implements Token<ZSTokenType> {
 	public final ZSTokenType type;
 	public final String content;
-	public final String whitespaceBefore;
 	
-	public ZSToken(ZSTokenType type, String content, String whitespaceBefore) {
+	public ZSToken(ZSTokenType type, String content) {
 		this.type = type;
 		this.content = content;
-		this.whitespaceBefore = whitespaceBefore;
 	}
 
 	@Override
@@ -29,9 +27,4 @@ public class ZSToken implements Token<ZSTokenType> {
 	public String getContent() {
 		return content;
 	}
-
-	@Override
-	public String getWhitespaceBefore() {
-		return whitespaceBefore;
-	}
 }

Parser/src/main/java/org/openzen/zenscript/lexer/ZSTokenFactory.java

@@ -96,10 +96,10 @@ public class ZSTokenFactory implements TokenFactory<ZSToken, ZSTokenType> {
 	}
 
 	@Override
-	public ZSToken create(ZSTokenType type, String content, String whitespaceBefore) {
+	public ZSToken create(ZSTokenType type, String content) {
 		if (type == T_IDENTIFIER && KEYWORDS.containsKey(content))
 			type = KEYWORDS.get(content);
 		
-		return new ZSToken(type, content, whitespaceBefore);
+		return new ZSToken(type, content);
 	}
 }

Parser/src/main/java/org/openzen/zenscript/parser/ParsedFile.java

@@ -120,7 +120,7 @@ public class ParsedFile {
 			}
 		}
 		
-		result.postComment = WhitespacePostComment.fromWhitespace(eof.whitespaceBefore);
+		result.postComment = WhitespacePostComment.fromWhitespace(tokens.getLastWhitespace());
 		return result;
 	}
 	

Parser/src/main/java/org/openzen/zenscript/parser/statements/ParsedStatement.java

@@ -59,7 +59,7 @@ public abstract class ParsedStatement {
 		}
 
 		WhitespaceInfo whitespace = parser.collectWhitespaceInfo(ws, isFirst);
-		WhitespacePostComment postComment = WhitespacePostComment.fromWhitespace(last.whitespaceBefore);
+		WhitespacePostComment postComment = WhitespacePostComment.fromWhitespace(parser.getLastWhitespace());
 		return new ParsedStatementBlock(position, whitespace, postComment, statements);
 	}