Separating parser responsibilities in multiple classes. (temporarily breaks whitespace preservation)

Parser/src/main/java/org/openzen/zenscript/lexer/MemoryTokenStream.java

@@ -0,0 +1,19 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.openzen.zenscript.lexer;
+
+/**
+ *
+ * @author Hoofdgebruiker
+ */
+public interface MemoryTokenStream<TT extends TokenType, T extends Token<TT>>
+		extends TokenStream<TT, T> {
+	void pushMark();
+	
+	void popMark();
+	
+	void reset();
+}

Parser/src/main/java/org/openzen/zenscript/lexer/MemoryTokenStreamImpl.java

@@ -0,0 +1,84 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.openzen.zenscript.lexer;
+
+import java.util.LinkedList;
+import java.util.Stack;
+import org.openzen.zenscript.shared.CodePosition;
+
+/**
+ *
+ * @author Hoofdgebruiker
+ */
+public abstract class MemoryTokenStreamImpl<TT extends TokenType, T extends Token<TT>> implements TokenStream<TT, T> {
+	private final TokenStream<TT, T> stream;
+    private final LinkedList<TokenParser.PositionedToken<T>> tokenMemory = new LinkedList<>();
+    private final Stack<Integer> marks = new Stack<>();
+	
+    private int tokenMemoryOffset = 0;
+    private int tokenMemoryCurrent = 0;
+	
+	public MemoryTokenStreamImpl(TokenStream<TT, T> stream) {
+		this.stream = stream;
+	}
+	
+	public void pushMark() {
+        marks.push(tokenMemoryCurrent);
+	}
+	
+	public void popMark() {
+        marks.pop();
+
+        if (marks.isEmpty()) {
+            tokenMemoryOffset = tokenMemoryCurrent;
+            tokenMemory.clear();
+        }
+	}
+	
+	public void reset() {
+        tokenMemoryCurrent = marks.pop();
+	}
+
+	@Override
+	public T peek() {
+		if (tokenMemoryCurrent < tokenMemoryOffset + tokenMemory.size()) {
+            return tokenMemory.get((tokenMemoryCurrent) - tokenMemoryOffset).token;
+        } else {
+            return stream.peek();
+        }
+	}
+
+	@Override
+	public void replace(TT other) {
+		stream.replace(other);
+	}
+
+	@Override
+	public T next() {
+		if (tokenMemoryCurrent < tokenMemoryOffset + tokenMemory.size()) {
+            return tokenMemory.get((tokenMemoryCurrent++) - tokenMemoryOffset).token;
+        } else {
+            T result = stream.peek();
+            if (marks.isEmpty()) {
+                tokenMemoryOffset++;
+            } else {
+                tokenMemory.add(new TokenParser.PositionedToken<>(stream.getPosition(), stream.peek()));
+            }
+            tokenMemoryCurrent++;
+            stream.next();
+			return result;
+        }
+	}
+
+	@Override
+	public CodePosition getPosition() {
+		if (tokenMemoryCurrent < tokenMemoryOffset + tokenMemory.size()) {
+            return tokenMemory.get((tokenMemoryCurrent) - tokenMemoryOffset).position;
+        } else {
+            return stream.getPosition();
+        }
+	}
+}

Parser/src/main/java/org/openzen/zenscript/lexer/TokenFactory.java

@@ -0,0 +1,14 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.openzen.zenscript.lexer;
+
+/**
+ *
+ * @author Hoofdgebruiker
+ */
+public interface TokenFactory<T, TT> {
+	T create(TT type, String content, String whitespaceBefore);
+}

Parser/src/main/java/org/openzen/zenscript/lexer/TokenParser.java

@@ -4,10 +4,9 @@ package org.openzen.zenscript.lexer;
 import java.io.IOException;
 import java.io.Reader;
 import java.io.StringReader;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.Stack;
 import org.openzen.zenscript.shared.CodePosition;
+import org.openzen.zenscript.shared.CompileException;
+import org.openzen.zenscript.shared.CompileExceptionCode;
 
 /**
  * Represents a token stream. A token stream reads characters from a reader and
@@ -19,25 +18,22 @@ import org.openzen.zenscript.shared.CodePosition;
  * @param <T> token class
  * @param <TT> token type class
  */
-public abstract class TokenParser<T extends Token<TT>, TT extends TokenType> implements Iterator<T>
+public class TokenParser<T extends Token<TT>, TT extends TokenType> implements TokenStream<TT, T>
 {
 	private final String filename;
     private final CountingReader reader;
     private final CompiledDFA<TT> dfa;
-    private final LinkedList<PositionedToken> tokenMemory;
-    private final Stack<Integer> marks;
 	private final TT eof;
+	private final TT invalid;
 	private final int tabSize = 4;
+	private final TokenFactory<T, TT> factory;
 	
-    private PositionedToken next;
+    private PositionedToken<T> next;
 	
     private int nextChar;
     private int line;
     private int lineOffset;
 	
-    private int tokenMemoryOffset;
-    private int tokenMemoryCurrent;
-    
     /**
      * Creates a token stream using the specified reader and DFA.
      *
@@ -46,25 +42,28 @@ public abstract class TokenParser<T extends Token<TT>, TT extends TokenType> imp
      * @param dfa DFA to tokenize the stream
 	 * @param eof end of file token type
      */
-    public TokenParser(String filename, Reader reader, CompiledDFA<TT> dfa, TT eof)
+    public TokenParser(
+			String filename,
+			Reader reader, 
+			CompiledDFA<TT> dfa,
+			TT eof,
+			TT invalid,
+			TokenFactory<T, TT> factory)
 	{
 		if (eof.isWhitespace()) // important for the advance() method
 			throw new IllegalArgumentException("EOF cannot be whitespace");
 		
-        tokenMemoryOffset = 0;
-        tokenMemoryCurrent = 0;
-        tokenMemory = new LinkedList<>();
-        marks = new Stack<>();
-        
         this.reader = new CountingReader(reader);
         this.dfa = dfa;
 		this.filename = filename;
 		this.eof = eof;
+		this.invalid = invalid;
+		this.factory = factory;
 		
 		try {
 	        nextChar = this.reader.read();
 		} catch (IOException ex) {
-			ioException(ex);
+			throw new CompileException(getPosition(), CompileExceptionCode.INTERNAL_ERROR, ex.getMessage());
 		}
 		
         line = 1;
@@ -80,102 +79,23 @@ public abstract class TokenParser<T extends Token<TT>, TT extends TokenType> imp
      * @param dfa DFA to tokenize the stream
 	 * @param eof end of file token type
      */
-    public TokenParser(String filename, String data, CompiledDFA<TT> dfa, TT eof)
+    public TokenParser(String filename, String data, CompiledDFA<TT> dfa, TT eof, TT invalid, TokenFactory<T, TT> factory)
 	{
-        this(filename, new StringReader(data), dfa, eof);
+        this(filename, new StringReader(data), dfa, eof, invalid, factory);
     }
 	
-	public String getFilename()
-	{
-		return filename;
-	}
-	
-	public int getLine()
-	{
-		return line;
-	}
-	
-	public int getLineOffset()
-	{
-		return lineOffset;
-	}
-    
+	@Override
     public T peek()
 	{
-        if (tokenMemoryCurrent < tokenMemoryOffset + tokenMemory.size()) {
-            return tokenMemory.get((tokenMemoryCurrent) - tokenMemoryOffset).token;
-        } else {
-            return next.token;
-        }
-    }
-
-    public boolean isNext(TT type)
-	{
-        return peek().getType() == type;
-    }
-
-    public T optional(TT type)
-	{
-        if (peek().getType() == type) {
-            return next();
-        } else {
-            return null;
-        }
-    }
-
-    public T required(TT type, String error)
-	{
-		T t = peek();
-        if (t.getType() == type) {
-            return next();
-        } else {
-			requiredTokenNotFound(getPosition(), error, t);
-			return null;
-        }
+        return next.token;
     }
 	
+	@Override
 	public CodePosition getPosition()
 	{
-		if (tokenMemoryCurrent < tokenMemoryOffset + tokenMemory.size()) {
-            return tokenMemory.get((tokenMemoryCurrent) - tokenMemoryOffset).position;
-        } else {
-            return next.position;
-        }
+		return next.position;
 	}
 	
-	// =====================
-    // === LL(*) ability ===
-	// =====================
-
-    /**
-     * Pushes a mark on the mark stack.
-     */
-    public void pushMark()
-	{
-        marks.push(tokenMemoryCurrent);
-    }
-
-    /**
-     * Pops a mark from the mark stack without reset.
-     */
-    public void popMark()
-	{
-        marks.pop();
-
-        if (marks.isEmpty()) {
-            tokenMemoryOffset = tokenMemoryCurrent;
-            tokenMemory.clear();
-        }
-    }
-
-    /**
-     * Pops a mark from the mark stack and resets the stream's position to it
-     */
-    public void reset()
-	{
-        tokenMemoryCurrent = marks.pop();
-    }
-	
 	/**
 	 * Replaces the current token with another one. Used to split composite tokens.
 	 * 
@@ -184,99 +104,24 @@ public abstract class TokenParser<T extends Token<TT>, TT extends TokenType> imp
 	public void replace(TT other) {
 		next = new PositionedToken(
 				next.position,
-				createToken(
+				factory.create(
+						other,
 						next.token.getWhitespaceBefore(),
-						next.token.getContent(),
-						other));
+						next.token.getContent()));
 	}
 
-    // ===============================
-    // === Iterator implementation ===
-    // ===============================
-
-	@Override
-    public boolean hasNext()
-	{
-        return next.token.getType() != eof;
-    }
-
 	@Override
     public T next()
 	{
-        if (tokenMemoryCurrent < tokenMemoryOffset + tokenMemory.size()) {
-            return tokenMemory.get((tokenMemoryCurrent++) - tokenMemoryOffset).token;
-        } else {
-            PositionedToken result = next;
-
-            if (marks.isEmpty()) {
-                tokenMemoryOffset++;
-            } else {
-                tokenMemory.add(result);
-            }
-            tokenMemoryCurrent++;
-
-            advance();
-            return result.token;
-        }
-    }
-
-	@Override
-    public void remove()
-	{
-        throw new UnsupportedOperationException("Not supported.");
+        T result = next.token;
+		advance();
+		return result;
     }
-
-    // ==================================
-    // === Protected abstract methods ===
-    // ==================================
-	
-	/**
-	 * Called to create a token. May also be used to postprocess tokens while
-	 * generating them.
-	 * 
-	 * @param position token position (range)
-	 * @param value token value
-	 * @param tokenType token type
-	 * @return newly created token
-	 */
-	protected abstract T createToken(
-			String whitespaceBefore,
-			String value,
-			TT tokenType);
-	
-	/**
-	 * Called when a required token could not be found. Should log an error or
-	 * throw an exception. If no exception is thrown, the calling required
-	 * method will return null as token value.
-	 * 
-	 * @param position erroring position
-	 * @param error error to be logged
-	 * @param token incorrect token
-	 */
-	protected abstract void requiredTokenNotFound(
-			CodePosition position,
-			String error,
-			T token);
 	
-	/**
-	 * Called when the input contains an invalid token. Should either create
-	 * a token indicating an invalid token, or throw an exception.
-	 * 
-	 * @param position erroring position
-	 * @param token token value
-	 * @return a token marking an invalid token
-	 */
-	protected abstract T invalidToken(
-			String whitespaceBefore,
-			String token);
-	
-	/**
-	 * Called when an IO exception occurs. Should throw an exception of some
-	 * kind.
-	 * 
-	 * @param ex exception to be logged
-	 */
-	protected abstract void ioException(IOException ex);
+	@Override
+	public TT getEOF() {
+		return eof;
+	}
 	
     // =======================
     // === Private methods ===
@@ -311,7 +156,7 @@ public abstract class TokenParser<T extends Token<TT>, TT extends TokenType> imp
 					line,
 					lineOffset);
 			
-            next = new PositionedToken(position, createToken(whitespace, "", eof));
+            next = new PositionedToken(position, factory.create(eof, whitespace, ""));
             return;
         }
 		
@@ -335,27 +180,21 @@ public abstract class TokenParser<T extends Token<TT>, TT extends TokenType> imp
             if (dfa.finals[state] != null) {
                 if (state == 0) {
 					value.append((char) nextChar);
-					next = new PositionedToken(position, invalidToken(
-							whitespace,
-							value.toString()));
+					next = new PositionedToken(position, factory.create(invalid, value.toString(), whitespace));
 					nextChar = reader.read();
 				}
 				
-				next = new PositionedToken(position, createToken(
-						whitespace,
-						value.toString(), dfa.finals[state]));
+				next = new PositionedToken(position, factory.create(dfa.finals[state], value.toString(), whitespace));
             } else {
 				if (nextChar < 0 && value.length() == 0)
 					return; // happens on comments at the end of files
 				
 				value.append((char) nextChar);
-				next = new PositionedToken(position, invalidToken(
-						whitespace,
-						value.toString()));
+				next = new PositionedToken(position, factory.create(invalid, value.toString(), whitespace));
 				nextChar = reader.read();
             }
         } catch (IOException ex) {
-            ioException(ex);
+			throw new CompileException(getPosition(), CompileExceptionCode.INTERNAL_ERROR, ex.getMessage());
         }
     }
 
@@ -399,7 +238,7 @@ public abstract class TokenParser<T extends Token<TT>, TT extends TokenType> imp
         }
     }
 	
-	private class PositionedToken {
+	public static class PositionedToken<T> {
 		public final CodePosition position;
 		public final T token;
 		

Parser/src/main/java/org/openzen/zenscript/lexer/TokenStream.java

@@ -6,6 +6,8 @@
 package org.openzen.zenscript.lexer;
 
 import org.openzen.zenscript.shared.CodePosition;
+import org.openzen.zenscript.shared.CompileException;
+import org.openzen.zenscript.shared.CompileExceptionCode;
 
 /**
  *
@@ -14,23 +16,36 @@ import org.openzen.zenscript.shared.CodePosition;
 public interface TokenStream<TT extends TokenType, T extends Token<TT>> {
 	T peek();
 	
-	boolean isNext(TT type);
-	
-	T optional(TT type);
-	
-	T required(TT type, String error);
-	
-	void pushMark();
-	
-	void popMark();
-	
-	void reset();
+	default boolean isNext(TT type) {
+		return peek().getType() == type;
+	}
+	
+	default T optional(TT type) {
+		if (peek().getType() == type) {
+            return next();
+        } else {
+            return null;
+        }
+	}
+	
+	default T required(TT type, String error) throws CompileException {
+		T t = peek();
+        if (t.getType() == type) {
+            return next();
+        } else {
+			throw new CompileException(getPosition(), CompileExceptionCode.UNEXPECTED_TOKEN, error);
+        }
+    }
 	
 	void replace(TT other);
 	
-	boolean hasNext();
+	default boolean hasNext() {
+		return peek().getType() != getEOF();
+	}
 	
 	T next();
 	
 	CodePosition getPosition();
+	
+	TT getEOF();
 }

Parser/src/main/java/org/openzen/zenscript/lexer/WhitespaceFilteringParser.java

@@ -0,0 +1,85 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.openzen.zenscript.lexer;
+
+import org.openzen.zenscript.shared.CodePosition;
+
+/**
+ *
+ * @author Hoofdgebruiker
+ */
+public class WhitespaceFilteringParser<TT extends TokenType, T extends Token<TT>> implements TokenStream<TT, T> {
+	private final TokenStream<TT, T> stream;
+	private T next;
+	private CodePosition position;
+	private String whitespace;
+	
+	public WhitespaceFilteringParser(TokenStream<TT, T> stream) {
+		this.stream = stream;
+		advance();
+	}
+	
+	public String getLastWhitespace() {
+		return whitespace;
+	}
+	
+	public void skipWhitespaceNewline() {
+		int index = whitespace.indexOf('\n');
+		if (index >= 0)
+			whitespace = whitespace.substring(index + 1);
+	}
+	
+	public String grabWhitespaceLine() {
+		if (whitespace.contains("\n")) {
+			int index = whitespace.indexOf('\n');
+			String result = whitespace.substring(0, index);
+			whitespace = whitespace.substring(index + 1);
+			return result;
+		} else {
+			whitespace = "";
+			return whitespace;
+		}
+	}
+
+	@Override
+	public T peek() {
+		return next;
+	}
+
+	@Override
+	public void replace(TT other) {
+		stream.replace(other);
+		next = peek();
+	}
+
+	@Override
+	public T next() {
+		T result = next;
+		advance();
+		return result;
+	}
+
+	@Override
+	public CodePosition getPosition() {
+		return position;
+	}
+
+	@Override
+	public TT getEOF() {
+		return stream.getEOF();
+	}
+	
+	private void advance() {
+		whitespace = "";
+		position = stream.getPosition();
+		next = stream.next();
+		while (next.getType().isWhitespace()) {
+			whitespace += next.getContent();
+			position = stream.getPosition();
+			next = stream.next();
+		}
+	}
+}

Parser/src/main/java/org/openzen/zenscript/lexer/ZSTokenFactory.java

@@ -0,0 +1,105 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.openzen.zenscript.lexer;
+
+import java.util.HashMap;
+import java.util.Map;
+import static org.openzen.zenscript.lexer.ZSTokenType.*;
+
+/**
+ *
+ * @author Hoofdgebruiker
+ */
+public class ZSTokenFactory implements TokenFactory<ZSToken, ZSTokenType> {
+	public static final ZSTokenFactory INSTANCE = new ZSTokenFactory();
+	
+	private static final Map<String, ZSTokenType> KEYWORDS = new HashMap<>();
+	
+	static {
+		KEYWORDS.put("import", K_IMPORT);
+		KEYWORDS.put("alias", K_ALIAS);
+		KEYWORDS.put("class", K_CLASS);
+		KEYWORDS.put("interface", K_INTERFACE);
+		KEYWORDS.put("enum", K_ENUM);
+		KEYWORDS.put("struct", K_STRUCT);
+		KEYWORDS.put("expand", K_EXPAND);
+		KEYWORDS.put("function", K_FUNCTION);
+		KEYWORDS.put("variant", K_VARIANT);
+		
+		KEYWORDS.put("abstract", K_ABSTRACT);
+		KEYWORDS.put("final", K_FINAL);
+		KEYWORDS.put("override", K_OVERRIDE);
+		KEYWORDS.put("const", K_CONST);
+		KEYWORDS.put("private", K_PRIVATE);
+		KEYWORDS.put("public", K_PUBLIC);
+		KEYWORDS.put("export", K_EXPORT);
+		KEYWORDS.put("static", K_STATIC);
+		KEYWORDS.put("protected", K_PROTECTED);
+		KEYWORDS.put("implicit", K_IMPLICIT);
+		KEYWORDS.put("virtual", K_VIRTUAL);
+		KEYWORDS.put("extern", K_EXTERN);
+		
+		KEYWORDS.put("val", K_VAL);
+		KEYWORDS.put("var", K_VAR);
+		KEYWORDS.put("get", K_GET);
+		KEYWORDS.put("implements", K_IMPLEMENTS);
+		KEYWORDS.put("set", K_SET);
+		
+		KEYWORDS.put("void", K_VOID);
+		KEYWORDS.put("any", K_ANY);
+		KEYWORDS.put("bool", K_BOOL);
+		KEYWORDS.put("byte", K_BYTE);
+		KEYWORDS.put("sbyte", K_SBYTE);
+		KEYWORDS.put("short", K_SHORT);
+		KEYWORDS.put("ushort", K_USHORT);
+		KEYWORDS.put("int", K_INT);
+		KEYWORDS.put("uint", K_UINT);
+		KEYWORDS.put("long", K_LONG);
+		KEYWORDS.put("ulong", K_ULONG);
+		KEYWORDS.put("float", K_FLOAT);
+		KEYWORDS.put("double", K_DOUBLE);
+		KEYWORDS.put("char", K_CHAR);
+		KEYWORDS.put("string", K_STRING);
+		
+		KEYWORDS.put("if", K_IF);
+		KEYWORDS.put("else", K_ELSE);
+		KEYWORDS.put("do", K_DO);
+		KEYWORDS.put("while", K_WHILE);
+		KEYWORDS.put("for", K_FOR);
+		KEYWORDS.put("throw", K_THROW);
+		KEYWORDS.put("lock", K_LOCK);
+		KEYWORDS.put("try", K_TRY);
+		KEYWORDS.put("catch", K_CATCH);
+		KEYWORDS.put("finally", K_FINALLY);
+		KEYWORDS.put("return", K_RETURN);
+		KEYWORDS.put("break", K_BREAK);
+		KEYWORDS.put("continue", K_CONTINUE);
+		KEYWORDS.put("switch", K_SWITCH);
+		KEYWORDS.put("case", K_CASE);
+		KEYWORDS.put("default", K_DEFAULT);
+		
+		KEYWORDS.put("in", K_IN);
+		KEYWORDS.put("is", K_IS);
+		KEYWORDS.put("as", K_AS);
+		KEYWORDS.put("match", K_MATCH);
+		KEYWORDS.put("throws", K_THROWS);
+		
+		KEYWORDS.put("this", K_THIS);
+		KEYWORDS.put("super", K_SUPER);
+		KEYWORDS.put("null", K_NULL);
+		KEYWORDS.put("true", K_TRUE);
+		KEYWORDS.put("false", K_FALSE);
+		KEYWORDS.put("new", K_NEW);
+	}
+
+	@Override
+	public ZSToken create(ZSTokenType type, String content, String whitespaceBefore) {
+		if (type == T_IDENTIFIER && KEYWORDS.containsKey(content))
+			type = KEYWORDS.get(content);
+		
+		return new ZSToken(type, content, whitespaceBefore);
+	}
+}

Parser/src/main/java/org/openzen/zenscript/lexer/ZSTokenParser.java

@@ -5,172 +5,67 @@
  */
 package org.openzen.zenscript.lexer;
 
-import java.io.IOException;
 import java.io.Reader;
-import java.util.HashMap;
-import java.util.Map;
 import org.openzen.zenscript.codemodel.WhitespaceInfo;
-import org.openzen.zenscript.shared.CompileException;
-import org.openzen.zenscript.shared.CompileExceptionCode;
-import static org.openzen.zenscript.lexer.ZSTokenType.*;
-import org.openzen.zenscript.shared.CodePosition;
 
 /**
  *
  * @author Hoofdgebruiker
  */
-public class ZSTokenParser extends TokenParser<ZSToken, ZSTokenType> implements ZSTokenStream {
-	private static final CompiledDFA DFA = CompiledDFA.createLexerDFA(ZSTokenType.values(), ZSTokenType.class);
-	private static final Map<String, ZSTokenType> KEYWORDS = new HashMap<>();
-	
-	static {
-		KEYWORDS.put("import", K_IMPORT);
-		KEYWORDS.put("alias", K_ALIAS);
-		KEYWORDS.put("class", K_CLASS);
-		KEYWORDS.put("interface", K_INTERFACE);
-		KEYWORDS.put("enum", K_ENUM);
-		KEYWORDS.put("struct", K_STRUCT);
-		KEYWORDS.put("expand", K_EXPAND);
-		KEYWORDS.put("function", K_FUNCTION);
-		KEYWORDS.put("variant", K_VARIANT);
-		
-		KEYWORDS.put("abstract", K_ABSTRACT);
-		KEYWORDS.put("final", K_FINAL);
-		KEYWORDS.put("override", K_OVERRIDE);
-		KEYWORDS.put("const", K_CONST);
-		KEYWORDS.put("private", K_PRIVATE);
-		KEYWORDS.put("public", K_PUBLIC);
-		KEYWORDS.put("export", K_EXPORT);
-		KEYWORDS.put("static", K_STATIC);
-		KEYWORDS.put("protected", K_PROTECTED);
-		KEYWORDS.put("implicit", K_IMPLICIT);
-		KEYWORDS.put("virtual", K_VIRTUAL);
-		KEYWORDS.put("extern", K_EXTERN);
-		
-		KEYWORDS.put("val", K_VAL);
-		KEYWORDS.put("var", K_VAR);
-		KEYWORDS.put("get", K_GET);
-		KEYWORDS.put("implements", K_IMPLEMENTS);
-		KEYWORDS.put("set", K_SET);
-		
-		KEYWORDS.put("void", K_VOID);
-		KEYWORDS.put("any", K_ANY);
-		KEYWORDS.put("bool", K_BOOL);
-		KEYWORDS.put("byte", K_BYTE);
-		KEYWORDS.put("sbyte", K_SBYTE);
-		KEYWORDS.put("short", K_SHORT);
-		KEYWORDS.put("ushort", K_USHORT);
-		KEYWORDS.put("int", K_INT);
-		KEYWORDS.put("uint", K_UINT);
-		KEYWORDS.put("long", K_LONG);
-		KEYWORDS.put("ulong", K_ULONG);
-		KEYWORDS.put("float", K_FLOAT);
-		KEYWORDS.put("double", K_DOUBLE);
-		KEYWORDS.put("char", K_CHAR);
-		KEYWORDS.put("string", K_STRING);
-		
-		KEYWORDS.put("if", K_IF);
-		KEYWORDS.put("else", K_ELSE);
-		KEYWORDS.put("do", K_DO);
-		KEYWORDS.put("while", K_WHILE);
-		KEYWORDS.put("for", K_FOR);
-		KEYWORDS.put("throw", K_THROW);
-		KEYWORDS.put("lock", K_LOCK);
-		KEYWORDS.put("try", K_TRY);
-		KEYWORDS.put("catch", K_CATCH);
-		KEYWORDS.put("finally", K_FINALLY);
-		KEYWORDS.put("return", K_RETURN);
-		KEYWORDS.put("break", K_BREAK);
-		KEYWORDS.put("continue", K_CONTINUE);
-		KEYWORDS.put("switch", K_SWITCH);
-		KEYWORDS.put("case", K_CASE);
-		KEYWORDS.put("default", K_DEFAULT);
-		
-		KEYWORDS.put("in", K_IN);
-		KEYWORDS.put("is", K_IS);
-		KEYWORDS.put("as", K_AS);
-		KEYWORDS.put("match", K_MATCH);
-		KEYWORDS.put("throws", K_THROWS);
-		
-		KEYWORDS.put("this", K_THIS);
-		KEYWORDS.put("super", K_SUPER);
-		KEYWORDS.put("null", K_NULL);
-		KEYWORDS.put("true", K_TRUE);
-		KEYWORDS.put("false", K_FALSE);
-		KEYWORDS.put("new", K_NEW);
+public class ZSTokenParser extends MemoryTokenStreamImpl<ZSTokenType, ZSToken>
+		implements ZSTokenStream {
+	public static TokenParser<ZSToken, ZSTokenType> createRaw(String filename, Reader reader) {
+		return new TokenParser<>(
+				filename,
+				reader,
+				DFA,
+				ZSTokenType.EOF,
+				ZSTokenType.INVALID,
+				ZSTokenFactory.INSTANCE);
 	}
 	
-	private String whitespaceBuffer = null;
+	public static ZSTokenParser create(String filename, Reader reader) {
+		return create(filename, new WhitespaceFilteringParser(createRaw(filename, reader)));
+	}
 	
-	public ZSTokenParser(String filename, Reader reader) {
-		super(filename, reader, DFA, ZSTokenType.EOF);
+	public static ZSTokenParser create(String filename, WhitespaceFilteringParser<ZSTokenType, ZSToken> parser) {
+		return new ZSTokenParser(filename, parser);
 	}
 	
-	@Override
-	public String loadWhitespace() {
-		if (whitespaceBuffer == null)
-			whitespaceBuffer = peek().whitespaceBefore;
+	private static final CompiledDFA DFA = CompiledDFA.createLexerDFA(ZSTokenType.values(), ZSTokenType.class);
+	
+	private final String filename;
+	private final WhitespaceFilteringParser whitespaceFilteringParser;
+	
+	private ZSTokenParser(String filename, WhitespaceFilteringParser<ZSTokenType, ZSToken> parser) {
+		super(parser);
 		
-		return whitespaceBuffer;
+		this.whitespaceFilteringParser = parser;
+		this.filename = filename;
 	}
 	
 	@Override
-	public void reloadWhitespace() {
-		whitespaceBuffer = peek().whitespaceBefore;
+	public String getFilename() {
+		return filename;
 	}
 	
 	@Override
-	public String grabWhitespace() {
-		String result = loadWhitespace();
-		whitespaceBuffer = null;
-		return result;
-	}
-	
-	public String grabWhitespaceLine() {
-		String whitespace = loadWhitespace();
-		if (whitespace.contains("\n")) {
-			int index = whitespace.indexOf('\n');
-			whitespaceBuffer = whitespace.substring(index + 1);
-			return whitespace.substring(0, index);
-		} else {
-			whitespaceBuffer = "";
-			return whitespace;
-		}
+	public String getLastWhitespace() {
+		return whitespaceFilteringParser.getLastWhitespace();
 	}
 	
 	@Override
 	public void skipWhitespaceNewline() {
-		loadWhitespace();
-		int index = whitespaceBuffer.indexOf('\n');
-		if (index >= 0)
-			whitespaceBuffer = whitespaceBuffer.substring(index + 1);
+		whitespaceFilteringParser.skipWhitespaceNewline();
 	}
 	
 	@Override
 	public WhitespaceInfo collectWhitespaceInfo(String whitespace, boolean skipLineBefore) {
-		return WhitespaceInfo.from(whitespace, grabWhitespaceLine(), skipLineBefore);
+		return WhitespaceInfo.from(whitespace, whitespaceFilteringParser.grabWhitespaceLine(), skipLineBefore);
 	}
 	
 	@Override
-	protected ZSToken createToken(String whitespaceBefore, String value, ZSTokenType tokenType) {
-		if (tokenType == T_IDENTIFIER && KEYWORDS.containsKey(value))
-			tokenType = KEYWORDS.get(value);
-		
-		return new ZSToken(tokenType, value, whitespaceBefore);
-	}
-
-	@Override
-	protected void requiredTokenNotFound(CodePosition position, String error, ZSToken token) {
-		throw new CompileException(position, CompileExceptionCode.UNEXPECTED_TOKEN, error);
-	}
-
-	@Override
-	protected ZSToken invalidToken(String whitespaceBefore, String token) {
-		return new ZSToken(ZSTokenType.INVALID, token, whitespaceBefore);
-	}
-
-	@Override
-	protected void ioException(IOException ex) {
-		throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+	public ZSTokenType getEOF() {
+		return ZSTokenType.EOF;
 	}
 }

Parser/src/main/java/org/openzen/zenscript/lexer/ZSTokenStream.java

@@ -11,14 +11,10 @@ import org.openzen.zenscript.codemodel.WhitespaceInfo;
  *
  * @author Hoofdgebruiker
  */
-public interface ZSTokenStream extends TokenStream<ZSTokenType, ZSToken> {
+public interface ZSTokenStream extends MemoryTokenStream<ZSTokenType, ZSToken> {
 	String getFilename();
 	
-	String loadWhitespace();
-	
-	void reloadWhitespace();
-	
-	String grabWhitespace();
+	String getLastWhitespace();
 	
 	void skipWhitespaceNewline();
 	

Parser/src/main/java/org/openzen/zenscript/parser/ParsedFile.java

@@ -58,7 +58,7 @@ public class ParsedFile {
 	}
 	
 	public static ParsedFile parse(ZSPackage pkg, String filename, Reader reader) throws IOException {
-		ZSTokenParser tokens = new ZSTokenParser(filename, reader);
+		ZSTokenParser tokens = ZSTokenParser.create(filename, reader);
 		return parse(pkg, tokens);
 	}
 	

Parser/src/main/java/org/openzen/zenscript/parser/statements/ParsedStatement.java

@@ -45,11 +45,9 @@ public abstract class ParsedStatement {
 	}
 	
 	public static ParsedStatementBlock parseBlock(ZSTokenStream parser, boolean isFirst) {
-		String ws = parser.grabWhitespace();
+		String ws = parser.getLastWhitespace();
 		CodePosition position = parser.getPosition();
 		parser.required(T_AOPEN, "{ expected");
-
-		parser.reloadWhitespace();
 		parser.skipWhitespaceNewline();
 
 		ArrayList<ParsedStatement> statements = new ArrayList<>();
@@ -70,7 +68,7 @@ public abstract class ParsedStatement {
 	}
 	
 	public static ParsedStatement parse(ZSTokenStream parser, boolean isFirst) {
-		String ws = parser.grabWhitespace();
+		String ws = parser.getLastWhitespace();
 		CodePosition position = parser.getPosition();
 		ZSToken next = parser.peek();
 		switch (next.getType()) {
@@ -113,7 +111,6 @@ public abstract class ParsedStatement {
 				ParsedStatement onIf = parse(parser);
 				ParsedStatement onElse = null;
 				if (parser.optional(K_ELSE) != null) {
-					parser.reloadWhitespace();
 					parser.skipWhitespaceNewline();
 					onElse = parse(parser);
 				}