- Simplified parser classes, removed unnecessary classes, improved code quality

- the TokenStream now no longer has any memory at all; its input is a CharStream which has peek() and next() methods, the TokenStream doesn't store a single character, making it stateless
- Added flyweight token support, decreasing memory usage

Parser/src/main/java/org/openzen/zenscript/lexer/MemoryTokenStream.java → Parser/src/main/java/org/openzen/zenscript/lexer/CharReader.java

@@ -5,15 +5,14 @@
  */
 package org.openzen.zenscript.lexer;
 
+import java.io.IOException;
+
 /**
  *
  * @author Hoofdgebruiker
  */
-public interface MemoryTokenStream<TT extends TokenType, T extends Token<TT>>
-		extends TokenStream<TT, T> {
-	void pushMark();
-	
-	void popMark();
+public interface CharReader {
+	int peek() throws IOException;
 	
-	void reset();
+	int next() throws IOException;
 }

Parser/src/main/java/org/openzen/zenscript/lexer/CountingCharReader.java

@@ -0,0 +1,61 @@
+package org.openzen.zenscript.lexer;
+
+import java.io.IOException;
+import org.openzen.zenscript.shared.CodePosition;
+
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+
+/**
+ *
+ * @author Hoofdgebruiker
+ */
+public class CountingCharReader implements CharReader {
+	private final CharReader reader;
+	private final String filename;
+	private final int tabSize;
+	
+	private int line;
+	private int lineOffset;
+
+	public CountingCharReader(CharReader reader, String filename, int tabSize)
+	{
+		this.reader = reader;
+		this.filename = filename;
+		this.tabSize = tabSize;
+		
+		line = 1;
+		lineOffset = 1;
+	}
+	
+	public CodePosition getPosition() {
+		return new CodePosition(filename, line, lineOffset, line, lineOffset);
+	}
+
+	@Override
+	public int peek() throws IOException
+	{
+		return reader.peek();
+	}
+
+	@Override
+	public int next() throws IOException
+	{
+		int ch = reader.next();
+		if (ch == -1)
+			return ch;
+		
+		if (ch == '\n') {
+			line++;
+			lineOffset = 1;
+		} else if (ch == '\t') {
+			lineOffset += tabSize;
+		} else {
+			lineOffset++;
+		}
+		return ch;
+	}
+}

Parser/src/main/java/org/openzen/zenscript/lexer/MemoryTokenStreamImpl.java → Parser/src/main/java/org/openzen/zenscript/lexer/LLParserTokenStream.java

@@ -8,21 +8,25 @@ package org.openzen.zenscript.lexer;
 import java.util.LinkedList;
 import java.util.Stack;
 import org.openzen.zenscript.shared.CodePosition;
+import org.openzen.zenscript.shared.CompileException;
+import org.openzen.zenscript.shared.CompileExceptionCode;
 
 /**
  *
  * @author Hoofdgebruiker
  */
-public abstract class MemoryTokenStreamImpl<TT extends TokenType, T extends Token<TT>> implements TokenStream<TT, T> {
+public abstract class LLParserTokenStream<TT extends TokenType, T extends Token<TT>> {
 	private final TokenStream<TT, T> stream;
-    private final LinkedList<TokenParser.PositionedToken<T>> tokenMemory = new LinkedList<>();
+    private final LinkedList<PositionedToken> tokenMemory = new LinkedList<>();
     private final Stack<Integer> marks = new Stack<>();
 	
+	private T next;
     private int tokenMemoryOffset = 0;
     private int tokenMemoryCurrent = 0;
 	
-	public MemoryTokenStreamImpl(TokenStream<TT, T> stream) {
+	public LLParserTokenStream(TokenStream<TT, T> stream) {
 		this.stream = stream;
+		next = stream.next();
 	}
 	
 	public void pushMark() {
@@ -42,38 +46,34 @@ public abstract class MemoryTokenStreamImpl<TT extends TokenType, T extends Toke
         tokenMemoryCurrent = marks.pop();
 	}
 
-	@Override
 	public T peek() {
 		if (tokenMemoryCurrent < tokenMemoryOffset + tokenMemory.size()) {
             return tokenMemory.get((tokenMemoryCurrent) - tokenMemoryOffset).token;
         } else {
-            return stream.peek();
+            return next;
         }
 	}
-
-	@Override
-	public void replace(TT other) {
-		stream.replace(other);
+	
+	public void replace(T other) {
+		next = other;
 	}
-
-	@Override
+	
 	public T next() {
 		if (tokenMemoryCurrent < tokenMemoryOffset + tokenMemory.size()) {
             return tokenMemory.get((tokenMemoryCurrent++) - tokenMemoryOffset).token;
         } else {
-            T result = stream.peek();
+            T result = next;
+			next = stream.next();
             if (marks.isEmpty()) {
                 tokenMemoryOffset++;
             } else {
-                tokenMemory.add(new TokenParser.PositionedToken<>(stream.getPosition(), stream.peek()));
+                tokenMemory.add(new PositionedToken(getPosition(), result));
             }
             tokenMemoryCurrent++;
-            stream.next();
 			return result;
         }
 	}
-
-	@Override
+	
 	public CodePosition getPosition() {
 		if (tokenMemoryCurrent < tokenMemoryOffset + tokenMemory.size()) {
             return tokenMemory.get((tokenMemoryCurrent) - tokenMemoryOffset).position;
@@ -81,4 +81,39 @@ public abstract class MemoryTokenStreamImpl<TT extends TokenType, T extends Toke
             return stream.getPosition();
         }
 	}
+	
+	public boolean isNext(TT type) {
+		return peek().getType() == type;
+	}
+	
+	public T optional(TT type) {
+		if (peek().getType() == type) {
+            return next();
+        } else {
+            return null;
+        }
+	}
+	
+	public T required(TT type, String error) throws CompileException {
+		T t = peek();
+        if (t.getType() == type) {
+            return next();
+        } else {
+			throw new CompileException(getPosition(), CompileExceptionCode.UNEXPECTED_TOKEN, error);
+        }
+    }
+	
+	public boolean hasNext() {
+		return peek().getType() != stream.getEOF();
+	}
+	
+	private class PositionedToken {
+		public final CodePosition position;
+		public final T token;
+		
+		public PositionedToken(CodePosition position, T token) {
+			this.position = position;
+			this.token = token;
+		}
+	}
 }

Parser/src/main/java/org/openzen/zenscript/lexer/ReaderCharReader.java

@@ -0,0 +1,35 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.openzen.zenscript.lexer;
+
+import java.io.IOException;
+import java.io.Reader;
+
+/**
+ *
+ * @author Hoofdgebruiker
+ */
+public class ReaderCharReader implements CharReader {
+	private final Reader reader;
+	private int next;
+	
+	public ReaderCharReader(Reader reader) throws IOException {
+		this.reader = reader;
+		next = reader.read();
+	}
+	
+	@Override
+	public int peek() {
+		return next;
+	}
+
+	@Override
+	public int next() throws IOException {
+		int result = next;
+		next = reader.read();
+		return result;
+	}
+}

Parser/src/main/java/org/openzen/zenscript/lexer/StringCharReader.java

@@ -0,0 +1,32 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.openzen.zenscript.lexer;
+
+import java.io.IOException;
+
+/**
+ *
+ * @author Hoofdgebruiker
+ */
+public class StringCharReader implements CharReader {
+	private final char[] data;
+	private int index;
+	
+	public StringCharReader(String data) {
+		this.data = data.toCharArray();
+		this.index = 0;
+	}
+	
+	@Override
+	public int peek() {
+		return index >= data.length ? -1 : data[index];
+	}
+
+	@Override
+	public int next() throws IOException {
+		return index >= data.length ? -1 : data[index++];
+	}
+}

Parser/src/main/java/org/openzen/zenscript/lexer/TokenParser.java

@@ -2,8 +2,6 @@
 package org.openzen.zenscript.lexer;
 
 import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
 import org.openzen.zenscript.shared.CodePosition;
 import org.openzen.zenscript.shared.CompileException;
 import org.openzen.zenscript.shared.CompileExceptionCode;
@@ -20,20 +18,12 @@ import org.openzen.zenscript.shared.CompileExceptionCode;
  */
 public class TokenParser<T extends Token<TT>, TT extends TokenType> implements TokenStream<TT, T>
 {
-	private final String filename;
-    private final CountingReader reader;
+    private final CountingCharReader reader;
     private final CompiledDFA<TT> dfa;
 	private final TT eof;
 	private final TT invalid;
-	private final int tabSize = 4;
 	private final TokenFactory<T, TT> factory;
 	
-    private PositionedToken<T> next;
-	
-    private int nextChar;
-    private int line;
-    private int lineOffset;
-	
     /**
      * Creates a token stream using the specified reader and DFA.
      *
@@ -44,7 +34,7 @@ public class TokenParser<T extends Token<TT>, TT extends TokenType> implements T
      */
     public TokenParser(
 			String filename,
-			Reader reader, 
+			CharReader reader, 
 			CompiledDFA<TT> dfa,
 			TT eof,
 			TT invalid,
@@ -53,22 +43,11 @@ public class TokenParser<T extends Token<TT>, TT extends TokenType> implements T
 		if (eof.isWhitespace()) // important for the advance() method
 			throw new IllegalArgumentException("EOF cannot be whitespace");
 		
-        this.reader = new CountingReader(reader);
+        this.reader = new CountingCharReader(reader, filename, 4);
         this.dfa = dfa;
-		this.filename = filename;
 		this.eof = eof;
 		this.invalid = invalid;
 		this.factory = factory;
-		
-		try {
-	        nextChar = this.reader.read();
-		} catch (IOException ex) {
-			throw new CompileException(getPosition(), CompileExceptionCode.INTERNAL_ERROR, ex.getMessage());
-		}
-		
-        line = 1;
-        lineOffset = 1;
-        advance();
     }
 
     /**
@@ -81,150 +60,59 @@ public class TokenParser<T extends Token<TT>, TT extends TokenType> implements T
      */
     public TokenParser(String filename, String data, CompiledDFA<TT> dfa, TT eof, TT invalid, TokenFactory<T, TT> factory)
 	{
-        this(filename, new StringReader(data), dfa, eof, invalid, factory);
-    }
-	
-	@Override
-    public T peek()
-	{
-        return next.token;
+        this(filename, new StringCharReader(data), dfa, eof, invalid, factory);
     }
 	
 	@Override
 	public CodePosition getPosition()
 	{
-		return next.position;
+		return reader.getPosition();
 	}
 	
-	/**
-	 * Replaces the current token with another one. Used to split composite tokens.
-	 * 
-	 * @param other 
-	 */
-	public void replace(TT other) {
-		next = new PositionedToken(
-				next.position,
-				factory.create(
-						other,
-						next.token.getContent()));
+	public boolean hasNext() {
+		try {
+			return reader.peek() >= 0;
+		} catch (IOException ex) {
+			return false;
+		}
 	}
-
-	@Override
-    public T next()
-	{
-        T result = next.token;
-		advance();
-		return result;
-    }
 	
 	@Override
 	public TT getEOF() {
 		return eof;
 	}
 	
-    // =======================
-    // === Private methods ===
-    // =======================
-	
-    private void advance()
+	@Override
+    public T next()
 	{
-        if (nextChar < 0) {
-			CodePosition position = new CodePosition(
-					filename,
-					line,
-					lineOffset,
-					line,
-					lineOffset);
+		try {
+			if (reader.peek() < 0)
+				return factory.create(eof, "");
 			
-            next = new PositionedToken(position, factory.create(eof, ""));
-            return;
-        }
-		
-        try {
             int state = 0;
             StringBuilder value = new StringBuilder();
-            int fromLine = line;
-            int fromLineOffset = lineOffset;
-            while (dfa.transitions[state].containsKey(nextChar)) {
-                value.append((char)nextChar);
-                state = dfa.transitions[state].get(nextChar);
-                line = reader.line;
-                lineOffset = reader.lineOffset;
-                nextChar = reader.read();
+            while (dfa.transitions[state].containsKey(reader.peek())) {
+				char c = (char) reader.next();
+                value.append(c);
+                state = dfa.transitions[state].get(c);
             }
 			
-			if (line < 0)
-				throw new IllegalStateException("Line cannot be negative");
-			
-			CodePosition position = new CodePosition(filename, fromLine, fromLineOffset, line, lineOffset);
             if (dfa.finals[state] != null) {
                 if (state == 0) {
-					value.append((char) nextChar);
-					next = new PositionedToken(position, factory.create(invalid, value.toString()));
-					nextChar = reader.read();
+					value.append((char) reader.next());
+					return factory.create(invalid, value.toString());
 				}
 				
-				next = new PositionedToken(position, factory.create(dfa.finals[state], value.toString()));
+				return factory.create(dfa.finals[state], value.toString());
             } else {
-				if (nextChar < 0 && value.length() == 0)
-					return; // happens on comments at the end of files
+				if (reader.peek() < 0 && value.length() == 0)
+					return factory.create(eof, ""); // happens on comments at the end of files
 				
-				value.append((char) nextChar);
-				next = new PositionedToken(position, factory.create(invalid, value.toString()));
-				nextChar = reader.read();
+				value.append((char) reader.next());
+				return factory.create(invalid, value.toString());
             }
         } catch (IOException ex) {
 			throw new CompileException(getPosition(), CompileExceptionCode.INTERNAL_ERROR, ex.getMessage());
         }
     }
-
-    // =============================
-    // === Private inner classes ===
-    // =============================
-
-    /**
-     * Keeps a line and line offset count.
-     */
-    private class CountingReader
-	{
-        private int line;
-        private int lineOffset;
-        private Reader reader;
-        private boolean eof;
-
-        public CountingReader(Reader reader)
-		{
-            this.reader = reader;
-            line = 1;
-            lineOffset = 1;
-        }
-
-        public int read() throws IOException
-		{
-            int ch = reader.read();
-            if (ch == -1) {
-                eof = true;
-                return ch;
-            }
-            if (ch == '\n') {
-                line++;
-                lineOffset = 1;
-            } else if (ch == '\t') {
-				lineOffset += tabSize;
-			} else {
-                lineOffset++;
-            }
-            return ch;
-        }
-    }
-	
-	public static class PositionedToken<T> {
-		public final CodePosition position;
-		public final T token;
-		
-		public PositionedToken(CodePosition position, T token) {
-			this.position = position;
-			this.token = token;
-		}
-	}
 }

Parser/src/main/java/org/openzen/zenscript/lexer/TokenStream.java

@@ -6,43 +6,12 @@
 package org.openzen.zenscript.lexer;
 
 import org.openzen.zenscript.shared.CodePosition;
-import org.openzen.zenscript.shared.CompileException;
-import org.openzen.zenscript.shared.CompileExceptionCode;
 
 /**
  *
  * @author Hoofdgebruiker
  */
 public interface TokenStream<TT extends TokenType, T extends Token<TT>> {
-	T peek();
-	
-	default boolean isNext(TT type) {
-		return peek().getType() == type;
-	}
-	
-	default T optional(TT type) {
-		if (peek().getType() == type) {
-            return next();
-        } else {
-            return null;
-        }
-	}
-	
-	default T required(TT type, String error) throws CompileException {
-		T t = peek();
-        if (t.getType() == type) {
-            return next();
-        } else {
-			throw new CompileException(getPosition(), CompileExceptionCode.UNEXPECTED_TOKEN, error);
-        }
-    }
-	
-	void replace(TT other);
-	
-	default boolean hasNext() {
-		return peek().getType() != getEOF();
-	}
-	
 	T next();
 	
 	CodePosition getPosition();

Parser/src/main/java/org/openzen/zenscript/lexer/WhitespaceFilteringParser.java

@@ -39,22 +39,12 @@ public class WhitespaceFilteringParser<TT extends TokenType, T extends Token<TT>
 			whitespace = whitespace.substring(index + 1);
 			return result;
 		} else {
+			String result = whitespace;
 			whitespace = "";
-			return whitespace;
+			return result;
 		}
 	}
 
-	@Override
-	public T peek() {
-		return next;
-	}
-
-	@Override
-	public void replace(TT other) {
-		stream.replace(other);
-		next = peek();
-	}
-
 	@Override
 	public T next() {
 		T result = next;

Parser/src/main/java/org/openzen/zenscript/lexer/ZSTokenFactory.java

@@ -16,89 +16,20 @@ import static org.openzen.zenscript.lexer.ZSTokenType.*;
 public class ZSTokenFactory implements TokenFactory<ZSToken, ZSTokenType> {
 	public static final ZSTokenFactory INSTANCE = new ZSTokenFactory();
 	
-	private static final Map<String, ZSTokenType> KEYWORDS = new HashMap<>();
+	private static final Map<String, ZSToken> KEYWORDS = new HashMap<>();
 	
 	static {
-		KEYWORDS.put("import", K_IMPORT);
-		KEYWORDS.put("alias", K_ALIAS);
-		KEYWORDS.put("class", K_CLASS);
-		KEYWORDS.put("interface", K_INTERFACE);
-		KEYWORDS.put("enum", K_ENUM);
-		KEYWORDS.put("struct", K_STRUCT);
-		KEYWORDS.put("expand", K_EXPAND);
-		KEYWORDS.put("function", K_FUNCTION);
-		KEYWORDS.put("variant", K_VARIANT);
-		
-		KEYWORDS.put("abstract", K_ABSTRACT);
-		KEYWORDS.put("final", K_FINAL);
-		KEYWORDS.put("override", K_OVERRIDE);
-		KEYWORDS.put("const", K_CONST);
-		KEYWORDS.put("private", K_PRIVATE);
-		KEYWORDS.put("public", K_PUBLIC);
-		KEYWORDS.put("export", K_EXPORT);
-		KEYWORDS.put("static", K_STATIC);
-		KEYWORDS.put("protected", K_PROTECTED);
-		KEYWORDS.put("implicit", K_IMPLICIT);
-		KEYWORDS.put("virtual", K_VIRTUAL);
-		KEYWORDS.put("extern", K_EXTERN);
-		
-		KEYWORDS.put("val", K_VAL);
-		KEYWORDS.put("var", K_VAR);
-		KEYWORDS.put("get", K_GET);
-		KEYWORDS.put("implements", K_IMPLEMENTS);
-		KEYWORDS.put("set", K_SET);
-		
-		KEYWORDS.put("void", K_VOID);
-		KEYWORDS.put("any", K_ANY);
-		KEYWORDS.put("bool", K_BOOL);
-		KEYWORDS.put("byte", K_BYTE);
-		KEYWORDS.put("sbyte", K_SBYTE);
-		KEYWORDS.put("short", K_SHORT);
-		KEYWORDS.put("ushort", K_USHORT);
-		KEYWORDS.put("int", K_INT);
-		KEYWORDS.put("uint", K_UINT);
-		KEYWORDS.put("long", K_LONG);
-		KEYWORDS.put("ulong", K_ULONG);
-		KEYWORDS.put("float", K_FLOAT);
-		KEYWORDS.put("double", K_DOUBLE);
-		KEYWORDS.put("char", K_CHAR);
-		KEYWORDS.put("string", K_STRING);
-		
-		KEYWORDS.put("if", K_IF);
-		KEYWORDS.put("else", K_ELSE);
-		KEYWORDS.put("do", K_DO);
-		KEYWORDS.put("while", K_WHILE);
-		KEYWORDS.put("for", K_FOR);
-		KEYWORDS.put("throw", K_THROW);
-		KEYWORDS.put("lock", K_LOCK);
-		KEYWORDS.put("try", K_TRY);
-		KEYWORDS.put("catch", K_CATCH);
-		KEYWORDS.put("finally", K_FINALLY);
-		KEYWORDS.put("return", K_RETURN);
-		KEYWORDS.put("break", K_BREAK);
-		KEYWORDS.put("continue", K_CONTINUE);
-		KEYWORDS.put("switch", K_SWITCH);
-		KEYWORDS.put("case", K_CASE);
-		KEYWORDS.put("default", K_DEFAULT);
-		
-		KEYWORDS.put("in", K_IN);
-		KEYWORDS.put("is", K_IS);
-		KEYWORDS.put("as", K_AS);
-		KEYWORDS.put("match", K_MATCH);
-		KEYWORDS.put("throws", K_THROWS);
-		
-		KEYWORDS.put("this", K_THIS);
-		KEYWORDS.put("super", K_SUPER);
-		KEYWORDS.put("null", K_NULL);
-		KEYWORDS.put("true", K_TRUE);
-		KEYWORDS.put("false", K_FALSE);
-		KEYWORDS.put("new", K_NEW);
+		for (ZSTokenType type : ZSTokenType.values())
+			if (type.isKeyword)
+				KEYWORDS.put(type.flyweight.content, type.flyweight);
 	}
 
 	@Override
 	public ZSToken create(ZSTokenType type, String content) {
 		if (type == T_IDENTIFIER && KEYWORDS.containsKey(content))
-			type = KEYWORDS.get(content);
+			return KEYWORDS.get(content);
+		else if (type.flyweight != null)
+			return type.flyweight;
 		
 		return new ZSToken(type, content);
 	}

Parser/src/main/java/org/openzen/zenscript/lexer/ZSTokenParser.java

@@ -5,6 +5,7 @@
  */
 package org.openzen.zenscript.lexer;
 
+import java.io.IOException;
 import java.io.Reader;
 import org.openzen.zenscript.codemodel.WhitespaceInfo;
 
@@ -12,9 +13,8 @@ import org.openzen.zenscript.codemodel.WhitespaceInfo;
  *
  * @author Hoofdgebruiker
  */
-public class ZSTokenParser extends MemoryTokenStreamImpl<ZSTokenType, ZSToken>
-		implements ZSTokenStream {
-	public static TokenParser<ZSToken, ZSTokenType> createRaw(String filename, Reader reader) {
+public class ZSTokenParser extends LLParserTokenStream<ZSTokenType, ZSToken> {
+	public static TokenParser<ZSToken, ZSTokenType> createRaw(String filename, CharReader reader) {
 		return new TokenParser<>(
 				filename,
 				reader,
@@ -24,12 +24,12 @@ public class ZSTokenParser extends MemoryTokenStreamImpl<ZSTokenType, ZSToken>
 				ZSTokenFactory.INSTANCE);
 	}
 	
-	public static ZSTokenParser create(String filename, Reader reader) {
-		return create(filename, new WhitespaceFilteringParser(createRaw(filename, reader)));
+	public static ZSTokenParser create(String filename, Reader reader) throws IOException {
+		return create(filename, createRaw(filename, new ReaderCharReader(reader)));
 	}
 	
-	public static ZSTokenParser create(String filename, WhitespaceFilteringParser<ZSTokenType, ZSToken> parser) {
-		return new ZSTokenParser(filename, parser);
+	public static ZSTokenParser create(String filename, TokenStream<ZSTokenType, ZSToken> tokens) {
+		return new ZSTokenParser(filename, new WhitespaceFilteringParser(tokens));
 	}
 	
 	private static final CompiledDFA DFA = CompiledDFA.createLexerDFA(ZSTokenType.values(), ZSTokenType.class);
@@ -44,28 +44,19 @@ public class ZSTokenParser extends MemoryTokenStreamImpl<ZSTokenType, ZSToken>
 		this.filename = filename;
 	}
 	
-	@Override
 	public String getFilename() {
 		return filename;
 	}
 	
-	@Override
 	public String getLastWhitespace() {
 		return whitespaceFilteringParser.getLastWhitespace();
 	}
 	
-	@Override
 	public void skipWhitespaceNewline() {
 		whitespaceFilteringParser.skipWhitespaceNewline();
 	}
 	
-	@Override
 	public WhitespaceInfo collectWhitespaceInfo(String whitespace, boolean skipLineBefore) {
 		return WhitespaceInfo.from(whitespace, whitespaceFilteringParser.grabWhitespaceLine(), skipLineBefore);
 	}
-	
-	@Override
-	public ZSTokenType getEOF() {
-		return ZSTokenType.EOF;
-	}
 }

Parser/src/main/java/org/openzen/zenscript/lexer/ZSTokenStream.java

@@ -1,22 +0,0 @@
-/*
- * To change this license header, choose License Headers in Project Properties.
- * To change this template file, choose Tools | Templates
- * and open the template in the editor.
- */
-package org.openzen.zenscript.lexer;
-
-import org.openzen.zenscript.codemodel.WhitespaceInfo;
-
-/**
- *
- * @author Hoofdgebruiker
- */
-public interface ZSTokenStream extends MemoryTokenStream<ZSTokenType, ZSToken> {
-	String getFilename();
-	
-	String getLastWhitespace();
-	
-	void skipWhitespaceNewline();
-	
-	WhitespaceInfo collectWhitespaceInfo(String whitespace, boolean skipLineBefore);
-}

Parser/src/main/java/org/openzen/zenscript/lexer/ZSTokenType.java

@@ -19,135 +19,135 @@ public enum ZSTokenType implements TokenType {
 	T_INT("\\-?(0|[1-9][0-9]*)"),
 	T_STRING_SQ("\"([^\"\\\\]|\\\\([\'\"\\\\/bfnrt&]|u[0-9a-fA-F]{4}))*\""),
 	T_STRING_DQ("\'([^\'\\\\]|\\\\([\'\"\\\\/bfnrt&]|u[0-9a-fA-F]{4}))*\'"),
-	T_AOPEN("\\{"),
-	T_ACLOSE("\\}"),
-	T_SQOPEN("\\["),
-	T_SQCLOSE("\\]"),
-	T_DOT3("\\.\\.\\."),
-	T_DOT2("\\.\\."),
-	T_DOT("\\."),
-	T_COMMA(","),
-	T_INCREMENT("\\+\\+"),
-	T_ADDASSIGN("\\+="),
-	T_ADD("\\+"),
-	T_DECREMENT("\\-\\-"),
-	T_SUBASSIGN("\\-="),
-	T_SUB("\\-"),
-	T_CATASSIGN("~="),
-	T_CAT("~"),
-	T_MULASSIGN("\\*="),
-	T_MUL("\\*"),
-	T_DIVASSIGN("/="),
-	T_DIV("/"),
-	T_MODASSIGN("%="),
-	T_MOD("%"),
-	T_ORASSIGN("\\|="),
-	T_OROR("\\|\\|"),
-	T_OR("\\|"),
-	T_ANDASSIGN("&="),
-	T_ANDAND("&&"),
-	T_AND("&"),
-	T_XORASSIGN("\\^="),
-	T_XOR("\\^"),
-	T_COALESCE("\\?\\?"),
-	T_QUEST("\\?"),
-	T_COLON(":"),
-	T_BROPEN("\\("),
-	T_BRCLOSE("\\)"),
-	T_SEMICOLON(";"),
-	T_LESSEQ("<="),
-	T_SHLASSIGN("<<="),
-	T_SHL("<<"),
-	T_LESS("<"),
-	T_GREATEREQ(">="),
-	T_USHR(">>>"),
-	T_USHRASSIGN(">>>="),
-	T_SHRASSIGN(">>="),
-	T_SHR(">>"),
-	T_GREATER(">"),
-	T_LAMBDA("=>"),
-	T_EQUAL3("==="),
-	T_EQUAL2("=="),
-	T_ASSIGN("="),
-	T_NOTEQUAL2("!=="),
-	T_NOTEQUAL("!="),
-	T_NOT("!"),
-	T_DOLLAR("$"),
+	T_AOPEN("\\{", "{"),
+	T_ACLOSE("\\}", "}"),
+	T_SQOPEN("\\[", "["),
+	T_SQCLOSE("\\]", "]"),
+	T_DOT3("\\.\\.\\.", "..."),
+	T_DOT2("\\.\\.", ".."),
+	T_DOT("\\.", "."),
+	T_COMMA(",", ","),
+	T_INCREMENT("\\+\\+", "++"),
+	T_ADDASSIGN("\\+=", "+="),
+	T_ADD("\\+", "+"),
+	T_DECREMENT("\\-\\-", "--"),
+	T_SUBASSIGN("\\-=", "-="),
+	T_SUB("\\-", "-"),
+	T_CATASSIGN("~=", "~="),
+	T_CAT("~", "~"),
+	T_MULASSIGN("\\*=", "*="),
+	T_MUL("\\*", "*"),
+	T_DIVASSIGN("/=", "/="),
+	T_DIV("/", "/"),
+	T_MODASSIGN("%=", "%="),
+	T_MOD("%", "%"),
+	T_ORASSIGN("\\|=", "|="),
+	T_OROR("\\|\\|", "||"),
+	T_OR("\\|", "|"),
+	T_ANDASSIGN("&=", "&="),
+	T_ANDAND("&&", "&&"),
+	T_AND("&", "&"),
+	T_XORASSIGN("\\^=", "^="),
+	T_XOR("\\^", "^"),
+	T_COALESCE("\\?\\?", "??"),
+	T_QUEST("\\?", "?"),
+	T_COLON(":", ":"),
+	T_BROPEN("\\(", "("),
+	T_BRCLOSE("\\)", ")"),
+	T_SEMICOLON(";", ";"),
+	T_LESSEQ("<=", "<="),
+	T_SHLASSIGN("<<=", "<<="),
+	T_SHL("<<", "<<"),
+	T_LESS("<", "<"),
+	T_GREATEREQ(">=", ">="),
+	T_USHR(">>>", ">>>"),
+	T_USHRASSIGN(">>>=", ">>>="),
+	T_SHRASSIGN(">>=", ">>="),
+	T_SHR(">>", ">>"),
+	T_GREATER(">", ">"),
+	T_LAMBDA("=>", "=>"),
+	T_EQUAL3("===", "==="),
+	T_EQUAL2("==", "=="),
+	T_ASSIGN("=", "="),
+	T_NOTEQUAL2("!==", "!=="),
+	T_NOTEQUAL("!=", "!="),
+	T_NOT("!", "!"),
+	T_DOLLAR("$", "$"),
 	
-	K_IMPORT,
-	K_ALIAS,
-	K_CLASS,
-	K_FUNCTION,
-	K_INTERFACE,
-	K_ENUM,
-	K_STRUCT,
-	K_EXPAND,
-	K_VARIANT,
+	K_IMPORT(true, "import"),
+	K_ALIAS(true, "alias"),
+	K_CLASS(true, "class"),
+	K_FUNCTION(true, "function"),
+	K_INTERFACE(true, "interface"),
+	K_ENUM(true, "enum"),
+	K_STRUCT(true, "struct"),
+	K_EXPAND(true, "expand"),
+	K_VARIANT(true, "variant"),
 	
-	K_ABSTRACT,
-	K_FINAL,
-	K_OVERRIDE,
-	K_CONST,
-	K_PRIVATE,
-	K_PUBLIC,
-	K_EXPORT,
-	K_STATIC,
-	K_PROTECTED,
-	K_IMPLICIT,
-	K_VIRTUAL,
-	K_EXTERN,
+	K_ABSTRACT(true, "abstract"),
+	K_FINAL(true, "final"),
+	K_OVERRIDE(true, "override"),
+	K_CONST(true, "const"),
+	K_PRIVATE(true, "private"),
+	K_PUBLIC(true, "public"),
+	K_EXPORT(true, "export"),
+	K_STATIC(true, "static"),
+	K_PROTECTED(true, "protected"),
+	K_IMPLICIT(true, "implicit"),
+	K_VIRTUAL(true, "virtual"),
+	K_EXTERN(true, "extern"),
 	
-	K_VAL,
-	K_VAR,
-	K_GET,
-	K_IMPLEMENTS,
-	K_SET,
+	K_VAL(true, "val"),
+	K_VAR(true, "var"),
+	K_GET(true, "get"),
+	K_IMPLEMENTS(true, "implements"),
+	K_SET(true, "set"),
 	
-	K_VOID,
-	K_ANY,
-	K_BOOL,
-	K_BYTE,
-	K_SBYTE,
-	K_SHORT,
-	K_USHORT,
-	K_INT,
-	K_UINT,
-	K_LONG,
-	K_ULONG,
-	K_FLOAT,
-	K_DOUBLE,
-	K_CHAR,
-	K_STRING,
+	K_VOID(true, "void"),
+	K_ANY(true, "any"),
+	K_BOOL(true, "bool"),
+	K_BYTE(true, "byte"),
+	K_SBYTE(true, "sbyte"),
+	K_SHORT(true, "short"),
+	K_USHORT(true, "ushort"),
+	K_INT(true, "int"),
+	K_UINT(true, "uint"),
+	K_LONG(true, "long"),
+	K_ULONG(true, "ulong"),
+	K_FLOAT(true, "float"),
+	K_DOUBLE(true, "double"),
+	K_CHAR(true, "char"),
+	K_STRING(true, "string"),
 	
-	K_IF,
-	K_ELSE,
-	K_DO,
-	K_WHILE,
-	K_FOR,
-	K_THROW,
-	K_LOCK,
-	K_TRY,
-	K_CATCH,
-	K_FINALLY,
-	K_RETURN,
-	K_BREAK,
-	K_CONTINUE,
-	K_SWITCH,
-	K_CASE,
-	K_DEFAULT,
+	K_IF(true, "if"),
+	K_ELSE(true, "else"),
+	K_DO(true, "do"),
+	K_WHILE(true, "while"),
+	K_FOR(true, "for"),
+	K_THROW(true, "throw"),
+	K_LOCK(true, "lock"),
+	K_TRY(true, "try"),
+	K_CATCH(true, "catch"),
+	K_FINALLY(true, "finally"),
+	K_RETURN(true, "return"),
+	K_BREAK(true, "break"),
+	K_CONTINUE(true, "continue"),
+	K_SWITCH(true, "switch"),
+	K_CASE(true, "case"),
+	K_DEFAULT(true, "default"),
 	
-	K_IN,
-	K_IS,
-	K_AS,
-	K_MATCH,
-	K_THROWS,
+	K_IN(true, "in"),
+	K_IS(true, "is"),
+	K_AS(true, "as"),
+	K_MATCH(true, "match"),
+	K_THROWS(true, "throws"),
 	
-	K_SUPER,
-	K_THIS,
-	K_NULL,
-	K_TRUE,
-	K_FALSE,
-	K_NEW,
+	K_SUPER(true, "super"),
+	K_THIS(true, "this"),
+	K_NULL(true, "null"),
+	K_TRUE(true, "true"),
+	K_FALSE(true, "false"),
+	K_NEW(true, "new"),
 	
 	INVALID,
 	EOF
@@ -156,19 +156,42 @@ public enum ZSTokenType implements TokenType {
 	private final String regexp;
 	private final boolean whitespace;
 	
+	public final ZSToken flyweight;
+	public final boolean isKeyword;
+	
 	private ZSTokenType() {
 		this.regexp = null;
 		this.whitespace = false;
+		this.isKeyword = false;
+		this.flyweight = null;
 	}
 	
 	private ZSTokenType(String regexp) {
 		this.regexp = regexp;
 		this.whitespace = false;
+		this.isKeyword = false;
+		this.flyweight = null;
 	}
 	
 	private ZSTokenType(String regexp, boolean whitespace) {
 		this.regexp = regexp;
 		this.whitespace = whitespace;
+		this.isKeyword = false;
+		this.flyweight = null;
+	}
+	
+	private ZSTokenType(String regexp, String content) {
+		this.regexp = regexp;
+		this.whitespace = false;
+		this.isKeyword = false;
+		this.flyweight = new ZSToken(this, content);
+	}
+	
+	private ZSTokenType(boolean isKeyword, String content) {
+		this.regexp = null;
+		this.whitespace = false;
+		this.isKeyword = isKeyword;
+		this.flyweight = new ZSToken(this, content);
 	}
 
 	@Override

Parser/src/main/java/org/openzen/zenscript/parser/ParsedDefinition.java

@@ -7,8 +7,8 @@ package org.openzen.zenscript.parser;
 
 import org.openzen.zenscript.codemodel.HighLevelDefinition;
 import org.openzen.zenscript.codemodel.definition.ZSPackage;
+import org.openzen.zenscript.lexer.ZSTokenParser;
 import org.openzen.zenscript.shared.CodePosition;
-import org.openzen.zenscript.lexer.ZSTokenStream;
 import static org.openzen.zenscript.lexer.ZSTokenType.*;
 import org.openzen.zenscript.linker.BaseScope;
 import org.openzen.zenscript.parser.definitions.ParsedAlias;
@@ -25,7 +25,7 @@ import org.openzen.zenscript.parser.definitions.ParsedVariant;
  * @author Hoofdgebruiker
  */
 public abstract class ParsedDefinition {
-	public static ParsedDefinition parse(ZSPackage pkg, CodePosition position, int modifiers, ZSTokenStream tokens, HighLevelDefinition outerDefinition) {
+	public static ParsedDefinition parse(ZSPackage pkg, CodePosition position, int modifiers, ZSTokenParser tokens, HighLevelDefinition outerDefinition) {
 		if (tokens.optional(K_CLASS) != null) {
 			return ParsedClass.parseClass(pkg, position, modifiers, tokens, outerDefinition);
 		} else if (tokens.optional(K_INTERFACE) != null) {

Parser/src/main/java/org/openzen/zenscript/parser/ParsedFile.java

@@ -28,7 +28,6 @@ import org.openzen.zenscript.shared.CodePosition;
 import org.openzen.zenscript.shared.CompileException;
 import org.openzen.zenscript.shared.CompileExceptionCode;
 import org.openzen.zenscript.lexer.ZSTokenParser;
-import org.openzen.zenscript.lexer.ZSTokenStream;
 import static org.openzen.zenscript.lexer.ZSTokenType.*;
 import org.openzen.zenscript.linker.FileScope;
 import org.openzen.zenscript.linker.GlobalScriptScope;
@@ -62,7 +61,7 @@ public class ParsedFile {
 		return parse(pkg, tokens);
 	}
 	
-	public static ParsedFile parse(ZSPackage pkg, ZSTokenStream tokens) {
+	public static ParsedFile parse(ZSPackage pkg, ZSTokenParser tokens) {
 		ParsedFile result = new ParsedFile(tokens.getFilename());
 		ZSToken eof = null;
 

Parser/src/main/java/org/openzen/zenscript/parser/ParsedImport.java

@@ -9,7 +9,7 @@ import java.util.ArrayList;
 import java.util.List;
 import org.openzen.zenscript.shared.CodePosition;
 import org.openzen.zenscript.lexer.ZSToken;
-import org.openzen.zenscript.lexer.ZSTokenStream;
+import org.openzen.zenscript.lexer.ZSTokenParser;
 import static org.openzen.zenscript.lexer.ZSTokenType.*;
 
 /**
@@ -17,7 +17,7 @@ import static org.openzen.zenscript.lexer.ZSTokenType.*;
  * @author Hoofdgebruiker
  */
 public class ParsedImport {
-	public static ParsedImport parse(CodePosition position, ZSTokenStream tokens) {
+	public static ParsedImport parse(CodePosition position, ZSTokenParser tokens) {
 		boolean relative = tokens.optional(T_DOT) != null;
 		
 		List<String> importName = new ArrayList<>();

Parser/src/main/java/org/openzen/zenscript/parser/definitions/ParsedAlias.java

@@ -10,7 +10,7 @@ import org.openzen.zenscript.codemodel.HighLevelDefinition;
 import org.openzen.zenscript.codemodel.definition.AliasDefinition;
 import org.openzen.zenscript.codemodel.definition.ZSPackage;
 import org.openzen.zenscript.codemodel.generic.TypeParameter;
-import org.openzen.zenscript.lexer.ZSTokenStream;
+import org.openzen.zenscript.lexer.ZSTokenParser;
 import org.openzen.zenscript.lexer.ZSTokenType;
 import org.openzen.zenscript.linker.BaseScope;
 import org.openzen.zenscript.linker.DefinitionScope;
@@ -23,7 +23,7 @@ import org.openzen.zenscript.shared.CodePosition;
  * @author Hoofdgebruiker
  */
 public class ParsedAlias extends ParsedDefinition {
-	public static ParsedAlias parseAlias(ZSPackage pkg, CodePosition position, int modifiers, ZSTokenStream tokens, HighLevelDefinition outerDefinition) {
+	public static ParsedAlias parseAlias(ZSPackage pkg, CodePosition position, int modifiers, ZSTokenParser tokens, HighLevelDefinition outerDefinition) {
 		String name = tokens.required(ZSTokenType.T_IDENTIFIER, "identifier expected").content;
 		List<ParsedGenericParameter> parameters = ParsedGenericParameter.parseAll(tokens);
 		IParsedType type = IParsedType.parse(tokens);

Parser/src/main/java/org/openzen/zenscript/parser/definitions/ParsedClass.java

@@ -9,7 +9,7 @@ import java.util.List;
 import org.openzen.zenscript.codemodel.HighLevelDefinition;
 import org.openzen.zenscript.codemodel.definition.ClassDefinition;
 import org.openzen.zenscript.codemodel.definition.ZSPackage;
-import org.openzen.zenscript.lexer.ZSTokenStream;
+import org.openzen.zenscript.lexer.ZSTokenParser;
 import org.openzen.zenscript.lexer.ZSTokenType;
 import org.openzen.zenscript.linker.BaseScope;
 import org.openzen.zenscript.linker.GenericFunctionScope;
@@ -22,7 +22,7 @@ import org.openzen.zenscript.shared.CodePosition;
  * @author Stan Hebben
  */
 public class ParsedClass extends BaseParsedDefinition {
-	public static ParsedClass parseClass(ZSPackage pkg, CodePosition position, int modifiers, ZSTokenStream tokens, HighLevelDefinition outerDefinition) {
+	public static ParsedClass parseClass(ZSPackage pkg, CodePosition position, int modifiers, ZSTokenParser tokens, HighLevelDefinition outerDefinition) {
 		String name = tokens.required(ZSTokenType.T_IDENTIFIER, "identifier expected").content;
 		List<ParsedGenericParameter> genericParameters = ParsedGenericParameter.parseAll(tokens);
 		

Parser/src/main/java/org/openzen/zenscript/parser/definitions/ParsedEnum.java

@@ -12,7 +12,7 @@ import org.openzen.zenscript.codemodel.definition.EnumDefinition;
 import org.openzen.zenscript.codemodel.definition.ZSPackage;
 import org.openzen.zenscript.codemodel.type.DefinitionTypeID;
 import org.openzen.zenscript.codemodel.type.ITypeID;
-import org.openzen.zenscript.lexer.ZSTokenStream;
+import org.openzen.zenscript.lexer.ZSTokenParser;
 import org.openzen.zenscript.lexer.ZSTokenType;
 import org.openzen.zenscript.linker.BaseScope;
 import org.openzen.zenscript.linker.ExpressionScope;
@@ -24,7 +24,7 @@ import org.openzen.zenscript.shared.CodePosition;
  * @author Hoofdgebruiker
  */
 public class ParsedEnum extends BaseParsedDefinition {
-	public static ParsedEnum parseEnum(ZSPackage pkg, CodePosition position, int modifiers, ZSTokenStream tokens, HighLevelDefinition outerDefinition) {
+	public static ParsedEnum parseEnum(ZSPackage pkg, CodePosition position, int modifiers, ZSTokenParser tokens, HighLevelDefinition outerDefinition) {
 		String name = tokens.required(ZSTokenType.T_IDENTIFIER, "identifier expected").content;
 		tokens.required(ZSTokenType.T_AOPEN, "{ expected");
 		

Parser/src/main/java/org/openzen/zenscript/parser/definitions/ParsedEnumConstant.java

@@ -12,7 +12,7 @@ import org.openzen.zenscript.codemodel.definition.EnumDefinition;
 import org.openzen.zenscript.codemodel.member.EnumConstantMember;
 import org.openzen.zenscript.codemodel.type.DefinitionTypeID;
 import org.openzen.zenscript.lexer.ZSToken;
-import org.openzen.zenscript.lexer.ZSTokenStream;
+import org.openzen.zenscript.lexer.ZSTokenParser;
 import org.openzen.zenscript.lexer.ZSTokenType;
 import org.openzen.zenscript.linker.ExpressionScope;
 import org.openzen.zenscript.parser.expression.ParsedCallArguments;
@@ -25,7 +25,7 @@ import org.openzen.zenscript.shared.CodePosition;
  * @author Hoofdgebruiker
  */
 public class ParsedEnumConstant {
-	public static ParsedEnumConstant parse(ZSTokenStream tokens, EnumDefinition definition, int value) {
+	public static ParsedEnumConstant parse(ZSTokenParser tokens, EnumDefinition definition, int value) {
 		CodePosition position = tokens.getPosition();
 		ZSToken name = tokens.required(ZSTokenType.T_IDENTIFIER, "identifier expected");
 		List<ParsedExpression> arguments = new ArrayList<>();

Parser/src/main/java/org/openzen/zenscript/parser/definitions/ParsedExpansion.java

@@ -9,7 +9,7 @@ import java.util.List;
 import org.openzen.zenscript.codemodel.HighLevelDefinition;
 import org.openzen.zenscript.codemodel.definition.ExpansionDefinition;
 import org.openzen.zenscript.codemodel.definition.ZSPackage;
-import org.openzen.zenscript.lexer.ZSTokenStream;
+import org.openzen.zenscript.lexer.ZSTokenParser;
 import org.openzen.zenscript.lexer.ZSTokenType;
 import org.openzen.zenscript.linker.BaseScope;
 import org.openzen.zenscript.linker.GenericFunctionScope;
@@ -22,7 +22,7 @@ import org.openzen.zenscript.shared.CodePosition;
  * @author Hoofdgebruiker
  */
 public class ParsedExpansion extends BaseParsedDefinition {
-	public static ParsedExpansion parseExpansion(ZSPackage pkg, CodePosition position, int modifiers, ZSTokenStream tokens, HighLevelDefinition outerDefinition) {
+	public static ParsedExpansion parseExpansion(ZSPackage pkg, CodePosition position, int modifiers, ZSTokenParser tokens, HighLevelDefinition outerDefinition) {
 		List<ParsedGenericParameter> parameters = ParsedGenericParameter.parseAll(tokens);
 		IParsedType target = IParsedType.parse(tokens);
 		tokens.required(ZSTokenType.T_AOPEN, "{ expected");

Parser/src/main/java/org/openzen/zenscript/parser/definitions/ParsedFunction.java

@@ -10,7 +10,7 @@ import static org.openzen.zenscript.lexer.ZSTokenType.*;
 import org.openzen.zenscript.codemodel.HighLevelDefinition;
 import org.openzen.zenscript.codemodel.definition.FunctionDefinition;
 import org.openzen.zenscript.codemodel.definition.ZSPackage;
-import org.openzen.zenscript.lexer.ZSTokenStream;
+import org.openzen.zenscript.lexer.ZSTokenParser;
 import org.openzen.zenscript.linker.BaseScope;
 import org.openzen.zenscript.linker.FunctionScope;
 import org.openzen.zenscript.parser.ParsedDefinition;
@@ -23,7 +23,7 @@ import org.openzen.zenscript.shared.CodePosition;
  * @author Stanneke
  */
 public class ParsedFunction extends ParsedDefinition {
-	public static ParsedFunction parseFunction(ZSPackage pkg, CodePosition position, int modifiers, ZSTokenStream parser, HighLevelDefinition outerDefinition) {
+	public static ParsedFunction parseFunction(ZSPackage pkg, CodePosition position, int modifiers, ZSTokenParser parser, HighLevelDefinition outerDefinition) {
 		String name = parser.required(T_IDENTIFIER, "identifier expected").content;
 		ParsedFunctionHeader header = ParsedFunctionHeader.parse(parser);
 		ParsedFunctionBody body = ParsedStatement.parseFunctionBody(parser);

Parser/src/main/java/org/openzen/zenscript/parser/definitions/ParsedFunctionHeader.java

@@ -13,7 +13,7 @@ import org.openzen.zenscript.codemodel.FunctionParameter;
 import org.openzen.zenscript.codemodel.generic.TypeParameter;
 import org.openzen.zenscript.codemodel.type.ITypeID;
 import org.openzen.zenscript.lexer.ZSToken;
-import org.openzen.zenscript.lexer.ZSTokenStream;
+import org.openzen.zenscript.lexer.ZSTokenParser;
 import org.openzen.zenscript.lexer.ZSTokenType;
 import static org.openzen.zenscript.lexer.ZSTokenType.*;
 import org.openzen.zenscript.linker.BaseScope;
@@ -27,7 +27,7 @@ import org.openzen.zenscript.parser.type.ParsedTypeBasic;
  * @author Hoofdgebruiker
  */
 public class ParsedFunctionHeader {
-	public static ParsedFunctionHeader parse(ZSTokenStream tokens) {
+	public static ParsedFunctionHeader parse(ZSTokenParser tokens) {
 		List<ParsedGenericParameter> genericParameters = null;
 		if (tokens.optional(ZSTokenType.T_LESS) != null) {
 			genericParameters = new ArrayList<>();

Parser/src/main/java/org/openzen/zenscript/parser/definitions/ParsedGenericParameter.java

@@ -10,7 +10,6 @@ import java.util.List;
 import org.openzen.zenscript.codemodel.generic.TypeParameter;
 import org.openzen.zenscript.lexer.ZSToken;
 import org.openzen.zenscript.lexer.ZSTokenParser;
-import org.openzen.zenscript.lexer.ZSTokenStream;
 import org.openzen.zenscript.lexer.ZSTokenType;
 import org.openzen.zenscript.linker.BaseScope;
 import org.openzen.zenscript.linker.GenericFunctionScope;
@@ -22,7 +21,7 @@ import org.openzen.zenscript.shared.CodePosition;
  * @author Hoofdgebruiker
  */
 public class ParsedGenericParameter {
-	public static ParsedGenericParameter parse(ZSTokenStream tokens) {
+	public static ParsedGenericParameter parse(ZSTokenParser tokens) {
 		CodePosition position = tokens.getPosition();
 		ZSToken name = tokens.required(ZSTokenType.T_IDENTIFIER, "identifier expected");
 		List<ParsedGenericBound> bounds = new ArrayList<>();
@@ -36,7 +35,7 @@ public class ParsedGenericParameter {
 		return new ParsedGenericParameter(position, name.content, bounds);
 	}
 	
-	public static List<ParsedGenericParameter> parseAll(ZSTokenStream tokens) {
+	public static List<ParsedGenericParameter> parseAll(ZSTokenParser tokens) {
 		if (tokens.optional(ZSTokenType.T_LESS) == null)
 			return null;
 		

Parser/src/main/java/org/openzen/zenscript/parser/definitions/ParsedInterface.java

@@ -11,7 +11,7 @@ import java.util.List;
 import org.openzen.zenscript.codemodel.HighLevelDefinition;
 import org.openzen.zenscript.codemodel.definition.InterfaceDefinition;
 import org.openzen.zenscript.codemodel.definition.ZSPackage;
-import org.openzen.zenscript.lexer.ZSTokenStream;
+import org.openzen.zenscript.lexer.ZSTokenParser;
 import org.openzen.zenscript.lexer.ZSTokenType;
 import org.openzen.zenscript.linker.BaseScope;
 import org.openzen.zenscript.parser.member.ParsedDefinitionMember;
@@ -23,7 +23,7 @@ import org.openzen.zenscript.shared.CodePosition;
  * @author Hoofdgebruiker
  */
 public class ParsedInterface extends BaseParsedDefinition {
-	public static ParsedInterface parseInterface(ZSPackage pkg, CodePosition position, int modifiers, ZSTokenStream tokens, HighLevelDefinition outerDefinition) {
+	public static ParsedInterface parseInterface(ZSPackage pkg, CodePosition position, int modifiers, ZSTokenParser tokens, HighLevelDefinition outerDefinition) {
 		String name = tokens.required(ZSTokenType.T_IDENTIFIER, "identifier expected").content;
 		List<ParsedGenericParameter> genericParameters = ParsedGenericParameter.parseAll(tokens);
 		List<IParsedType> superInterfaces = Collections.emptyList();

Parser/src/main/java/org/openzen/zenscript/parser/definitions/ParsedStruct.java

@@ -9,7 +9,7 @@ import java.util.List;
 import org.openzen.zenscript.codemodel.HighLevelDefinition;
 import org.openzen.zenscript.codemodel.definition.StructDefinition;
 import org.openzen.zenscript.codemodel.definition.ZSPackage;
-import org.openzen.zenscript.lexer.ZSTokenStream;
+import org.openzen.zenscript.lexer.ZSTokenParser;
 import org.openzen.zenscript.lexer.ZSTokenType;
 import org.openzen.zenscript.linker.BaseScope;
 import org.openzen.zenscript.parser.member.ParsedDefinitionMember;
@@ -20,7 +20,7 @@ import org.openzen.zenscript.shared.CodePosition;
  * @author Hoofdgebruiker
  */
 public class ParsedStruct extends BaseParsedDefinition {
-	public static ParsedStruct parseStruct(ZSPackage pkg, CodePosition position, int modifiers, ZSTokenStream tokens, HighLevelDefinition outerDefinition) {
+	public static ParsedStruct parseStruct(ZSPackage pkg, CodePosition position, int modifiers, ZSTokenParser tokens, HighLevelDefinition outerDefinition) {
 		String name = tokens.required(ZSTokenType.T_IDENTIFIER, "identifier expected").content;
 		List<ParsedGenericParameter> parameters = ParsedGenericParameter.parseAll(tokens);
 		

Parser/src/main/java/org/openzen/zenscript/parser/definitions/ParsedVariant.java

@@ -10,7 +10,7 @@ import java.util.List;
 import org.openzen.zenscript.codemodel.HighLevelDefinition;
 import org.openzen.zenscript.codemodel.definition.VariantDefinition;
 import org.openzen.zenscript.codemodel.definition.ZSPackage;
-import org.openzen.zenscript.lexer.ZSTokenStream;
+import org.openzen.zenscript.lexer.ZSTokenParser;
 import org.openzen.zenscript.lexer.ZSTokenType;
 import org.openzen.zenscript.linker.BaseScope;
 import org.openzen.zenscript.linker.DefinitionScope;
@@ -23,7 +23,7 @@ import org.openzen.zenscript.shared.CodePosition;
  * @author Hoofdgebruiker
  */
 public class ParsedVariant extends BaseParsedDefinition {
-	public static ParsedVariant parseVariant(ZSPackage pkg, CodePosition position, int modifiers, ZSTokenStream tokens, HighLevelDefinition outerDefinition) {
+	public static ParsedVariant parseVariant(ZSPackage pkg, CodePosition position, int modifiers, ZSTokenParser tokens, HighLevelDefinition outerDefinition) {
 		String name = tokens.required(ZSTokenType.T_IDENTIFIER, "identifier expected").content;
 		List<ParsedGenericParameter> typeParameters = ParsedGenericParameter.parseAll(tokens);
 		tokens.required(ZSTokenType.T_AOPEN, "{ expected");

Parser/src/main/java/org/openzen/zenscript/parser/expression/ParsedCallArguments.java

@@ -16,7 +16,7 @@ import org.openzen.zenscript.codemodel.expression.Expression;
 import org.openzen.zenscript.codemodel.partial.IPartialExpression;
 import org.openzen.zenscript.codemodel.type.ITypeID;
 import org.openzen.zenscript.codemodel.type.member.DefinitionMemberGroup;
-import org.openzen.zenscript.lexer.ZSTokenStream;
+import org.openzen.zenscript.lexer.ZSTokenParser;
 import org.openzen.zenscript.linker.BaseScope;
 import org.openzen.zenscript.linker.ExpressionScope;
 import org.openzen.zenscript.shared.CodePosition;
@@ -30,7 +30,7 @@ import org.openzen.zenscript.shared.CompileExceptionCode;
 public class ParsedCallArguments {
 	public static final ParsedCallArguments NONE = new ParsedCallArguments(Collections.emptyList());
 	
-	public static ParsedCallArguments parse(ZSTokenStream tokens) {
+	public static ParsedCallArguments parse(ZSTokenParser tokens) {
 		tokens.required(ZSTokenType.T_BROPEN, "( expected");
 		
 		List<ParsedExpression> arguments = new ArrayList<>();

Parser/src/main/java/org/openzen/zenscript/parser/expression/ParsedExpression.java

@@ -19,7 +19,7 @@ import org.openzen.zenscript.codemodel.expression.switchvalue.SwitchValue;
 import org.openzen.zenscript.codemodel.partial.IPartialExpression;
 import org.openzen.zenscript.codemodel.type.ITypeID;
 import org.openzen.zenscript.lexer.ZSToken;
-import org.openzen.zenscript.lexer.ZSTokenStream;
+import org.openzen.zenscript.lexer.ZSTokenParser;
 import org.openzen.zenscript.linker.BaseScope;
 import org.openzen.zenscript.parser.ParseException;
 import org.openzen.zenscript.parser.definitions.ParsedFunctionHeader;
@@ -47,15 +47,15 @@ public abstract class ParsedExpression {
 		}
 	}
 	
-	public static ParsedExpression parse(ZSTokenStream parser) {
+	public static ParsedExpression parse(ZSTokenParser parser) {
 		return readAssignExpression(parser, ParsingOptions.DEFAULT);
 	}
 	
-	public static ParsedExpression parse(ZSTokenStream parser, ParsingOptions options) {
+	public static ParsedExpression parse(ZSTokenParser parser, ParsingOptions options) {
 		return readAssignExpression(parser, options);
 	}
 
-	private static ParsedExpression readAssignExpression(ZSTokenStream parser, ParsingOptions options) {
+	private static ParsedExpression readAssignExpression(ZSTokenParser parser, ParsingOptions options) {
 		CodePosition position = parser.getPosition();
 		ParsedExpression left = readConditionalExpression(position, parser, options);
 
@@ -104,7 +104,7 @@ public abstract class ParsedExpression {
 		return left;
 	}
 
-	private static ParsedExpression readConditionalExpression(CodePosition position, ZSTokenStream parser, ParsingOptions options) {
+	private static ParsedExpression readConditionalExpression(CodePosition position, ZSTokenParser parser, ParsingOptions options) {
 		ParsedExpression left = readOrOrExpression(position, parser, options);
 
 		if (parser.optional(T_QUEST) != null) {
@@ -117,7 +117,7 @@ public abstract class ParsedExpression {
 		return left;
 	}
 
-	private static ParsedExpression readOrOrExpression(CodePosition position, ZSTokenStream parser, ParsingOptions options) {
+	private static ParsedExpression readOrOrExpression(CodePosition position, ZSTokenParser parser, ParsingOptions options) {
 		ParsedExpression left = readAndAndExpression(position, parser, options);
 
 		while (parser.optional(T_OROR) != null) {
@@ -133,7 +133,7 @@ public abstract class ParsedExpression {
 		return left;
 	}
 
-	private static ParsedExpression readAndAndExpression(CodePosition position, ZSTokenStream parser, ParsingOptions options) {
+	private static ParsedExpression readAndAndExpression(CodePosition position, ZSTokenParser parser, ParsingOptions options) {
 		ParsedExpression left = readOrExpression(position, parser, options);
 
 		while (parser.optional(T_ANDAND) != null) {
@@ -143,7 +143,7 @@ public abstract class ParsedExpression {
 		return left;
 	}
 
-	private static ParsedExpression readOrExpression(CodePosition position, ZSTokenStream parser, ParsingOptions options) {
+	private static ParsedExpression readOrExpression(CodePosition position, ZSTokenParser parser, ParsingOptions options) {
 		ParsedExpression left = readXorExpression(position, parser, options);
 
 		while (parser.optional(T_OR) != null) {
@@ -153,7 +153,7 @@ public abstract class ParsedExpression {
 		return left;
 	}
 
-	private static ParsedExpression readXorExpression(CodePosition position, ZSTokenStream parser, ParsingOptions options) {
+	private static ParsedExpression readXorExpression(CodePosition position, ZSTokenParser parser, ParsingOptions options) {
 		ParsedExpression left = readAndExpression(position, parser, options);
 
 		while (parser.optional(T_XOR) != null) {
@@ -163,7 +163,7 @@ public abstract class ParsedExpression {
 		return left;
 	}
 
-	private static ParsedExpression readAndExpression(CodePosition position, ZSTokenStream parser, ParsingOptions options) {
+	private static ParsedExpression readAndExpression(CodePosition position, ZSTokenParser parser, ParsingOptions options) {
 		ParsedExpression left = readCompareExpression(position, parser, options);
 
 		while (parser.optional(T_AND) != null) {
@@ -173,7 +173,7 @@ public abstract class ParsedExpression {
 		return left;
 	}
 
-	private static ParsedExpression readCompareExpression(CodePosition position, ZSTokenStream parser, ParsingOptions options) {
+	private static ParsedExpression readCompareExpression(CodePosition position, ZSTokenParser parser, ParsingOptions options) {
 		ParsedExpression left = readShiftExpression(position, parser, options);
 
 		switch (parser.peek().getType()) {
@@ -244,7 +244,7 @@ public abstract class ParsedExpression {
 		return left;
 	}
 	
-	private static ParsedExpression readShiftExpression(CodePosition position, ZSTokenStream parser, ParsingOptions options) {
+	private static ParsedExpression readShiftExpression(CodePosition position, ZSTokenParser parser, ParsingOptions options) {
 		ParsedExpression left = readAddExpression(position, parser, options);
 		
 		while (true) {
@@ -265,7 +265,7 @@ public abstract class ParsedExpression {
 		return left;
 	}
 
-	private static ParsedExpression readAddExpression(CodePosition position, ZSTokenStream parser, ParsingOptions options) {
+	private static ParsedExpression readAddExpression(CodePosition position, ZSTokenParser parser, ParsingOptions options) {
 		ParsedExpression left = readMulExpression(position, parser, options);
 		
 		while (true) {
@@ -285,7 +285,7 @@ public abstract class ParsedExpression {
 		return left;
 	}
 
-	private static ParsedExpression readMulExpression(CodePosition position, ZSTokenStream parser, ParsingOptions options) {
+	private static ParsedExpression readMulExpression(CodePosition position, ZSTokenParser parser, ParsingOptions options) {
 		ParsedExpression left = readUnaryExpression(position, parser, options);
 
 		while (true) {
@@ -306,7 +306,7 @@ public abstract class ParsedExpression {
 		return left;
 	}
 
-	private static ParsedExpression readUnaryExpression(CodePosition position, ZSTokenStream parser, ParsingOptions options) {
+	private static ParsedExpression readUnaryExpression(CodePosition position, ZSTokenParser parser, ParsingOptions options) {
 		switch (parser.peek().getType()) {
 			case T_NOT:
 				parser.next();
@@ -361,7 +361,7 @@ public abstract class ParsedExpression {
 		}
 	}
 
-	private static ParsedExpression readPostfixExpression(CodePosition position, ZSTokenStream parser, ParsingOptions options) {
+	private static ParsedExpression readPostfixExpression(CodePosition position, ZSTokenParser parser, ParsingOptions options) {
 		ParsedExpression base = readPrimaryExpression(position, parser, options);
 
 		while (true) {
@@ -415,7 +415,7 @@ public abstract class ParsedExpression {
 		return base;
 	}
 	
-	private static ParsedExpression readPrimaryExpression(CodePosition position, ZSTokenStream parser, ParsingOptions options) {
+	private static ParsedExpression readPrimaryExpression(CodePosition position, ZSTokenParser parser, ParsingOptions options) {
 		switch (parser.peek().getType()) {
 			case T_INT:
 				return new ParsedExpressionInt(position, Long.parseLong(parser.next().content));

Parser/src/main/java/org/openzen/zenscript/parser/member/ParsedDefinitionMember.java

@@ -17,7 +17,7 @@ import org.openzen.zenscript.codemodel.Modifiers;
 import org.openzen.zenscript.codemodel.OperatorType;
 import org.openzen.zenscript.codemodel.member.IDefinitionMember;
 import org.openzen.zenscript.lexer.ZSToken;
-import org.openzen.zenscript.lexer.ZSTokenStream;
+import org.openzen.zenscript.lexer.ZSTokenParser;
 import org.openzen.zenscript.lexer.ZSTokenType;
 import org.openzen.zenscript.linker.BaseScope;
 import org.openzen.zenscript.parser.definitions.ParsedFunctionHeader;
@@ -32,7 +32,7 @@ import org.openzen.zenscript.shared.CodePosition;
  * @author Hoofdgebruiker
  */
 public abstract class ParsedDefinitionMember {
-	public static ParsedDefinitionMember parse(ZSTokenStream tokens, HighLevelDefinition forDefinition) {
+	public static ParsedDefinitionMember parse(ZSTokenParser tokens, HighLevelDefinition forDefinition) {
 		CodePosition start = tokens.getPosition();
 		int modifiers = 0;
 		outer: while (true) {

Parser/src/main/java/org/openzen/zenscript/parser/statements/ParsedStatement.java

@@ -6,7 +6,7 @@ import org.openzen.zenscript.codemodel.WhitespaceInfo;
 import org.openzen.zenscript.codemodel.WhitespacePostComment;
 import org.openzen.zenscript.codemodel.statement.Statement;
 import org.openzen.zenscript.lexer.ZSToken;
-import org.openzen.zenscript.lexer.ZSTokenStream;
+import org.openzen.zenscript.lexer.ZSTokenParser;
 import org.openzen.zenscript.lexer.ZSTokenType;
 import static org.openzen.zenscript.lexer.ZSTokenType.*;
 import org.openzen.zenscript.linker.StatementScope;
@@ -18,7 +18,7 @@ import org.openzen.zenscript.shared.CompileException;
 import org.openzen.zenscript.shared.CompileExceptionCode;
 
 public abstract class ParsedStatement {
-	public static ParsedFunctionBody parseLambdaBody(ZSTokenStream tokens, boolean inExpression) {
+	public static ParsedFunctionBody parseLambdaBody(ZSTokenParser tokens, boolean inExpression) {
 		CodePosition position = tokens.getPosition();
 		ZSToken start = tokens.peek();
 		if (tokens.optional(T_AOPEN) != null) {
@@ -35,7 +35,7 @@ public abstract class ParsedStatement {
 		}
 	}
 	
-	public static ParsedFunctionBody parseFunctionBody(ZSTokenStream tokens) {
+	public static ParsedFunctionBody parseFunctionBody(ZSTokenParser tokens) {
 		if (tokens.optional(T_LAMBDA) != null)
 			return parseLambdaBody(tokens, false);
 		else if (tokens.optional(T_SEMICOLON) != null)
@@ -44,7 +44,7 @@ public abstract class ParsedStatement {
 			return new ParsedStatementsFunctionBody(parseBlock(tokens, true));
 	}
 	
-	public static ParsedStatementBlock parseBlock(ZSTokenStream parser, boolean isFirst) {
+	public static ParsedStatementBlock parseBlock(ZSTokenParser parser, boolean isFirst) {
 		String ws = parser.getLastWhitespace();
 		CodePosition position = parser.getPosition();
 		parser.required(T_AOPEN, "{ expected");
@@ -63,11 +63,11 @@ public abstract class ParsedStatement {
 		return new ParsedStatementBlock(position, whitespace, postComment, statements);
 	}
 	
-	public static ParsedStatement parse(ZSTokenStream parser) {
+	public static ParsedStatement parse(ZSTokenParser parser) {
 		return parse(parser, false);
 	}
 	
-	public static ParsedStatement parse(ZSTokenStream parser, boolean isFirst) {
+	public static ParsedStatement parse(ZSTokenParser parser, boolean isFirst) {
 		String ws = parser.getLastWhitespace();
 		CodePosition position = parser.getPosition();
 		ZSToken next = parser.peek();

Parser/src/main/java/org/openzen/zenscript/parser/type/IParsedType.java

@@ -10,7 +10,7 @@ import java.util.Collections;
 import java.util.List;
 import org.openzen.zenscript.codemodel.type.ITypeID;
 import org.openzen.zenscript.codemodel.type.member.TypeMembers;
-import org.openzen.zenscript.lexer.ZSTokenStream;
+import org.openzen.zenscript.lexer.ZSTokenParser;
 import org.openzen.zenscript.lexer.ZSTokenType;
 import static org.openzen.zenscript.lexer.ZSTokenType.*;
 import org.openzen.zenscript.linker.BaseScope;
@@ -24,7 +24,7 @@ import org.openzen.zenscript.shared.CodePosition;
  * @author Hoofdgebruiker
  */
 public interface IParsedType {
-	public static IParsedType parse(ZSTokenStream tokens) {
+	public static IParsedType parse(ZSTokenParser tokens) {
 		IParsedType result = tryParse(tokens);
 		if (result == null)
 			throw new ParseException(tokens.getPosition(), "Type expected (got " + tokens.peek().content + ")");
@@ -32,7 +32,7 @@ public interface IParsedType {
 		return result;
 	}
 	
-	public static IParsedType tryParse(ZSTokenStream tokens) {
+	public static IParsedType tryParse(ZSTokenParser tokens) {
 		int modifiers = 0;
 		while (true) {
 			if (tokens.optional(ZSTokenType.K_CONST) != null) {
@@ -168,7 +168,7 @@ public interface IParsedType {
 		return result;
 	}
 	
-	public static List<IParsedType> parseGenericParameters(ZSTokenStream tokens) {
+	public static List<IParsedType> parseGenericParameters(ZSTokenParser tokens) {
 		if (!tokens.isNext(T_LESS))
 			return null;
 		
@@ -186,13 +186,13 @@ public interface IParsedType {
 		} while (tokens.optional(T_COMMA) != null);
 		
 		if (tokens.isNext(T_SHR)) {
-			tokens.replace(T_GREATER);
+			tokens.replace(T_GREATER.flyweight);
 		} else if (tokens.isNext(T_USHR)) {
-			tokens.replace(T_SHR);
+			tokens.replace(T_SHR.flyweight);
 		} else if (tokens.isNext(T_SHRASSIGN)) {
-			tokens.replace(T_GREATEREQ);
+			tokens.replace(T_GREATEREQ.flyweight);
 		} else if (tokens.isNext(T_USHRASSIGN)) {
-			tokens.replace(T_SHRASSIGN);
+			tokens.replace(T_SHRASSIGN.flyweight);
 		} else if (tokens.optional(T_GREATER) == null) {
 			tokens.reset();
 			return Collections.emptyList();