- Split token stream and parser (will be important for the IDE)

- Can now parse directly from a token stream (doesn't have to be a file)
- Changed the way libraries are stored in projects
- Tokens no longer contain their position (will be important for the IDE)

Constructor/src/main/java/org/openzen/zenscript/constructor/Library.java

@@ -0,0 +1,34 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.openzen.zenscript.constructor;
+
+import java.io.File;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.openzen.zenscript.constructor.module.DirectoryModuleLoader;
+import org.openzen.zenscript.constructor.module.ModuleReference;
+
+/**
+ *
+ * @author Hoofdgebruiker
+ */
+public class Library {
+	public final String name;
+	public final File directory;
+	public final ModuleReference[] modules;
+	
+	public Library(ModuleLoader loader, File directory, String name, JSONObject json) {
+		this.name = name;
+		this.directory = new File(directory, json.getString("directory"));
+		
+		JSONArray modulesJson = json.getJSONArray("modules");
+		modules = new ModuleReference[modulesJson.length()];
+		for (int i = 0; i < modulesJson.length(); i++) {
+			String moduleName = modulesJson.getString(i);
+			modules[i] = new DirectoryModuleLoader(loader, moduleName, new File(this.directory, moduleName), false);
+		}
+	}
+}

Constructor/src/main/java/org/openzen/zenscript/constructor/Main.java

@@ -22,6 +22,10 @@ public class Main {
 		moduleLoader.register("stdlib", new DirectoryModuleLoader(moduleLoader, "stdlib", new File("libraries/stdlib"), true));
 		
 		Project project = new Project(moduleLoader, currentDirectory);
+		for (Library library : project.libraries) {
+			for (ModuleReference module : library.modules)
+				moduleLoader.register(module.getName(), module);
+		}
 		for (ModuleReference module : project.modules) {
 			moduleLoader.register(module.getName(), module);
 		}

Constructor/src/main/java/org/openzen/zenscript/constructor/Project.java

@@ -17,9 +17,13 @@ import org.openzen.zenscript.constructor.module.ModuleReference;
  * @author Hoofdgebruiker
  */
 public class Project {
+	public final String name;
 	public final ModuleReference[] modules;
+	public final Library[] libraries;
 	
 	public Project(ModuleLoader loader, File directory) throws IOException {
+		name = directory.getName();
+		
 		if (!directory.exists())
 			throw new ConstructorException("Project directory doesn't exist");
 		if (!directory.isDirectory())
@@ -31,6 +35,14 @@ public class Project {
 		
 		JSONObject json = JSONUtils.load(projectFile);
 		
+		JSONObject jsonLibraries = json.getJSONObject("libraries");
+		libraries = new Library[jsonLibraries.length()];
+		int k = 0;
+		for (String key : jsonLibraries.keySet()) {
+			libraries[k] = new Library(loader, directory, key, jsonLibraries.getJSONObject(key));
+			k++;
+		}
+		
 		JSONArray jsonModules = json.getJSONArray("modules");
 		modules = new ModuleReference[jsonModules.length()];
 		for (int i = 0; i < jsonModules.length(); i++) {

Constructor/src/main/java/org/openzen/zenscript/constructor/module/DirectoryModuleLoader.java

@@ -28,6 +28,7 @@ public class DirectoryModuleLoader implements ModuleReference {
 	private final String moduleName;
 	private final File directory;
 	private final boolean isStdlib;
+	private SourcePackage rootPackage = null;
 	
 	public DirectoryModuleLoader(ModuleLoader loader, String moduleName, File directory, boolean isStdlib) {
 		this.loader = loader;
@@ -93,4 +94,26 @@ public class DirectoryModuleLoader implements ModuleReference {
 			throw new ConstructorException("Loading module files failed: " + ex.getMessage(), ex);
 		}
 	}
+	
+	@Override
+	public SourcePackage getRootPackage() {
+		if (rootPackage == null)
+			rootPackage = loadPackage("", new File(directory, "src"));
+		
+		return rootPackage;
+	}
+	
+	private SourcePackage loadPackage(String name, File directory) {
+		SourcePackage pkg = new SourcePackage(name);
+		
+		for (File file : directory.listFiles()) {
+			if (file.isDirectory()) {
+				pkg.addPackage(loadPackage(file.getName(), file));
+			} else if (file.isFile() && file.getName().toLowerCase().endsWith(".zs")) {
+				pkg.addFile(new SourceFile(file.getName(), file));
+			}
+		}
+		
+		return pkg;
+	}
 }

Constructor/src/main/java/org/openzen/zenscript/constructor/module/ModuleReference.java

@@ -13,4 +13,6 @@ public interface ModuleReference {
 	public String getName();
 	
 	public SemanticModule load();
+	
+	public SourcePackage getRootPackage();
 }

Constructor/src/main/java/org/openzen/zenscript/constructor/module/SourceFile.java

@@ -0,0 +1,22 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.openzen.zenscript.constructor.module;
+
+import java.io.File;
+
+/**
+ *
+ * @author Hoofdgebruiker
+ */
+public class SourceFile {
+	public final String name;
+	public final File file;
+	
+	public SourceFile(String name, File file) {
+		this.name = name;
+		this.file = file;
+	}
+}

Constructor/src/main/java/org/openzen/zenscript/constructor/module/SourcePackage.java

@@ -0,0 +1,31 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.openzen.zenscript.constructor.module;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ *
+ * @author Hoofdgebruiker
+ */
+public class SourcePackage {
+	public final String name;
+	public final Map<String, SourcePackage> subPackages = new HashMap<>();
+	public final Map<String, SourceFile> sourceFiles = new HashMap<>();
+	
+	public SourcePackage(String name) {
+		this.name = name;
+	}
+	
+	public void addPackage(SourcePackage pkg) {
+		subPackages.put(pkg.name, pkg);
+	}
+	
+	public void addFile(SourceFile file) {
+		sourceFiles.put(file.name, file);
+	}
+}

Parser/src/main/java/org/openzen/zenscript/lexer/Token.java

@@ -1,8 +1,6 @@
 /* Licensed under GPLv3 - https://opensource.org/licenses/GPL-3.0 */
 package org.openzen.zenscript.lexer;
 
-import org.openzen.zenscript.shared.CodePosition;
-
 /**
  * Represents a token.
  * 
@@ -11,8 +9,6 @@ import org.openzen.zenscript.shared.CodePosition;
  */
 public interface Token<TT extends TokenType>
 {
-	CodePosition getPosition();
-	
 	TT getType();
 	
 	String getContent();

Parser/src/main/java/org/openzen/zenscript/lexer/TokenParser.java

@@ -0,0 +1,411 @@
+/* Licensed under GPLv3 - https://opensource.org/licenses/GPL-3.0 */
+package org.openzen.zenscript.lexer;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Stack;
+import org.openzen.zenscript.shared.CodePosition;
+
+/**
+ * Represents a token stream. A token stream reads characters from a reader and
+ * presents it as a series of tokens. Can be used to implement LL(*) parsers.
+ *
+ * Token classes with a negative class are considered to be whitespace.
+ *
+ * @author Stan Hebben
+ * @param <T> token class
+ * @param <TT> token type class
+ */
+public abstract class TokenParser<T extends Token<TT>, TT extends TokenType> implements Iterator<T>
+{
+	private final String filename;
+    private final CountingReader reader;
+    private final CompiledDFA<TT> dfa;
+    private final LinkedList<PositionedToken> tokenMemory;
+    private final Stack<Integer> marks;
+	private final TT eof;
+	private final int tabSize = 4;
+	
+    private PositionedToken next;
+	
+    private int nextChar;
+    private int line;
+    private int lineOffset;
+	
+    private int tokenMemoryOffset;
+    private int tokenMemoryCurrent;
+    
+    /**
+     * Creates a token stream using the specified reader and DFA.
+     *
+	 * @param filename filename
+     * @param reader reader to read characters from
+     * @param dfa DFA to tokenize the stream
+	 * @param eof end of file token type
+     */
+    public TokenParser(String filename, Reader reader, CompiledDFA<TT> dfa, TT eof)
+	{
+		if (eof.isWhitespace()) // important for the advance() method
+			throw new IllegalArgumentException("EOF cannot be whitespace");
+		
+        tokenMemoryOffset = 0;
+        tokenMemoryCurrent = 0;
+        tokenMemory = new LinkedList<>();
+        marks = new Stack<>();
+        
+        this.reader = new CountingReader(reader);
+        this.dfa = dfa;
+		this.filename = filename;
+		this.eof = eof;
+		
+		try {
+	        nextChar = this.reader.read();
+		} catch (IOException ex) {
+			ioException(ex);
+		}
+		
+        line = 1;
+        lineOffset = 1;
+        advance();
+    }
+
+    /**
+     * Creates a token stream which reads data from the specified string.
+     *
+	 * @param filename filename
+     * @param data data to read
+     * @param dfa DFA to tokenize the stream
+	 * @param eof end of file token type
+     */
+    public TokenParser(String filename, String data, CompiledDFA<TT> dfa, TT eof)
+	{
+        this(filename, new StringReader(data), dfa, eof);
+    }
+	
+	public String getFilename()
+	{
+		return filename;
+	}
+	
+	public int getLine()
+	{
+		return line;
+	}
+	
+	public int getLineOffset()
+	{
+		return lineOffset;
+	}
+    
+    public T peek()
+	{
+        if (tokenMemoryCurrent < tokenMemoryOffset + tokenMemory.size()) {
+            return tokenMemory.get((tokenMemoryCurrent) - tokenMemoryOffset).token;
+        } else {
+            return next.token;
+        }
+    }
+
+    public boolean isNext(TT type)
+	{
+        return peek().getType() == type;
+    }
+
+    public T optional(TT type)
+	{
+        if (peek().getType() == type) {
+            return next();
+        } else {
+            return null;
+        }
+    }
+
+    public T required(TT type, String error)
+	{
+		T t = peek();
+        if (t.getType() == type) {
+            return next();
+        } else {
+			requiredTokenNotFound(getPosition(), error, t);
+			return null;
+        }
+    }
+	
+	public CodePosition getPosition()
+	{
+		if (tokenMemoryCurrent < tokenMemoryOffset + tokenMemory.size()) {
+            return tokenMemory.get((tokenMemoryCurrent) - tokenMemoryOffset).position;
+        } else {
+            return next.position;
+        }
+	}
+	
+	// =====================
+    // === LL(*) ability ===
+	// =====================
+
+    /**
+     * Pushes a mark on the mark stack.
+     */
+    public void pushMark()
+	{
+        marks.push(tokenMemoryCurrent);
+    }
+
+    /**
+     * Pops a mark from the mark stack without reset.
+     */
+    public void popMark()
+	{
+        marks.pop();
+
+        if (marks.isEmpty()) {
+            tokenMemoryOffset = tokenMemoryCurrent;
+            tokenMemory.clear();
+        }
+    }
+
+    /**
+     * Pops a mark from the mark stack and resets the stream's position to it
+     */
+    public void reset()
+	{
+        tokenMemoryCurrent = marks.pop();
+    }
+	
+	/**
+	 * Replaces the current token with another one. Used to split composite tokens.
+	 * 
+	 * @param other 
+	 */
+	public void replace(TT other) {
+		next = new PositionedToken(
+				next.position,
+				createToken(
+						next.token.getWhitespaceBefore(),
+						next.token.getContent(),
+						other));
+	}
+
+    // ===============================
+    // === Iterator implementation ===
+    // ===============================
+
+	@Override
+    public boolean hasNext()
+	{
+        return next.token.getType() != eof;
+    }
+
+	@Override
+    public T next()
+	{
+        if (tokenMemoryCurrent < tokenMemoryOffset + tokenMemory.size()) {
+            return tokenMemory.get((tokenMemoryCurrent++) - tokenMemoryOffset).token;
+        } else {
+            PositionedToken result = next;
+
+            if (marks.isEmpty()) {
+                tokenMemoryOffset++;
+            } else {
+                tokenMemory.add(result);
+            }
+            tokenMemoryCurrent++;
+
+            advance();
+            return result.token;
+        }
+    }
+
+	@Override
+    public void remove()
+	{
+        throw new UnsupportedOperationException("Not supported.");
+    }
+
+    // ==================================
+    // === Protected abstract methods ===
+    // ==================================
+	
+	/**
+	 * Called to create a token. May also be used to postprocess tokens while
+	 * generating them.
+	 * 
+	 * @param position token position (range)
+	 * @param value token value
+	 * @param tokenType token type
+	 * @return newly created token
+	 */
+	protected abstract T createToken(
+			String whitespaceBefore,
+			String value,
+			TT tokenType);
+	
+	/**
+	 * Called when a required token could not be found. Should log an error or
+	 * throw an exception. If no exception is thrown, the calling required
+	 * method will return null as token value.
+	 * 
+	 * @param position erroring position
+	 * @param error error to be logged
+	 * @param token incorrect token
+	 */
+	protected abstract void requiredTokenNotFound(
+			CodePosition position,
+			String error,
+			T token);
+	
+	/**
+	 * Called when the input contains an invalid token. Should either create
+	 * a token indicating an invalid token, or throw an exception.
+	 * 
+	 * @param position erroring position
+	 * @param token token value
+	 * @return a token marking an invalid token
+	 */
+	protected abstract T invalidToken(
+			String whitespaceBefore,
+			String token);
+	
+	/**
+	 * Called when an IO exception occurs. Should throw an exception of some
+	 * kind.
+	 * 
+	 * @param ex exception to be logged
+	 */
+	protected abstract void ioException(IOException ex);
+	
+    // =======================
+    // === Private methods ===
+    // =======================
+
+    /**
+     * Advances to the next non - whitespace token.
+     */
+    private void advance()
+	{
+		StringBuilder whitespace = new StringBuilder();
+        while (true) {
+            advanceToken(whitespace.toString());
+			if (next.token.getType().isWhitespace()) {
+				whitespace.append(next.token.getContent());
+			} else {
+				break;
+			}
+        }
+    }
+
+    /**
+     * Advances to the next token.
+     */
+    private void advanceToken(String whitespace)
+	{
+        if (nextChar < 0) {
+			CodePosition position = new CodePosition(
+					filename,
+					line,
+					lineOffset,
+					line,
+					lineOffset);
+			
+            next = new PositionedToken(position, createToken(whitespace, "", eof));
+            return;
+        }
+		
+        try {
+            int state = 0;
+            StringBuilder value = new StringBuilder();
+            int fromLine = line;
+            int fromLineOffset = lineOffset;
+            while (dfa.transitions[state].containsKey(nextChar)) {
+                value.append((char)nextChar);
+                state = dfa.transitions[state].get(nextChar);
+                line = reader.line;
+                lineOffset = reader.lineOffset;
+                nextChar = reader.read();
+            }
+			
+			if (line < 0)
+				throw new IllegalStateException("Line cannot be negative");
+			
+			CodePosition position = new CodePosition(filename, fromLine, fromLineOffset, line, lineOffset);
+            if (dfa.finals[state] != null) {
+                if (state == 0) {
+					value.append((char) nextChar);
+					next = new PositionedToken(position, invalidToken(
+							whitespace,
+							value.toString()));
+					nextChar = reader.read();
+				}
+				
+				next = new PositionedToken(position, createToken(
+						whitespace,
+						value.toString(), dfa.finals[state]));
+            } else {
+				if (nextChar < 0 && value.length() == 0)
+					return; // happens on comments at the end of files
+				
+				value.append((char) nextChar);
+				next = new PositionedToken(position, invalidToken(
+						whitespace,
+						value.toString()));
+				nextChar = reader.read();
+            }
+        } catch (IOException ex) {
+            ioException(ex);
+        }
+    }
+
+    // =============================
+    // === Private inner classes ===
+    // =============================
+
+    /**
+     * Keeps a line and line offset count.
+     */
+    private class CountingReader
+	{
+        private int line;
+        private int lineOffset;
+        private Reader reader;
+        private boolean eof;
+
+        public CountingReader(Reader reader)
+		{
+            this.reader = reader;
+            line = 1;
+            lineOffset = 1;
+        }
+
+        public int read() throws IOException
+		{
+            int ch = reader.read();
+            if (ch == -1) {
+                eof = true;
+                return ch;
+            }
+            if (ch == '\n') {
+                line++;
+                lineOffset = 1;
+            } else if (ch == '\t') {
+				lineOffset += tabSize;
+			} else {
+                lineOffset++;
+            }
+            return ch;
+        }
+    }
+	
+	private class PositionedToken {
+		public final CodePosition position;
+		public final T token;
+		
+		public PositionedToken(CodePosition position, T token) {
+			this.position = position;
+			this.token = token;
+		}
+	}
+}

Parser/src/main/java/org/openzen/zenscript/lexer/TokenStream.java

@@ -1,392 +1,36 @@
-/* Licensed under GPLv3 - https://opensource.org/licenses/GPL-3.0 */
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
 package org.openzen.zenscript.lexer;
 
-import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.Stack;
 import org.openzen.zenscript.shared.CodePosition;
 
 /**
- * Represents a token stream. A token stream reads characters from a reader and
- * presents it as a series of tokens. Can be used to implement LL(*) parsers.
- *
- * Token classes with a negative class are considered to be whitespace.
  *
- * @author Stan Hebben
- * @param <T> token class
- * @param <TT> token type class
+ * @author Hoofdgebruiker
  */
-public abstract class TokenStream<T extends Token<TT>, TT extends TokenType> implements Iterator<T>
-{
-	private final String filename;
-    private final CountingReader reader;
-    private final CompiledDFA<TT> dfa;
-    private final LinkedList<T> tokenMemory;
-    private final Stack<Integer> marks;
-	private final TT eof;
-	private final int tabSize = 4;
+public interface TokenStream<TT extends TokenType, T extends Token<TT>> {
+	T peek();
 	
-    private T next;
-    private int nextChar;
-    private int line;
-    private int lineOffset;
-	
-    private int tokenMemoryOffset;
-    private int tokenMemoryCurrent;
-    
-    /**
-     * Creates a token stream using the specified reader and DFA.
-     *
-	 * @param filename filename
-     * @param reader reader to read characters from
-     * @param dfa DFA to tokenize the stream
-	 * @param eof end of file token type
-     */
-    public TokenStream(String filename, Reader reader, CompiledDFA<TT> dfa, TT eof)
-	{
-		if (eof.isWhitespace()) // important for the advance() method
-			throw new IllegalArgumentException("EOF cannot be whitespace");
-		
-        tokenMemoryOffset = 0;
-        tokenMemoryCurrent = 0;
-        tokenMemory = new LinkedList<>();
-        marks = new Stack<>();
-        
-        this.reader = new CountingReader(reader);
-        this.dfa = dfa;
-		this.filename = filename;
-		this.eof = eof;
-		
-		try {
-	        nextChar = this.reader.read();
-		} catch (IOException ex) {
-			ioException(ex);
-		}
-		
-        line = 1;
-        lineOffset = 1;
-        advance();
-    }
-
-    /**
-     * Creates a token stream which reads data from the specified string.
-     *
-	 * @param filename filename
-     * @param data data to read
-     * @param dfa DFA to tokenize the stream
-	 * @param eof end of file token type
-     */
-    public TokenStream(String filename, String data, CompiledDFA<TT> dfa, TT eof)
-	{
-        this(filename, new StringReader(data), dfa, eof);
-    }
+	boolean isNext(TT type);
 	
-	public String getFile()
-	{
-		return filename;
-	}
+	T optional(TT type);
 	
-	public int getLine()
-	{
-		return line;
-	}
+	T required(TT type, String error);
 	
-	public int getLineOffset()
-	{
-		return lineOffset;
-	}
-    
-    public T peek()
-	{
-        if (tokenMemoryCurrent < tokenMemoryOffset + tokenMemory.size()) {
-            return tokenMemory.get((tokenMemoryCurrent) - tokenMemoryOffset);
-        } else {
-            return next;
-        }
-    }
-
-    public boolean isNext(TT type)
-	{
-        return peek().getType() == type;
-    }
-
-    public T optional(TT type)
-	{
-        if (peek().getType() == type) {
-            return next();
-        } else {
-            return null;
-        }
-    }
-
-    public T required(TT type, String error)
-	{
-		T t = peek();
-        if (t.getType() == type) {
-            return next();
-        } else {
-			requiredTokenNotFound(getPosition(), error, t);
-			return null;
-        }
-    }
-	
-	public CodePosition getPosition()
-	{
-		return peek().getPosition();
-	}
-	
-	// =====================
-    // === LL(*) ability ===
-	// =====================
-
-    /**
-     * Pushes a mark on the mark stack.
-     */
-    public void pushMark()
-	{
-        marks.push(tokenMemoryCurrent);
-    }
-
-    /**
-     * Pops a mark from the mark stack without reset.
-     */
-    public void popMark()
-	{
-        marks.pop();
-
-        if (marks.isEmpty()) {
-            tokenMemoryOffset = tokenMemoryCurrent;
-            tokenMemory.clear();
-        }
-    }
-
-    /**
-     * Pops a mark from the mark stack and resets the stream's position to it
-     */
-    public void reset()
-	{
-        tokenMemoryCurrent = marks.pop();
-    }
+	void pushMark();
 	
-	/**
-	 * Replaces the current token with another one. Used to split composite tokens.
-	 * 
-	 * @param other 
-	 */
-	public void replace(TT other) {
-		next = createToken(next.getPosition(), next.getWhitespaceBefore(), next.getContent(), other);
-	}
-
-    // ===============================
-    // === Iterator implementation ===
-    // ===============================
-
-	@Override
-    public boolean hasNext()
-	{
-        return next.getType() != eof;
-    }
-
-	@Override
-    public T next()
-	{
-        if (tokenMemoryCurrent < tokenMemoryOffset + tokenMemory.size()) {
-            return tokenMemory.get((tokenMemoryCurrent++) - tokenMemoryOffset);
-        } else {
-            T result = next;
-
-            if (marks.isEmpty()) {
-                tokenMemoryOffset++;
-            } else {
-                tokenMemory.add(result);
-            }
-            tokenMemoryCurrent++;
-
-            advance();
-            return result;
-        }
-    }
-
-	@Override
-    public void remove()
-	{
-        throw new UnsupportedOperationException("Not supported.");
-    }
-
-    // ==================================
-    // === Protected abstract methods ===
-    // ==================================
+	void popMark();
 	
-	/**
-	 * Called to create a token. May also be used to postprocess tokens while
-	 * generating them.
-	 * 
-	 * @param position token position (range)
-	 * @param value token value
-	 * @param tokenType token type
-	 * @return newly created token
-	 */
-	protected abstract T createToken(
-			CodePosition position,
-			String whitespaceBefore,
-			String value,
-			TT tokenType);
+	void reset();
 	
-	/**
-	 * Called when a required token could not be found. Should log an error or
-	 * throw an exception. If no exception is thrown, the calling required
-	 * method will return null as token value.
-	 * 
-	 * @param position erroring position
-	 * @param error error to be logged
-	 * @param token incorrect token
-	 */
-	protected abstract void requiredTokenNotFound(
-			CodePosition position,
-			String error,
-			T token);
+	void replace(TT other);
 	
-	/**
-	 * Called when the input contains an invalid token. Should either create
-	 * a token indicating an invalid token, or throw an exception.
-	 * 
-	 * @param position erroring position
-	 * @param token token value
-	 * @return a token marking an invalid token
-	 */
-	protected abstract T invalidToken(
-			CodePosition position,
-			String whitespaceBefore,
-			String token);
+	boolean hasNext();
 	
-	/**
-	 * Called when an IO exception occurs. Should throw an exception of some
-	 * kind.
-	 * 
-	 * @param ex exception to be logged
-	 */
-	protected abstract void ioException(IOException ex);
+	T next();
 	
-    // =======================
-    // === Private methods ===
-    // =======================
-
-    /**
-     * Advances to the next non - whitespace token.
-     */
-    private void advance()
-	{
-		StringBuilder whitespace = new StringBuilder();
-        while (true) {
-            advanceToken(whitespace.toString());
-			if (next.getType().isWhitespace()) {
-				whitespace.append(next.getContent());
-			} else {
-				break;
-			}
-        }
-    }
-
-    /**
-     * Advances to the next token.
-     */
-    private void advanceToken(String whitespace)
-	{
-        if (nextChar < 0) {
-			CodePosition position = new CodePosition(
-					filename,
-					line,
-					lineOffset,
-					line,
-					lineOffset);
-			
-            next = createToken(position, whitespace, "", eof);
-            return;
-        }
-		
-        try {
-            int state = 0;
-            StringBuilder value = new StringBuilder();
-            int fromLine = line;
-            int fromLineOffset = lineOffset;
-            while (dfa.transitions[state].containsKey(nextChar)) {
-                value.append((char)nextChar);
-                state = dfa.transitions[state].get(nextChar);
-                line = reader.line;
-                lineOffset = reader.lineOffset;
-                nextChar = reader.read();
-            }
-			
-			if (line < 0)
-				throw new IllegalStateException("Line cannot be negative");
-			
-            if (dfa.finals[state] != null) {
-                if (state == 0) {
-					value.append((char) nextChar);
-					next = invalidToken(new CodePosition(filename, fromLine, fromLineOffset, line, lineOffset),
-							whitespace,
-							value.toString());
-					nextChar = reader.read();
-				}
-				
-				next = createToken(new CodePosition(filename, fromLine, fromLineOffset, line, lineOffset),
-						whitespace,
-						value.toString(), dfa.finals[state]);
-            } else {
-				if (nextChar < 0 && value.length() == 0)
-					return; // happens on comments at the end of files
-				
-				value.append((char) nextChar);
-				next = invalidToken(new CodePosition(filename, fromLine, fromLineOffset, line, lineOffset),
-						whitespace,
-						value.toString());
-				nextChar = reader.read();
-            }
-        } catch (IOException ex) {
-            ioException(ex);
-        }
-    }
-
-    // =============================
-    // === Private inner classes ===
-    // =============================
-
-    /**
-     * Keeps a line and line offset count.
-     */
-    private class CountingReader
-	{
-        private int line;
-        private int lineOffset;
-        private Reader reader;
-        private boolean eof;
-
-        public CountingReader(Reader reader)
-		{
-            this.reader = reader;
-            line = 1;
-            lineOffset = 1;
-        }
-
-        public int read() throws IOException
-		{
-            int ch = reader.read();
-            if (ch == -1) {
-                eof = true;
-                return ch;
-            }
-            if (ch == '\n') {
-                line++;
-                lineOffset = 1;
-            } else if (ch == '\t') {
-				lineOffset += tabSize;
-			} else {
-                lineOffset++;
-            }
-            return ch;
-        }
-    }
+	CodePosition getPosition();
 }

Parser/src/main/java/org/openzen/zenscript/lexer/ZSToken.java

@@ -5,29 +5,20 @@
  */
 package org.openzen.zenscript.lexer;
 
-import org.openzen.zenscript.shared.CodePosition;
-
 /**
  *
  * @author Hoofdgebruiker
  */
 public class ZSToken implements Token<ZSTokenType> {
-	public final CodePosition position;
 	public final ZSTokenType type;
 	public final String content;
 	public final String whitespaceBefore;
 	
-	public ZSToken(CodePosition position, ZSTokenType type, String content, String whitespaceBefore) {
-		this.position = position;
+	public ZSToken(ZSTokenType type, String content, String whitespaceBefore) {
 		this.type = type;
 		this.content = content;
 		this.whitespaceBefore = whitespaceBefore;
 	}
-	
-	@Override
-	public CodePosition getPosition() {
-		return position;
-	}
 
 	@Override
 	public ZSTokenType getType() {

Parser/src/main/java/org/openzen/zenscript/lexer/ZSTokenParser.java

@@ -0,0 +1,176 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.openzen.zenscript.lexer;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.HashMap;
+import java.util.Map;
+import org.openzen.zenscript.codemodel.WhitespaceInfo;
+import org.openzen.zenscript.shared.CompileException;
+import org.openzen.zenscript.shared.CompileExceptionCode;
+import static org.openzen.zenscript.lexer.ZSTokenType.*;
+import org.openzen.zenscript.shared.CodePosition;
+
+/**
+ *
+ * @author Hoofdgebruiker
+ */
+public class ZSTokenParser extends TokenParser<ZSToken, ZSTokenType> implements ZSTokenStream {
+	private static final CompiledDFA DFA = CompiledDFA.createLexerDFA(ZSTokenType.values(), ZSTokenType.class);
+	private static final Map<String, ZSTokenType> KEYWORDS = new HashMap<>();
+	
+	static {
+		KEYWORDS.put("import", K_IMPORT);
+		KEYWORDS.put("alias", K_ALIAS);
+		KEYWORDS.put("class", K_CLASS);
+		KEYWORDS.put("interface", K_INTERFACE);
+		KEYWORDS.put("enum", K_ENUM);
+		KEYWORDS.put("struct", K_STRUCT);
+		KEYWORDS.put("expand", K_EXPAND);
+		KEYWORDS.put("function", K_FUNCTION);
+		KEYWORDS.put("variant", K_VARIANT);
+		
+		KEYWORDS.put("abstract", K_ABSTRACT);
+		KEYWORDS.put("final", K_FINAL);
+		KEYWORDS.put("override", K_OVERRIDE);
+		KEYWORDS.put("const", K_CONST);
+		KEYWORDS.put("private", K_PRIVATE);
+		KEYWORDS.put("public", K_PUBLIC);
+		KEYWORDS.put("export", K_EXPORT);
+		KEYWORDS.put("static", K_STATIC);
+		KEYWORDS.put("protected", K_PROTECTED);
+		KEYWORDS.put("implicit", K_IMPLICIT);
+		KEYWORDS.put("virtual", K_VIRTUAL);
+		KEYWORDS.put("extern", K_EXTERN);
+		
+		KEYWORDS.put("val", K_VAL);
+		KEYWORDS.put("var", K_VAR);
+		KEYWORDS.put("get", K_GET);
+		KEYWORDS.put("implements", K_IMPLEMENTS);
+		KEYWORDS.put("set", K_SET);
+		
+		KEYWORDS.put("void", K_VOID);
+		KEYWORDS.put("any", K_ANY);
+		KEYWORDS.put("bool", K_BOOL);
+		KEYWORDS.put("byte", K_BYTE);
+		KEYWORDS.put("sbyte", K_SBYTE);
+		KEYWORDS.put("short", K_SHORT);
+		KEYWORDS.put("ushort", K_USHORT);
+		KEYWORDS.put("int", K_INT);
+		KEYWORDS.put("uint", K_UINT);
+		KEYWORDS.put("long", K_LONG);
+		KEYWORDS.put("ulong", K_ULONG);
+		KEYWORDS.put("float", K_FLOAT);
+		KEYWORDS.put("double", K_DOUBLE);
+		KEYWORDS.put("char", K_CHAR);
+		KEYWORDS.put("string", K_STRING);
+		
+		KEYWORDS.put("if", K_IF);
+		KEYWORDS.put("else", K_ELSE);
+		KEYWORDS.put("do", K_DO);
+		KEYWORDS.put("while", K_WHILE);
+		KEYWORDS.put("for", K_FOR);
+		KEYWORDS.put("throw", K_THROW);
+		KEYWORDS.put("lock", K_LOCK);
+		KEYWORDS.put("try", K_TRY);
+		KEYWORDS.put("catch", K_CATCH);
+		KEYWORDS.put("finally", K_FINALLY);
+		KEYWORDS.put("return", K_RETURN);
+		KEYWORDS.put("break", K_BREAK);
+		KEYWORDS.put("continue", K_CONTINUE);
+		KEYWORDS.put("switch", K_SWITCH);
+		KEYWORDS.put("case", K_CASE);
+		KEYWORDS.put("default", K_DEFAULT);
+		
+		KEYWORDS.put("in", K_IN);
+		KEYWORDS.put("is", K_IS);
+		KEYWORDS.put("as", K_AS);
+		KEYWORDS.put("match", K_MATCH);
+		KEYWORDS.put("throws", K_THROWS);
+		
+		KEYWORDS.put("this", K_THIS);
+		KEYWORDS.put("super", K_SUPER);
+		KEYWORDS.put("null", K_NULL);
+		KEYWORDS.put("true", K_TRUE);
+		KEYWORDS.put("false", K_FALSE);
+		KEYWORDS.put("new", K_NEW);
+	}
+	
+	private String whitespaceBuffer = null;
+	
+	public ZSTokenParser(String filename, Reader reader) {
+		super(filename, reader, DFA, ZSTokenType.EOF);
+	}
+	
+	@Override
+	public String loadWhitespace() {
+		if (whitespaceBuffer == null)
+			whitespaceBuffer = peek().whitespaceBefore;
+		
+		return whitespaceBuffer;
+	}
+	
+	@Override
+	public void reloadWhitespace() {
+		whitespaceBuffer = peek().whitespaceBefore;
+	}
+	
+	@Override
+	public String grabWhitespace() {
+		String result = loadWhitespace();
+		whitespaceBuffer = null;
+		return result;
+	}
+	
+	public String grabWhitespaceLine() {
+		String whitespace = loadWhitespace();
+		if (whitespace.contains("\n")) {
+			int index = whitespace.indexOf('\n');
+			whitespaceBuffer = whitespace.substring(index + 1);
+			return whitespace.substring(0, index);
+		} else {
+			whitespaceBuffer = "";
+			return whitespace;
+		}
+	}
+	
+	@Override
+	public void skipWhitespaceNewline() {
+		loadWhitespace();
+		int index = whitespaceBuffer.indexOf('\n');
+		if (index >= 0)
+			whitespaceBuffer = whitespaceBuffer.substring(index + 1);
+	}
+	
+	@Override
+	public WhitespaceInfo collectWhitespaceInfo(String whitespace, boolean skipLineBefore) {
+		return WhitespaceInfo.from(whitespace, grabWhitespaceLine(), skipLineBefore);
+	}
+	
+	@Override
+	protected ZSToken createToken(String whitespaceBefore, String value, ZSTokenType tokenType) {
+		if (tokenType == T_IDENTIFIER && KEYWORDS.containsKey(value))
+			tokenType = KEYWORDS.get(value);
+		
+		return new ZSToken(tokenType, value, whitespaceBefore);
+	}
+
+	@Override
+	protected void requiredTokenNotFound(CodePosition position, String error, ZSToken token) {
+		throw new CompileException(position, CompileExceptionCode.UNEXPECTED_TOKEN, error);
+	}
+
+	@Override
+	protected ZSToken invalidToken(String whitespaceBefore, String token) {
+		return new ZSToken(ZSTokenType.INVALID, token, whitespaceBefore);
+	}
+
+	@Override
+	protected void ioException(IOException ex) {
+		throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+	}
+}

Parser/src/main/java/org/openzen/zenscript/lexer/ZSTokenStream.java

@@ -5,171 +5,22 @@
  */
 package org.openzen.zenscript.lexer;
 
-import java.io.IOException;
-import java.io.Reader;
-import java.util.HashMap;
-import java.util.Map;
 import org.openzen.zenscript.codemodel.WhitespaceInfo;
-import org.openzen.zenscript.shared.CompileException;
-import org.openzen.zenscript.shared.CompileExceptionCode;
-import static org.openzen.zenscript.lexer.ZSTokenType.*;
-import org.openzen.zenscript.shared.CodePosition;
 
 /**
  *
  * @author Hoofdgebruiker
  */
-public class ZSTokenStream extends TokenStream<ZSToken, ZSTokenType> {
-	private static final CompiledDFA DFA = CompiledDFA.createLexerDFA(ZSTokenType.values(), ZSTokenType.class);
-	private static final Map<String, ZSTokenType> KEYWORDS = new HashMap<>();
+public interface ZSTokenStream extends TokenStream<ZSTokenType, ZSToken> {
+	String getFilename();
 	
-	static {
-		KEYWORDS.put("import", K_IMPORT);
-		KEYWORDS.put("alias", K_ALIAS);
-		KEYWORDS.put("class", K_CLASS);
-		KEYWORDS.put("interface", K_INTERFACE);
-		KEYWORDS.put("enum", K_ENUM);
-		KEYWORDS.put("struct", K_STRUCT);
-		KEYWORDS.put("expand", K_EXPAND);
-		KEYWORDS.put("function", K_FUNCTION);
-		KEYWORDS.put("variant", K_VARIANT);
-		
-		KEYWORDS.put("abstract", K_ABSTRACT);
-		KEYWORDS.put("final", K_FINAL);
-		KEYWORDS.put("override", K_OVERRIDE);
-		KEYWORDS.put("const", K_CONST);
-		KEYWORDS.put("private", K_PRIVATE);
-		KEYWORDS.put("public", K_PUBLIC);
-		KEYWORDS.put("export", K_EXPORT);
-		KEYWORDS.put("static", K_STATIC);
-		KEYWORDS.put("protected", K_PROTECTED);
-		KEYWORDS.put("implicit", K_IMPLICIT);
-		KEYWORDS.put("virtual", K_VIRTUAL);
-		KEYWORDS.put("extern", K_EXTERN);
-		
-		KEYWORDS.put("val", K_VAL);
-		KEYWORDS.put("var", K_VAR);
-		KEYWORDS.put("get", K_GET);
-		KEYWORDS.put("implements", K_IMPLEMENTS);
-		KEYWORDS.put("set", K_SET);
-		
-		KEYWORDS.put("void", K_VOID);
-		KEYWORDS.put("any", K_ANY);
-		KEYWORDS.put("bool", K_BOOL);
-		KEYWORDS.put("byte", K_BYTE);
-		KEYWORDS.put("sbyte", K_SBYTE);
-		KEYWORDS.put("short", K_SHORT);
-		KEYWORDS.put("ushort", K_USHORT);
-		KEYWORDS.put("int", K_INT);
-		KEYWORDS.put("uint", K_UINT);
-		KEYWORDS.put("long", K_LONG);
-		KEYWORDS.put("ulong", K_ULONG);
-		KEYWORDS.put("float", K_FLOAT);
-		KEYWORDS.put("double", K_DOUBLE);
-		KEYWORDS.put("char", K_CHAR);
-		KEYWORDS.put("string", K_STRING);
-		
-		KEYWORDS.put("if", K_IF);
-		KEYWORDS.put("else", K_ELSE);
-		KEYWORDS.put("do", K_DO);
-		KEYWORDS.put("while", K_WHILE);
-		KEYWORDS.put("for", K_FOR);
-		KEYWORDS.put("throw", K_THROW);
-		KEYWORDS.put("lock", K_LOCK);
-		KEYWORDS.put("try", K_TRY);
-		KEYWORDS.put("catch", K_CATCH);
-		KEYWORDS.put("finally", K_FINALLY);
-		KEYWORDS.put("return", K_RETURN);
-		KEYWORDS.put("break", K_BREAK);
-		KEYWORDS.put("continue", K_CONTINUE);
-		KEYWORDS.put("switch", K_SWITCH);
-		KEYWORDS.put("case", K_CASE);
-		KEYWORDS.put("default", K_DEFAULT);
-		
-		KEYWORDS.put("in", K_IN);
-		KEYWORDS.put("is", K_IS);
-		KEYWORDS.put("as", K_AS);
-		KEYWORDS.put("match", K_MATCH);
-		KEYWORDS.put("throws", K_THROWS);
-		
-		KEYWORDS.put("this", K_THIS);
-		KEYWORDS.put("super", K_SUPER);
-		KEYWORDS.put("null", K_NULL);
-		KEYWORDS.put("true", K_TRUE);
-		KEYWORDS.put("false", K_FALSE);
-		KEYWORDS.put("new", K_NEW);
-	}
+	String loadWhitespace();
 	
-	private String whitespaceBuffer = null;
+	void reloadWhitespace();
 	
-	public ZSTokenStream(String filename, Reader reader) {
-		super(filename, reader, DFA, ZSTokenType.EOF);
-	}
+	String grabWhitespace();
 	
-	public String loadWhitespace() {
-		if (whitespaceBuffer == null)
-			whitespaceBuffer = peek().whitespaceBefore;
-		
-		return whitespaceBuffer;
-	}
+	void skipWhitespaceNewline();
 	
-	public void reloadWhitespace() {
-		whitespaceBuffer = peek().whitespaceBefore;
-	}
-	
-	public String grabWhitespace() {
-		String result = loadWhitespace();
-		whitespaceBuffer = null;
-		return result;
-	}
-	
-	public String grabWhitespaceLine() {
-		String whitespace = loadWhitespace();
-		if (whitespace.contains("\n")) {
-			int index = whitespace.indexOf('\n');
-			whitespaceBuffer = whitespace.substring(index + 1);
-			return whitespace.substring(0, index);
-		} else {
-			whitespaceBuffer = "";
-			return whitespace;
-		}
-	}
-	
-	public void skipWhitespaceNewline() {
-		loadWhitespace();
-		int index = whitespaceBuffer.indexOf('\n');
-		if (index >= 0)
-			whitespaceBuffer = whitespaceBuffer.substring(index + 1);
-	}
-	
-	public WhitespaceInfo collectWhitespaceInfoForBlock(String whitespace) {
-		return WhitespaceInfo.from(whitespace, "", false);
-	}
-	
-	public WhitespaceInfo collectWhitespaceInfo(String whitespace, boolean skipLineBefore) {
-		return WhitespaceInfo.from(whitespace, grabWhitespaceLine(), skipLineBefore);
-	}
-	
-	@Override
-	protected ZSToken createToken(CodePosition position, String whitespaceBefore, String value, ZSTokenType tokenType) {
-		if (tokenType == T_IDENTIFIER && KEYWORDS.containsKey(value))
-			tokenType = KEYWORDS.get(value);
-		
-		return new ZSToken(position, tokenType, value, whitespaceBefore);
-	}
-
-	@Override
-	protected void requiredTokenNotFound(CodePosition position, String error, ZSToken token) {
-		throw new CompileException(position, CompileExceptionCode.UNEXPECTED_TOKEN, error);
-	}
-
-	@Override
-	protected ZSToken invalidToken(CodePosition position, String whitespaceBefore, String token) {
-		return new ZSToken(position, ZSTokenType.INVALID, token, whitespaceBefore);
-	}
-
-	@Override
-	protected void ioException(IOException ex) {
-		throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
-	}
+	WhitespaceInfo collectWhitespaceInfo(String whitespace, boolean skipLineBefore);
 }

Parser/src/main/java/org/openzen/zenscript/parser/ParseException.java

@@ -6,17 +6,12 @@
 package org.openzen.zenscript.parser;
 
 import org.openzen.zenscript.shared.CodePosition;
-import org.openzen.zenscript.lexer.ZSToken;
 
 /**
  *
  * @author Hoofdgebruiker
  */
 public class ParseException extends RuntimeException {
-	public ParseException(ZSToken token, String message) {
-		this(token.position, message);
-	}
-	
 	public ParseException(CodePosition position, String message) {
 		super(position.toString() + ": " + message);
 	}

Parser/src/main/java/org/openzen/zenscript/parser/ParsedFile.java

@@ -27,6 +27,7 @@ import org.openzen.zenscript.lexer.ZSToken;
 import org.openzen.zenscript.shared.CodePosition;
 import org.openzen.zenscript.shared.CompileException;
 import org.openzen.zenscript.shared.CompileExceptionCode;
+import org.openzen.zenscript.lexer.ZSTokenParser;
 import org.openzen.zenscript.lexer.ZSTokenStream;
 import static org.openzen.zenscript.lexer.ZSTokenType.*;
 import org.openzen.zenscript.linker.FileScope;
@@ -57,13 +58,16 @@ public class ParsedFile {
 	}
 	
 	public static ParsedFile parse(ZSPackage pkg, String filename, Reader reader) throws IOException {
-		ParsedFile result = new ParsedFile(filename);
-		
-		ZSTokenStream tokens = new ZSTokenStream(filename, reader);
+		ZSTokenParser tokens = new ZSTokenParser(filename, reader);
+		return parse(pkg, tokens);
+	}
+	
+	public static ParsedFile parse(ZSPackage pkg, ZSTokenStream tokens) {
+		ParsedFile result = new ParsedFile(tokens.getFilename());
 		ZSToken eof = null;
 
 		while (true) {
-			CodePosition position = tokens.peek().position;
+			CodePosition position = tokens.getPosition();
 			int modifiers = 0;
 			outer: while (true) {
 				switch (tokens.peek().type) {

Parser/src/main/java/org/openzen/zenscript/parser/definitions/ParsedEnumConstant.java

@@ -26,6 +26,7 @@ import org.openzen.zenscript.shared.CodePosition;
  */
 public class ParsedEnumConstant {
 	public static ParsedEnumConstant parse(ZSTokenStream tokens, EnumDefinition definition, int value) {
+		CodePosition position = tokens.getPosition();
 		ZSToken name = tokens.required(ZSTokenType.T_IDENTIFIER, "identifier expected");
 		List<ParsedExpression> arguments = new ArrayList<>();
 		if (tokens.optional(ZSTokenType.T_BROPEN) != null) {
@@ -35,7 +36,7 @@ public class ParsedEnumConstant {
 			tokens.required(ZSTokenType.T_BRCLOSE, ") expected");
 		}
 		
-		return new ParsedEnumConstant(name.position, definition, name.content, value, arguments);
+		return new ParsedEnumConstant(position, definition, name.content, value, arguments);
 	}
 	
 	public final CodePosition position;

Parser/src/main/java/org/openzen/zenscript/parser/definitions/ParsedGenericParameter.java

@@ -9,6 +9,7 @@ import java.util.ArrayList;
 import java.util.List;
 import org.openzen.zenscript.codemodel.generic.TypeParameter;
 import org.openzen.zenscript.lexer.ZSToken;
+import org.openzen.zenscript.lexer.ZSTokenParser;
 import org.openzen.zenscript.lexer.ZSTokenStream;
 import org.openzen.zenscript.lexer.ZSTokenType;
 import org.openzen.zenscript.linker.BaseScope;
@@ -22,6 +23,7 @@ import org.openzen.zenscript.shared.CodePosition;
  */
 public class ParsedGenericParameter {
 	public static ParsedGenericParameter parse(ZSTokenStream tokens) {
+		CodePosition position = tokens.getPosition();
 		ZSToken name = tokens.required(ZSTokenType.T_IDENTIFIER, "identifier expected");
 		List<ParsedGenericBound> bounds = new ArrayList<>();
 		while (tokens.optional(ZSTokenType.T_COLON) != null) {
@@ -31,7 +33,7 @@ public class ParsedGenericParameter {
 				bounds.add(new ParsedTypeBound(tokens.getPosition(), IParsedType.parse(tokens)));
 			}
 		}
-		return new ParsedGenericParameter(name.position, name.content, bounds);
+		return new ParsedGenericParameter(position, name.content, bounds);
 	}
 	
 	public static List<ParsedGenericParameter> parseAll(ZSTokenStream tokens) {

Parser/src/main/java/org/openzen/zenscript/parser/definitions/ParsedVariant.java

@@ -10,7 +10,6 @@ import java.util.List;
 import org.openzen.zenscript.codemodel.HighLevelDefinition;
 import org.openzen.zenscript.codemodel.definition.VariantDefinition;
 import org.openzen.zenscript.codemodel.definition.ZSPackage;
-import org.openzen.zenscript.codemodel.scope.TypeScope;
 import org.openzen.zenscript.lexer.ZSTokenStream;
 import org.openzen.zenscript.lexer.ZSTokenType;
 import org.openzen.zenscript.linker.BaseScope;

Parser/src/main/java/org/openzen/zenscript/parser/expression/ParsedCallArguments.java

@@ -9,7 +9,6 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
-import org.openzen.zenscript.lexer.ZSTokenStream;
 import org.openzen.zenscript.lexer.ZSTokenType;
 import org.openzen.zenscript.codemodel.FunctionHeader;
 import org.openzen.zenscript.codemodel.expression.CallArguments;
@@ -17,6 +16,7 @@ import org.openzen.zenscript.codemodel.expression.Expression;
 import org.openzen.zenscript.codemodel.partial.IPartialExpression;
 import org.openzen.zenscript.codemodel.type.ITypeID;
 import org.openzen.zenscript.codemodel.type.member.DefinitionMemberGroup;
+import org.openzen.zenscript.lexer.ZSTokenStream;
 import org.openzen.zenscript.linker.BaseScope;
 import org.openzen.zenscript.linker.ExpressionScope;
 import org.openzen.zenscript.shared.CodePosition;

Parser/src/main/java/org/openzen/zenscript/parser/expression/ParsedExpression.java

@@ -11,7 +11,6 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import static org.openzen.zenscript.lexer.ZSTokenType.*;
-import org.openzen.zenscript.lexer.ZSTokenStream;
 import org.openzen.zenscript.lexer.ZSTokenType;
 import org.openzen.zenscript.codemodel.CompareType;
 import org.openzen.zenscript.codemodel.OperatorType;
@@ -20,6 +19,7 @@ import org.openzen.zenscript.codemodel.expression.switchvalue.SwitchValue;
 import org.openzen.zenscript.codemodel.partial.IPartialExpression;
 import org.openzen.zenscript.codemodel.type.ITypeID;
 import org.openzen.zenscript.lexer.ZSToken;
+import org.openzen.zenscript.lexer.ZSTokenStream;
 import org.openzen.zenscript.linker.BaseScope;
 import org.openzen.zenscript.parser.ParseException;
 import org.openzen.zenscript.parser.definitions.ParsedFunctionHeader;
@@ -108,9 +108,9 @@ public abstract class ParsedExpression {
 		ParsedExpression left = readOrOrExpression(position, parser, options);
 
 		if (parser.optional(T_QUEST) != null) {
-			ParsedExpression onIf = readOrOrExpression(parser.peek().position, parser, options);
+			ParsedExpression onIf = readOrOrExpression(parser.getPosition(), parser, options);
 			parser.required(T_COLON, ": expected");
-			ParsedExpression onElse = readConditionalExpression(parser.peek().position, parser, options);
+			ParsedExpression onElse = readConditionalExpression(parser.getPosition(), parser, options);
 			return new ParsedExpressionConditional(position, left, onIf, onElse);
 		}
 
@@ -121,12 +121,12 @@ public abstract class ParsedExpression {
 		ParsedExpression left = readAndAndExpression(position, parser, options);
 
 		while (parser.optional(T_OROR) != null) {
-			ParsedExpression right = readAndAndExpression(parser.peek().position, parser, options);
+			ParsedExpression right = readAndAndExpression(parser.getPosition(), parser, options);
 			left = new ParsedExpressionOrOr(position, left, right);
 		}
 		
 		while (parser.optional(T_COALESCE) != null) {
-			ParsedExpression right = readAndAndExpression(parser.peek().position, parser, options);
+			ParsedExpression right = readAndAndExpression(parser.getPosition(), parser, options);
 			left = new ParsedExpressionCoalesce(position, left, right);
 		}
 		
@@ -137,7 +137,7 @@ public abstract class ParsedExpression {
 		ParsedExpression left = readOrExpression(position, parser, options);
 
 		while (parser.optional(T_ANDAND) != null) {
-			ParsedExpression right = readOrExpression(parser.peek().position, parser, options);
+			ParsedExpression right = readOrExpression(parser.getPosition(), parser, options);
 			left = new ParsedExpressionAndAnd(position, left, right);
 		}
 		return left;
@@ -147,7 +147,7 @@ public abstract class ParsedExpression {
 		ParsedExpression left = readXorExpression(position, parser, options);
 
 		while (parser.optional(T_OR) != null) {
-			ParsedExpression right = readXorExpression(parser.peek().position, parser, options);
+			ParsedExpression right = readXorExpression(parser.getPosition(), parser, options);
 			left = new ParsedExpressionBinary(position, left, right, OperatorType.OR);
 		}
 		return left;
@@ -157,7 +157,7 @@ public abstract class ParsedExpression {
 		ParsedExpression left = readAndExpression(position, parser, options);
 
 		while (parser.optional(T_XOR) != null) {
-			ParsedExpression right = readAndExpression(parser.peek().position, parser, options);
+			ParsedExpression right = readAndExpression(parser.getPosition(), parser, options);
 			left = new ParsedExpressionBinary(position, left, right, OperatorType.XOR);
 		}
 		return left;
@@ -167,7 +167,7 @@ public abstract class ParsedExpression {
 		ParsedExpression left = readCompareExpression(position, parser, options);
 
 		while (parser.optional(T_AND) != null) {
-			ParsedExpression right = readCompareExpression(parser.peek().position, parser, options);
+			ParsedExpression right = readCompareExpression(parser.getPosition(), parser, options);
 			left = new ParsedExpressionBinary(position, left, right, OperatorType.AND);
 		}
 		return left;
@@ -179,47 +179,47 @@ public abstract class ParsedExpression {
 		switch (parser.peek().getType()) {
 			case T_EQUAL2: {
 				parser.next();
-				ParsedExpression right = readShiftExpression(parser.peek().position, parser, options);
+				ParsedExpression right = readShiftExpression(parser.getPosition(), parser, options);
 				return new ParsedExpressionCompare(position, left, right, CompareType.EQ);
 			}
 			case T_EQUAL3: {
 				parser.next();
-				ParsedExpression right = readShiftExpression(parser.peek().position, parser, options);
+				ParsedExpression right = readShiftExpression(parser.getPosition(), parser, options);
 				return new ParsedExpressionCompare(position, left, right, CompareType.SAME);
 			}
 			case T_NOTEQUAL: {
 				parser.next();
-				ParsedExpression right = readShiftExpression(parser.peek().position, parser, options);
+				ParsedExpression right = readShiftExpression(parser.getPosition(), parser, options);
 				return new ParsedExpressionCompare(position, left, right, CompareType.NE);
 			}
 			case T_NOTEQUAL2: {
 				parser.next();
-				ParsedExpression right = readShiftExpression(parser.peek().position, parser, options);
+				ParsedExpression right = readShiftExpression(parser.getPosition(), parser, options);
 				return new ParsedExpressionCompare(position, left, right, CompareType.NOTSAME);
 			}
 			case T_LESS: {
 				parser.next();
-				ParsedExpression right = readShiftExpression(parser.peek().position, parser, options);
+				ParsedExpression right = readShiftExpression(parser.getPosition(), parser, options);
 				return new ParsedExpressionCompare(position, left, right, CompareType.LT);
 			}
 			case T_LESSEQ: {
 				parser.next();
-				ParsedExpression right = readShiftExpression(parser.peek().position, parser, options);
+				ParsedExpression right = readShiftExpression(parser.getPosition(), parser, options);
 				return new ParsedExpressionCompare(position, left, right, CompareType.LE);
 			}
 			case T_GREATER: {
 				parser.next();
-				ParsedExpression right = readShiftExpression(parser.peek().position, parser, options);
+				ParsedExpression right = readShiftExpression(parser.getPosition(), parser, options);
 				return new ParsedExpressionCompare(position, left, right, CompareType.GT);
 			}
 			case T_GREATEREQ: {
 				parser.next();
-				ParsedExpression right = readShiftExpression(parser.peek().position, parser, options);
+				ParsedExpression right = readShiftExpression(parser.getPosition(), parser, options);
 				return new ParsedExpressionCompare(position, left, right, CompareType.GE);
 			}
 			case K_IN: {
 				parser.next();
-				ParsedExpression right = readShiftExpression(parser.peek().position, parser, options);
+				ParsedExpression right = readShiftExpression(parser.getPosition(), parser, options);
 				return new ParsedExpressionBinary(position, right, left, OperatorType.CONTAINS);
 			}
 			case K_IS: {
@@ -230,7 +230,7 @@ public abstract class ParsedExpression {
 			case T_NOT: {
 				parser.next();
 				if (parser.optional(K_IN) != null) {
-					ParsedExpression right = readShiftExpression(parser.peek().position, parser, options);
+					ParsedExpression right = readShiftExpression(parser.getPosition(), parser, options);
 					return new ParsedExpressionUnary(position, new ParsedExpressionBinary(position, right, left, OperatorType.CONTAINS), OperatorType.NOT);
 				} else if (parser.optional(K_IS) != null) {
 					IParsedType type = IParsedType.parse(parser);
@@ -249,13 +249,13 @@ public abstract class ParsedExpression {
 		
 		while (true) {
 			if (parser.optional(T_SHL) != null) {
-				ParsedExpression right = readAddExpression(parser.peek().position, parser, options);
+				ParsedExpression right = readAddExpression(parser.getPosition(), parser, options);
 				left = new ParsedExpressionBinary(position, left, right, OperatorType.SHL);
 			} else if (parser.optional(T_SHR) != null) {
-				ParsedExpression right = readAddExpression(parser.peek().position, parser, options);
+				ParsedExpression right = readAddExpression(parser.getPosition(), parser, options);
 				left = new ParsedExpressionBinary(position, left, right, OperatorType.SHR);
 			} else if (parser.optional(T_USHR) != null) {
-				ParsedExpression right = readAddExpression(parser.peek().position, parser, options);
+				ParsedExpression right = readAddExpression(parser.getPosition(), parser, options);
 				left = new ParsedExpressionBinary(position, left, right, OperatorType.USHR);
 			} else {
 				break;
@@ -270,13 +270,13 @@ public abstract class ParsedExpression {
 		
 		while (true) {
 			if (parser.optional(T_ADD) != null) {
-				ParsedExpression right = readMulExpression(parser.peek().position, parser, options);
+				ParsedExpression right = readMulExpression(parser.getPosition(), parser, options);
 				left = new ParsedExpressionBinary(position, left, right, OperatorType.ADD);
 			} else if (parser.optional(T_SUB) != null) {
-				ParsedExpression right = readMulExpression(parser.peek().position, parser, options);
+				ParsedExpression right = readMulExpression(parser.getPosition(), parser, options);
 				left = new ParsedExpressionBinary(position, left, right, OperatorType.SUB);
 			} else if (parser.optional(T_CAT) != null) {
-				ParsedExpression right = readMulExpression(parser.peek().position, parser, options);
+				ParsedExpression right = readMulExpression(parser.getPosition(), parser, options);
 				left = new ParsedExpressionBinary(position, left, right, OperatorType.CAT);
 			} else {
 				break;
@@ -290,13 +290,13 @@ public abstract class ParsedExpression {
 
 		while (true) {
 			if (parser.optional(T_MUL) != null) {
-				ParsedExpression right = readUnaryExpression(parser.peek().position, parser, options);
+				ParsedExpression right = readUnaryExpression(parser.getPosition(), parser, options);
 				left = new ParsedExpressionBinary(position, left, right, OperatorType.MUL);
 			} else if (parser.optional(T_DIV) != null) {
-				ParsedExpression right = readUnaryExpression(parser.peek().position, parser, options);
+				ParsedExpression right = readUnaryExpression(parser.getPosition(), parser, options);
 				left = new ParsedExpressionBinary(position, left, right, OperatorType.DIV);
 			} else if (parser.optional(T_MOD) != null) {
-				ParsedExpression right = readUnaryExpression(parser.peek().position, parser, options);
+				ParsedExpression right = readUnaryExpression(parser.getPosition(), parser, options);
 				left = new ParsedExpressionBinary(position, left, right, OperatorType.MOD);
 			} else {
 				break;
@@ -312,31 +312,31 @@ public abstract class ParsedExpression {
 				parser.next();
 				return new ParsedExpressionUnary(
 						position,
-						readUnaryExpression(parser.peek().position, parser, options),
+						readUnaryExpression(parser.getPosition(), parser, options),
 						OperatorType.NOT);
 			case T_SUB:
 				parser.next();
 				return new ParsedExpressionUnary(
 						position,
-						readUnaryExpression(parser.peek().position, parser, options),
+						readUnaryExpression(parser.getPosition(), parser, options),
 						OperatorType.NEG);
 			case T_CAT:
 				parser.next();
 				return new ParsedExpressionUnary(
 						position,
-						readUnaryExpression(parser.peek().position, parser, options),
+						readUnaryExpression(parser.getPosition(), parser, options),
 						OperatorType.CAT);
 			case T_INCREMENT:
 				parser.next();
 				return new ParsedExpressionUnary(
 						position,
-						readUnaryExpression(parser.peek().position, parser, options),
+						readUnaryExpression(parser.getPosition(), parser, options),
 						OperatorType.INCREMENT);
 			case T_DECREMENT:
 				parser.next();
 				return new ParsedExpressionUnary(
 						position,
-						readUnaryExpression(parser.peek().position, parser, options),
+						readUnaryExpression(parser.getPosition(), parser, options),
 						OperatorType.DECREMENT);
 			case K_TRY:
 				parser.next();
@@ -381,7 +381,7 @@ public abstract class ParsedExpression {
 						base = new ParsedExpressionMember(position, base, unescape(indexString2.content), Collections.emptyList());
 					} else {
 						ZSToken last = parser.next();
-						throw new ParseException(last, "Invalid expression, last token: " + last.content);
+						throw new ParseException(parser.getPosition(), "Invalid expression, last token: " + last.content);
 					}
 				}
 			} else if (parser.optional(T_DOT2) != null) {
@@ -532,7 +532,7 @@ public abstract class ParsedExpression {
 				IParsedType type = IParsedType.parse(parser);
 				if (type == null) {
 					ZSToken last = parser.next();
-					throw new ParseException(last, "Invalid expression, last token: " + last.content);
+					throw new ParseException(parser.getPosition(), "Invalid expression, last token: " + last.content);
 				} else {
 					return new ParsedTypeExpression(position, type);
 				}

Parser/src/main/java/org/openzen/zenscript/parser/member/ParsedDefinitionMember.java

@@ -265,9 +265,9 @@ public abstract class ParsedDefinitionMember {
 			default:
 				if (modifiers == Modifiers.STATIC && tokens.peek().type == ZSTokenType.T_AOPEN) {
 					ParsedStatementBlock body = ParsedStatementBlock.parseBlock(tokens, true);
-					return new ParsedStaticInitializer(forDefinition, tokens.peek().position, body);
+					return new ParsedStaticInitializer(forDefinition, tokens.getPosition(), body);
 				}
-				throw new CompileException(tokens.peek().position, CompileExceptionCode.UNEXPECTED_TOKEN, "Unexpected token: " + tokens.peek().content);
+				throw new CompileException(tokens.getPosition(), CompileExceptionCode.UNEXPECTED_TOKEN, "Unexpected token: " + tokens.peek().content);
 		}
 	}
 	

Parser/src/main/java/org/openzen/zenscript/parser/statements/ParsedStatement.java

@@ -19,13 +19,14 @@ import org.openzen.zenscript.shared.CompileExceptionCode;
 
 public abstract class ParsedStatement {
 	public static ParsedFunctionBody parseLambdaBody(ZSTokenStream tokens, boolean inExpression) {
+		CodePosition position = tokens.getPosition();
 		ZSToken start = tokens.peek();
 		if (tokens.optional(T_AOPEN) != null) {
 			List<ParsedStatement> statements = new ArrayList<>();
 			while (tokens.optional(T_ACLOSE) == null)
 				statements.add(ParsedStatement.parse(tokens));
 			
-			return new ParsedStatementsFunctionBody(new ParsedStatementBlock(start.position, null, null, statements));
+			return new ParsedStatementsFunctionBody(new ParsedStatementBlock(position, null, null, statements));
 		} else {
 			ParsedFunctionBody result = new ParsedLambdaFunctionBody(ParsedExpression.parse(tokens));
 			if (!inExpression)
@@ -45,7 +46,8 @@ public abstract class ParsedStatement {
 	
 	public static ParsedStatementBlock parseBlock(ZSTokenStream parser, boolean isFirst) {
 		String ws = parser.grabWhitespace();
-		ZSToken t = parser.required(T_AOPEN, "{ expected");
+		CodePosition position = parser.getPosition();
+		parser.required(T_AOPEN, "{ expected");
 
 		parser.reloadWhitespace();
 		parser.skipWhitespaceNewline();
@@ -60,7 +62,7 @@ public abstract class ParsedStatement {
 
 		WhitespaceInfo whitespace = parser.collectWhitespaceInfo(ws, isFirst);
 		WhitespacePostComment postComment = WhitespacePostComment.fromWhitespace(last.whitespaceBefore);
-		return new ParsedStatementBlock(t.getPosition(), whitespace, postComment, statements);
+		return new ParsedStatementBlock(position, whitespace, postComment, statements);
 	}
 	
 	public static ParsedStatement parse(ZSTokenStream parser) {
@@ -69,6 +71,7 @@ public abstract class ParsedStatement {
 	
 	public static ParsedStatement parse(ZSTokenStream parser, boolean isFirst) {
 		String ws = parser.grabWhitespace();
+		CodePosition position = parser.getPosition();
 		ZSToken next = parser.peek();
 		switch (next.getType()) {
 			case T_AOPEN:
@@ -83,7 +86,7 @@ public abstract class ParsedStatement {
 				parser.required(T_SEMICOLON, "; expected");
 				
 				WhitespaceInfo whitespace = parser.collectWhitespaceInfo(ws, isFirst);
-				return new ParsedStatementReturn(next.getPosition(), whitespace, expression);
+				return new ParsedStatementReturn(position, whitespace, expression);
 			}
 			case K_VAR:
 			case K_VAL: {
@@ -101,10 +104,10 @@ public abstract class ParsedStatement {
 				parser.required(T_SEMICOLON, "; expected");
 				
 				WhitespaceInfo whitespace = parser.collectWhitespaceInfo(ws, isFirst);
-				return new ParsedStatementVar(start.getPosition(), whitespace, name, type, initializer, start.getType() == K_VAL);
+				return new ParsedStatementVar(position, whitespace, name, type, initializer, start.getType() == K_VAL);
 			}
 			case K_IF: {
-				ZSToken t = parser.next();
+				parser.next();
 				ParsedExpression expression = ParsedExpression.parse(parser);
 				parser.skipWhitespaceNewline();
 				ParsedStatement onIf = parse(parser);
@@ -116,10 +119,10 @@ public abstract class ParsedStatement {
 				}
 				
 				WhitespaceInfo whitespace = parser.collectWhitespaceInfo(ws, isFirst);
-				return new ParsedStatementIf(t.getPosition(), whitespace, expression, onIf, onElse);
+				return new ParsedStatementIf(position, whitespace, expression, onIf, onElse);
 			}
 			case K_FOR: {
-				ZSToken t = parser.next();
+				parser.next();
 				String name = parser.required(T_IDENTIFIER, "identifier expected").content;
 				List<String> names = new ArrayList<>();
 				names.add(name);
@@ -134,7 +137,7 @@ public abstract class ParsedStatement {
 				
 				WhitespaceInfo whitespace = parser.collectWhitespaceInfo(ws, isFirst);
 				return new ParsedStatementForeach(
-						t.getPosition(),
+						position,
 						whitespace,
 						names.toArray(new String[names.size()]),
 						source,
@@ -152,7 +155,7 @@ public abstract class ParsedStatement {
 				parser.required(T_SEMICOLON, "; expected");
 				
 				WhitespaceInfo whitespace = parser.collectWhitespaceInfo(ws, isFirst);
-				return new ParsedStatementDoWhile(t.position, whitespace, label, content, condition);
+				return new ParsedStatementDoWhile(position, whitespace, label, content, condition);
 			}
 			case K_WHILE: {
 				ZSToken t = parser.next();
@@ -164,7 +167,7 @@ public abstract class ParsedStatement {
 				ParsedStatement content = parse(parser);
 				
 				WhitespaceInfo whitespace = parser.collectWhitespaceInfo(ws, isFirst);
-				return new ParsedStatementWhile(t.position, whitespace, label, condition, content);
+				return new ParsedStatementWhile(position, whitespace, label, condition, content);
 			}
 			case K_LOCK: {
 				ZSToken t = parser.next();
@@ -172,7 +175,7 @@ public abstract class ParsedStatement {
 				ParsedStatement content = parse(parser);
 				
 				WhitespaceInfo whitespace = parser.collectWhitespaceInfo(ws, isFirst);
-				return new ParsedStatementLock(t.position, whitespace, object, content);
+				return new ParsedStatementLock(position, whitespace, object, content);
 			}
 			case K_THROW: {
 				ZSToken t = parser.next();
@@ -180,7 +183,7 @@ public abstract class ParsedStatement {
 				parser.required(T_SEMICOLON, "; expected");
 				
 				WhitespaceInfo whitespace = parser.collectWhitespaceInfo(ws, isFirst);
-				return new ParsedStatementThrow(t.position, whitespace, value);
+				return new ParsedStatementThrow(position, whitespace, value);
 			}
 			case K_TRY: {
 				parser.pushMark();
@@ -206,7 +209,7 @@ public abstract class ParsedStatement {
 				ParsedStatement content = parse(parser);
 				List<ParsedCatchClause> catchClauses = new ArrayList<>();
 				while (parser.optional(K_CATCH) != null) {
-					CodePosition position = parser.getPosition();
+					CodePosition catchPosition = parser.getPosition();
 					
 					String catchName = null;
 					if (parser.isNext(ZSTokenType.T_IDENTIFIER))
@@ -217,7 +220,7 @@ public abstract class ParsedStatement {
 						catchType = IParsedType.parse(parser);
 					
 					ParsedStatement catchContent = ParsedStatement.parse(parser);
-					catchClauses.add(new ParsedCatchClause(position, catchName, catchType, catchContent));
+					catchClauses.add(new ParsedCatchClause(catchPosition, catchName, catchType, catchContent));
 				}
 				
 				ParsedStatement finallyClause = null;
@@ -225,7 +228,7 @@ public abstract class ParsedStatement {
 					finallyClause = ParsedStatement.parse(parser);
 				
 				WhitespaceInfo whitespace = parser.collectWhitespaceInfo(ws, isFirst);
-				return new ParsedStatementTryCatch(t.position, whitespace, name, initializer, content, catchClauses, finallyClause);
+				return new ParsedStatementTryCatch(position, whitespace, name, initializer, content, catchClauses, finallyClause);
 			}
 			case K_CONTINUE: {
 				ZSToken t = parser.next();
@@ -236,7 +239,7 @@ public abstract class ParsedStatement {
 				parser.required(T_SEMICOLON, "; expected");
 				
 				WhitespaceInfo whitespace = parser.collectWhitespaceInfo(ws, isFirst);
-				return new ParsedStatementContinue(t.position, whitespace, name);
+				return new ParsedStatementContinue(position, whitespace, name);
 			}
 			case K_BREAK: {
 				ZSToken t = parser.next();
@@ -247,7 +250,7 @@ public abstract class ParsedStatement {
 				parser.required(T_SEMICOLON, "; expected");
 				
 				WhitespaceInfo whitespace = parser.collectWhitespaceInfo(ws, isFirst);
-				return new ParsedStatementBreak(t.position, whitespace, name);
+				return new ParsedStatementBreak(position, whitespace, name);
 			}
 			case K_SWITCH: {
 				ZSToken t = parser.next();
@@ -277,11 +280,10 @@ public abstract class ParsedStatement {
 				}
 				
 				WhitespaceInfo whitespace = parser.collectWhitespaceInfo(ws, isFirst);
-				return new ParsedStatementSwitch(t.position, whitespace, name, value, cases);
+				return new ParsedStatementSwitch(position, whitespace, name, value, cases);
 			}
 		}
-
-		CodePosition position = parser.peek().getPosition();
+		
 		ParsedExpression expression = ParsedExpression.parse(parser);
 		parser.required(T_SEMICOLON, "; expected");
 		WhitespaceInfo whitespace = parser.collectWhitespaceInfo(ws, isFirst);

Parser/src/main/java/org/openzen/zenscript/parser/type/IParsedType.java

@@ -105,13 +105,14 @@ public interface IParsedType {
 				result = ParsedTypeBasic.STRING;
 				break;
 			case K_FUNCTION: {
-				CodePosition position = tokens.next().position;
+				CodePosition position = tokens.getPosition();
+				tokens.next();
 				ParsedFunctionHeader header = ParsedFunctionHeader.parse(tokens);
 				result = new ParsedFunctionType(position, header);
 				break;
 			}
 			case T_IDENTIFIER: {
-				CodePosition position = tokens.peek().position;
+				CodePosition position = tokens.getPosition();
 				List<ParsedNamedType.ParsedNamePart> name = new ArrayList<>();
 				do {
 					String namePart = tokens.required(ZSTokenType.T_IDENTIFIER, "identifier expected").content;