- Fixed various bugs in the source editor

- Added ability to remove a selected block of text

DrawableGui/src/main/java/org/openzen/drawablegui/DDrawingContext.java

@@ -20,6 +20,8 @@ public interface DDrawingContext {
 	
 	void scrollInView(int x, int y, int width, int height);
 	
+	DTimerHandle setTimer(int millis, Runnable target);
+	
 	DFontMetrics getFontMetrics(DFont font);
 	
 	enum Cursor {

DrawableGui/src/main/java/org/openzen/drawablegui/DTimerHandle.java

@@ -0,0 +1,17 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.openzen.drawablegui;
+
+import java.io.Closeable;
+
+/**
+ *
+ * @author Hoofdgebruiker
+ */
+public interface DTimerHandle extends Closeable {
+	@Override
+	public void close();
+}

DrawableGui/src/main/java/org/openzen/drawablegui/scroll/DScrollPane.java

@@ -14,6 +14,7 @@ import org.openzen.drawablegui.DFont;
 import org.openzen.drawablegui.DFontMetrics;
 import org.openzen.drawablegui.DIRectangle;
 import org.openzen.drawablegui.DMouseEvent;
+import org.openzen.drawablegui.DTimerHandle;
 import org.openzen.drawablegui.listeners.ListenerHandle;
 import org.openzen.drawablegui.live.LiveInt;
 import org.openzen.drawablegui.live.LiveObject;
@@ -248,6 +249,11 @@ public class DScrollPane implements DComponent {
 			if (y + height > offsetY.getValue() + bounds.height)
 				offsetY.setValue(y + height - bounds.height);
 		}
+
+		@Override
+		public DTimerHandle setTimer(int millis, Runnable target) {
+			return context.setTimer(millis, target);
+		}
 	}
 	
 	private class ScrollListener implements LiveInt.Listener {

DrawableGui/src/main/java/org/openzen/drawablegui/swing/SwingGraphicsContext.java

@@ -9,12 +9,14 @@ import java.awt.Font;
 import java.awt.Graphics;
 import java.awt.geom.GeneralPath;
 import java.util.WeakHashMap;
+import javax.swing.Timer;
 import org.openzen.drawablegui.DComponent;
 import org.openzen.drawablegui.DPath;
 import org.openzen.drawablegui.DDrawingContext;
 import org.openzen.drawablegui.DFont;
 import org.openzen.drawablegui.DFontMetrics;
 import org.openzen.drawablegui.DPathTracer;
+import org.openzen.drawablegui.DTimerHandle;
 
 /**
  *
@@ -101,6 +103,13 @@ public class SwingGraphicsContext implements DDrawingContext {
 	public void scrollInView(int x, int y, int width, int height) {
 		// not in a scrollable context
 	}
+
+	@Override
+	public DTimerHandle setTimer(int millis, Runnable target) {
+		Timer timer = new Timer(millis, e -> target.run());
+		timer.start();
+		return () -> timer.stop();
+	}
 	
 	private class PathTracer implements DPathTracer {
 		private final GeneralPath path;

IDE/src/main/java/org/openzen/zenscript/ide/ui/view/editor/SourceEditor.java

@@ -6,8 +6,6 @@
 package org.openzen.zenscript.ide.ui.view.editor;
 
 import java.io.IOException;
-import java.util.Timer;
-import java.util.TimerTask;
 import org.openzen.drawablegui.DCanvas;
 import org.openzen.drawablegui.DComponent;
 import org.openzen.drawablegui.DDimensionPreferences;
@@ -52,10 +50,9 @@ public class SourceEditor implements DComponent {
 	private int selectionLineHeight;
 	
 	private int lineBarWidth;
-	private CursorPosition cursorStart = null;
-	private CursorPosition cursorEnd = null;
+	private SourcePosition cursorStart = null;
+	private SourcePosition cursorEnd = null;
 	
-	private Timer blink = new Timer();
 	private boolean cursorBlink = true;
 	
 	private int mouseDownX = -1;
@@ -67,13 +64,6 @@ public class SourceEditor implements DComponent {
 		tokens = new TokenModel(sourceFile.getName(), tab.length());
 		tokenListener = tokens.addListener(new TokenListener());
 		
-		blink.scheduleAtFixedRate(new TimerTask() {
-			@Override
-			public void run() {
-				blink();
-			}
-		}, 300, 300);
-		
 		try {
 			TokenParser<ZSToken, ZSTokenType> parser = ZSTokenParser.createRaw(sourceFile.getName(), new ReaderCharReader(sourceFile.read()), tab.length());
 			tokens.set(parser);
@@ -91,6 +81,7 @@ public class SourceEditor implements DComponent {
 		selectionLineHeight = textLineHeight + style.selectionPaddingTop + style.selectionPaddingBottom;
 		
 		dimensionPreferences.setValue(new DDimensionPreferences(0, fullLineHeight * tokens.getLineCount()));
+		context.setTimer(300, this::blink);
 	}
 
 	@Override
@@ -126,25 +117,25 @@ public class SourceEditor implements DComponent {
 		
 		if (cursorStart != null && !cursorStart.equals(cursorEnd)) {
 			if (cursorStart.line == cursorEnd.line) {
-				int y = cursorStart.getY();
-				int x1 = cursorStart.getX();
-				int x2 = cursorEnd.getX();
+				int y = getY(cursorStart);
+				int x1 = getX(cursorStart);
+				int x2 = getX(cursorEnd);
 				int fromX = Math.min(x1, x2);
 				int toX = Math.max(x1, x2);
 				canvas.fillRectangle(fromX, y, toX - fromX, selectionLineHeight, style.selectionColor);
 			} else {
-				CursorPosition from = cursorStart.line < cursorEnd.line ? cursorStart : cursorEnd;
-				CursorPosition to = cursorStart.line < cursorEnd.line ? cursorEnd : cursorStart;
+				SourcePosition from = SourcePosition.min(cursorStart, cursorEnd);
+				SourcePosition to = SourcePosition.max(cursorStart, cursorEnd);
 				
-				int fromX = from.getX();
-				canvas.fillRectangle(fromX, from.getY(), bounds.width - fromX, selectionLineHeight, style.selectionColor);
+				int fromX = getX(from);
+				canvas.fillRectangle(fromX, getY(from), bounds.width - fromX, selectionLineHeight, style.selectionColor);
 				
 				for (int i = from.line + 1; i < to.line; i++) {
 					canvas.fillRectangle(x, lineToY(i), bounds.width - x, selectionLineHeight, style.selectionColor);
 				}
 				
-				int toX = to.getX();
-				canvas.fillRectangle(x, to.getY(), toX - x, selectionLineHeight, style.selectionColor);
+				int toX = getX(to);
+				canvas.fillRectangle(x, getY(to), toX - x, selectionLineHeight, style.selectionColor);
 			}
 		}
 		
@@ -169,8 +160,8 @@ public class SourceEditor implements DComponent {
 		}
 		
 		if (cursorEnd != null && cursorBlink) {
-			int cursorX = cursorEnd.getX();
-			int cursorY = cursorEnd.getY();
+			int cursorX = getX(cursorEnd);
+			int cursorY = getY(cursorEnd);
 			canvas.fillRectangle(cursorX, cursorY, 2, selectionLineHeight, 0xFF000000);
 		}
 	}
@@ -189,23 +180,23 @@ public class SourceEditor implements DComponent {
 	public void onMouseClick(DMouseEvent e) {
 		context.focus(this);
 		
-		CursorPosition position = getPositionAt(e.x, e.y);
+		SourcePosition position = getPositionAt(e.x, e.y);
 		if (e.isDoubleClick()) {
 			// select entire word
-			TokenPosition token = getTokenAt(position);
+			TokenModel.Position tokenPosition = tokens.getPosition(position.line, position.offset);
 			setCursor(
-					new CursorPosition(position.line, position.offset - token.offset),
-					new CursorPosition(position.line, position.offset + token.token.content.length() - token.offset));
+					new SourcePosition(tokens, position.line, position.offset - tokenPosition.offset),
+					new SourcePosition(tokens, position.line, position.offset - tokenPosition.offset + tokens.getTokenAt(tokenPosition).content.length()));
 		} else if (e.isTripleClick()) {
 			setCursor(
-					new CursorPosition(position.line, 0),
-					new CursorPosition(position.line + 1, 0));
+					new SourcePosition(tokens, position.line, 0),
+					new SourcePosition(tokens, position.line + 1, 0));
 		} else {
 			setCursor(position, position);
 		}
 	}
 	
-	private void setCursor(CursorPosition start, CursorPosition end) {
+	private void setCursor(SourcePosition start, SourcePosition end) {
 		if (cursorStart != null)
 			repaint(cursorStart, cursorEnd);
 		
@@ -231,7 +222,7 @@ public class SourceEditor implements DComponent {
 	
 	@Override
 	public void onMouseDrag(DMouseEvent e) {
-		CursorPosition start = cursorStart;
+		SourcePosition start = cursorStart;
 		if (!dragging)
 			start = getPositionAt(mouseDownX, mouseDownY);
 		
@@ -248,7 +239,8 @@ public class SourceEditor implements DComponent {
 				
 				{
 					int line = cursorEnd.line - 1;
-					CursorPosition position = new CursorPosition(
+					SourcePosition position = new SourcePosition(
+							tokens,
 							line,
 							Math.min(tokens.getLineLength(line), cursorEnd.offset));
 					setCursor(shift ? cursorStart : position, position);
@@ -260,7 +252,8 @@ public class SourceEditor implements DComponent {
 				
 				{
 					int line = cursorEnd.line + 1;
-					CursorPosition position = new CursorPosition(
+					SourcePosition position = new SourcePosition(
+							tokens,
 							line,
 							Math.min(tokens.getLineLength(line), cursorEnd.offset));
 					setCursor(shift ? cursorStart : position, position);
@@ -271,12 +264,12 @@ public class SourceEditor implements DComponent {
 					return;
 				
 				{
-					CursorPosition position;
+					SourcePosition position;
 					if (cursorEnd.offset == 0) {
 						int line = cursorEnd.line - 1;
-						position = new CursorPosition(line, tokens.getLineLength(line));
+						position = new SourcePosition(tokens, line, tokens.getLineLength(line));
 					} else {
-						position = new CursorPosition(cursorEnd.line, cursorEnd.offset - 1);
+						position = new SourcePosition(tokens, cursorEnd.line, cursorEnd.offset - 1);
 					}
 					setCursor(shift ? cursorStart : position, position);
 				}
@@ -286,11 +279,11 @@ public class SourceEditor implements DComponent {
 					return;
 				
 				{
-					CursorPosition position;
+					SourcePosition position;
 					if (cursorEnd.offset == tokens.getLineLength(cursorEnd.line)) {
-						position = new CursorPosition(cursorEnd.line + 1, 0);
+						position = new SourcePosition(tokens, cursorEnd.line + 1, 0);
 					} else {
-						position = new CursorPosition(cursorEnd.line, cursorEnd.offset + 1);
+						position = new SourcePosition(tokens, cursorEnd.line, cursorEnd.offset + 1);
 					}
 					setCursor(shift ? cursorStart : position, position);
 				}
@@ -344,10 +337,17 @@ public class SourceEditor implements DComponent {
 		}
 		
 		tokens.deleteCharacter(cursorEnd.line, cursorEnd.offset);
-		repaintLine(cursorEnd.line);
 	}
 	
 	private void backspace() {
+		if (!cursorEnd.equals(cursorStart)) {
+			SourcePosition min = SourcePosition.min(cursorStart, cursorEnd);
+			SourcePosition max = SourcePosition.max(cursorStart, cursorEnd);
+			tokens.delete(min.line, min.offset, max.line, max.offset);
+			setCursor(min, min);
+			return;
+		}
+		
 		if (cursorEnd.offset == 0) {
 			if (cursorEnd.line == 0)
 				return;
@@ -355,56 +355,40 @@ public class SourceEditor implements DComponent {
 			int length = tokens.getLineLength(cursorEnd.line - 1);
 			tokens.deleteNewline(cursorEnd.line - 1);
 			
-			CursorPosition position = new CursorPosition(cursorEnd.line - 1, length);
+			SourcePosition position = new SourcePosition(tokens, cursorEnd.line - 1, length);
 			setCursor(position, position);
 			return;
 		}
 		
-		tokens.deleteCharacter(cursorEnd.line, cursorEnd.offset - 1);
-		CursorPosition position = new CursorPosition(cursorEnd.line, cursorEnd.offset - 1);
-		setCursor(position, position);
+		try {
+			tokens.deleteCharacter(cursorEnd.line, cursorEnd.offset - 1);
+			SourcePosition position = new SourcePosition(tokens, cursorEnd.line, cursorEnd.offset - 1);
+			setCursor(position, position);
+		} catch (Exception ex) {
+			ex.printStackTrace();
+		}
 	}
 	
 	private void type(String value) {
 		tokens.insert(cursorEnd.line, cursorEnd.offset, value);
-		CursorPosition position = new CursorPosition(cursorEnd.line, cursorEnd.offset + value.length());
+		SourcePosition position = new SourcePosition(tokens, cursorEnd.line, cursorEnd.offset + value.length());
 		setCursor(position, position);
 	}
 	
 	private void newline() {
 		String indent = tokens.getLine(cursorEnd.line).getIndent();
 		tokens.insert(cursorEnd.line, cursorEnd.offset, "\n" + indent);
-		CursorPosition position = new CursorPosition(cursorEnd.line + 1, indent.length());
+		SourcePosition position = new SourcePosition(tokens, cursorEnd.line + 1, indent.length());
 		setCursor(position, position);
 	}
 	
-	private TokenPosition getTokenAt(CursorPosition position) {
-		TokenLine line = tokens.getLine(position.line);
-		int offset = 0;
-		for (ZSToken token : line.getTokens()) {
-			if (offset + token.content.length() > position.offset)
-				return new TokenPosition(token, position.offset - offset);
-			offset += token.content.length();
-		}
-		
-		return new TokenPosition(line.getLastToken(), position.offset - offset);
-	}
-	
-	private class TokenPosition {
-		public final ZSToken token;
-		public final int offset;
-		
-		public TokenPosition(ZSToken token, int offset) {
-			this.token = token;
-			this.offset = offset;
-		}
-	}
-	
-	private void repaint(CursorPosition from, CursorPosition to) {
+	private void repaint(SourcePosition from, SourcePosition to) {
 		if (from.line == to.line) {
 			int y = lineToY(from.line);
-			int fromX = offsetToX(from.line, Math.min(from.offset, to.offset));
-			int toX = offsetToX(from.line, Math.max(from.offset, to.offset)) + 2;
+			int x1 = getX(from);
+			int x2 = getX(to);
+			int fromX = Math.min(x1, x2);
+			int toX = Math.max(x1, x2) + 2;
 			context.repaint(fromX, y, toX - fromX, selectionLineHeight);
 		} else {
 			int fromY = lineToY(Math.min(from.line, to.line));
@@ -420,10 +404,8 @@ public class SourceEditor implements DComponent {
 		context.repaint(bounds.x, lineToY(line), bounds.width, selectionLineHeight);
 	}
 	
-	private void scrollTo(CursorPosition position) {
-		int y = lineToY(position.line);
-		int x = offsetToX(position.line, position.offset);
-		context.scrollInView(x, y, 2, selectionLineHeight);
+	public void scrollTo(SourcePosition position) {
+		context.scrollInView(getX(position), getY(position), 2, selectionLineHeight);
 	}
 	
 	private void blink() {
@@ -433,7 +415,7 @@ public class SourceEditor implements DComponent {
 		}
 	}
 	
-	private CursorPosition getPositionAt(int x, int y) {
+	public SourcePosition getPositionAt(int x, int y) {
 		int line = yToLine(y);
 		int offset = xToOffset(line, x);
 		
@@ -442,7 +424,7 @@ public class SourceEditor implements DComponent {
 		if (line >= tokens.getLineCount())
 			line = tokens.getLineCount() - 1;
 		
-		return new CursorPosition(line, offset);
+		return new SourcePosition(tokens, line, offset);
 	}
 	
 	private int xToOffset(int lineIndex, int x) {
@@ -493,28 +475,23 @@ public class SourceEditor implements DComponent {
 		return startY + line * fullLineHeight;
 	}
 	
-	private int offsetToX(int line, int offset) {
-		if (line >= tokens.getLineCount())
-			return 0;
-		
-		int tokensOffset = 0;
+	public int getX(SourcePosition position) {
 		int x = bounds.x + lineBarWidth + 10;
-		TokenLine lineData = tokens.getLine(line);
-		for (ZSToken token : lineData.getTokens()) {
-			String content = getDisplayContent(token);
-			if (tokensOffset + token.content.length() >= offset) {
-				if (token.type == ZSTokenType.T_WHITESPACE_TAB)
-					return offset == tokensOffset ? x : x + fontMetrics.getWidth(tab);
-				
-				return x + fontMetrics.getWidth(token.content, 0, offset - tokensOffset);
-			}
-			
-			x += fontMetrics.getWidth(content);
-			tokensOffset += token.content.length();
-		}
+		TokenLine lineData = tokens.getLine(position.line);
+		
+		TokenModel.Position tokenPosition = position.asTokenPosition();
+		for (int i = 0; i < tokenPosition.token; i++)
+			x += fontMetrics.getWidth(getDisplayContent(lineData.getToken(i)));
+		if (tokenPosition.offset > 0)
+			x += fontMetrics.getWidth(getDisplayContent(lineData.getToken(tokenPosition.token)), 0, tokenPosition.offset);
+		
 		return x;
 	}
 	
+	public int getY(SourcePosition position) {
+		return lineToY(position.line);
+	}
+	
 	private String getDisplayContent(ZSToken token) {
 		return token.type == ZSTokenType.T_WHITESPACE_TAB ? tab : token.content;
 	}
@@ -658,26 +635,4 @@ public class SourceEditor implements DComponent {
 			}
 		}
 	}
-	
-	public class CursorPosition {
-		public final int line;
-		public final int offset;
-		
-		public CursorPosition(int line, int offset) {
-			this.line = line;
-			this.offset = offset;
-		}
-		
-		public int getX() {
-			return offsetToX(line, offset);
-		}
-		
-		public int getY() {
-			return lineToY(line);
-		}
-		
-		public boolean equals(CursorPosition other) {
-			return line == other.line && offset == other.offset;
-		}
-	}
 }

IDE/src/main/java/org/openzen/zenscript/ide/ui/view/editor/SourcePosition.java

@@ -0,0 +1,64 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.openzen.zenscript.ide.ui.view.editor;
+
+/**
+ *
+ * @author Hoofdgebruiker
+ */
+public class SourcePosition {
+	public final TokenModel tokens;
+	public final int line;
+	public final int offset;
+	
+	private long positionVersion;
+	private TokenModel.Position position;
+
+	public SourcePosition(TokenModel tokens, int line, int offset) {
+		this.tokens = tokens;
+		this.line = line;
+		this.offset = offset;
+	}
+	
+	public static SourcePosition min(SourcePosition a, SourcePosition b) {
+		if (a.line < b.line)
+			return a;
+		if (a.line > b.line)
+			return b;
+		if (a.offset < b.offset)
+			return a;
+		if (a.offset > b.offset)
+			return b;
+
+		return a;
+	}
+	
+	public static SourcePosition max(SourcePosition a, SourcePosition b) {
+		if (a.line < b.line)
+			return b;
+		if (a.line > b.line)
+			return a;
+		if (a.offset < b.offset)
+			return b;
+		if (a.offset > b.offset)
+			return a;
+
+		return b;
+	}
+
+	public boolean equals(SourcePosition other) {
+		return line == other.line && offset == other.offset;
+	}
+	
+	public TokenModel.Position asTokenPosition() {
+		if (position == null || positionVersion != tokens.getVersion()) {
+			positionVersion = tokens.getVersion();
+			position = tokens.getPosition(line, offset);
+		}
+		
+		return position;
+	}
+}

IDE/src/main/java/org/openzen/zenscript/ide/ui/view/editor/TokenLine.java

@@ -40,7 +40,7 @@ public final class TokenLine {
 	}
 	
 	public ZSToken getLastToken() {
-		return tokens.get(tokens.size() - 1);
+		return tokens.isEmpty() ? null : tokens.get(tokens.size() - 1);
 	}
 	
 	public String getIndent() {

IDE/src/main/java/org/openzen/zenscript/ide/ui/view/editor/TokenModel.java

@@ -26,6 +26,7 @@ public class TokenModel {
 	private final String filename;
 	private final int spacesPerTab;
 	private final List<TokenLine> lines = new ArrayList<>();
+	private long version = 0;
 	
 	public TokenModel(String filename, int spacesPerTab) {
 		this.filename = filename;
@@ -52,6 +53,42 @@ public class TokenModel {
 		return lines.get(line).length();
 	}
 	
+	public ZSToken getTokenAt(Position position) {
+		TokenLine line = lines.get(position.line);
+		return position.token >= line.getTokenCount() ? null : line.getToken(position.token);
+	}
+	
+	/**
+	 * Version can be used to check if the token model has been updated. If the
+	 * version is unchanged, the contents of the token model will be unchanged
+	 * too.
+	 * 
+	 * @return version number
+	 */
+	public long getVersion() {
+		return version;
+	}
+	
+	public Position getPosition(int line, int offset) {
+		if (line < 0)
+			line = 0;
+		if (line >= lines.size())
+			return new Position(lines.size(), lines.get(lines.size() - 1).getTokenCount(), 0);
+		
+		int token = 0;
+		int tokenOffset = 0;
+		TokenLine tokenLine = getLine(line);
+		while (token < tokenLine.getTokenCount()) {
+			ZSToken t = tokenLine.getToken(token);
+			if (tokenOffset + t.content.length() > offset)
+				return new Position(line, token, offset - tokenOffset);
+			
+			tokenOffset += t.content.length();
+			token++;
+		}
+		return new Position(line, tokenLine.getTokenCount(), 0);
+	}
+	
 	public void set(Iterator<ZSToken> tokens) {
 		lines.clear();
 		lines.add(new TokenLine());
@@ -64,6 +101,7 @@ public class TokenModel {
 		merge(line, lineIndex, getLine(lineIndex + 1));
 		lines.remove(lineIndex + 1);
 		
+		version++;
 		listeners.accept(listener -> listener.onLineDeleted(lineIndex + 1));
 	}
 	
@@ -75,48 +113,67 @@ public class TokenModel {
 			if (tokenOffset + token.content.length() > offset) {
 				if (token.content.length() == 1) {
 					line.remove(i);
-					i--; // make sure previous token is reparsed too
+					if (i > 0)
+						i--; // make sure previous token is reparsed too
 				} else {
 					token = token.delete(offset - tokenOffset, 1);
 					line.replace(i, token);
 				}
 
-				reparse(lineIndex, i, lineIndex, i + 1);
+				relex(lineIndex, i, lineIndex, i + 1);
 				return;
 			}
 			tokenOffset += token.content.length();
 		}
 	}
 	
+	public void delete(int fromLineIndex, int fromOffset, int toLineIndex, int toOffset) {
+		Position from = getPosition(fromLineIndex, fromOffset);
+		Position to = getPosition(toLineIndex, toOffset);
+		
+		ZSToken fromToken = getTokenAt(from);
+		ZSToken toToken = getTokenAt(to);
+		
+		String remainder = "";
+		if (fromToken != null)
+			remainder = fromToken.content.substring(0, from.offset);
+		if (toToken != null && to.offset > 0)
+			remainder += toToken.content.substring(to.offset);
+		
+		removeTokens(from.line, from.token, to.line, to.offset > 0 ? to.token + 1 : to.token);
+		if (!remainder.isEmpty()) {
+			getLine(from.line).insert(from.token, new ZSToken(ZSTokenType.INVALID, remainder));
+			relex(from.line, Math.max(0, from.token - 1), from.line, from.token + 1);
+		}
+	}
+	
 	public void insert(int lineIndex, int offset, String value) {
 		TokenLine line = lines.get(lineIndex);
 		int tokenOffset = 0;
-		if (line.isEmpty()) {
-			line.add(new ZSToken(ZSTokenType.INVALID, value));
-			reparse(lineIndex, 0, lineIndex, 1);
-			return;
-		}
-
 		for (int i = 0; i < line.getTokenCount(); i++) {
 			ZSToken token = line.getToken(i);
 			if (tokenOffset + token.content.length() > offset) {
 				token = token.insert(offset - tokenOffset, value);
 				line.replace(i, token);
-				reparse(lineIndex, i, lineIndex, i + 1);
+				relex(lineIndex, i, lineIndex, i + 1);
 				return;
 			}
 			tokenOffset += token.content.length();
 		}
 
 		ZSToken token = line.getLastToken();
-		token = new ZSToken(token.type, token.content + value);
-		line.replace(line.getTokenCount() - 1, token);
-		reparse(lineIndex, line.getTokenCount() - 1, lineIndex, line.getTokenCount());
+		if (token == null) {
+			line.add(new ZSToken(ZSTokenType.INVALID, value));
+		} else {
+			token = new ZSToken(token.type, token.content + value);
+			line.replace(line.getTokenCount() - 1, token);
+		}
+		relex(lineIndex, line.getTokenCount() - 1, lineIndex, line.getTokenCount());
 	}
 	
-	private void reparse(int fromLine, int fromToken, int toLine, int toToken) {
-		TokenReparser reparser = new TokenReparser(filename, lines, fromLine, fromToken, toLine, toToken, spacesPerTab);
-		List<ZSToken> tokens = reparser.reparse();
+	private void relex(int fromLine, int fromToken, int toLine, int toToken) {
+		TokenRelexer reparser = new TokenRelexer(filename, lines, fromLine, fromToken, toLine, toToken, spacesPerTab);
+		List<ZSToken> tokens = reparser.relex();
 		replaceTokens(fromLine, fromToken, reparser.getLine(), reparser.getToken(), tokens);
 	}
 	
@@ -130,19 +187,22 @@ public class TokenModel {
 			TokenLine fromLineObject = lines.get(fromLine);
 			fromLineObject.removeRange(fromToken, fromLineObject.getTokenCount());
 			
-			listeners.accept(listener -> listener.onLineChanged(fromLine));
-			
 			TokenLine toLineObject = lines.get(toLine);
 			for (int i = toToken - 1; i >= 0; i--)
 				toLineObject.remove(i);
 			
-			listeners.accept(listener -> listener.onLineChanged(toLine));
-			
 			merge(lines.get(fromLine), fromLine, lines.remove(toLine));
+			
+			version++;
+			listeners.accept(listener -> listener.onLineChanged(fromLine));
+			listeners.accept(listener -> listener.onLineDeleted(toLine));
+			
 			for (int i = toLine - 1; i > fromLine; i--) {
 				lines.remove(i);
 				
 				int ix = i;
+				
+				version++;
 				listeners.accept(listener -> listener.onLineDeleted(ix));
 			}
 		} else {
@@ -150,6 +210,7 @@ public class TokenModel {
 			for (int i = toToken - 1; i >= fromToken; i--)
 				line.remove(i);
 			
+			version++;
 			listeners.accept(listener -> listener.onLineChanged(fromLine));
 		}
 	}
@@ -195,6 +256,7 @@ public class TokenModel {
 			}
 		}
 		
+		version++;
 		listeners.accept(listener -> {
 			for (Integer inserted : insertedLines)
 				listener.onLineInserted(inserted);
@@ -228,7 +290,19 @@ public class TokenModel {
 		int fromToken = line.getTokenCount() - 1;
 		int toToken = !other.isEmpty() ? fromToken + 2 : fromToken + 1;
 		line.addAll(other.getTokens());
-		reparse(lineIndex, fromToken, lineIndex, toToken);
+		relex(lineIndex, fromToken, lineIndex, toToken);
+	}
+	
+	public static class Position {
+		public final int line;
+		public final int token;
+		public final int offset;
+		
+		public Position(int line, int token, int offset) {
+			this.line = line;
+			this.token = token;
+			this.offset = offset;
+		}
 	}
 	
 	public interface Listener {

IDE/src/main/java/org/openzen/zenscript/ide/ui/view/editor/TokenReparser.java → IDE/src/main/java/org/openzen/zenscript/ide/ui/view/editor/TokenRelexer.java

@@ -18,7 +18,7 @@ import org.openzen.zenscript.lexer.ZSTokenType;
  *
  * @author Hoofdgebruiker
  */
-public class TokenReparser {
+public class TokenRelexer {
 	private final List<TokenLine> lines;
 	private final String filename;
 	private final int toLine;
@@ -28,7 +28,7 @@ public class TokenReparser {
 	private int lineIndex;
 	private int token;
 	
-	public TokenReparser(String filename, List<TokenLine> lines, int fromLine, int fromToken, int toLine, int toToken, int spacesPerTab) {
+	public TokenRelexer(String filename, List<TokenLine> lines, int fromLine, int fromToken, int toLine, int toToken, int spacesPerTab) {
 		if (fromToken < 0 || toToken < 0)
 			throw new IllegalArgumentException("fromToken or toToken cannot be < 0");
 		
@@ -42,8 +42,8 @@ public class TokenReparser {
 		this.spacesPerTab = spacesPerTab;
 	}
 	
-	public List<ZSToken> reparse() {
-		ReparseCharReader reader = new ReparseCharReader();
+	public List<ZSToken> relex() {
+		RelexCharReader reader = new RelexCharReader();
 		TokenParser<ZSToken, ZSTokenType> reparser = ZSTokenParser.createRaw(filename, reader, spacesPerTab);
 		List<ZSToken> result = new ArrayList<>();
 		while ((lineIndex < toLine || token < toToken || !reader.isAtTokenBoundary()) && reparser.hasNext())
@@ -79,11 +79,11 @@ public class TokenReparser {
 		return token;
 	}
 	
-	private class ReparseCharReader implements CharReader {
+	private class RelexCharReader implements CharReader {
 		private String token;
 		private int tokenOffset;
 		
-		public ReparseCharReader() {
+		public RelexCharReader() {
 			token = get();
 		}
 		

Parser/src/main/java/org/openzen/zenscript/lexer/ZSToken.java

@@ -5,10 +5,6 @@
  */
 package org.openzen.zenscript.lexer;
 
-import java.util.HashMap;
-import java.util.Map;
-import org.openzen.zenscript.shared.StringUtils;
-
 /**
  *
  * @author Hoofdgebruiker