Browse Source

- Removed whitespace handling from the bare parser

- Tokens no longer store whitespace
Stan Hebben 6 years ago
parent
commit
18566301a3

+ 0
- 2
Parser/src/main/java/org/openzen/zenscript/lexer/Token.java View File

12
 	TT getType();
12
 	TT getType();
13
 	
13
 	
14
 	String getContent();
14
 	String getContent();
15
-	
16
-	String getWhitespaceBefore();
17
 }
15
 }

+ 1
- 1
Parser/src/main/java/org/openzen/zenscript/lexer/TokenFactory.java View File

10
  * @author Hoofdgebruiker
10
  * @author Hoofdgebruiker
11
  */
11
  */
12
 public interface TokenFactory<T, TT> {
12
 public interface TokenFactory<T, TT> {
13
-	T create(TT type, String content, String whitespaceBefore);
13
+	T create(TT type, String content);
14
 }
14
 }

+ 5
- 25
Parser/src/main/java/org/openzen/zenscript/lexer/TokenParser.java View File

106
 				next.position,
106
 				next.position,
107
 				factory.create(
107
 				factory.create(
108
 						other,
108
 						other,
109
-						next.token.getWhitespaceBefore(),
110
 						next.token.getContent()));
109
 						next.token.getContent()));
111
 	}
110
 	}
112
 
111
 
126
     // =======================
125
     // =======================
127
     // === Private methods ===
126
     // === Private methods ===
128
     // =======================
127
     // =======================
129
-
130
-    /**
131
-     * Advances to the next non - whitespace token.
132
-     */
128
+	
133
     private void advance()
129
     private void advance()
134
-	{
135
-		StringBuilder whitespace = new StringBuilder();
136
-        while (true) {
137
-            advanceToken(whitespace.toString());
138
-			if (next.token.getType().isWhitespace()) {
139
-				whitespace.append(next.token.getContent());
140
-			} else {
141
-				break;
142
-			}
143
-        }
144
-    }
145
-
146
-    /**
147
-     * Advances to the next token.
148
-     */
149
-    private void advanceToken(String whitespace)
150
 	{
130
 	{
151
         if (nextChar < 0) {
131
         if (nextChar < 0) {
152
 			CodePosition position = new CodePosition(
132
 			CodePosition position = new CodePosition(
156
 					line,
136
 					line,
157
 					lineOffset);
137
 					lineOffset);
158
 			
138
 			
159
-            next = new PositionedToken(position, factory.create(eof, whitespace, ""));
139
+            next = new PositionedToken(position, factory.create(eof, ""));
160
             return;
140
             return;
161
         }
141
         }
162
 		
142
 		
180
             if (dfa.finals[state] != null) {
160
             if (dfa.finals[state] != null) {
181
                 if (state == 0) {
161
                 if (state == 0) {
182
 					value.append((char) nextChar);
162
 					value.append((char) nextChar);
183
-					next = new PositionedToken(position, factory.create(invalid, value.toString(), whitespace));
163
+					next = new PositionedToken(position, factory.create(invalid, value.toString()));
184
 					nextChar = reader.read();
164
 					nextChar = reader.read();
185
 				}
165
 				}
186
 				
166
 				
187
-				next = new PositionedToken(position, factory.create(dfa.finals[state], value.toString(), whitespace));
167
+				next = new PositionedToken(position, factory.create(dfa.finals[state], value.toString()));
188
             } else {
168
             } else {
189
 				if (nextChar < 0 && value.length() == 0)
169
 				if (nextChar < 0 && value.length() == 0)
190
 					return; // happens on comments at the end of files
170
 					return; // happens on comments at the end of files
191
 				
171
 				
192
 				value.append((char) nextChar);
172
 				value.append((char) nextChar);
193
-				next = new PositionedToken(position, factory.create(invalid, value.toString(), whitespace));
173
+				next = new PositionedToken(position, factory.create(invalid, value.toString()));
194
 				nextChar = reader.read();
174
 				nextChar = reader.read();
195
             }
175
             }
196
         } catch (IOException ex) {
176
         } catch (IOException ex) {

+ 1
- 8
Parser/src/main/java/org/openzen/zenscript/lexer/ZSToken.java View File

12
 public class ZSToken implements Token<ZSTokenType> {
12
 public class ZSToken implements Token<ZSTokenType> {
13
 	public final ZSTokenType type;
13
 	public final ZSTokenType type;
14
 	public final String content;
14
 	public final String content;
15
-	public final String whitespaceBefore;
16
 	
15
 	
17
-	public ZSToken(ZSTokenType type, String content, String whitespaceBefore) {
16
+	public ZSToken(ZSTokenType type, String content) {
18
 		this.type = type;
17
 		this.type = type;
19
 		this.content = content;
18
 		this.content = content;
20
-		this.whitespaceBefore = whitespaceBefore;
21
 	}
19
 	}
22
 
20
 
23
 	@Override
21
 	@Override
29
 	public String getContent() {
27
 	public String getContent() {
30
 		return content;
28
 		return content;
31
 	}
29
 	}
32
-
33
-	@Override
34
-	public String getWhitespaceBefore() {
35
-		return whitespaceBefore;
36
-	}
37
 }
30
 }

+ 2
- 2
Parser/src/main/java/org/openzen/zenscript/lexer/ZSTokenFactory.java View File

96
 	}
96
 	}
97
 
97
 
98
 	@Override
98
 	@Override
99
-	public ZSToken create(ZSTokenType type, String content, String whitespaceBefore) {
99
+	public ZSToken create(ZSTokenType type, String content) {
100
 		if (type == T_IDENTIFIER && KEYWORDS.containsKey(content))
100
 		if (type == T_IDENTIFIER && KEYWORDS.containsKey(content))
101
 			type = KEYWORDS.get(content);
101
 			type = KEYWORDS.get(content);
102
 		
102
 		
103
-		return new ZSToken(type, content, whitespaceBefore);
103
+		return new ZSToken(type, content);
104
 	}
104
 	}
105
 }
105
 }

+ 1
- 1
Parser/src/main/java/org/openzen/zenscript/parser/ParsedFile.java View File

120
 			}
120
 			}
121
 		}
121
 		}
122
 		
122
 		
123
-		result.postComment = WhitespacePostComment.fromWhitespace(eof.whitespaceBefore);
123
+		result.postComment = WhitespacePostComment.fromWhitespace(tokens.getLastWhitespace());
124
 		return result;
124
 		return result;
125
 	}
125
 	}
126
 	
126
 	

+ 1
- 1
Parser/src/main/java/org/openzen/zenscript/parser/statements/ParsedStatement.java View File

59
 		}
59
 		}
60
 
60
 
61
 		WhitespaceInfo whitespace = parser.collectWhitespaceInfo(ws, isFirst);
61
 		WhitespaceInfo whitespace = parser.collectWhitespaceInfo(ws, isFirst);
62
-		WhitespacePostComment postComment = WhitespacePostComment.fromWhitespace(last.whitespaceBefore);
62
+		WhitespacePostComment postComment = WhitespacePostComment.fromWhitespace(parser.getLastWhitespace());
63
 		return new ParsedStatementBlock(position, whitespace, postComment, statements);
63
 		return new ParsedStatementBlock(position, whitespace, postComment, statements);
64
 	}
64
 	}
65
 	
65
 	

Loading…
Cancel
Save