1 | // Copyright (C) 2005 Google Inc. |
2 | // |
3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
4 | // you may not use this file except in compliance with the License. |
5 | // You may obtain a copy of the License at |
6 | // |
7 | // http://www.apache.org/licenses/LICENSE-2.0 |
8 | // |
9 | // Unless required by applicable law or agreed to in writing, software |
10 | // distributed under the License is distributed on an "AS IS" BASIS, |
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | // See the License for the specific language governing permissions and |
13 | // limitations under the License. |
14 | |
15 | package com.google.caja.lexer; |
16 | |
17 | import com.google.caja.util.CajaTestCase; |
18 | import com.google.caja.util.FailureIsAnOption; |
19 | |
20 | import java.io.BufferedReader; |
21 | import java.io.InputStreamReader; |
22 | import java.net.URI; |
23 | |
24 | import junit.framework.AssertionFailedError; |
25 | |
26 | /** |
27 | * testcases for {@link JsLexer}. |
28 | * |
29 | * @author mikesamuel@gmail.com |
30 | */ |
31 | public class JsLexerTest extends CajaTestCase { |
32 | |
33 | public final void testLexer() throws Exception { |
34 | StringBuilder output = new StringBuilder(); |
35 | JsLexer t = new JsLexer(fromResource("lexertest1.js")); |
36 | while (t.hasNext()) { |
37 | Token<JsTokenType> tok = t.next(); |
38 | output.append(tok.type.toString().substring(0, 4) |
39 | + " [" + tok.text + "]: " + tok.pos + "\n"); |
40 | } |
41 | |
42 | String golden = fromResource("lexergolden1.txt").toString(); |
43 | |
44 | assertEquals(golden, output.toString()); |
45 | } |
46 | |
47 | public final void testLexer2() throws Exception { |
48 | StringBuilder output = new StringBuilder(); |
49 | |
50 | JsLexer t = new JsLexer(fromResource("lexertest2.js")); |
51 | while (t.hasNext()) { |
52 | Token<JsTokenType> tok = t.next(); |
53 | output.append(tok.type.toString().substring(0, 4) |
54 | + " [" + tok.text + "]: " + tok.pos + "\n"); |
55 | } |
56 | |
57 | String golden = fromResource("lexergolden2.txt").toString(); |
58 | |
59 | assertEquals(golden, output.toString()); |
60 | } |
61 | |
62 | public final void testRegexLiterals() { |
63 | JsLexer lexer = createLexer("foo.replace(/[A-Z]/g, '#')"); |
64 | assertNext(lexer, JsTokenType.WORD, "foo"); |
65 | assertNext(lexer, JsTokenType.PUNCTUATION, "."); |
66 | assertNext(lexer, JsTokenType.WORD, "replace"); |
67 | assertNext(lexer, JsTokenType.PUNCTUATION, "("); |
68 | assertNext(lexer, JsTokenType.REGEXP, "/[A-Z]/g"); |
69 | assertNext(lexer, JsTokenType.PUNCTUATION, ","); |
70 | assertNext(lexer, JsTokenType.STRING, "'#'"); |
71 | assertNext(lexer, JsTokenType.PUNCTUATION, ")"); |
72 | assertEmpty(lexer); |
73 | } |
74 | |
75 | public final void testSimpleExpression() { |
76 | JsLexer lexer = createLexer("while (foo) { 1; }"); |
77 | assertNext(lexer, JsTokenType.KEYWORD, "while"); |
78 | assertNext(lexer, JsTokenType.PUNCTUATION, "("); |
79 | assertNext(lexer, JsTokenType.WORD, "foo"); |
80 | assertNext(lexer, JsTokenType.PUNCTUATION, ")"); |
81 | assertNext(lexer, JsTokenType.PUNCTUATION, "{"); |
82 | assertNext(lexer, JsTokenType.INTEGER, "1"); |
83 | assertNext(lexer, JsTokenType.PUNCTUATION, ";"); |
84 | assertNext(lexer, JsTokenType.PUNCTUATION, "}"); |
85 | assertEmpty(lexer); |
86 | } |
87 | |
88 | public final void testQuasiExpressionSingle() { |
89 | JsLexer lexer = createLexer("@foo * 1;", true); |
90 | assertNext(lexer, JsTokenType.WORD, "@foo"); |
91 | assertNext(lexer, JsTokenType.PUNCTUATION, "*"); |
92 | assertNext(lexer, JsTokenType.INTEGER, "1"); |
93 | assertNext(lexer, JsTokenType.PUNCTUATION, ";"); |
94 | assertEmpty(lexer); |
95 | } |
96 | |
97 | public final void testQuasiExpressionStar() { |
98 | JsLexer lexer = createLexer("@foo* * 1;", true); |
99 | assertNext(lexer, JsTokenType.WORD, "@foo*"); |
100 | assertNext(lexer, JsTokenType.PUNCTUATION, "*"); |
101 | assertNext(lexer, JsTokenType.INTEGER, "1"); |
102 | assertNext(lexer, JsTokenType.PUNCTUATION, ";"); |
103 | assertEmpty(lexer); |
104 | } |
105 | |
106 | public final void testQuasiExpressionPlus() { |
107 | JsLexer lexer = createLexer("@foo+ + 1;", true); |
108 | assertNext(lexer, JsTokenType.WORD, "@foo+"); |
109 | assertNext(lexer, JsTokenType.PUNCTUATION, "+"); |
110 | assertNext(lexer, JsTokenType.INTEGER, "1"); |
111 | assertNext(lexer, JsTokenType.PUNCTUATION, ";"); |
112 | assertEmpty(lexer); |
113 | } |
114 | |
115 | public final void testQuasiExpressionSingleParens() { |
116 | JsLexer lexer = createLexer("(@foo)", true); |
117 | assertNext(lexer, JsTokenType.PUNCTUATION, "("); |
118 | assertNext(lexer, JsTokenType.WORD, "@foo"); |
119 | assertNext(lexer, JsTokenType.PUNCTUATION, ")"); |
120 | assertEmpty(lexer); |
121 | } |
122 | |
123 | public final void testQuasiExpressionStarParens() { |
124 | JsLexer lexer = createLexer("(@foo*)", true); |
125 | assertNext(lexer, JsTokenType.PUNCTUATION, "("); |
126 | assertNext(lexer, JsTokenType.WORD, "@foo*"); |
127 | assertNext(lexer, JsTokenType.PUNCTUATION, ")"); |
128 | assertEmpty(lexer); |
129 | } |
130 | |
131 | public final void testQuasiExpressionPlusParens() { |
132 | JsLexer lexer = createLexer("(@foo+)", true); |
133 | assertNext(lexer, JsTokenType.PUNCTUATION, "("); |
134 | assertNext(lexer, JsTokenType.WORD, "@foo+"); |
135 | assertNext(lexer, JsTokenType.PUNCTUATION, ")"); |
136 | assertEmpty(lexer); |
137 | } |
138 | |
139 | public final void testDoubleDot() { |
140 | JsLexer lexer = createLexer( |
141 | "a == = function () {..} ... .. . .... foo", true); |
142 | assertNext(lexer, JsTokenType.WORD, "a"); |
143 | assertNext(lexer, JsTokenType.PUNCTUATION, "=="); |
144 | assertNext(lexer, JsTokenType.PUNCTUATION, "="); |
145 | assertNext(lexer, JsTokenType.KEYWORD, "function"); |
146 | assertNext(lexer, JsTokenType.PUNCTUATION, "("); |
147 | assertNext(lexer, JsTokenType.PUNCTUATION, ")"); |
148 | assertNext(lexer, JsTokenType.PUNCTUATION, "{"); |
149 | assertNext(lexer, JsTokenType.PUNCTUATION, ".."); |
150 | assertNext(lexer, JsTokenType.PUNCTUATION, "}"); |
151 | assertNext(lexer, JsTokenType.PUNCTUATION, "..."); |
152 | assertNext(lexer, JsTokenType.PUNCTUATION, ".."); |
153 | assertNext(lexer, JsTokenType.PUNCTUATION, "."); |
154 | assertNext(lexer, JsTokenType.PUNCTUATION, "..."); |
155 | assertNext(lexer, JsTokenType.PUNCTUATION, "."); |
156 | assertNext(lexer, JsTokenType.WORD, "foo"); |
157 | assertEmpty(lexer); |
158 | } |
159 | |
160 | public final void testNumberDotWord() { |
161 | JsLexer lexer = createLexer("0..toString()", false); // evaluates to "0" |
162 | assertNext(lexer, JsTokenType.FLOAT, "0."); |
163 | assertNext(lexer, JsTokenType.PUNCTUATION, "."); |
164 | assertNext(lexer, JsTokenType.WORD, "toString"); |
165 | assertNext(lexer, JsTokenType.PUNCTUATION, "("); |
166 | assertNext(lexer, JsTokenType.PUNCTUATION, ")"); |
167 | assertEmpty(lexer); |
168 | } |
169 | |
170 | public final void testByteOrderMarkersAtBeginning() { |
171 | JsLexer lexer = createLexer("\uFEFFvar foo", false); |
172 | assertNext(lexer, JsTokenType.KEYWORD, "var"); |
173 | assertNext(lexer, JsTokenType.WORD, "foo"); |
174 | assertEmpty(lexer); |
175 | } |
176 | |
177 | public final void testByteOrderMarkersBetweenTokens() { |
178 | JsLexer lexer = createLexer("1.\uFEFF3", false); |
179 | assertNext(lexer, JsTokenType.FLOAT, "1."); |
180 | assertNext(lexer, JsTokenType.INTEGER, "3"); |
181 | assertEmpty(lexer); |
182 | } |
183 | |
184 | public final void testByteOrderMarkersInStrings() { |
185 | JsLexer lexer = createLexer("'\uFEFF'", false); |
186 | assertNext(lexer, JsTokenType.STRING, "'\uFEFF'"); |
187 | assertEmpty(lexer); |
188 | } |
189 | |
190 | public final void testEllipsisAndNumber() { |
191 | JsLexer lexer = createLexer("...0x01", false); |
192 | assertNext(lexer, JsTokenType.PUNCTUATION, "..."); |
193 | assertNext(lexer, JsTokenType.INTEGER, "0x01"); |
194 | assertEmpty(lexer); |
195 | } |
196 | |
197 | public final void testEmphaticallyDecremented() { |
198 | JsLexer lexer = createLexer("i---j", false); |
199 | assertNext(lexer, JsTokenType.WORD, "i"); |
200 | assertNext(lexer, JsTokenType.PUNCTUATION, "--"); |
201 | assertNext(lexer, JsTokenType.PUNCTUATION, "-"); |
202 | assertNext(lexer, JsTokenType.WORD, "j"); |
203 | assertEmpty(lexer); |
204 | } |
205 | |
206 | public final void testIsRegexpFollowingWord() { |
207 | { |
208 | JsLexer lexer = createLexer("min / max /*/**/", false); |
209 | assertNext(lexer, JsTokenType.WORD, "min"); |
210 | assertNext(lexer, JsTokenType.PUNCTUATION, "/"); |
211 | assertNext(lexer, JsTokenType.WORD, "max"); |
212 | assertNext(lexer, JsTokenType.COMMENT, "/*/**/"); |
213 | assertEmpty(lexer); |
214 | } |
215 | { |
216 | JsLexer lexer = createLexer("in / max /*/**/", false); |
217 | assertNext(lexer, JsTokenType.KEYWORD, "in"); |
218 | assertNext(lexer, JsTokenType.REGEXP, "/ max /"); |
219 | assertNext(lexer, JsTokenType.PUNCTUATION, "*"); |
220 | assertNext(lexer, JsTokenType.COMMENT, "/**/"); |
221 | assertEmpty(lexer); |
222 | } |
223 | } |
224 | |
225 | public final void testRegexpFollowingVoid() { |
226 | JsLexer lexer = createLexer("void /./", false); |
227 | assertNext(lexer, JsTokenType.KEYWORD, "void"); |
228 | assertNext(lexer, JsTokenType.REGEXP, "/./"); |
229 | assertEmpty(lexer); |
230 | } |
231 | |
232 | @FailureIsAnOption |
233 | public final void testRegexpFollowingPreincrement() { |
234 | JsLexer lexer = createLexer("x = ++/x/m", false); |
235 | assertNext(lexer, JsTokenType.WORD, "x"); |
236 | assertNext(lexer, JsTokenType.PUNCTUATION, "="); |
237 | assertNext(lexer, JsTokenType.PUNCTUATION, "++"); |
238 | assertNext(lexer, JsTokenType.REGEXP, "/x/m"); |
239 | assertEmpty(lexer); |
240 | } |
241 | |
242 | public final void testRegexpFollowingPostincrement() { |
243 | JsLexer lexer = createLexer("x++/y/m", false); |
244 | assertNext(lexer, JsTokenType.WORD, "x"); |
245 | assertNext(lexer, JsTokenType.PUNCTUATION, "++"); |
246 | assertNext(lexer, JsTokenType.PUNCTUATION, "/"); |
247 | assertNext(lexer, JsTokenType.WORD, "y"); |
248 | assertNext(lexer, JsTokenType.PUNCTUATION, "/"); |
249 | assertNext(lexer, JsTokenType.WORD, "m"); |
250 | assertEmpty(lexer); |
251 | } |
252 | |
253 | private JsLexer createLexer(String src) { |
254 | return createLexer(src, false); |
255 | } |
256 | |
257 | private JsLexer createLexer(String src, boolean isQuasiliteral) { |
258 | return new JsLexer(fromString(src), isQuasiliteral); |
259 | } |
260 | |
261 | private void assertNext(JsLexer lexer, JsTokenType type, String text) { |
262 | Token<JsTokenType> tok = null; |
263 | try { |
264 | tok = lexer.next(); |
265 | } catch (ParseException e) { |
266 | throw (AssertionFailedError) |
267 | new AssertionFailedError(e.getMessage()).initCause(e); |
268 | } |
269 | assertEquals(type, tok.type); |
270 | assertEquals("was '" + tok.text + "', expected '" + text + "'", |
271 | text, tok.text); |
272 | } |
273 | |
274 | public void assertEmpty(JsLexer lexer) { |
275 | try { |
276 | assertFalse(lexer.hasNext()); |
277 | } catch (ParseException e) { |
278 | fail(e.toString()); |
279 | } |
280 | } |
281 | |
282 | public static void main(String[] args) throws Exception { |
283 | InputSource input = new InputSource(URI.create("file:///proc/self/fd/0")); |
284 | |
285 | BufferedReader in = new BufferedReader( |
286 | new InputStreamReader(System.in, "UTF-8")); |
287 | JsLexer t = new JsLexer(CharProducer.Factory.create(in, input)); |
288 | while (t.hasNext()) { |
289 | Token<JsTokenType> tok = t.next(); |
290 | System.out.append(tok.type.toString().substring(0, 4) |
291 | + " [" + tok.text + "]: " + tok.pos + "\n"); |
292 | } |
293 | } |
294 | } |