source: josm/trunk/src/org/glassfish/json/JsonTokenizer.java@ 10399

Last change on this file since 10399 was 6756, checked in by Don-vip, 11 years ago

fix #9590 - replace org.json with GPL-compliant jsonp + remove mention of old world image removed in r1680
File size: 17.1 KB
Line 
1/*
2 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
3 *
4 * Copyright (c) 2012-2013 Oracle and/or its affiliates. All rights reserved.
5 *
6 * The contents of this file are subject to the terms of either the GNU
7 * General Public License Version 2 only ("GPL") or the Common Development
8 * and Distribution License("CDDL") (collectively, the "License"). You
9 * may not use this file except in compliance with the License. You can
10 * obtain a copy of the License at
11 * https://glassfish.dev.java.net/public/CDDL+GPL_1_1.html
12 * or packager/legal/LICENSE.txt. See the License for the specific
13 * language governing permissions and limitations under the License.
14 *
15 * When distributing the software, include this License Header Notice in each
16 * file and include the License file at packager/legal/LICENSE.txt.
17 *
18 * GPL Classpath Exception:
19 * Oracle designates this particular file as subject to the "Classpath"
20 * exception as provided by Oracle in the GPL Version 2 section of the License
21 * file that accompanied this code.
22 *
23 * Modifications:
24 * If applicable, add the following below the License Header, with the fields
25 * enclosed by brackets [] replaced by your own identifying information:
26 * "Portions Copyright [year] [name of copyright owner]"
27 *
28 * Contributor(s):
29 * If you wish your version of this file to be governed by only the CDDL or
30 * only the GPL Version 2, indicate your decision by adding "[Contributor]
31 * elects to include this software in this distribution under the [CDDL or GPL
32 * Version 2] license." If you don't indicate a single choice of license, a
33 * recipient has the option to distribute your version of this file under
34 * either the CDDL, the GPL Version 2 or to extend the choice of license to
35 * its licensees as provided above. However, if you add GPL Version 2 code
36 * and therefore, elected the GPL Version 2 license, then the option applies
37 * only if the new code is made subject to such option by the copyright
38 * holder.
39 */
40
41package org.glassfish.json;
42
43import org.glassfish.json.api.BufferPool;
44
45import javax.json.JsonException;
46import javax.json.stream.JsonLocation;
47import javax.json.stream.JsonParser;
48import javax.json.stream.JsonParsingException;
49import java.io.*;
50import java.math.BigDecimal;
51import java.util.Arrays;
52
53import javax.json.stream.JsonParser.Event;
54
55/**
56 * JSON Tokenizer
57 *
58 * @author Jitendra Kotamraju
59 */
60final class JsonTokenizer implements Closeable {
61 // Table to look up hex ch -> value (for e.g HEX['F'] = 15, HEX['5'] = 5)
62 private final static int[] HEX = new int[128];
63 static {
64 Arrays.fill(HEX, -1);
65 for (int i='0'; i <= '9'; i++) {
66 HEX[i] = i-'0';
67 }
68 for (int i='A'; i <= 'F'; i++) {
69 HEX[i] = 10+i-'A';
70 }
71 for (int i='a'; i <= 'f'; i++) {
72 HEX[i] = 10+i-'a';
73 }
74 }
75 private final static int HEX_LENGTH = HEX.length;
76
77 private final BufferPool bufferPool;
78
79 private final Reader reader;
80
81 // Internal buffer that is used for parsing. It is also used
82 // for storing current string and number value token
83 private char[] buf;
84
85 // Indexes in buffer
86 //
87 // XXXssssssssssssXXXXXXXXXXXXXXXXXXXXXXrrrrrrrrrrrrrrXXXXXX
88 // ^ ^ ^ ^
89 // | | | |
90 // storeBegin storeEnd readBegin readEnd
91 private int readBegin;
92 private int readEnd;
93 private int storeBegin;
94 private int storeEnd;
95
96 // line number of the current pointer of parsing char
97 private long lineNo = 1;
98
99 // XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
100 // ^
101 // |
102 // bufferOffset
103 //
104 // offset of the last \r\n or \n. will be used to calculate column number
105 // of a token or an error. This may be outside of the buffer.
106 private long lastLineOffset = 0;
107 // offset in the stream for the start of the buffer, will be used in
108 // calculating JsonLocation's stream offset, column no.
109 private long bufferOffset = 0;
110
111 private boolean minus;
112 private boolean fracOrExp;
113 private BigDecimal bd;
114
115 enum JsonToken {
116 CURLYOPEN(Event.START_OBJECT, false),
117 SQUAREOPEN(Event.START_ARRAY, false),
118 COLON(null, false),
119 COMMA(null, false),
120 STRING(Event.VALUE_STRING, true),
121 NUMBER(Event.VALUE_NUMBER, true),
122 TRUE(Event.VALUE_TRUE, true),
123 FALSE(Event.VALUE_FALSE, true),
124 NULL(Event.VALUE_NULL, true),
125 CURLYCLOSE(Event.END_OBJECT, false),
126 SQUARECLOSE(Event.END_ARRAY, false),
127 EOF(null, false);
128
129 private final JsonParser.Event event;
130 private final boolean value;
131
132 JsonToken(JsonParser.Event event, boolean value) {
133 this.event = event;
134 this.value = value;
135 }
136
137 JsonParser.Event getEvent() {
138 return event;
139 }
140
141 boolean isValue() {
142 return value;
143 }
144 }
145
146 JsonTokenizer(Reader reader, BufferPool bufferPool) {
147 this.reader = reader;
148 this.bufferPool = bufferPool;
149 buf = bufferPool.take();
150 }
151
152 private void readString() {
153 // when inPlace is true, no need to copy chars
154 boolean inPlace = true;
155 storeBegin = storeEnd = readBegin;
156
157 do {
158 // Write unescaped char block within the current buffer
159 if (inPlace) {
160 int ch;
161 while(readBegin < readEnd && ((ch=buf[readBegin]) >= 0x20) && ch != '\\') {
162 if (ch == '"') {
163 storeEnd = readBegin++; // ++ to consume quote char
164 return; // Got the entire string
165 }
166 readBegin++; // consume unescaped char
167 }
168 storeEnd = readBegin;
169 }
170
171 // string may be crossing buffer boundaries and may contain
172 // escaped characters.
173 int ch = read();
174 if (ch >= 0x20 && ch != 0x22 && ch != 0x5c) {
175 if (!inPlace) {
176 buf[storeEnd] = (char)ch;
177 }
178 storeEnd++;
179 continue;
180 }
181 switch (ch) {
182 case '\\':
183 inPlace = false; // Now onwards need to copy chars
184 unescape();
185 break;
186 case '"':
187 return;
188 default:
189 throw unexpectedChar(ch);
190 }
191 } while (true);
192 }
193
194 private void unescape() {
195 int ch = read();
196 switch (ch) {
197 case 'b':
198 buf[storeEnd++] = '\b';
199 break;
200 case 't':
201 buf[storeEnd++] = '\t';
202 break;
203 case 'n':
204 buf[storeEnd++] = '\n';
205 break;
206 case 'f':
207 buf[storeEnd++] = '\f';
208 break;
209 case 'r':
210 buf[storeEnd++] = '\r';
211 break;
212 case '"':
213 case '\\':
214 case '/':
215 buf[storeEnd++] = (char)ch;
216 break;
217 case 'u': {
218 int unicode = 0;
219 for (int i = 0; i < 4; i++) {
220 int ch3 = read();
221 int digit = (ch3 >= 0 && ch3 < HEX_LENGTH) ? HEX[ch3] : -1;
222 if (digit < 0) {
223 throw unexpectedChar(ch3);
224 }
225 unicode = (unicode << 4)|digit;
226 }
227 buf[storeEnd++] = (char)unicode;
228 break;
229 }
230 default:
231 throw unexpectedChar(ch);
232 }
233 }
234
235 // Reads a number char. If the char is within the buffer, directly
236 // reads from the buffer. Otherwise, uses read() which takes care
237 // of resizing, filling up the buf, adjusting the pointers
238 private int readNumberChar() {
239 if (readBegin < readEnd) {
240 return buf[readBegin++];
241 } else {
242 storeEnd = readBegin;
243 return read();
244 }
245 }
246
247 private void readNumber(int ch) {
248 storeBegin = storeEnd = readBegin-1;
249 // sign
250 if (ch == '-') {
251 this.minus = true;
252 ch = readNumberChar();
253 if (ch < '0' || ch >'9') {
254 throw unexpectedChar(ch);
255 }
256 }
257
258 // int
259 if (ch == '0') {
260 ch = readNumberChar();
261 } else {
262 do {
263 ch = readNumberChar();
264 } while (ch >= '0' && ch <= '9');
265 }
266
267 // frac
268 if (ch == '.') {
269 this.fracOrExp = true;
270 int count = 0;
271 do {
272 ch = readNumberChar();
273 count++;
274 } while (ch >= '0' && ch <= '9');
275 if (count == 1) {
276 throw unexpectedChar(ch);
277 }
278 }
279
280 // exp
281 if (ch == 'e' || ch == 'E') {
282 this.fracOrExp = true;
283 ch = readNumberChar();
284 if (ch == '+' || ch == '-') {
285 ch = readNumberChar();
286 }
287 int count;
288 for (count = 0; ch >= '0' && ch <= '9'; count++) {
289 ch = readNumberChar();
290 }
291 if (count == 0) {
292 throw unexpectedChar(ch);
293 }
294 }
295 readBegin--;
296 storeEnd = readBegin;
297 }
298
299 private void readTrue() {
300 int ch1 = read();
301 if (ch1 != 'r') {
302 throw expectedChar(ch1, 'r');
303 }
304 int ch2 = read();
305 if (ch2 != 'u') {
306 throw expectedChar(ch2, 'u');
307 }
308 int ch3 = read();
309 if (ch3 != 'e') {
310 throw expectedChar(ch3, 'e');
311 }
312 }
313
314 private void readFalse() {
315 int ch1 = read();
316 if (ch1 != 'a') {
317 throw expectedChar(ch1, 'a');
318 }
319 int ch2 = read();
320 if (ch2 != 'l') {
321 throw expectedChar(ch2, 'l');
322 }
323 int ch3 = read();
324 if (ch3 != 's') {
325 throw expectedChar(ch3, 's');
326 }
327 int ch4 = read();
328 if (ch4 != 'e') {
329 throw expectedChar(ch4, 'e');
330 }
331 }
332
333 private void readNull() {
334 int ch1 = read();
335 if (ch1 != 'u') {
336 throw expectedChar(ch1, 'u');
337 }
338 int ch2 = read();
339 if (ch2 != 'l') {
340 throw expectedChar(ch2, 'l');
341 }
342 int ch3 = read();
343 if (ch3 != 'l') {
344 throw expectedChar(ch3, 'l');
345 }
346 }
347
348 /*
349 * Could be optimized if the parser uses separate methods to match colon
350 * etc (that would avoid the switch statement cost in certain cases)
351 */
352 JsonToken nextToken() {
353 reset();
354 int ch = read();
355
356 // whitespace
357 while (ch == 0x20 || ch == 0x09 || ch == 0x0a || ch == 0x0d) {
358 if (ch == '\r') {
359 ++lineNo;
360 ch = read();
361 if (ch == '\n') {
362 lastLineOffset = bufferOffset+readBegin;
363 } else {
364 lastLineOffset = bufferOffset+readBegin-1;
365 continue;
366 }
367 } else if (ch == '\n') {
368 ++lineNo;
369 lastLineOffset = bufferOffset+readBegin;
370 }
371 ch = read();
372 }
373
374 switch (ch) {
375 case '"':
376 readString();
377 return JsonToken.STRING;
378 case '{':
379 return JsonToken.CURLYOPEN;
380 case '[':
381 return JsonToken.SQUAREOPEN;
382 case ':':
383 return JsonToken.COLON;
384 case ',':
385 return JsonToken.COMMA;
386 case 't':
387 readTrue();
388 return JsonToken.TRUE;
389 case 'f':
390 readFalse();
391 return JsonToken.FALSE;
392 case 'n':
393 readNull();
394 return JsonToken.NULL;
395 case ']':
396 return JsonToken.SQUARECLOSE;
397 case '}':
398 return JsonToken.CURLYCLOSE;
399 case '0':
400 case '1':
401 case '2':
402 case '3':
403 case '4':
404 case '5':
405 case '6':
406 case '7':
407 case '8':
408 case '9':
409 case '-':
410 readNumber(ch);
411 return JsonToken.NUMBER;
412 case -1:
413 return JsonToken.EOF;
414 default:
415 throw unexpectedChar(ch);
416 }
417 }
418
419 // Gives the location of the last char. Used for
420 // JsonParsingException.getLocation
421 JsonLocation getLastCharLocation() {
422 // Already read the char, so subtracting -1
423 return new JsonLocationImpl(lineNo, bufferOffset +readBegin-lastLineOffset, bufferOffset +readBegin-1);
424 }
425
426 // Gives the parser location. Used for JsonParser.getLocation
427 JsonLocation getLocation() {
428 return new JsonLocationImpl(lineNo, bufferOffset +readBegin-lastLineOffset+1, bufferOffset +readBegin);
429 }
430
431 private int read() {
432 try {
433 if (readBegin == readEnd) { // need to fill the buffer
434 int len = fillBuf();
435 if (len == -1) {
436 return -1;
437 }
438 assert len != 0;
439 readBegin = storeEnd;
440 readEnd = readBegin+len;
441 }
442 return buf[readBegin++];
443 } catch (IOException ioe) {
444 throw new JsonException(JsonMessages.TOKENIZER_IO_ERR(), ioe);
445 }
446 }
447
448 private int fillBuf() throws IOException {
449 if (storeEnd != 0) {
450 int storeLen = storeEnd-storeBegin;
451 if (storeLen > 0) {
452 // there is some store data
453 if (storeLen == buf.length) {
454 // buffer is full, double the capacity
455 char[] doubleBuf = Arrays.copyOf(buf, 2 * buf.length);
456 bufferPool.recycle(buf);
457 buf = doubleBuf;
458 } else {
459 // Left shift all the stored data to make space
460 System.arraycopy(buf, storeBegin, buf, 0, storeLen);
461 storeEnd = storeLen;
462 storeBegin = 0;
463 bufferOffset += readBegin-storeEnd;
464 }
465 } else {
466 storeBegin = storeEnd = 0;
467 bufferOffset += readBegin;
468 }
469 } else {
470 bufferOffset += readBegin;
471 }
472 // Fill the rest of the buf
473 return reader.read(buf, storeEnd, buf.length-storeEnd);
474 }
475
476 // state associated with the current token is no more valid
477 private void reset() {
478 if (storeEnd != 0) {
479 storeBegin = 0;
480 storeEnd = 0;
481 bd = null;
482 minus = false;
483 fracOrExp = false;
484 }
485 }
486
487 String getValue() {
488 return new String(buf, storeBegin, storeEnd-storeBegin);
489 }
490
491 BigDecimal getBigDecimal() {
492 if (bd == null) {
493 bd = new BigDecimal(buf, storeBegin, storeEnd-storeBegin);
494 }
495 return bd;
496 }
497
498 int getInt() {
499 // no need to create BigDecimal for common integer values (1-9 digits)
500 int storeLen = storeEnd-storeBegin;
501 if (!fracOrExp && (storeLen <= 9 || (minus && storeLen == 10))) {
502 int num = 0;
503 int i = minus ? 1 : 0;
504 for(; i < storeLen; i++) {
505 num = num * 10 + (buf[storeBegin+i] - '0');
506 }
507 return minus ? -num : num;
508 } else {
509 return getBigDecimal().intValue();
510 }
511 }
512
513 // returns true for common integer values (1-9 digits).
514 // So there are cases it will return false even though the number is int
515 boolean isDefinitelyInt() {
516 int storeLen = storeEnd-storeBegin;
517 return !fracOrExp && (storeLen <= 9 || (minus && storeLen == 10));
518 }
519
520 boolean isIntegral() {
521 return !fracOrExp || getBigDecimal().scale() == 0;
522 }
523
524 @Override
525 public void close() throws IOException {
526 reader.close();
527 bufferPool.recycle(buf);
528 }
529
530 private JsonParsingException unexpectedChar(int ch) {
531 JsonLocation location = getLastCharLocation();
532 return new JsonParsingException(
533 JsonMessages.TOKENIZER_UNEXPECTED_CHAR(ch, location), location);
534 }
535
536 private JsonParsingException expectedChar(int unexpected, char expected) {
537 JsonLocation location = getLastCharLocation();
538 return new JsonParsingException(
539 JsonMessages.TOKENIZER_EXPECTED_CHAR(unexpected, location, expected), location);
540 }
541
542}
Note: See TracBrowser for help on using the repository browser.