001/* 002// This software is subject to the terms of the Eclipse Public License v1.0 003// Agreement, available at the following URL: 004// http://www.eclipse.org/legal/epl-v10.html. 005// You must accept the terms of that agreement to use this software. 006// 007// Copyright (C) 1998-2005 Julian Hyde 008// Copyright (C) 2005-2011 Pentaho and others 009// All Rights Reserved. 010*/ 011package mondrian.olap; 012 013import org.apache.log4j.Logger; 014 015import java_cup.runtime.Symbol; 016 017import java.io.IOException; 018import java.math.BigDecimal; 019import java.util.*; 020 021/** 022 * Lexical analyzer for MDX. 023 * 024 * @author jhyde, 20 January, 1999 025 */ 026public class Scanner { 027 private static final Logger LOGGER = Logger.getLogger(Scanner.class); 028 029 /** single lookahead character */ 030 protected int nextChar; 031 /** next lookahead character */ 032 private int lookaheadChars[] = new int[16]; 033 private int firstLookaheadChar = 0; 034 private int lastLookaheadChar = 0; 035 private Hashtable<String, Integer> m_resWordsTable; 036 private int iMaxResword; 037 private String m_aResWords[]; 038 protected boolean debug; 039 040 /** lines[x] is the start of the x'th line */ 041 private List<Integer> lines; 042 043 /** number of times advance() has been called */ 044 private int iChar; 045 046 /** end of previous token */ 047 private int iPrevChar; 048 049 /** previous symbol returned */ 050 private int previousSymbol; 051 private boolean inFormula; 052 053 /** 054 * Comment delimiters. Modify this list to support other comment styles. 055 */ 056 private static final String[][] commentDelim = { 057 new String[] {"//", null}, 058 new String[] {"--", null}, 059 new String[] {"/*", "*/"} 060 }; 061 062 /** 063 * Whether to allow nested comments. 064 */ 065 private static final boolean allowNestedComments = true; 066 067 /** 068 * The {@link java.math.BigDecimal} value 0. 069 * Note that BigDecimal.ZERO does not exist until JDK 1.5. 070 */ 071 private static final BigDecimal BigDecimalZero = BigDecimal.valueOf(0); 072 073 /** 074 * Creates a Scanner. 075 * 076 * @param debug Whether to emit debug messages. 077 */ 078 Scanner(boolean debug) { 079 this.debug = debug; 080 } 081 082 /** 083 * Returns the current nested comments state. 084 */ 085 public static boolean getNestedCommentsState() { 086 return allowNestedComments; 087 } 088 089 /** 090 * Returns the list of comment delimiters. 091 */ 092 public static String[][] getCommentDelimiters() { 093 return commentDelim; 094 } 095 096 /** 097 * Advance input by one character, setting {@link #nextChar}. 098 */ 099 private void advance() throws IOException { 100 if (firstLookaheadChar == lastLookaheadChar) { 101 // We have nothing in the lookahead buffer. 102 nextChar = getChar(); 103 } else { 104 // We have called lookahead(); advance to the next character it got. 105 nextChar = lookaheadChars[firstLookaheadChar++]; 106 if (firstLookaheadChar == lastLookaheadChar) { 107 firstLookaheadChar = 0; 108 lastLookaheadChar = 0; 109 } 110 } 111 if (nextChar == '\012') { 112 lines.add(iChar); 113 } 114 iChar++; 115 } 116 117 /** Peek at the character after {@link #nextChar} without advancing. */ 118 private int lookahead() throws IOException { 119 return lookahead(1); 120 } 121 122 /** 123 * Peeks at the character n after {@link #nextChar} without advancing. 124 * 125 * <p>lookahead(0) returns the current char (nextChar). 126 * lookahead(1) returns the next char (was lookaheadChar, same as 127 * lookahead()); 128 */ 129 private int lookahead(int n) throws IOException { 130 if (n == 0) { 131 return nextChar; 132 } else { 133 // if the desired character not in lookahead buffer, read it in 134 if (n > lastLookaheadChar - firstLookaheadChar) { 135 int len = lastLookaheadChar - firstLookaheadChar; 136 int t[]; 137 138 // make sure we do not go off the end of the buffer 139 if (n + firstLookaheadChar > lookaheadChars.length) { 140 if (n > lookaheadChars.length) { 141 // the array is too small; make it bigger and shift 142 // everything to the beginning. 143 t = new int[n * 2]; 144 } else { 145 // the array is big enough, so just shift everything 146 // to the beginning of it. 147 t = lookaheadChars; 148 } 149 150 System.arraycopy( 151 lookaheadChars, firstLookaheadChar, t, 0, len); 152 lookaheadChars = t; 153 firstLookaheadChar = 0; 154 lastLookaheadChar = len; 155 } 156 157 // read ahead enough 158 while (n > lastLookaheadChar - firstLookaheadChar) { 159 lookaheadChars[lastLookaheadChar++] = getChar(); 160 } 161 } 162 163 return lookaheadChars[n - 1 + firstLookaheadChar]; 164 } 165 } 166 167 /** Read a character from input, returning -1 if end of input. */ 168 protected int getChar() throws IOException { 169 return System.in.read(); 170 } 171 172 /** Initialize the scanner */ 173 public void init() throws IOException { 174 initReswords(); 175 lines = new ArrayList<Integer>(); 176 iChar = iPrevChar = 0; 177 advance(); 178 } 179 180 /** 181 * Deduces the line and column (0-based) of a symbol. 182 * Called by {@link Parser#syntax_error}. 183 */ 184 void getLocation(Symbol symbol, int[] loc) { 185 int iTarget = symbol.left; 186 int iLine = -1; 187 int iLineEnd = 0; 188 int iLineStart; 189 do { 190 iLine++; 191 iLineStart = iLineEnd; 192 iLineEnd = Integer.MAX_VALUE; 193 if (iLine < lines.size()) { 194 iLineEnd = lines.get(iLine); 195 } 196 } while (iLineEnd < iTarget); 197 198 loc[0] = iLine; // line 199 loc[1] = iTarget - iLineStart; // column 200 } 201 202 private Symbol trace(Symbol s) { 203 if (debug) { 204 String name = null; 205 if (s.sym < m_aResWords.length) { 206 name = m_aResWords[s.sym]; 207 } 208 209 LOGGER.error( 210 "Scanner returns #" + s.sym 211 + (name == null ? "" : ":" + name) 212 + (s.value == null ? "" : "(" + s.value.toString() + ")")); 213 } 214 return s; 215 } 216 217 private void initResword(int id, String s) { 218 m_resWordsTable.put(s, id); 219 if (id > iMaxResword) { 220 iMaxResword = id; 221 } 222 } 223 224 /** 225 * Initializes the table of reserved words. 226 */ 227 private void initReswords() { 228 // This list generated by piping the 'terminal' declaration in mdx.cup 229 // through: 230 // grep -list // | 231 // sed -e 's/,//' | 232 // awk '{printf "initResword(%20s,%c%s%c);",$1,34,$1,34}' 233 m_resWordsTable = new Hashtable<String, Integer>(); 234 iMaxResword = 0; 235// initResword(ParserSym.ALL, "ALL"); 236 initResword(ParserSym.AND, "AND"); 237 initResword(ParserSym.AS, "AS"); 238// initResword(ParserSym.ASC, "ASC"); 239 initResword(ParserSym.AXIS, "AXIS"); 240// initResword(ParserSym.BACK_COLOR, "BACK_COLOR"); 241// initResword(ParserSym.BASC, "BASC"); 242// initResword(ParserSym.BDESC, "BDESC"); 243 // CAST is a mondrian extension 244 initResword(ParserSym.CAST, "CAST"); 245 initResword(ParserSym.CASE, "CASE"); 246 initResword(ParserSym.CELL, "CELL"); 247// initResword(ParserSym.CELL_ORDINAL, "CELL_ORDINAL"); 248 initResword(ParserSym.CHAPTERS, "CHAPTERS"); 249// initResword(ParserSym.CHILDREN, "CHILDREN"); 250 initResword(ParserSym.COLUMNS, "COLUMNS"); 251// initResword(ParserSym.DESC, "DESC"); 252 initResword(ParserSym.DIMENSION, "DIMENSION"); 253 initResword(ParserSym.DRILLTHROUGH, "DRILLTHROUGH"); 254 initResword(ParserSym.ELSE, "ELSE"); 255 initResword(ParserSym.EMPTY, "EMPTY"); 256 initResword(ParserSym.END, "END"); 257 initResword(ParserSym.EXPLAIN, "EXPLAIN"); 258// initResword(ParserSym.FIRSTCHILD, "FIRSTCHILD"); 259 initResword(ParserSym.FIRSTROWSET, "FIRSTROWSET"); 260// initResword(ParserSym.FIRSTSIBLING, "FIRSTSIBLING"); 261// initResword(ParserSym.FONT_FLAGS, "FONT_FLAGS"); 262// initResword(ParserSym.FONT_NAME, "FONT_NAME"); 263// initResword(ParserSym.FONT_SIZE, "FONT_SIZE"); 264// initResword(ParserSym.FORE_COLOR, "FORE_COLOR"); 265// initResword(ParserSym.FORMATTED_VALUE, "FORMATTED_VALUE"); 266// initResword(ParserSym.FORMAT_STRING, "FORMAT_STRING"); 267 initResword(ParserSym.FOR, "FOR"); 268 initResword(ParserSym.FROM, "FROM"); 269 initResword(ParserSym.IS, "IS"); 270 initResword(ParserSym.IN, "IN"); 271// initResword(ParserSym.LAG, "LAG"); 272// initResword(ParserSym.LASTCHILD, "LASTCHILD"); 273// initResword(ParserSym.LASTSIBLING, "LASTSIBLING"); 274// initResword(ParserSym.LEAD, "LEAD"); 275 initResword(ParserSym.MATCHES, "MATCHES"); 276 initResword(ParserSym.MAXROWS, "MAXROWS"); 277 initResword(ParserSym.MEMBER, "MEMBER"); 278// initResword(ParserSym.MEMBERS, "MEMBERS"); 279// initResword(ParserSym.NEXTMEMBER, "NEXTMEMBER"); 280 initResword(ParserSym.NON, "NON"); 281 initResword(ParserSym.NOT, "NOT"); 282 initResword(ParserSym.NULL, "NULL"); 283 initResword(ParserSym.ON, "ON"); 284 initResword(ParserSym.OR, "OR"); 285 initResword(ParserSym.PAGES, "PAGES"); 286// initResword(ParserSym.PARENT, "PARENT"); 287 initResword(ParserSym.PLAN, "PLAN"); 288// initResword(ParserSym.PREVMEMBER, "PREVMEMBER"); 289 initResword(ParserSym.PROPERTIES, "PROPERTIES"); 290// initResword(ParserSym.RECURSIVE, "RECURSIVE"); 291 initResword(ParserSym.RETURN, "RETURN"); 292 initResword(ParserSym.ROWS, "ROWS"); 293 initResword(ParserSym.SECTIONS, "SECTIONS"); 294 initResword(ParserSym.SELECT, "SELECT"); 295 initResword(ParserSym.SET, "SET"); 296// initResword(ParserSym.SOLVE_ORDER, "SOLVE_ORDER"); 297 initResword(ParserSym.THEN, "THEN"); 298// initResword(ParserSym.VALUE, "VALUE"); 299 initResword(ParserSym.WHEN, "WHEN"); 300 initResword(ParserSym.WHERE, "WHERE"); 301 initResword(ParserSym.WITH, "WITH"); 302 initResword(ParserSym.XOR, "XOR"); 303 304 m_aResWords = new String[iMaxResword + 1]; 305 Enumeration<String> e = m_resWordsTable.keys(); 306 while (e.hasMoreElements()) { 307 Object o = e.nextElement(); 308 String s = (String) o; 309 int i = (m_resWordsTable.get(s)).intValue(); 310 m_aResWords[i] = s; 311 } 312 } 313 314 /** return the name of the reserved word whose token code is "i" */ 315 public String lookupReserved(int i) { 316 return m_aResWords[i]; 317 } 318 319 private Symbol makeSymbol(int id, Object o) { 320 int iPrevPrevChar = iPrevChar; 321 this.iPrevChar = iChar; 322 this.previousSymbol = id; 323 return trace(new Symbol(id, iPrevPrevChar, iChar, o)); 324 } 325 326 /** 327 * Creates a token representing a numeric literal. 328 * 329 * @param mantissa The digits of the number 330 * @param exponent The base-10 exponent of the number 331 * @return number literal token 332 */ 333 private Symbol makeNumber(BigDecimal mantissa, int exponent) { 334 BigDecimal d = mantissa.movePointRight(exponent); 335 return makeSymbol(ParserSym.NUMBER, d); 336 } 337 338 private Symbol makeId(String s, boolean quoted, boolean ampersand) { 339 return makeSymbol( 340 quoted && ampersand 341 ? ParserSym.AMP_QUOTED_ID 342 : quoted 343 ? ParserSym.QUOTED_ID 344 : ParserSym.ID, 345 s); 346 } 347 348 /** 349 * Creates a token representing a reserved word. 350 * 351 * @param i Token code 352 * @return Token 353 */ 354 private Symbol makeRes(int i) { 355 return makeSymbol(i, m_aResWords[i]); 356 } 357 358 /** 359 * Creates a token. 360 * 361 * @param i Token code 362 * @param s Text of the token 363 * @return Token 364 */ 365 private Symbol makeToken(int i, String s) { 366 return makeSymbol(i, s); 367 } 368 369 /** 370 * Creates a token representing a string literal. 371 * 372 * @param s String 373 * @return String token 374 */ 375 private Symbol makeString(String s) { 376 if (inFormula) { 377 inFormula = false; 378 return makeSymbol(ParserSym.FORMULA_STRING, s); 379 } else { 380 return makeSymbol(ParserSym.STRING, s); 381 } 382 } 383 384 /** 385 * Discards all characters until the end of the current line. 386 */ 387 private void skipToEOL() throws IOException { 388 while (nextChar != -1 && nextChar != '\012') { 389 advance(); 390 } 391 } 392 393 /** 394 * Eats a delimited comment. 395 * The type of delimiters are kept in commentDelim. The current 396 * comment type is indicated by commentType. 397 * end of file terminates a comment without error. 398 */ 399 private void skipComment( 400 final String startDelim, 401 final String endDelim) throws IOException 402 { 403 int depth = 1; 404 405 // skip the starting delimiter 406 for (int x = 0; x < startDelim.length(); x++) { 407 advance(); 408 } 409 410 for (;;) { 411 if (nextChar == -1) { 412 return; 413 } else if (checkForSymbol(endDelim)) { 414 // eat the end delimiter 415 for (int x = 0; x < endDelim.length(); x++) { 416 advance(); 417 } 418 if (--depth == 0) { 419 return; 420 } 421 } else if (allowNestedComments && checkForSymbol(startDelim)) { 422 // eat the nested start delimiter 423 for (int x = 0; x < startDelim.length(); x++) { 424 advance(); 425 } 426 depth++; 427 } else { 428 advance(); 429 } 430 } 431 } 432 433 /** 434 * If the next tokens are comments, skip over them. 435 */ 436 private void searchForComments() throws IOException { 437 // eat all following comments 438 boolean foundComment; 439 do { 440 foundComment = false; 441 for (String[] aCommentDelim : commentDelim) { 442 if (checkForSymbol(aCommentDelim[0])) { 443 if (aCommentDelim[1] == null) { 444 foundComment = true; 445 skipToEOL(); 446 } else { 447 foundComment = true; 448 skipComment(aCommentDelim[0], aCommentDelim[1]); 449 } 450 } 451 } 452 } while (foundComment); 453 } 454 455 /** 456 * Checks if the next symbol is the supplied string 457 */ 458 private boolean checkForSymbol(final String symb) throws IOException { 459 for (int x = 0; x < symb.length(); x++) { 460 if (symb.charAt(x) != lookahead(x)) { 461 return false; 462 } 463 } 464 return true; 465 } 466 467 /** 468 * Recognizes and returns the next complete token. 469 */ 470 public Symbol next_token() throws IOException { 471 StringBuilder id; 472 boolean ampersandId = false; 473 for (;;) { 474 searchForComments(); 475 switch (nextChar) { 476 case '.': 477 switch (lookahead()) { 478 case '0': case '1': case '2': case '3': case '4': 479 case '5': case '6': case '7': case '8': case '9': 480 // We're looking at the '.' on the start of a number, 481 // e.g. .1; fall through to parse a number. 482 break; 483 default: 484 advance(); 485 return makeToken(ParserSym.DOT, "."); 486 } 487 // fall through 488 489 case '0': case '1': case '2': case '3': case '4': 490 case '5': case '6': case '7': case '8': case '9': 491 492 // Parse a number. Valid examples include 1, 1.2, 0.1, .1, 493 // 1e2, 1E2, 1e-2, 1e+2. Invalid examples include e2, 1.2.3, 494 // 1e2e3, 1e2.3. 495 // 496 // Signs preceding numbers (e.g. -1, + 1E-5) are valid, but are 497 // handled by the parser. 498 // 499 BigDecimal n = BigDecimalZero; 500 int digitCount = 0, exponent = 0; 501 boolean positive = true; 502 BigDecimal mantissa = BigDecimalZero; 503 State state = State.leftOfPoint; 504 505 for (;;) { 506 switch (nextChar) { 507 case '.': 508 switch (state) { 509 case leftOfPoint: 510 state = State.rightOfPoint; 511 mantissa = n; 512 n = BigDecimalZero; 513 digitCount = 0; 514 positive = true; 515 advance(); 516 break; 517 // Error: we are seeing a point in the exponent 518 // (e.g. 1E2.3 or 1.2E3.4) or a second point in the 519 // mantissa (e.g. 1.2.3). Return what we've got 520 // and let the parser raise the error. 521 case rightOfPoint: 522 mantissa = 523 mantissa.add( 524 n.movePointRight(-digitCount)); 525 return makeNumber(mantissa, exponent); 526 case inExponent: 527 if (!positive) { 528 n = n.negate(); 529 } 530 exponent = n.intValue(); 531 return makeNumber(mantissa, exponent); 532 } 533 break; 534 535 case 'E': 536 case 'e': 537 switch (state) { 538 case inExponent: 539 // Error: we are seeing an 'e' in the exponent 540 // (e.g. 1.2e3e4). Return what we've got and let 541 // the parser raise the error. 542 if (!positive) { 543 n = n.negate(); 544 } 545 exponent = n.intValue(); 546 return makeNumber(mantissa, exponent); 547 case leftOfPoint: 548 mantissa = n; 549 break; 550 default: 551 mantissa = 552 mantissa.add( 553 n.movePointRight(-digitCount)); 554 break; 555 } 556 557 digitCount = 0; 558 n = BigDecimalZero; 559 positive = true; 560 advance(); 561 state = State.inExponent; 562 break; 563 564 case'0': case'1': case'2': case'3': case'4': 565 case'5': case'6': case'7': case'8': case'9': 566 n = n.movePointRight(1); 567 n = n.add(BigDecimal.valueOf(nextChar - '0')); 568 digitCount++; 569 advance(); 570 break; 571 572 case '+': 573 case '-': 574 if (state == State.inExponent 575 && digitCount == 0) 576 { 577 // We're looking at the sign after the 'e'. 578 positive = !positive; 579 advance(); 580 break; 581 } 582 // fall through - end of number 583 584 default: 585 // Reached end of number. 586 switch (state) { 587 case leftOfPoint: 588 mantissa = n; 589 break; 590 case rightOfPoint: 591 mantissa = 592 mantissa.add( 593 n.movePointRight(-digitCount)); 594 break; 595 default: 596 if (!positive) { 597 n = n.negate(); 598 } 599 exponent = n.intValue(); 600 break; 601 } 602 return makeNumber(mantissa, exponent); 603 } 604 } 605 606 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': 607 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': 608 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': 609 case 's': case 't': case 'u': case 'v': case 'w': case 'x': 610 case 'y': case 'z': 611 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': 612 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': 613 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': 614 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': 615 case 'Y': case 'Z': 616 case '_': case '$': 617 /* parse an identifier */ 618 id = new StringBuilder(); 619 for (;;) { 620 id.append((char)nextChar); 621 advance(); 622 switch (nextChar) { 623 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': 624 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': 625 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': 626 case 's': case 't': case 'u': case 'v': case 'w': case 'x': 627 case 'y': case 'z': 628 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': 629 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': 630 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': 631 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': 632 case 'Y': case 'Z': 633 case '0': case '1': case '2': case '3': case '4': 634 case '5': case '6': case '7': case '8': case '9': 635 case '_': case '$': 636 break; 637 default: 638 String strId = id.toString(); 639 Integer i = m_resWordsTable.get( 640 strId.toUpperCase()); 641 if (i == null) { 642 // identifier 643 return makeId(strId, false, false); 644 } else { 645 // reserved word 646 return makeRes(i); 647 } 648 } 649 } 650 651 case '&': 652 advance(); 653 if (nextChar == '[') { 654 ampersandId = true; 655 // fall through 656 } else { 657 return makeToken(ParserSym.UNKNOWN, "&"); 658 } 659 660 case '[': 661 /* parse a delimited identifier */ 662 id = new StringBuilder(); 663 for (;;) { 664 advance(); 665 switch (nextChar) { 666 case ']': 667 advance(); 668 if (nextChar == ']') { 669 // ] escaped with ] - just take one 670 id.append(']'); 671 break; 672 } else { 673 // end of identifier 674 if (ampersandId) { 675 ampersandId = false; 676 return makeId(id.toString(), true, true); 677 } else { 678 return makeId(id.toString(), true, false); 679 } 680 } 681 case -1: 682 if (ampersandId) { 683 ampersandId = false; 684 return makeId(id.toString(), true, true); 685 } else { 686 return makeId(id.toString(), true, false); 687 } 688 default: 689 id.append((char)nextChar); 690 } 691 } 692 693 case ':': 694 advance(); 695 return makeToken(ParserSym.COLON, ":"); 696 case ',': 697 advance(); 698 return makeToken(ParserSym.COMMA, ","); 699 case '=': 700 advance(); 701 return makeToken(ParserSym.EQ, "="); 702 case '<': 703 advance(); 704 switch (nextChar) { 705 case '>': 706 advance(); 707 return makeToken(ParserSym.NE, "<>"); 708 case '=': 709 advance(); 710 return makeToken(ParserSym.LE, "<="); 711 default: 712 return makeToken(ParserSym.LT, "<"); 713 } 714 case '>': 715 advance(); 716 switch (nextChar) { 717 case '=': 718 advance(); 719 return makeToken(ParserSym.GE, ">="); 720 default: 721 return makeToken(ParserSym.GT, ">"); 722 } 723 case '{': 724 advance(); 725 return makeToken(ParserSym.LBRACE, "{"); 726 case '(': 727 advance(); 728 return makeToken(ParserSym.LPAREN, "("); 729 case '}': 730 advance(); 731 return makeToken(ParserSym.RBRACE, "}"); 732 case ')': 733 advance(); 734 return makeToken(ParserSym.RPAREN, ")"); 735 case '+': 736 advance(); 737 return makeToken(ParserSym.PLUS, "+"); 738 case '-': 739 advance(); 740 return makeToken(ParserSym.MINUS, "-"); 741 case '*': 742 advance(); 743 return makeToken(ParserSym.ASTERISK, "*"); 744 case '/': 745 advance(); 746 return makeToken(ParserSym.SOLIDUS, "/"); 747 case '!': 748 advance(); 749 return makeToken(ParserSym.BANG, "!"); 750 case '|': 751 advance(); 752 switch (nextChar) { 753 case '|': 754 advance(); 755 return makeToken(ParserSym.CONCAT, "||"); 756 default: 757 return makeToken(ParserSym.UNKNOWN, "|"); 758 } 759 760 case '"': 761 /* parse a double-quoted string */ 762 id = new StringBuilder(); 763 for (;;) { 764 advance(); 765 switch (nextChar) { 766 case '"': 767 advance(); 768 if (nextChar == '"') { 769 // " escaped with " 770 id.append('"'); 771 break; 772 } else { 773 // end of string 774 return makeString(id.toString()); 775 } 776 case -1: 777 return makeString(id.toString()); 778 default: 779 id.append((char)nextChar); 780 } 781 } 782 783 case '\'': 784 if (previousSymbol == ParserSym.AS) { 785 inFormula = true; 786 } 787 788 /* parse a single-quoted string */ 789 id = new StringBuilder(); 790 for (;;) { 791 advance(); 792 switch (nextChar) { 793 case '\'': 794 advance(); 795 if (nextChar == '\'') { 796 // " escaped with " 797 id.append('\''); 798 break; 799 } else { 800 // end of string 801 return makeString(id.toString()); 802 } 803 case -1: 804 return makeString(id.toString()); 805 default: 806 id.append((char)nextChar); 807 } 808 } 809 810 case -1: 811 // we're done 812 return makeToken(ParserSym.EOF, "EOF"); 813 814 default: 815 // If it's whitespace, skip over it. 816 // (When we switch to JDK 1.5, use Character.isWhitespace(int); 817 // til then, there's just Character.isWhitespace(char).) 818 if (nextChar <= Character.MAX_VALUE 819 && Character.isWhitespace((char) nextChar)) 820 { 821 // fall through 822 } else { 823 // everything else is an error 824 throw new RuntimeException( 825 "Unexpected character '" + (char) nextChar + "'"); 826 } 827 828 case ' ': 829 case '\t': 830 case '\n': 831 case '\r': 832 // whitespace can be ignored 833 iPrevChar = iChar; 834 advance(); 835 break; 836 } 837 } 838 } 839 840 private enum State { 841 leftOfPoint, 842 rightOfPoint, 843 inExponent, 844 } 845} 846 847// End Scanner.java