1. /*
  2. * @(#)StreamTokenizer.java 1.44 04/05/18
  3. *
  4. * Copyright 2004 Sun Microsystems, Inc. All rights reserved.
  5. * SUN PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
  6. */
  7. package java.io;
  8. /**
  9. * The <code>StreamTokenizer</code> class takes an input stream and
  10. * parses it into "tokens", allowing the tokens to be
  11. * read one at a time. The parsing process is controlled by a table
  12. * and a number of flags that can be set to various states. The
  13. * stream tokenizer can recognize identifiers, numbers, quoted
  14. * strings, and various comment styles.
  15. * <p>
  16. * Each byte read from the input stream is regarded as a character
  17. * in the range <code>'\u0000'</code> through <code>'\u00FF'</code>.
  18. * The character value is used to look up five possible attributes of
  19. * the character: <i>white space</i>, <i>alphabetic</i>,
  20. * <i>numeric</i>, <i>string quote</i>, and <i>comment character</i>.
  21. * Each character can have zero or more of these attributes.
  22. * <p>
  23. * In addition, an instance has four flags. These flags indicate:
  24. * <ul>
  25. * <li>Whether line terminators are to be returned as tokens or treated
  26. * as white space that merely separates tokens.
  27. * <li>Whether C-style comments are to be recognized and skipped.
  28. * <li>Whether C++-style comments are to be recognized and skipped.
  29. * <li>Whether the characters of identifiers are converted to lowercase.
  30. * </ul>
  31. * <p>
  32. * A typical application first constructs an instance of this class,
  33. * sets up the syntax tables, and then repeatedly loops calling the
  34. * <code>nextToken</code> method in each iteration of the loop until
  35. * it returns the value <code>TT_EOF</code>.
  36. *
  37. * @author James Gosling
  38. * @version 1.44, 05/18/04
  39. * @see java.io.StreamTokenizer#nextToken()
  40. * @see java.io.StreamTokenizer#TT_EOF
  41. * @since JDK1.0
  42. */
  43. public class StreamTokenizer {
  44. /* Only one of these will be non-null */
  45. private Reader reader = null;
  46. private InputStream input = null;
  47. private char buf[] = new char[20];
  48. /**
  49. * The next character to be considered by the nextToken method. May also
  50. * be NEED_CHAR to indicate that a new character should be read, or SKIP_LF
  51. * to indicate that a new character should be read and, if it is a '\n'
  52. * character, it should be discarded and a second new character should be
  53. * read.
  54. */
  55. private int peekc = NEED_CHAR;
  56. private static final int NEED_CHAR = Integer.MAX_VALUE;
  57. private static final int SKIP_LF = Integer.MAX_VALUE - 1;
  58. private boolean pushedBack;
  59. private boolean forceLower;
  60. /** The line number of the last token read */
  61. private int LINENO = 1;
  62. private boolean eolIsSignificantP = false;
  63. private boolean slashSlashCommentsP = false;
  64. private boolean slashStarCommentsP = false;
  65. private byte ctype[] = new byte[256];
  66. private static final byte CT_WHITESPACE = 1;
  67. private static final byte CT_DIGIT = 2;
  68. private static final byte CT_ALPHA = 4;
  69. private static final byte CT_QUOTE = 8;
  70. private static final byte CT_COMMENT = 16;
  71. /**
  72. * After a call to the <code>nextToken</code> method, this field
  73. * contains the type of the token just read. For a single character
  74. * token, its value is the single character, converted to an integer.
  75. * For a quoted string token (see , its value is the quote character.
  76. * Otherwise, its value is one of the following:
  77. * <ul>
  78. * <li><code>TT_WORD</code> indicates that the token is a word.
  79. * <li><code>TT_NUMBER</code> indicates that the token is a number.
  80. * <li><code>TT_EOL</code> indicates that the end of line has been read.
  81. * The field can only have this value if the
  82. * <code>eolIsSignificant</code> method has been called with the
  83. * argument <code>true</code>.
  84. * <li><code>TT_EOF</code> indicates that the end of the input stream
  85. * has been reached.
  86. * </ul>
  87. * <p>
  88. * The initial value of this field is -4.
  89. *
  90. * @see java.io.StreamTokenizer#eolIsSignificant(boolean)
  91. * @see java.io.StreamTokenizer#nextToken()
  92. * @see java.io.StreamTokenizer#quoteChar(int)
  93. * @see java.io.StreamTokenizer#TT_EOF
  94. * @see java.io.StreamTokenizer#TT_EOL
  95. * @see java.io.StreamTokenizer#TT_NUMBER
  96. * @see java.io.StreamTokenizer#TT_WORD
  97. */
  98. public int ttype = TT_NOTHING;
  99. /**
  100. * A constant indicating that the end of the stream has been read.
  101. */
  102. public static final int TT_EOF = -1;
  103. /**
  104. * A constant indicating that the end of the line has been read.
  105. */
  106. public static final int TT_EOL = '\n';
  107. /**
  108. * A constant indicating that a number token has been read.
  109. */
  110. public static final int TT_NUMBER = -2;
  111. /**
  112. * A constant indicating that a word token has been read.
  113. */
  114. public static final int TT_WORD = -3;
  115. /* A constant indicating that no token has been read, used for
  116. * initializing ttype. FIXME This could be made public and
  117. * made available as the part of the API in a future release.
  118. */
  119. private static final int TT_NOTHING = -4;
  120. /**
  121. * If the current token is a word token, this field contains a
  122. * string giving the characters of the word token. When the current
  123. * token is a quoted string token, this field contains the body of
  124. * the string.
  125. * <p>
  126. * The current token is a word when the value of the
  127. * <code>ttype</code> field is <code>TT_WORD</code>. The current token is
  128. * a quoted string token when the value of the <code>ttype</code> field is
  129. * a quote character.
  130. * <p>
  131. * The initial value of this field is null.
  132. *
  133. * @see java.io.StreamTokenizer#quoteChar(int)
  134. * @see java.io.StreamTokenizer#TT_WORD
  135. * @see java.io.StreamTokenizer#ttype
  136. */
  137. public String sval;
  138. /**
  139. * If the current token is a number, this field contains the value
  140. * of that number. The current token is a number when the value of
  141. * the <code>ttype</code> field is <code>TT_NUMBER</code>.
  142. * <p>
  143. * The initial value of this field is 0.0.
  144. *
  145. * @see java.io.StreamTokenizer#TT_NUMBER
  146. * @see java.io.StreamTokenizer#ttype
  147. */
  148. public double nval;
  149. /** Private constructor that initializes everything except the streams. */
  150. private StreamTokenizer() {
  151. wordChars('a', 'z');
  152. wordChars('A', 'Z');
  153. wordChars(128 + 32, 255);
  154. whitespaceChars(0, ' ');
  155. commentChar('/');
  156. quoteChar('"');
  157. quoteChar('\'');
  158. parseNumbers();
  159. }
  160. /**
  161. * Creates a stream tokenizer that parses the specified input
  162. * stream. The stream tokenizer is initialized to the following
  163. * default state:
  164. * <ul>
  165. * <li>All byte values <code>'A'</code> through <code>'Z'</code>,
  166. * <code>'a'</code> through <code>'z'</code>, and
  167. * <code>'\u00A0'</code> through <code>'\u00FF'</code> are
  168. * considered to be alphabetic.
  169. * <li>All byte values <code>'\u0000'</code> through
  170. * <code>'\u0020'</code> are considered to be white space.
  171. * <li><code>'/'</code> is a comment character.
  172. * <li>Single quote <code>'\''</code> and double quote <code>'"'</code>
  173. * are string quote characters.
  174. * <li>Numbers are parsed.
  175. * <li>Ends of lines are treated as white space, not as separate tokens.
  176. * <li>C-style and C++-style comments are not recognized.
  177. * </ul>
  178. *
  179. * @deprecated As of JDK version 1.1, the preferred way to tokenize an
  180. * input stream is to convert it into a character stream, for example:
  181. * <blockquote><pre>
  182. * Reader r = new BufferedReader(new InputStreamReader(is));
  183. * StreamTokenizer st = new StreamTokenizer(r);
  184. * </pre></blockquote>
  185. *
  186. * @param is an input stream.
  187. * @see java.io.BufferedReader
  188. * @see java.io.InputStreamReader
  189. * @see java.io.StreamTokenizer#StreamTokenizer(java.io.Reader)
  190. */
  191. @Deprecated
  192. public StreamTokenizer(InputStream is) {
  193. this();
  194. if (is == null) {
  195. throw new NullPointerException();
  196. }
  197. input = is;
  198. }
  199. /**
  200. * Create a tokenizer that parses the given character stream.
  201. *
  202. * @param r a Reader object providing the input stream.
  203. * @since JDK1.1
  204. */
  205. public StreamTokenizer(Reader r) {
  206. this();
  207. if (r == null) {
  208. throw new NullPointerException();
  209. }
  210. reader = r;
  211. }
  212. /**
  213. * Resets this tokenizer's syntax table so that all characters are
  214. * "ordinary." See the <code>ordinaryChar</code> method
  215. * for more information on a character being ordinary.
  216. *
  217. * @see java.io.StreamTokenizer#ordinaryChar(int)
  218. */
  219. public void resetSyntax() {
  220. for (int i = ctype.length; --i >= 0;)
  221. ctype[i] = 0;
  222. }
  223. /**
  224. * Specifies that all characters <i>c</i> in the range
  225. * <code>low <= <i>c</i> <= high</code>
  226. * are word constituents. A word token consists of a word constituent
  227. * followed by zero or more word constituents or number constituents.
  228. *
  229. * @param low the low end of the range.
  230. * @param hi the high end of the range.
  231. */
  232. public void wordChars(int low, int hi) {
  233. if (low < 0)
  234. low = 0;
  235. if (hi >= ctype.length)
  236. hi = ctype.length - 1;
  237. while (low <= hi)
  238. ctype[low++] |= CT_ALPHA;
  239. }
  240. /**
  241. * Specifies that all characters <i>c</i> in the range
  242. * <code>low <= <i>c</i> <= high</code>
  243. * are white space characters. White space characters serve only to
  244. * separate tokens in the input stream.
  245. *
  246. * <p>Any other attribute settings for the characters in the specified
  247. * range are cleared.
  248. *
  249. * @param low the low end of the range.
  250. * @param hi the high end of the range.
  251. */
  252. public void whitespaceChars(int low, int hi) {
  253. if (low < 0)
  254. low = 0;
  255. if (hi >= ctype.length)
  256. hi = ctype.length - 1;
  257. while (low <= hi)
  258. ctype[low++] = CT_WHITESPACE;
  259. }
  260. /**
  261. * Specifies that all characters <i>c</i> in the range
  262. * <code>low <= <i>c</i> <= high</code>
  263. * are "ordinary" in this tokenizer. See the
  264. * <code>ordinaryChar</code> method for more information on a
  265. * character being ordinary.
  266. *
  267. * @param low the low end of the range.
  268. * @param hi the high end of the range.
  269. * @see java.io.StreamTokenizer#ordinaryChar(int)
  270. */
  271. public void ordinaryChars(int low, int hi) {
  272. if (low < 0)
  273. low = 0;
  274. if (hi >= ctype.length)
  275. hi = ctype.length - 1;
  276. while (low <= hi)
  277. ctype[low++] = 0;
  278. }
  279. /**
  280. * Specifies that the character argument is "ordinary"
  281. * in this tokenizer. It removes any special significance the
  282. * character has as a comment character, word component, string
  283. * delimiter, white space, or number character. When such a character
  284. * is encountered by the parser, the parser treats it as a
  285. * single-character token and sets <code>ttype</code> field to the
  286. * character value.
  287. *
  288. * <p>Making a line terminator character "ordinary" may interfere
  289. * with the ability of a <code>StreamTokenizer</code> to count
  290. * lines. The <code>lineno</code> method may no longer reflect
  291. * the presence of such terminator characters in its line count.
  292. *
  293. * @param ch the character.
  294. * @see java.io.StreamTokenizer#ttype
  295. */
  296. public void ordinaryChar(int ch) {
  297. if (ch >= 0 && ch < ctype.length)
  298. ctype[ch] = 0;
  299. }
  300. /**
  301. * Specified that the character argument starts a single-line
  302. * comment. All characters from the comment character to the end of
  303. * the line are ignored by this stream tokenizer.
  304. *
  305. * <p>Any other attribute settings for the specified character are cleared.
  306. *
  307. * @param ch the character.
  308. */
  309. public void commentChar(int ch) {
  310. if (ch >= 0 && ch < ctype.length)
  311. ctype[ch] = CT_COMMENT;
  312. }
  313. /**
  314. * Specifies that matching pairs of this character delimit string
  315. * constants in this tokenizer.
  316. * <p>
  317. * When the <code>nextToken</code> method encounters a string
  318. * constant, the <code>ttype</code> field is set to the string
  319. * delimiter and the <code>sval</code> field is set to the body of
  320. * the string.
  321. * <p>
  322. * If a string quote character is encountered, then a string is
  323. * recognized, consisting of all characters after (but not including)
  324. * the string quote character, up to (but not including) the next
  325. * occurrence of that same string quote character, or a line
  326. * terminator, or end of file. The usual escape sequences such as
  327. * <code>"\n"</code> and <code>"\t"</code> are recognized and
  328. * converted to single characters as the string is parsed.
  329. *
  330. * <p>Any other attribute settings for the specified character are cleared.
  331. *
  332. * @param ch the character.
  333. * @see java.io.StreamTokenizer#nextToken()
  334. * @see java.io.StreamTokenizer#sval
  335. * @see java.io.StreamTokenizer#ttype
  336. */
  337. public void quoteChar(int ch) {
  338. if (ch >= 0 && ch < ctype.length)
  339. ctype[ch] = CT_QUOTE;
  340. }
  341. /**
  342. * Specifies that numbers should be parsed by this tokenizer. The
  343. * syntax table of this tokenizer is modified so that each of the twelve
  344. * characters:
  345. * <blockquote><pre>
  346. * 0 1 2 3 4 5 6 7 8 9 . -
  347. * </pre></blockquote>
  348. * <p>
  349. * has the "numeric" attribute.
  350. * <p>
  351. * When the parser encounters a word token that has the format of a
  352. * double precision floating-point number, it treats the token as a
  353. * number rather than a word, by setting the <code>ttype</code>
  354. * field to the value <code>TT_NUMBER</code> and putting the numeric
  355. * value of the token into the <code>nval</code> field.
  356. *
  357. * @see java.io.StreamTokenizer#nval
  358. * @see java.io.StreamTokenizer#TT_NUMBER
  359. * @see java.io.StreamTokenizer#ttype
  360. */
  361. public void parseNumbers() {
  362. for (int i = '0'; i <= '9'; i++)
  363. ctype[i] |= CT_DIGIT;
  364. ctype['.'] |= CT_DIGIT;
  365. ctype['-'] |= CT_DIGIT;
  366. }
  367. /**
  368. * Determines whether or not ends of line are treated as tokens.
  369. * If the flag argument is true, this tokenizer treats end of lines
  370. * as tokens; the <code>nextToken</code> method returns
  371. * <code>TT_EOL</code> and also sets the <code>ttype</code> field to
  372. * this value when an end of line is read.
  373. * <p>
  374. * A line is a sequence of characters ending with either a
  375. * carriage-return character (<code>'\r'</code>) or a newline
  376. * character (<code>'\n'</code>). In addition, a carriage-return
  377. * character followed immediately by a newline character is treated
  378. * as a single end-of-line token.
  379. * <p>
  380. * If the <code>flag</code> is false, end-of-line characters are
  381. * treated as white space and serve only to separate tokens.
  382. *
  383. * @param flag <code>true</code> indicates that end-of-line characters
  384. * are separate tokens; <code>false</code> indicates that
  385. * end-of-line characters are white space.
  386. * @see java.io.StreamTokenizer#nextToken()
  387. * @see java.io.StreamTokenizer#ttype
  388. * @see java.io.StreamTokenizer#TT_EOL
  389. */
  390. public void eolIsSignificant(boolean flag) {
  391. eolIsSignificantP = flag;
  392. }
  393. /**
  394. * Determines whether or not the tokenizer recognizes C-style comments.
  395. * If the flag argument is <code>true</code>, this stream tokenizer
  396. * recognizes C-style comments. All text between successive
  397. * occurrences of <code>/*</code> and <code>*/</code> are discarded.
  398. * <p>
  399. * If the flag argument is <code>false</code>, then C-style comments
  400. * are not treated specially.
  401. *
  402. * @param flag <code>true</code> indicates to recognize and ignore
  403. * C-style comments.
  404. */
  405. public void slashStarComments(boolean flag) {
  406. slashStarCommentsP = flag;
  407. }
  408. /**
  409. * Determines whether or not the tokenizer recognizes C++-style comments.
  410. * If the flag argument is <code>true</code>, this stream tokenizer
  411. * recognizes C++-style comments. Any occurrence of two consecutive
  412. * slash characters (<code>'/'</code>) is treated as the beginning of
  413. * a comment that extends to the end of the line.
  414. * <p>
  415. * If the flag argument is <code>false</code>, then C++-style
  416. * comments are not treated specially.
  417. *
  418. * @param flag <code>true</code> indicates to recognize and ignore
  419. * C++-style comments.
  420. */
  421. public void slashSlashComments(boolean flag) {
  422. slashSlashCommentsP = flag;
  423. }
  424. /**
  425. * Determines whether or not word token are automatically lowercased.
  426. * If the flag argument is <code>true</code>, then the value in the
  427. * <code>sval</code> field is lowercased whenever a word token is
  428. * returned (the <code>ttype</code> field has the
  429. * value <code>TT_WORD</code> by the <code>nextToken</code> method
  430. * of this tokenizer.
  431. * <p>
  432. * If the flag argument is <code>false</code>, then the
  433. * <code>sval</code> field is not modified.
  434. *
  435. * @param fl <code>true</code> indicates that all word tokens should
  436. * be lowercased.
  437. * @see java.io.StreamTokenizer#nextToken()
  438. * @see java.io.StreamTokenizer#ttype
  439. * @see java.io.StreamTokenizer#TT_WORD
  440. */
  441. public void lowerCaseMode(boolean fl) {
  442. forceLower = fl;
  443. }
  444. /** Read the next character */
  445. private int read() throws IOException {
  446. if (reader != null)
  447. return reader.read();
  448. else if (input != null)
  449. return input.read();
  450. else
  451. throw new IllegalStateException();
  452. }
  453. /**
  454. * Parses the next token from the input stream of this tokenizer.
  455. * The type of the next token is returned in the <code>ttype</code>
  456. * field. Additional information about the token may be in the
  457. * <code>nval</code> field or the <code>sval</code> field of this
  458. * tokenizer.
  459. * <p>
  460. * Typical clients of this
  461. * class first set up the syntax tables and then sit in a loop
  462. * calling nextToken to parse successive tokens until TT_EOF
  463. * is returned.
  464. *
  465. * @return the value of the <code>ttype</code> field.
  466. * @exception IOException if an I/O error occurs.
  467. * @see java.io.StreamTokenizer#nval
  468. * @see java.io.StreamTokenizer#sval
  469. * @see java.io.StreamTokenizer#ttype
  470. */
  471. public int nextToken() throws IOException {
  472. if (pushedBack) {
  473. pushedBack = false;
  474. return ttype;
  475. }
  476. byte ct[] = ctype;
  477. sval = null;
  478. int c = peekc;
  479. if (c < 0)
  480. c = NEED_CHAR;
  481. if (c == SKIP_LF) {
  482. c = read();
  483. if (c < 0)
  484. return ttype = TT_EOF;
  485. if (c == '\n')
  486. c = NEED_CHAR;
  487. }
  488. if (c == NEED_CHAR) {
  489. c = read();
  490. if (c < 0)
  491. return ttype = TT_EOF;
  492. }
  493. ttype = c; /* Just to be safe */
  494. /* Set peekc so that the next invocation of nextToken will read
  495. * another character unless peekc is reset in this invocation
  496. */
  497. peekc = NEED_CHAR;
  498. int ctype = c < 256 ? ct[c] : CT_ALPHA;
  499. while ((ctype & CT_WHITESPACE) != 0) {
  500. if (c == '\r') {
  501. LINENO++;
  502. if (eolIsSignificantP) {
  503. peekc = SKIP_LF;
  504. return ttype = TT_EOL;
  505. }
  506. c = read();
  507. if (c == '\n')
  508. c = read();
  509. } else {
  510. if (c == '\n') {
  511. LINENO++;
  512. if (eolIsSignificantP) {
  513. return ttype = TT_EOL;
  514. }
  515. }
  516. c = read();
  517. }
  518. if (c < 0)
  519. return ttype = TT_EOF;
  520. ctype = c < 256 ? ct[c] : CT_ALPHA;
  521. }
  522. if ((ctype & CT_DIGIT) != 0) {
  523. boolean neg = false;
  524. if (c == '-') {
  525. c = read();
  526. if (c != '.' && (c < '0' || c > '9')) {
  527. peekc = c;
  528. return ttype = '-';
  529. }
  530. neg = true;
  531. }
  532. double v = 0;
  533. int decexp = 0;
  534. int seendot = 0;
  535. while (true) {
  536. if (c == '.' && seendot == 0)
  537. seendot = 1;
  538. else if ('0' <= c && c <= '9') {
  539. v = v * 10 + (c - '0');
  540. decexp += seendot;
  541. } else
  542. break;
  543. c = read();
  544. }
  545. peekc = c;
  546. if (decexp != 0) {
  547. double denom = 10;
  548. decexp--;
  549. while (decexp > 0) {
  550. denom *= 10;
  551. decexp--;
  552. }
  553. /* Do one division of a likely-to-be-more-accurate number */
  554. v = v / denom;
  555. }
  556. nval = neg ? -v : v;
  557. return ttype = TT_NUMBER;
  558. }
  559. if ((ctype & CT_ALPHA) != 0) {
  560. int i = 0;
  561. do {
  562. if (i >= buf.length) {
  563. char nb[] = new char[buf.length * 2];
  564. System.arraycopy(buf, 0, nb, 0, buf.length);
  565. buf = nb;
  566. }
  567. buf[i++] = (char) c;
  568. c = read();
  569. ctype = c < 0 ? CT_WHITESPACE : c < 256 ? ct[c] : CT_ALPHA;
  570. } while ((ctype & (CT_ALPHA | CT_DIGIT)) != 0);
  571. peekc = c;
  572. sval = String.copyValueOf(buf, 0, i);
  573. if (forceLower)
  574. sval = sval.toLowerCase();
  575. return ttype = TT_WORD;
  576. }
  577. if ((ctype & CT_QUOTE) != 0) {
  578. ttype = c;
  579. int i = 0;
  580. /* Invariants (because \Octal needs a lookahead):
  581. * (i) c contains char value
  582. * (ii) d contains the lookahead
  583. */
  584. int d = read();
  585. while (d >= 0 && d != ttype && d != '\n' && d != '\r') {
  586. if (d == '\\') {
  587. c = read();
  588. int first = c; /* To allow \377, but not \477 */
  589. if (c >= '0' && c <= '7') {
  590. c = c - '0';
  591. int c2 = read();
  592. if ('0' <= c2 && c2 <= '7') {
  593. c = (c << 3) + (c2 - '0');
  594. c2 = read();
  595. if ('0' <= c2 && c2 <= '7' && first <= '3') {
  596. c = (c << 3) + (c2 - '0');
  597. d = read();
  598. } else
  599. d = c2;
  600. } else
  601. d = c2;
  602. } else {
  603. switch (c) {
  604. case 'a':
  605. c = 0x7;
  606. break;
  607. case 'b':
  608. c = '\b';
  609. break;
  610. case 'f':
  611. c = 0xC;
  612. break;
  613. case 'n':
  614. c = '\n';
  615. break;
  616. case 'r':
  617. c = '\r';
  618. break;
  619. case 't':
  620. c = '\t';
  621. break;
  622. case 'v':
  623. c = 0xB;
  624. break;
  625. }
  626. d = read();
  627. }
  628. } else {
  629. c = d;
  630. d = read();
  631. }
  632. if (i >= buf.length) {
  633. char nb[] = new char[buf.length * 2];
  634. System.arraycopy(buf, 0, nb, 0, buf.length);
  635. buf = nb;
  636. }
  637. buf[i++] = (char)c;
  638. }
  639. /* If we broke out of the loop because we found a matching quote
  640. * character then arrange to read a new character next time
  641. * around; otherwise, save the character.
  642. */
  643. peekc = (d == ttype) ? NEED_CHAR : d;
  644. sval = String.copyValueOf(buf, 0, i);
  645. return ttype;
  646. }
  647. if (c == '/' && (slashSlashCommentsP || slashStarCommentsP)) {
  648. c = read();
  649. if (c == '*' && slashStarCommentsP) {
  650. int prevc = 0;
  651. while ((c = read()) != '/' || prevc != '*') {
  652. if (c == '\r') {
  653. LINENO++;
  654. c = read();
  655. if (c == '\n') {
  656. c = read();
  657. }
  658. } else {
  659. if (c == '\n') {
  660. LINENO++;
  661. c = read();
  662. }
  663. }
  664. if (c < 0)
  665. return ttype = TT_EOF;
  666. prevc = c;
  667. }
  668. return nextToken();
  669. } else if (c == '/' && slashSlashCommentsP) {
  670. while ((c = read()) != '\n' && c != '\r' && c >= 0);
  671. peekc = c;
  672. return nextToken();
  673. } else {
  674. /* Now see if it is still a single line comment */
  675. if ((ct['/'] & CT_COMMENT) != 0) {
  676. while ((c = read()) != '\n' && c != '\r' && c >= 0);
  677. peekc = c;
  678. return nextToken();
  679. } else {
  680. peekc = c;
  681. return ttype = '/';
  682. }
  683. }
  684. }
  685. if ((ctype & CT_COMMENT) != 0) {
  686. while ((c = read()) != '\n' && c != '\r' && c >= 0);
  687. peekc = c;
  688. return nextToken();
  689. }
  690. return ttype = c;
  691. }
  692. /**
  693. * Causes the next call to the <code>nextToken</code> method of this
  694. * tokenizer to return the current value in the <code>ttype</code>
  695. * field, and not to modify the value in the <code>nval</code> or
  696. * <code>sval</code> field.
  697. *
  698. * @see java.io.StreamTokenizer#nextToken()
  699. * @see java.io.StreamTokenizer#nval
  700. * @see java.io.StreamTokenizer#sval
  701. * @see java.io.StreamTokenizer#ttype
  702. */
  703. public void pushBack() {
  704. if (ttype != TT_NOTHING) /* No-op if nextToken() not called */
  705. pushedBack = true;
  706. }
  707. /**
  708. * Return the current line number.
  709. *
  710. * @return the current line number of this stream tokenizer.
  711. */
  712. public int lineno() {
  713. return LINENO;
  714. }
  715. /**
  716. * Returns the string representation of the current stream token and
  717. * the line number it occurs on.
  718. *
  719. * <p>The precise string returned is unspecified, although the following
  720. * example can be considered typical:
  721. *
  722. * <blockquote><pre>Token['a'], line 10</pre></blockquote>
  723. *
  724. * @return a string representation of the token
  725. * @see java.io.StreamTokenizer#nval
  726. * @see java.io.StreamTokenizer#sval
  727. * @see java.io.StreamTokenizer#ttype
  728. */
  729. public String toString() {
  730. String ret;
  731. switch (ttype) {
  732. case TT_EOF:
  733. ret = "EOF";
  734. break;
  735. case TT_EOL:
  736. ret = "EOL";
  737. break;
  738. case TT_WORD:
  739. ret = sval;
  740. break;
  741. case TT_NUMBER:
  742. ret = "n=" + nval;
  743. break;
  744. case TT_NOTHING:
  745. ret = "NOTHING";
  746. break;
  747. default: {
  748. /*
  749. * ttype is the first character of either a quoted string or
  750. * is an ordinary character. ttype can definitely not be less
  751. * than 0, since those are reserved values used in the previous
  752. * case statements
  753. */
  754. if (ttype < 256 &&
  755. ((ctype[ttype] & CT_QUOTE) != 0)) {
  756. ret = sval;
  757. break;
  758. }
  759. char s[] = new char[3];
  760. s[0] = s[2] = '\'';
  761. s[1] = (char) ttype;
  762. ret = new String(s);
  763. break;
  764. }
  765. }
  766. return "Token[" + ret + "], line " + LINENO;
  767. }
  768. }