1. /*
  2. * @(#)StreamTokenizer.java 1.36 00/02/02
  3. *
  4. * Copyright 1995-2000 Sun Microsystems, Inc. All Rights Reserved.
  5. *
  6. * This software is the proprietary information of Sun Microsystems, Inc.
  7. * Use is subject to license terms.
  8. *
  9. */
  10. package java.io;
  11. /**
  12. * The <code>StreamTokenizer</code> class takes an input stream and
  13. * parses it into "tokens", allowing the tokens to be
  14. * read one at a time. The parsing process is controlled by a table
  15. * and a number of flags that can be set to various states. The
  16. * stream tokenizer can recognize identifiers, numbers, quoted
  17. * strings, and various comment styles.
  18. * <p>
  19. * Each byte read from the input stream is regarded as a character
  20. * in the range <code>'\u0000'</code> through <code>'\u00FF'</code>.
  21. * The character value is used to look up five possible attributes of
  22. * the character: <i>white space</i>, <i>alphabetic</i>,
  23. * <i>numeric</i>, <i>string quote</i>, and <i>comment character</i>.
  24. * Each character can have zero or more of these attributes.
  25. * <p>
  26. * In addition, an instance has four flags. These flags indicate:
  27. * <ul>
  28. * <li>Whether line terminators are to be returned as tokens or treated
  29. * as white space that merely separates tokens.
  30. * <li>Whether C-style comments are to be recognized and skipped.
  31. * <li>Whether C++-style comments are to be recognized and skipped.
  32. * <li>Whether the characters of identifiers are converted to lowercase.
  33. * </ul>
  34. * <p>
  35. * A typical application first constructs an instance of this class,
  36. * sets up the syntax tables, and then repeatedly loops calling the
  37. * <code>nextToken</code> method in each iteration of the loop until
  38. * it returns the value <code>TT_EOF</code>.
  39. *
  40. * @author James Gosling
  41. * @version 1.36, 02/02/00
  42. * @see java.io.StreamTokenizer#nextToken()
  43. * @see java.io.StreamTokenizer#TT_EOF
  44. * @since JDK1.0
  45. */
  46. public class StreamTokenizer {
  47. /* Only one of these will be non-null */
  48. private Reader reader = null;
  49. private InputStream input = null;
  50. private char buf[] = new char[20];
  51. /**
  52. * The next character to be considered by the nextToken method. May also
  53. * be NEED_CHAR to indicate that a new character should be read, or SKIP_LF
  54. * to indicate that a new character should be read and, if it is a '\n'
  55. * character, it should be discarded and a second new character should be
  56. * read.
  57. */
  58. private int peekc = NEED_CHAR;
  59. private static final int NEED_CHAR = Integer.MAX_VALUE;
  60. private static final int SKIP_LF = Integer.MAX_VALUE - 1;
  61. private boolean pushedBack;
  62. private boolean forceLower;
  63. /** The line number of the last token read */
  64. private int LINENO = 1;
  65. private boolean eolIsSignificantP = false;
  66. private boolean slashSlashCommentsP = false;
  67. private boolean slashStarCommentsP = false;
  68. private byte ctype[] = new byte[256];
  69. private static final byte CT_WHITESPACE = 1;
  70. private static final byte CT_DIGIT = 2;
  71. private static final byte CT_ALPHA = 4;
  72. private static final byte CT_QUOTE = 8;
  73. private static final byte CT_COMMENT = 16;
  74. /**
  75. * After a call to the <code>nextToken</code> method, this field
  76. * contains the type of the token just read. For a single character
  77. * token, its value is the single character, converted to an integer.
  78. * For a quoted string token (see , its value is the quote character.
  79. * Otherwise, its value is one of the following:
  80. * <ul>
  81. * <li><code>TT_WORD</code> indicates that the token is a word.
  82. * <li><code>TT_NUMBER</code> indicates that the token is a number.
  83. * <li><code>TT_EOL</code> indicates that the end of line has been read.
  84. * The field can only have this value if the
  85. * <code>eolIsSignificant</code> method has been called with the
  86. * argument <code>true</code>.
  87. * <li><code>TT_EOF</code> indicates that the end of the input stream
  88. * has been reached.
  89. * </ul>
  90. * <p>
  91. * The initial value of this field is -4.
  92. *
  93. * @see java.io.StreamTokenizer#eolIsSignificant(boolean)
  94. * @see java.io.StreamTokenizer#nextToken()
  95. * @see java.io.StreamTokenizer#quoteChar(int)
  96. * @see java.io.StreamTokenizer#TT_EOF
  97. * @see java.io.StreamTokenizer#TT_EOL
  98. * @see java.io.StreamTokenizer#TT_NUMBER
  99. * @see java.io.StreamTokenizer#TT_WORD
  100. */
  101. public int ttype = TT_NOTHING;
  102. /**
  103. * A constant indicating that the end of the stream has been read.
  104. */
  105. public static final int TT_EOF = -1;
  106. /**
  107. * A constant indicating that the end of the line has been read.
  108. */
  109. public static final int TT_EOL = '\n';
  110. /**
  111. * A constant indicating that a number token has been read.
  112. */
  113. public static final int TT_NUMBER = -2;
  114. /**
  115. * A constant indicating that a word token has been read.
  116. */
  117. public static final int TT_WORD = -3;
  118. /* A constant indicating that no token has been read, used for
  119. * initializing ttype. FIXME This could be made public and
  120. * made available as the part of the API in a future release.
  121. */
  122. private static final int TT_NOTHING = -4;
  123. /**
  124. * If the current token is a word token, this field contains a
  125. * string giving the characters of the word token. When the current
  126. * token is a quoted string token, this field contains the body of
  127. * the string.
  128. * <p>
  129. * The current token is a word when the value of the
  130. * <code>ttype</code> field is <code>TT_WORD</code>. The current token is
  131. * a quoted string token when the value of the <code>ttype</code> field is
  132. * a quote character.
  133. * <p>
  134. * The initial value of this field is null.
  135. *
  136. * @see java.io.StreamTokenizer#quoteChar(int)
  137. * @see java.io.StreamTokenizer#TT_WORD
  138. * @see java.io.StreamTokenizer#ttype
  139. */
  140. public String sval;
  141. /**
  142. * If the current token is a number, this field contains the value
  143. * of that number. The current token is a number when the value of
  144. * the <code>ttype</code> field is <code>TT_NUMBER</code>.
  145. * <p>
  146. * The initial value of this field is 0.0.
  147. *
  148. * @see java.io.StreamTokenizer#TT_NUMBER
  149. * @see java.io.StreamTokenizer#ttype
  150. */
  151. public double nval;
  152. /** Private constructor that initializes everything except the streams. */
  153. private StreamTokenizer() {
  154. wordChars('a', 'z');
  155. wordChars('A', 'Z');
  156. wordChars(128 + 32, 255);
  157. whitespaceChars(0, ' ');
  158. commentChar('/');
  159. quoteChar('"');
  160. quoteChar('\'');
  161. parseNumbers();
  162. }
  163. /**
  164. * Creates a stream tokenizer that parses the specified input
  165. * stream. The stream tokenizer is initialized to the following
  166. * default state:
  167. * <ul>
  168. * <li>All byte values <code>'A'</code> through <code>'Z'</code>,
  169. * <code>'a'</code> through <code>'z'</code>, and
  170. * <code>'\u00A0'</code> through <code>'\u00FF'</code> are
  171. * considered to be alphabetic.
  172. * <li>All byte values <code>'\u0000'</code> through
  173. * <code>'\u0020'</code> are considered to be white space.
  174. * <li><code>'/'</code> is a comment character.
  175. * <li>Single quote <code>'\''</code> and double quote <code>'"'</code>
  176. * are string quote characters.
  177. * <li>Numbers are parsed.
  178. * <li>Ends of lines are treated as white space, not as separate tokens.
  179. * <li>C-style and C++-style comments are not recognized.
  180. * </ul>
  181. *
  182. * @deprecated As of JDK version 1.1, the preferred way to tokenize an
  183. * input stream is to convert it into a character stream, for example:
  184. * <blockquote><pre>
  185. * Reader r = new BufferedReader(new InputStreamReader(is));
  186. * StreamTokenizer st = new StreamTokenizer(r);
  187. * </pre></blockquote>
  188. *
  189. * @param is an input stream.
  190. * @see java.io.BufferedReader
  191. * @see java.io.InputStreamReader
  192. * @see java.io.StreamTokenizer#StreamTokenizer(java.io.Reader)
  193. */
  194. public StreamTokenizer(InputStream is) {
  195. this();
  196. if (is == null) {
  197. throw new NullPointerException();
  198. }
  199. input = is;
  200. }
  201. /**
  202. * Create a tokenizer that parses the given character stream.
  203. *
  204. * @param r a Reader object providing the input stream.
  205. * @since JDK1.1
  206. */
  207. public StreamTokenizer(Reader r) {
  208. this();
  209. if (r == null) {
  210. throw new NullPointerException();
  211. }
  212. reader = r;
  213. }
  214. /**
  215. * Resets this tokenizer's syntax table so that all characters are
  216. * "ordinary." See the <code>ordinaryChar</code> method
  217. * for more information on a character being ordinary.
  218. *
  219. * @see java.io.StreamTokenizer#ordinaryChar(int)
  220. */
  221. public void resetSyntax() {
  222. for (int i = ctype.length; --i >= 0;)
  223. ctype[i] = 0;
  224. }
  225. /**
  226. * Specifies that all characters <i>c</i> in the range
  227. * <code>low <= <i>c</i> <= high</code>
  228. * are word constituents. A word token consists of a word constituent
  229. * followed by zero or more word constituents or number constituents.
  230. *
  231. * @param low the low end of the range.
  232. * @param hi the high end of the range.
  233. */
  234. public void wordChars(int low, int hi) {
  235. if (low < 0)
  236. low = 0;
  237. if (hi >= ctype.length)
  238. hi = ctype.length - 1;
  239. while (low <= hi)
  240. ctype[low++] |= CT_ALPHA;
  241. }
  242. /**
  243. * Specifies that all characters <i>c</i> in the range
  244. * <code>low <= <i>c</i> <= high</code>
  245. * are white space characters. White space characters serve only to
  246. * separate tokens in the input stream.
  247. *
  248. * @param low the low end of the range.
  249. * @param hi the high end of the range.
  250. */
  251. public void whitespaceChars(int low, int hi) {
  252. if (low < 0)
  253. low = 0;
  254. if (hi >= ctype.length)
  255. hi = ctype.length - 1;
  256. while (low <= hi)
  257. ctype[low++] = CT_WHITESPACE;
  258. }
  259. /**
  260. * Specifies that all characters <i>c</i> in the range
  261. * <code>low <= <i>c</i> <= high</code>
  262. * are "ordinary" in this tokenizer. See the
  263. * <code>ordinaryChar</code> method for more information on a
  264. * character being ordinary.
  265. *
  266. * @param low the low end of the range.
  267. * @param hi the high end of the range.
  268. * @see java.io.StreamTokenizer#ordinaryChar(int)
  269. */
  270. public void ordinaryChars(int low, int hi) {
  271. if (low < 0)
  272. low = 0;
  273. if (hi >= ctype.length)
  274. hi = ctype.length - 1;
  275. while (low <= hi)
  276. ctype[low++] = 0;
  277. }
  278. /**
  279. * Specifies that the character argument is "ordinary"
  280. * in this tokenizer. It removes any special significance the
  281. * character has as a comment character, word component, string
  282. * delimiter, white space, or number character. When such a character
  283. * is encountered by the parser, the parser treates it as a
  284. * single-character token and sets <code>ttype</code> field to the
  285. * character value.
  286. *
  287. * @param ch the character.
  288. * @see java.io.StreamTokenizer#ttype
  289. */
  290. public void ordinaryChar(int ch) {
  291. if (ch >= 0 && ch < ctype.length)
  292. ctype[ch] = 0;
  293. }
  294. /**
  295. * Specified that the character argument starts a single-line
  296. * comment. All characters from the comment character to the end of
  297. * the line are ignored by this stream tokenizer.
  298. *
  299. * @param ch the character.
  300. */
  301. public void commentChar(int ch) {
  302. if (ch >= 0 && ch < ctype.length)
  303. ctype[ch] = CT_COMMENT;
  304. }
  305. /**
  306. * Specifies that matching pairs of this character delimit string
  307. * constants in this tokenizer.
  308. * <p>
  309. * When the <code>nextToken</code> method encounters a string
  310. * constant, the <code>ttype</code> field is set to the string
  311. * delimiter and the <code>sval</code> field is set to the body of
  312. * the string.
  313. * <p>
  314. * If a string quote character is encountered, then a string is
  315. * recognized, consisting of all characters after (but not including)
  316. * the string quote character, up to (but not including) the next
  317. * occurrence of that same string quote character, or a line
  318. * terminator, or end of file. The usual escape sequences such as
  319. * <code>"\n"</code> and <code>"\t"</code> are recognized and
  320. * converted to single characters as the string is parsed.
  321. *
  322. * @param ch the character.
  323. * @see java.io.StreamTokenizer#nextToken()
  324. * @see java.io.StreamTokenizer#sval
  325. * @see java.io.StreamTokenizer#ttype
  326. */
  327. public void quoteChar(int ch) {
  328. if (ch >= 0 && ch < ctype.length)
  329. ctype[ch] = CT_QUOTE;
  330. }
  331. /**
  332. * Specifies that numbers should be parsed by this tokenizer. The
  333. * syntax table of this tokenizer is modified so that each of the twelve
  334. * characters:
  335. * <blockquote><pre>
  336. * 0 1 2 3 4 5 6 7 8 9 . -
  337. * </pre></blockquote>
  338. * <p>
  339. * has the "numeric" attribute.
  340. * <p>
  341. * When the parser encounters a word token that has the format of a
  342. * double precision floating-point number, it treats the token as a
  343. * number rather than a word, by setting the the <code>ttype</code>
  344. * field to the value <code>TT_NUMBER</code> and putting the numeric
  345. * value of the token into the <code>nval</code> field.
  346. *
  347. * @see java.io.StreamTokenizer#nval
  348. * @see java.io.StreamTokenizer#TT_NUMBER
  349. * @see java.io.StreamTokenizer#ttype
  350. */
  351. public void parseNumbers() {
  352. for (int i = '0'; i <= '9'; i++)
  353. ctype[i] |= CT_DIGIT;
  354. ctype['.'] |= CT_DIGIT;
  355. ctype['-'] |= CT_DIGIT;
  356. }
  357. /**
  358. * Determines whether or not ends of line are treated as tokens.
  359. * If the flag argument is true, this tokenizer treats end of lines
  360. * as tokens; the <code>nextToken</code> method returns
  361. * <code>TT_EOL</code> and also sets the <code>ttype</code> field to
  362. * this value when an end of line is read.
  363. * <p>
  364. * A line is a sequence of characters ending with either a
  365. * carriage-return character (<code>'\r'</code>) or a newline
  366. * character (<code>'\n'</code>). In addition, a carriage-return
  367. * character followed immediately by a newline character is treated
  368. * as a single end-of-line token.
  369. * <p>
  370. * If the <code>flag</code> is false, end-of-line characters are
  371. * treated as white space and serve only to separate tokens.
  372. *
  373. * @param flag <code>true</code> indicates that end-of-line characters
  374. * are separate tokens; <code>false</code> indicates that
  375. * end-of-line characters are white space.
  376. * @see java.io.StreamTokenizer#nextToken()
  377. * @see java.io.StreamTokenizer#ttype
  378. * @see java.io.StreamTokenizer#TT_EOL
  379. */
  380. public void eolIsSignificant(boolean flag) {
  381. eolIsSignificantP = flag;
  382. }
  383. /**
  384. * Determines whether or not the tokenizer recognizes C-style comments.
  385. * If the flag argument is <code>true</code>, this stream tokenizer
  386. * recognizes C-style comments. All text between successive
  387. * occurrences of <code>/*</code> and <code>*/</code> are discarded.
  388. * <p>
  389. * If the flag argument is <code>false</code>, then C-style comments
  390. * are not treated specially.
  391. *
  392. * @param flag <code>true</code> indicates to recognize and ignore
  393. * C-style comments.
  394. */
  395. public void slashStarComments(boolean flag) {
  396. slashStarCommentsP = flag;
  397. }
  398. /**
  399. * Determines whether or not the tokenizer recognizes C++-style comments.
  400. * If the flag argument is <code>true</code>, this stream tokenizer
  401. * recognizes C++-style comments. Any occurrence of two consecutive
  402. * slash characters (<code>'/'</code>) is treated as the beginning of
  403. * a comment that extends to the end of the line.
  404. * <p>
  405. * If the flag argument is <code>false</code>, then C++-style
  406. * comments are not treated specially.
  407. *
  408. * @param flag <code>true</code> indicates to recognize and ignore
  409. * C++-style comments.
  410. */
  411. public void slashSlashComments(boolean flag) {
  412. slashSlashCommentsP = flag;
  413. }
  414. /**
  415. * Determines whether or not word token are automatically lowercased.
  416. * If the flag argument is <code>true</code>, then the value in the
  417. * <code>sval</code> field is lowercased whenever a word token is
  418. * returned (the <code>ttype</code> field has the
  419. * value <code>TT_WORD</code> by the <code>nextToken</code> method
  420. * of this tokenizer.
  421. * <p>
  422. * If the flag argument is <code>false</code>, then the
  423. * <code>sval</code> field is not modified.
  424. *
  425. * @param fl <code>true</code> indicates that all word tokens should
  426. * be lowercased.
  427. * @see java.io.StreamTokenizer#nextToken()
  428. * @see java.io.StreamTokenizer#ttype
  429. * @see java.io.StreamTokenizer#TT_WORD
  430. */
  431. public void lowerCaseMode(boolean fl) {
  432. forceLower = fl;
  433. }
  434. /** Read the next character */
  435. private int read() throws IOException {
  436. if (reader != null)
  437. return reader.read();
  438. else if (input != null)
  439. return input.read();
  440. else
  441. throw new IllegalStateException();
  442. }
  443. /**
  444. * Parses the next token from the input stream of this tokenizer.
  445. * The type of the next token is returned in the <code>ttype</code>
  446. * field. Additional information about the token may be in the
  447. * <code>nval</code> field or the <code>sval</code> field of this
  448. * tokenizer.
  449. * <p>
  450. * Typical clients of this
  451. * class first set up the syntax tables and then sit in a loop
  452. * calling nextToken to parse successive tokens until TT_EOF
  453. * is returned.
  454. *
  455. * @return the value of the <code>ttype</code> field.
  456. * @exception IOException if an I/O error occurs.
  457. * @see java.io.StreamTokenizer#nval
  458. * @see java.io.StreamTokenizer#sval
  459. * @see java.io.StreamTokenizer#ttype
  460. */
  461. public int nextToken() throws IOException {
  462. if (pushedBack) {
  463. pushedBack = false;
  464. return ttype;
  465. }
  466. byte ct[] = ctype;
  467. sval = null;
  468. int c = peekc;
  469. if (c < 0)
  470. c = NEED_CHAR;
  471. if (c == SKIP_LF) {
  472. c = read();
  473. if (c < 0)
  474. return ttype = TT_EOF;
  475. if (c == '\n')
  476. c = NEED_CHAR;
  477. }
  478. if (c == NEED_CHAR) {
  479. c = read();
  480. if (c < 0)
  481. return ttype = TT_EOF;
  482. }
  483. ttype = c; /* Just to be safe */
  484. /* Set peekc so that the next invocation of nextToken will read
  485. * another character unless peekc is reset in this invocation
  486. */
  487. peekc = NEED_CHAR;
  488. int ctype = c < 256 ? ct[c] : CT_ALPHA;
  489. while ((ctype & CT_WHITESPACE) != 0) {
  490. if (c == '\r') {
  491. LINENO++;
  492. if (eolIsSignificantP) {
  493. peekc = SKIP_LF;
  494. return ttype = TT_EOL;
  495. }
  496. c = read();
  497. if (c == '\n')
  498. c = read();
  499. } else {
  500. if (c == '\n') {
  501. LINENO++;
  502. if (eolIsSignificantP) {
  503. return ttype = TT_EOL;
  504. }
  505. }
  506. c = read();
  507. }
  508. if (c < 0)
  509. return ttype = TT_EOF;
  510. ctype = c < 256 ? ct[c] : CT_ALPHA;
  511. }
  512. if ((ctype & CT_DIGIT) != 0) {
  513. boolean neg = false;
  514. if (c == '-') {
  515. c = read();
  516. if (c != '.' && (c < '0' || c > '9')) {
  517. peekc = c;
  518. return ttype = '-';
  519. }
  520. neg = true;
  521. }
  522. double v = 0;
  523. int decexp = 0;
  524. int seendot = 0;
  525. while (true) {
  526. if (c == '.' && seendot == 0)
  527. seendot = 1;
  528. else if ('0' <= c && c <= '9') {
  529. v = v * 10 + (c - '0');
  530. decexp += seendot;
  531. } else
  532. break;
  533. c = read();
  534. }
  535. peekc = c;
  536. if (decexp != 0) {
  537. double denom = 10;
  538. decexp--;
  539. while (decexp > 0) {
  540. denom *= 10;
  541. decexp--;
  542. }
  543. /* Do one division of a likely-to-be-more-accurate number */
  544. v = v / denom;
  545. }
  546. nval = neg ? -v : v;
  547. return ttype = TT_NUMBER;
  548. }
  549. if ((ctype & CT_ALPHA) != 0) {
  550. int i = 0;
  551. do {
  552. if (i >= buf.length) {
  553. char nb[] = new char[buf.length * 2];
  554. System.arraycopy(buf, 0, nb, 0, buf.length);
  555. buf = nb;
  556. }
  557. buf[i++] = (char) c;
  558. c = read();
  559. ctype = c < 0 ? CT_WHITESPACE : c < 256 ? ct[c] : CT_ALPHA;
  560. } while ((ctype & (CT_ALPHA | CT_DIGIT)) != 0);
  561. peekc = c;
  562. sval = String.copyValueOf(buf, 0, i);
  563. if (forceLower)
  564. sval = sval.toLowerCase();
  565. return ttype = TT_WORD;
  566. }
  567. if ((ctype & CT_QUOTE) != 0) {
  568. ttype = c;
  569. int i = 0;
  570. /* Invariants (because \Octal needs a lookahead):
  571. * (i) c contains char value
  572. * (ii) d contains the lookahead
  573. */
  574. int d = read();
  575. while (d >= 0 && d != ttype && d != '\n' && d != '\r') {
  576. if (d == '\\') {
  577. c = read();
  578. int first = c; /* To allow \377, but not \477 */
  579. if (c >= '0' && c <= '7') {
  580. c = c - '0';
  581. int c2 = read();
  582. if ('0' <= c2 && c2 <= '7') {
  583. c = (c << 3) + (c2 - '0');
  584. c2 = read();
  585. if ('0' <= c2 && c2 <= '7' && first <= '3') {
  586. c = (c << 3) + (c2 - '0');
  587. d = read();
  588. } else
  589. d = c2;
  590. } else
  591. d = c2;
  592. } else {
  593. switch (c) {
  594. case 'a':
  595. c = 0x7;
  596. break;
  597. case 'b':
  598. c = '\b';
  599. break;
  600. case 'f':
  601. c = 0xC;
  602. break;
  603. case 'n':
  604. c = '\n';
  605. break;
  606. case 'r':
  607. c = '\r';
  608. break;
  609. case 't':
  610. c = '\t';
  611. break;
  612. case 'v':
  613. c = 0xB;
  614. break;
  615. }
  616. d = read();
  617. }
  618. } else {
  619. c = d;
  620. d = read();
  621. }
  622. if (i >= buf.length) {
  623. char nb[] = new char[buf.length * 2];
  624. System.arraycopy(buf, 0, nb, 0, buf.length);
  625. buf = nb;
  626. }
  627. buf[i++] = (char)c;
  628. }
  629. /* If we broke out of the loop because we found a matching quote
  630. * character then arrange to read a new character next time
  631. * around; otherwise, save the character.
  632. */
  633. peekc = (d == ttype) ? NEED_CHAR : d;
  634. sval = String.copyValueOf(buf, 0, i);
  635. return ttype;
  636. }
  637. if (c == '/' && (slashSlashCommentsP || slashStarCommentsP)) {
  638. c = read();
  639. if (c == '*' && slashStarCommentsP) {
  640. int prevc = 0;
  641. while ((c = read()) != '/' || prevc != '*') {
  642. if (c == '\r') {
  643. LINENO++;
  644. c = read();
  645. if (c == '\n') {
  646. c = read();
  647. }
  648. } else {
  649. if (c == '\n') {
  650. LINENO++;
  651. c = read();
  652. }
  653. }
  654. if (c < 0)
  655. return ttype = TT_EOF;
  656. prevc = c;
  657. }
  658. return nextToken();
  659. } else if (c == '/' && slashSlashCommentsP) {
  660. while ((c = read()) != '\n' && c != '\r' && c >= 0);
  661. peekc = c;
  662. return nextToken();
  663. } else {
  664. /* Now see if it is still a single line comment */
  665. if ((ct['/'] & CT_COMMENT) != 0) {
  666. while ((c = read()) != '\n' && c != '\r' && c >= 0);
  667. peekc = c;
  668. return nextToken();
  669. } else {
  670. peekc = c;
  671. return ttype = '/';
  672. }
  673. }
  674. }
  675. if ((ctype & CT_COMMENT) != 0) {
  676. while ((c = read()) != '\n' && c != '\r' && c >= 0);
  677. peekc = c;
  678. return nextToken();
  679. }
  680. return ttype = c;
  681. }
  682. /**
  683. * Causes the next call to the <code>nextToken</code> method of this
  684. * tokenizer to return the current value in the <code>ttype</code>
  685. * field, and not to modify the value in the <code>nval</code> or
  686. * <code>sval</code> field.
  687. *
  688. * @see java.io.StreamTokenizer#nextToken()
  689. * @see java.io.StreamTokenizer#nval
  690. * @see java.io.StreamTokenizer#sval
  691. * @see java.io.StreamTokenizer#ttype
  692. */
  693. public void pushBack() {
  694. if (ttype != TT_NOTHING) /* No-op if nextToken() not called */
  695. pushedBack = true;
  696. }
  697. /**
  698. * Return the current line number.
  699. *
  700. * @return the current line number of this stream tokenizer.
  701. */
  702. public int lineno() {
  703. return LINENO;
  704. }
  705. /**
  706. * Returns the string representation of the current stream token.
  707. *
  708. * @return a string representation of the token specified by the
  709. * <code>ttype</code>, <code>nval</code>, and <code>sval</code>
  710. * fields.
  711. * @see java.io.StreamTokenizer#nval
  712. * @see java.io.StreamTokenizer#sval
  713. * @see java.io.StreamTokenizer#ttype
  714. */
  715. public String toString() {
  716. String ret;
  717. switch (ttype) {
  718. case TT_EOF:
  719. ret = "EOF";
  720. break;
  721. case TT_EOL:
  722. ret = "EOL";
  723. break;
  724. case TT_WORD:
  725. ret = sval;
  726. break;
  727. case TT_NUMBER:
  728. ret = "n=" + nval;
  729. break;
  730. case TT_NOTHING:
  731. ret = "NOTHING";
  732. break;
  733. default: {
  734. /*
  735. * ttype is the first character of either a quoted string or
  736. * is an ordinary character. ttype can definitely not be less
  737. * than 0, since those are reserved values used in the previous
  738. * case statements
  739. */
  740. if (ttype < 256 &&
  741. ((ctype[ttype] & CT_QUOTE) != 0)) {
  742. ret = sval;
  743. break;
  744. }
  745. char s[] = new char[3];
  746. s[0] = s[2] = '\'';
  747. s[1] = (char) ttype;
  748. ret = new String(s);
  749. break;
  750. }
  751. }
  752. return "Token[" + ret + "], line " + LINENO;
  753. }
  754. }