Package: LexTokenReader$Position

LexTokenReader$Position

nameinstructionbranchcomplexitylinemethod
LexTokenReader.Position(LexTokenReader)
M: 0 C: 41
100%
M: 0 C: 0
100%
M: 0 C: 1
100%
M: 0 C: 10
100%
M: 0 C: 1
100%
set()
M: 0 C: 48
100%
M: 0 C: 0
100%
M: 0 C: 1
100%
M: 0 C: 9
100%
M: 0 C: 1
100%

Coverage

1: /*******************************************************************************
2: *
3: *        Copyright (c) 2008 Fujitsu Services Ltd.
4: *
5: *        Author: Nick Battle
6: *
7: *        This file is part of VDMJ.
8: *
9: *        VDMJ is free software: you can redistribute it and/or modify
10: *        it under the terms of the GNU General Public License as published by
11: *        the Free Software Foundation, either version 3 of the License, or
12: *        (at your option) any later version.
13: *
14: *        VDMJ is distributed in the hope that it will be useful,
15: *        but WITHOUT ANY WARRANTY; without even the implied warranty of
16: *        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17: *        GNU General Public License for more details.
18: *
19: *        You should have received a copy of the GNU General Public License
20: *        along with VDMJ. If not, see <http://www.gnu.org/licenses/>.
21: *
22: ******************************************************************************/
23:
24: package org.overture.parser.lex;
25:
26: import java.io.File;
27: import java.util.List;
28: import java.util.Stack;
29: import java.util.Vector;
30:
31: import org.overture.ast.intf.lex.ILexCommentList;
32: import org.overture.ast.intf.lex.ILexLocation;
33: import org.overture.ast.lex.Dialect;
34: import org.overture.ast.lex.LexBooleanToken;
35: import org.overture.ast.lex.LexCharacterToken;
36: import org.overture.ast.lex.LexCommentList;
37: import org.overture.ast.lex.LexIdentifierToken;
38: import org.overture.ast.lex.LexIntegerToken;
39: import org.overture.ast.lex.LexKeywordToken;
40: import org.overture.ast.lex.LexLocation;
41: import org.overture.ast.lex.LexNameToken;
42: import org.overture.ast.lex.LexQuoteToken;
43: import org.overture.ast.lex.LexRealToken;
44: import org.overture.ast.lex.LexStringToken;
45: import org.overture.ast.lex.LexToken;
46: import org.overture.ast.lex.VDMToken;
47: import org.overture.parser.config.Properties;
48:
49: /**
50: * The main lexical analyser class.
51: */
52:
53: public class LexTokenReader extends BacktrackInputReader
54: {
55:         /** The filename used for console expressions. */
56:         public static String consoleFileName = "console";
57:         /** The current module name, if parsing a module. */
58:         public String currentModule = "";
59:         /** The current file name. */
60:         public final File file;
61:         /** The VDM language dialect we're parsing. */
62:         public final Dialect dialect;
63:
64:         /** The current line, starting at line 1. */
65:         private int linecount;
66:         /** The current character position on the line, starting at 1. */
67:         private int charpos;
68:         /** The number of chars read since the last push. */
69:         private int charsread;
70:         /** The number of tokens read since the last push. */
71:         private int tokensread;
72:         /** The current offset */
73:         private int offset;
74:
75:         /** The next character to process. */
76:         private char ch;
77:         /** The last token returned. */
78:         private LexToken last = null;
79:         /** True if ch is a quoted double quote, ie. \" */
80:         private boolean quotedQuote = false;
81:
82:         /** Added to fix location on the traces **/
83:         private ILexLocation location = null;
84:
85:         /** Comments read since last getComments call */
86:         private LexCommentList comments = new LexCommentList();
87:
88:         
89:         /**
90:          * An inner class to hold all the position details that need to be saved and restored on push/pop.
91:          */
92:
93:         private class Position
94:         {
95:                 public int lc;
96:                 public int cc;
97:                 public int cr;
98:                 public int tr;
99:
100:                 public char c;
101:                 public LexToken l;
102:         public LexCommentList co = new LexCommentList();
103:
104:                 /**
105:                  * Create a Position from the outer class' current position details.
106:                  */
107:
108:                 @SuppressWarnings("synthetic-access")
109:                 public Position()
110:                 {
111:                         lc = linecount;
112:                         cc = charpos;
113:                         cr = charsread;
114:                         tr = tokensread;
115:                         co.addAll(comments);
116:
117:                         c = ch;
118:                         l = last;
119:                 }
120:
121:                 /**
122:                  * Set the outer class position details to those contained in this.
123:                  */
124:
125:                 @SuppressWarnings("synthetic-access")
126:                 public void set()
127:                 {
128:                         linecount = lc;
129:                         charpos = cc;
130:                         charsread = cr;
131:                         tokensread = tr;
132:
133:                         ch = c;
134:                         last = l;
135:                         comments.clear();
136:                         comments.addAll(co);
137:                 }
138:         }
139:
140:         /** A stack of Positions for backtracking. */
141:         private Stack<Position> stack = new Stack<Position>();
142:
143:         /** An end of file symbol. */
144:         private static final char EOF = (char) -1;
145:
146:         /** The assumed tab stop, for calculating positions. */
147:         public static/* final */int TABSTOP = 4;
148:
149:         /**
150:          * Create a LexTokenReader for the filename passed.
151:          *
152:          * @param file
153:          * The filename to parse.
154:          * @param dialect
155:          * if VDM-SL or VDM++ tokens should be processed.
156:          */
157:
158:         public LexTokenReader(File file, Dialect dialect)
159:         {
160:                 super(file);
161:                 this.file = file;
162:                 this.dialect = dialect;
163:                 init();
164:         }
165:
166:         /**
167:          * Create a LexTokenReader for the filename and charset passed.
168:          *
169:          * @param file
170:          * The filename to parse.
171:          * @param dialect
172:          * if VDM-SL or VDM++ tokens should be processed.
173:          * @param charset
174:          * The charset for the file.
175:          */
176:
177:         public LexTokenReader(File file, Dialect dialect, String charset)
178:         {
179:                 super(file, charset);
180:                 this.file = file;
181:                 this.dialect = dialect;
182:                 init();
183:         }
184:
185:         /**
186:          * Create a LexTokenReader for the string passed.
187:          *
188:          * @param expression
189:          * The string (expression) to parse.
190:          * @param dialect
191:          * Parse VDM++ or VDM-SL tokens.
192:          */
193:         public LexTokenReader(String expression, Dialect dialect)
194:         {
195:                 super(expression);
196:                 this.file = new File(consoleFileName);
197:                 this.dialect = dialect;
198:                 init();
199:         }
200:
201:         /**
202:          * Create a LexTokenReader for the string and charset passed.
203:          *
204:          * @param expression
205:          * The string (expression) to parse.
206:          * @param dialect
207:          * Parse VDM++ or VDM-SL tokens.
208:          * @param charset
209:          * The charset to use.
210:          */
211:         public LexTokenReader(String expression, Dialect dialect, String charset)
212:         {
213:                 super(expression, charset);
214:                 this.file = new File(consoleFileName);
215:                 this.dialect = dialect;
216:                 init();
217:         }
218:
219:         /**
220:          * Create a LexTokenReader to read content which originates from a file which is not yet saved and enable the source
221:          * of the file to be set. This is used in the IDE to provide while typing outline and parse error info.
222:          *
223:          * @param content
224:          * @param dialect
225:          * @param file
226:          */
227:
228:         public LexTokenReader(String content, Dialect dialect, File file)
229:         {
230:                 super(content);
231:                 this.file = file;
232:                 this.dialect = dialect;
233:                 init();
234:         }
235:
236:         /**
237:          * Create a LexTokenReader to read content which originates from a file which is not yet saved and enable the source
238:          * of the file to be set. This is used in the IDE to provide while typing outline and parse error info.
239:          *
240:          * @param content
241:          * @param dialect
242:          * @param file
243:          * @param charset
244:          * @param streamReaderType
245:          */
246:
247:         public LexTokenReader(String content, Dialect dialect, File file,
248:                         String charset, ReaderType streamReaderType)
249:         {
250:                 super(content, charset, file, streamReaderType);
251:                 this.file = file;
252:                 this.dialect = dialect;
253:                 init();
254:         }
255:
256:         /**
257:          * Added to fix the location on traces
258:          *
259:          * @param content
260:          * @param dialect
261:          * @param location
262:          */
263:         public LexTokenReader(String content, Dialect dialect, ILexLocation location)
264:         {
265:                 super(content);
266:                 this.file = location.getFile();
267:                 this.dialect = dialect;
268:                 this.location = location;
269:                 init();
270:         }
271:
272:         /**
273:          * Create a string based LexTokenReader, with the position details of another reader.
274:          */
275:         public LexTokenReader(String content, ILexLocation location, LexTokenReader reader)
276:         {
277:                 super(content);
278:                 this.currentModule = reader.currentModule;
279:                 this.file = location.getFile();
280:                 this.dialect = reader.dialect;
281:                 rdCh();
282:                 this.linecount = location.getStartLine();
283:                 this.charpos = location.getStartPos();
284:                 this.charsread = 0;
285:                 this.tokensread = 0;
286:                 this.last = null;
287:                 this.comments.clear();
288:         }
289:
290:         /**
291:          * A string representation of the current location.
292:          */
293:         @Override
294:         public String toString()
295:         {
296:                 return "Last token [" + last + "], last char [" +
297:                                 (ch == EOF ? "EOF" : ch) +
298:                                 "] in " + file + " at " + linecount + ":" + charpos;
299:         }
300:
301:         /**
302:          * Initialize the position counters for a new file/string.
303:          */
304:
305:         private void init()
306:         {
307:                 linecount = 1;
308:                 charpos = 0;
309:                 rdCh();
310:                 charsread = 0;
311:                 tokensread = 0;
312:                 comments.clear();
313:         }
314:
315:         /**
316:          * Throw a {@link LexException} with the given message and details of the current file and position appended.
317:          *
318:          * @param number
319:          * The error number.
320:          * @param msg
321:          * The basic error message.
322:          * @throws LexException
323:          */
324:
325:         private void throwMessage(int number, String msg) throws LexException
326:         {
327:                 throwMessage(number, linecount, charpos, msg);
328:         }
329:
330:         private void throwMessage(int number, int line, int pos, String msg)
331:                         throws LexException
332:         {
333:                 throw new LexException(number, msg, new LexLocation(file, currentModule, line, pos, line, pos, -1, -1));//
334:         }
335:
336:         /**
337:          * Check the next character is as expected. If the character is not as expected, throw the message as a
338:          * {@link LexException}.
339:          *
340:          * @param c
341:          * The expected next character.
342:          * @param number
343:          * The number of the error message.
344:          * @param message
345:          * The error message.
346:          * @throws LexException
347:          */
348:
349:         private void checkFor(char c, int number, String message)
350:                         throws LexException
351:         {
352:                 if (ch == c)
353:                 {
354:                         rdCh();
355:                 } else
356:                 {
357:                         throwMessage(number, message);
358:                 }
359:         }
360:
361:         /**
362:          * @see org.overture.parser.lex.BacktrackInputReader#push()
363:          */
364:
365:         @Override
366:         public void push()
367:         {
368:                 super.push();
369:                 stack.push(new Position());
370:                 charsread = 0;
371:                 tokensread = 0; // Restored on pop
372:         }
373:
374:         /**
375:          * @see org.overture.parser.lex.BacktrackInputReader#pop()
376:          */
377:
378:         @Override
379:         public void pop()
380:         {
381:                 super.pop();
382:                 stack.pop().set();
383:                 LexLocation.clearAfter(file, linecount, charpos);
384:         }
385:
386:         /**
387:          * @see org.overture.parser.lex.BacktrackInputReader#unpush()
388:          */
389:
390:         @Override
391:         public void unpush()
392:         {
393:                 super.unpush();
394:                 Position p = stack.pop(); // Note, don't set the position
395:                 charsread = charsread + p.cr;
396:                 tokensread = tokensread + p.tr;
397:         }
398:
399:         /**
400:          * Go back to the mark, and re-mark it. This just calls pop() followed by push().
401:          */
402:
403:         public void retry()
404:         {
405:                 pop();
406:                 push();
407:         }
408:
409:         /**
410:          * @return the number of characters read since the last push().
411:          */
412:
413:         public int getCharsRead()
414:         {
415:                 return charsread;
416:         }
417:         
418:         /**
419:          * @return the comments read since last getComments(), and clear.
420:          */
421:         public ILexCommentList getComments()
422:         {
423:                 LexCommentList list = new LexCommentList(comments);
424:                 comments.clear();
425:                 return list;
426:         }
427:
428:         /**
429:          * @return the number of tokens read since the last push().
430:          */
431:         public int getTokensRead()
432:         {
433:                 return tokensread;
434:         }
435:         
436:         /**
437:          * @return the current location.
438:          */
439:         public ILexLocation getLocation()
440:         {
441:                 return new LexLocation(file, currentModule, linecount, charpos, 0, 0, 0, 0);
442:         }
443:
444:         /**
445:          * Read the next character from the stream. The position details are updated, accounting for newlines and tab stops.
446:          * The next character is set in the "ch" field, as well as being returned for convenience.
447:          *
448:          * @return the next character.
449:          */
450:
451:         private char rdCh()
452:         {
453:                 char c = super.readCh();
454:
455:                 if (c == '\n')
456:                 {
457:                         linecount++;
458:                         charpos = 0;
459:                 } else if (c == '\t')
460:                 {
461:                         charpos += Properties.parser_tabstop - charpos
462:                                         % Properties.parser_tabstop;
463:                 } else if (c != (char) -1)
464:                 {
465:                         charpos++;
466:                 }
467:
468:                 ch = c;
469:                 charsread++;
470:                 offset = getCurrentRawReadOffset();
471:
472:                 // if(ch == '\r')
473:                 // {
474:                 // ch = rdCh();
475:                 // }else
476:                 // {
477:                 // ch = c;
478:                 // }
479:                 return ch;
480:         }
481:
482:         /**
483:          * Read a backslash quoted character from the stream. This method is used when parsing strings and individual quoted
484:          * characters, which may include things like "\n".
485:          *
486:          * @return The actual character value (eg. "\n" returns 10).
487:          * @throws LexException
488:          */
489:
490:         private char rdQuotedCh() throws LexException
491:         {
492:                 quotedQuote = false;
493:                 char c = rdCh();
494:
495:                 if (c == '\\')
496:                 {
497:                         rdCh();
498:
499:                         switch (ch)
500:                         {
501:                                 case 'r':
502:                                         ch = '\r';
503:                                         break;
504:                                 case 'n':
505:                                         ch = '\n';
506:                                         break;
507:                                 case 't':
508:                                         ch = '\t';
509:                                         break;
510:                                 case 'f':
511:                                         ch = '\f';
512:                                         break;
513:                                 case 'e':
514:                                         ch = '\033';
515:                                         break;
516:                                 case 'a':
517:                                         ch = '\007';
518:                                         break;
519:                                 case '\'':
520:                                         ch = '\'';
521:                                         break;
522:                                 case '\"':
523:                                         ch = '\"';
524:                                         quotedQuote = true;
525:                                         break;
526:                                 case '\\':
527:                                         ch = '\\';
528:                                         break;
529:
530:                                 case 'x':
531:                                         ch = (char) (value(rdCh()) * 16 + value(rdCh()));
532:                                         break;
533:
534:                                 case 'c':
535:                                         ch = (char) (rdCh() - 'A' + 1); // eg. CTRL-A = 1
536:                                         break;
537:
538:                                 case 'u':
539:                                         ch = (char) (value(rdCh()) * 4096 + value(rdCh()) * 256
540:                                                         + value(rdCh()) * 16 + value(rdCh()));
541:                                         break;
542:
543:                                 case '0':
544:                                 case '1':
545:                                 case '2':
546:                                 case '3':
547:                                 case '4':
548:                                 case '5':
549:                                 case '6':
550:                                 case '7':
551:                                         ch = (char) (value(ch) * 64 + value(rdCh()) * 8 + value(rdCh()));
552:                                         break;
553:
554:                                 default:
555:                                         throwMessage(1000, "Malformed quoted character");
556:                         }
557:                 }
558:
559:                 return ch;
560:         }
561:
562:         /**
563:          * Return the value of a character for parsing numbers. The ASCII characters 0-9 are turned into decimal 0-9, while
564:          * a-f and A-F are turned into the hex values 10-15. Characters outside these ranges return -1.
565:          *
566:          * @param c
567:          * The ASCII value to convert.
568:          * @return The converted value.
569:          * @see #rdNumber
570:          */
571:
572:         private int value(char c)
573:         {
574:                 switch (c)
575:                 {
576:                         case '0':
577:                         case '1':
578:                         case '2':
579:                         case '3':
580:                         case '4':
581:                         case '5':
582:                         case '6':
583:                         case '7':
584:                         case '8':
585:                         case '9':
586:                                 return c - '0';
587:
588:                         case 'a':
589:                         case 'b':
590:                         case 'c':
591:                         case 'd':
592:                         case 'e':
593:                         case 'f':
594:                                 return c - 'a' + 10;
595:
596:                         case 'A':
597:                         case 'B':
598:                         case 'C':
599:                         case 'D':
600:                         case 'E':
601:                         case 'F':
602:                                 return c - 'A' + 10;
603:
604:                         default:
605:                                 return -1;
606:                 }
607:         }
608:
609:         /**
610:          * Read a number of the given base. Parsing terminates when a character not within the number base is read.
611:          *
612:          * @param base
613:          * The base of the number (eg. 10 for reading decimals)
614:          * @return The integer value of the number read.
615:          * @throws LexException
616:          */
617:
618:         private String rdNumber(int base) throws LexException
619:         {
620:                 StringBuilder v = new StringBuilder();
621:                 int n = value(ch);
622:                 // long v = n;
623:                 v.append(ch);
624:
625:                 if (n < 0 || n >= base)
626:                 {
627:                         throwMessage(1001, "Invalid char [" + ch + "] in base " + base
628:                                         + " number");
629:                 }
630:
631:                 while (true)
632:                 {
633:                         rdCh();
634:                         n = value(ch);
635:
636:                         if (n < 0 || n >= base)
637:                         {
638:                                 return v.toString();
639:                         }
640:
641:                         // v = (v * base) + n;
642:                         v.append(ch);
643:                 }
644:         }
645:
646:         /**
647:          * Read the next complete token from the input stream. Whitespace is skipped, and the start line and position of the
648:          * token are noted from the current stream position. Then the next character to process is used to drive a large
649:          * switch statement to produce the right {@link VDMToken}. The lastToken field is updated and the result returned.
650:          *
651:          * @return The next token, or a LexToken of type EOF.
652:          * @throws LexException
653:          */
654:
655:         public LexToken nextToken() throws LexException
656:         {
657:                 while (Character.isWhitespace(ch))
658:                 {
659:                         rdCh();
660:                 }
661:
662:                 int tokline = linecount;
663:                 int tokpos = charpos;
664:                 int tokOffset = offset;
665:                 VDMToken type = null;
666:                 last = null;
667:                 boolean rdch = true;
668:
669:                 switch (ch)
670:                 {
671:                         case EOF:
672:                                 type = VDMToken.EOF;
673:                                 break;
674:
675:                         case '-':
676:                                 if (rdCh() == '-')
677:                                 {
678:                                         StringBuilder sb = new StringBuilder();
679:                                         ILexLocation here = location(linecount, charpos + 1, tokpos, tokOffset);
680:                                         
681:                                         while (ch != '\n' && ch != EOF)
682:                                         {
683:                                                 sb.append(ch);
684:                                                 rdCh();
685:                                         }
686:
687:                                         comments.add(here, sb.toString().substring(1), false);
688:                                         return nextToken();
689:                                 }
690:                                 else if (ch == '>')
691:                                 {
692:                                         type = VDMToken.ARROW;
693:                                 }
694:                                 else
695:                                 {
696:                                         rdch = false;
697:                                         type = VDMToken.MINUS;
698:                                 }
699:                                 break;
700:
701:                         case '+':
702:                                 if (rdCh() == '>')
703:                                 {
704:                                         type = VDMToken.TOTAL_FUNCTION;
705:                                 } else if (ch == '+')
706:                                 {
707:                                         type = VDMToken.PLUSPLUS;
708:                                 } else
709:                                 {
710:                                         rdch = false;
711:                                         type = VDMToken.PLUS;
712:                                 }
713:                                 break;
714:
715:                         case ':':
716:                                 if (rdCh() == '-')
717:                                 {
718:                                         if (rdCh() == '>')
719:                                         {
720:                                                 type = VDMToken.RANGERESBY;
721:                                         } else
722:                                         {
723:                                                 rdch = false;
724:                                                 type = VDMToken.EQABST;
725:                                         }
726:                                 } else if (ch == '>')
727:                                 {
728:                                         type = VDMToken.RANGERESTO;
729:                                 } else if (ch == '=')
730:                                 {
731:                                         type = VDMToken.ASSIGN;
732:                                 } else if (ch == ':')
733:                                 {
734:                                         type = VDMToken.COLONCOLON;
735:                                 } else
736:                                 {
737:                                         rdch = false;
738:                                         type = VDMToken.COLON;
739:                                 }
740:                                 break;
741:
742:                         case '|':
743:                                 if (rdCh() == '-')
744:                                 {
745:                                         if (rdCh() == '>')
746:                                         {
747:                                                 type = VDMToken.MAPLET;
748:                                         } else
749:                                         {
750:                                                 throwMessage(1002, "Expecting '|->'");
751:                                         }
752:                                 } else if (ch == '|')
753:                                 {
754:                                         type = VDMToken.PIPEPIPE;
755:                                 } else
756:                                 {
757:                                         rdch = false;
758:                                         type = VDMToken.PIPE;
759:                                 }
760:                                 break;
761:
762:                         case '.':
763:                                 if (rdCh() == '.')
764:                                 {
765:                                         if (rdCh() == '.')
766:                                         {
767:                                                 type = VDMToken.RANGE;
768:                                                 break;
769:                                         }
770:
771:                                         throwMessage(1003, "Expecting '...'");
772:                                 } else
773:                                 {
774:                                         rdch = false;
775:                                         type = VDMToken.POINT;
776:                                 }
777:                                 break;
778:
779:                         case '=':
780:                                 if (rdCh() == '=')
781:                                 {
782:                                         if (rdCh() == '>')
783:                                         {
784:                                                 type = VDMToken.OPDEF;
785:                                         } else
786:                                         {
787:                                                 rdch = false;
788:                                                 type = VDMToken.EQUALSEQUALS;
789:                                         }
790:                                 } else if (ch == '>')
791:                                 {
792:                                         type = VDMToken.IMPLIES;
793:                                 } else
794:                                 {
795:                                         rdch = false;
796:                                         type = VDMToken.EQUALS;
797:                                 }
798:                                 break;
799:
800:                         case '*':
801:                                 if (rdCh() == '*')
802:                                 {
803:                                         type = VDMToken.STARSTAR;
804:                                 } else
805:                                 {
806:                                         rdch = false;
807:                                         type = VDMToken.TIMES;
808:                                 }
809:                                 break;
810:
811:                         case '<':
812:                                 push();
813:                                 if (rdCh() == '=')
814:                                 {
815:                                         unpush();
816:                                         if (rdCh() == '>')
817:                                         {
818:                                                 type = VDMToken.EQUIVALENT;
819:                                         } else
820:                                         {
821:                                                 rdch = false;
822:                                                 type = VDMToken.LE;
823:                                         }
824:                                 } else if (ch == ':')
825:                                 {
826:                                         unpush();
827:                                         type = VDMToken.DOMRESTO;
828:                                 } else if (ch == '-')
829:                                 {
830:                                         unpush();
831:                                         if (rdCh() == ':')
832:                                         {
833:                                                 type = VDMToken.DOMRESBY;
834:                                         } else
835:                                         {
836:                                                 throwMessage(1004, "Expecting '<-:'");
837:                                         }
838:                                 } else if (ch == '>')
839:                                 {
840:                                         unpush();
841:                                         type = VDMToken.NE;
842:                                 } else if (startOfName(ch))
843:                                 {
844:                                         // <QuoteLiteral> or <x
845:                                         String name = rdIdentifier();
846:
847:                                         if (ch == '>')
848:                                         {
849:                                                 unpush();
850:                                                 last = new LexQuoteToken(name, new LexLocation(file, currentModule, tokline, tokpos, linecount, charpos + 1, tokOffset, offset + 1));
851:                                                 // location(tokline, tokpos, tokOffset, offset));
852:                                                 type = VDMToken.QUOTE;
853:                                         } else
854:                                         {
855:                                                 pop();
856:                                                 type = VDMToken.LT;
857:                                         }
858:                                 } else
859:                                 {
860:                                         unpush();
861:                                         rdch = false;
862:                                         type = VDMToken.LT;
863:                                 }
864:                                 break;
865:
866:                         case '>':
867:                                 if (rdCh() == '=')
868:                                 {
869:                                         type = VDMToken.GE;
870:                                 } else
871:                                 {
872:                                         rdch = false;
873:                                         type = VDMToken.GT;
874:                                 }
875:                                 break;
876:
877:                         case '"':
878:                                 rdQuotedCh();
879:                                 StringBuffer msg = new StringBuffer();
880:
881:                                 while ((ch != '"' || quotedQuote == true) && ch != EOF)
882:                                 {
883:                                         msg.append(ch);
884:                                         rdQuotedCh();
885:                                 }
886:
887:                                 checkFor('\"', 1005, "Expecting close double quote");
888:                                 last = new LexStringToken(msg.toString(), location(tokline, tokpos, tokOffset, offset));
889:                                 rdch = false;
890:                                 break;
891:
892:                         case '\'':
893:                                 last = new LexCharacterToken(rdQuotedCh(), location(tokline, tokpos, tokOffset, offset));
894:                                 rdCh();
895:                                 checkFor('\'', 1006, "Expecting close quote after character");
896:                                 rdch = false;
897:                                 break;
898:
899:                         case '#':
900:                                 if (Character.isLetter(rdCh()))
901:                                 {
902:                                         StringBuilder tag = new StringBuilder();
903:                                         tag.append('#');
904:
905:                                         do
906:                                         {
907:                                                 tag.append(ch);
908:                                                 rdCh();
909:                                         } while (Character.isLetter(ch));
910:
911:                                         type = VDMToken.lookup(tag.toString(), dialect);
912:
913:                                         if (type == null)
914:                                         {
915:                                                 throwMessage(1007, "Unexpected tag after '#'");
916:                                         }
917:
918:                                         rdch = false;
919:                                 } else
920:                                 {
921:                                         type = VDMToken.HASH;
922:                                         rdch = false;
923:                                 }
924:                                 break;
925:
926:                         case '/':
927:                                 if (rdCh() == '*') // Block comments
928:                                 {
929:                                         StringBuilder sb = new StringBuilder();
930:                                         rdCh();
931:                                         ILexLocation here = location(linecount, charpos, tokline, tokpos);
932:
933:                                         while (ch != EOF)
934:                                         {
935:                                          while (ch != '*' && ch != EOF)
936:                                          {
937:                                                  sb.append(ch);
938:                                                  rdCh();
939:                                          }
940:
941:                                          if (ch == EOF)
942:                                          {
943:                                                  throwMessage(1011, tokline, tokpos, "Unterminated block comment");
944:                                          }
945:                                          else
946:                                          {
947:                                                  if (rdCh() == '/')
948:                                                  {
949:                                                          break;
950:                                                  }
951:                                                  else
952:                                                  {
953:                                                          sb.append('*');
954:                                                  }
955:                                          }
956:                                         }
957:
958:                                         comments.add(here, sb.toString(), true);
959:                                         rdCh();
960:                                         return nextToken();
961:                                 }
962:                                 else
963:                                 {
964:                                         type = VDMToken.DIVIDE;
965:                                         rdch = false;
966:                                 }
967:                                 break;
968:
969:                         case ',':
970:                                 type = VDMToken.COMMA;
971:                                 break;
972:                         case ';':
973:                                 type = VDMToken.SEMICOLON;
974:                                 break;
975:                         case '?':
976:                                 type = VDMToken.QMARK;
977:                                 break;
978:                         case '@':
979:                                 type = VDMToken.AT;
980:                                 break;
981:                         case '&':
982:                                 type = VDMToken.AMPERSAND;
983:                                 break;
984:                         case '^':
985:                                 type = VDMToken.CONCATENATE;
986:                                 break;
987:                         case '(':
988:                                 type = VDMToken.BRA;
989:                                 break;
990:                         case ')':
991:                                 type = VDMToken.KET;
992:                                 break;
993:                         case '{':
994:                                 type = VDMToken.SET_OPEN;
995:                                 break;
996:                         case '}':
997:                                 type = VDMToken.SET_CLOSE;
998:                                 break;
999:                         case '[':
1000:                                 type = VDMToken.SEQ_OPEN;
1001:                                 break;
1002:                         case ']':
1003:                                 type = VDMToken.SEQ_CLOSE;
1004:                                 break;
1005:                         case '\\':
1006:                                 type = VDMToken.SETDIFF;
1007:                                 break;
1008:
1009:                         case '0':
1010:                                 push();
1011:                                 rdCh();
1012:
1013:                                 if (ch == 'x' || ch == 'X')
1014:                                 {
1015:                                         unpush();
1016:                                         rdCh();
1017:                                         String decimal = String.valueOf(Long.parseLong(rdNumber(16), 16));
1018:                                         last = new LexIntegerToken(decimal, location(tokline, tokpos, tokOffset, offset));
1019:                                 }
1020:                                 else
1021:                                 {
1022:                                         pop();
1023:                                         last = rdReal(tokline, tokpos, tokOffset);
1024:                                 }
1025:
1026:                                 rdch = false;
1027:                                 break;
1028:
1029:                         default:
1030:                                 if (ch >= '0' && ch <= '9')
1031:                                 {
1032:                                         last = rdReal(tokline, tokpos, tokOffset);
1033:                                         rdch = false;
1034:                                 } else if (startOfName(ch))
1035:                                 {
1036:                                         List<String> name = rdName(); // module`name parts
1037:                                         rdch = false;
1038:
1039:                                         switch (name.size())
1040:                                         {
1041:                                                 case 1:
1042:                                                         type = VDMToken.lookup(name.get(0), dialect);
1043:
1044:                                                         if (type == null)
1045:                                                         {
1046:                                                                 last = new LexIdentifierToken(name.get(0), ch == '~', location(tokline, tokpos, tokOffset, offset));
1047:                                                                 rdch = ch == '~';
1048:                                                         } else
1049:                                                         {
1050:                                                                 switch (type)
1051:                                                                 {
1052:                                                                         case TRUE:
1053:                                                                         case FALSE:
1054:                                                                                 last = new LexBooleanToken(type, location(tokline, tokpos, tokOffset, offset));
1055:                                                                                 break;
1056:
1057:                                                                         default:
1058:                                                                                 last = new LexKeywordToken(type, location(tokline, tokpos, tokOffset, offset));
1059:                                                                                 break;
1060:                                                                 }
1061:                                                         }
1062:                                                         break;
1063:
1064:                                                 case 2:
1065:                                                         last = new LexNameToken(name.get(0), name.get(1), location(tokline, tokpos, tokOffset, offset), false, true);
1066:                                                         break;
1067:
1068:                                                 default:
1069:                                                         throwMessage(1008, "Malformed module`name");
1070:                                         }
1071:                                 } else
1072:                                 {
1073:                                         char badch = ch;
1074:                                         rdCh();
1075:                                         throwMessage(1009, "Unexpected character '" + badch
1076:                                                         + "' (code 0x" + Integer.toHexString(badch) + ")");
1077:                                 }
1078:                                 break;
1079:                 }
1080:
1081:                 if (rdch)
1082:                 {
1083:                         rdCh();
1084:                 }
1085:
1086:                 if (last == null)
1087:                 {
1088:                         last = new LexKeywordToken(type, location(tokline, tokpos, tokOffset, offset));
1089:                 }
1090:
1091:                 tokensread++;
1092:                 return last;
1093:         }
1094:
1095:         /**
1096:          * Create a {@link LexLocation} object from the current stream position.
1097:          *
1098:          * @param tokline
1099:          * The token start line.
1100:          * @param tokpos
1101:          * The token start position.
1102:          * @param endOffset
1103:          * @return A new LexLocation.
1104:          */
1105:
1106:         private ILexLocation location(int tokline, int tokpos, int startOffset,
1107:                         int endOffset)
1108:         {
1109:                 // Fix for location on traces
1110:                 if (this.location != null)
1111:                 {
1112:                         return this.location;
1113:                 } else
1114:                 {
1115:                         return new LexLocation(file, currentModule, tokline, tokpos, linecount, charpos, startOffset, endOffset);
1116:                 }
1117:         }
1118:
1119:         /**
1120:          * Read a decimal floating point number.
1121:          *
1122:          * @param tokline
1123:          * The start line of the number.
1124:          * @param tokpos
1125:          * The start position of the number.
1126:          * @param tokOffset
1127:          * @return Either a LexRealToken or a LexIntegerToken.
1128:          * @throws LexException
1129:          */
1130:
1131:         private LexToken rdReal(int tokline, int tokpos, int tokOffset)
1132:                         throws LexException
1133:         {
1134:                 String floatSyntax = "Expecting <digits>[.<digits>][e<+-><digits>]";
1135:                 String value = rdNumber(10);
1136:                 String fraction = null;
1137:                 String exponent = null;
1138:                 boolean negative = false;
1139:
1140:                 push();
1141:
1142:                 if (ch == '.')
1143:                 {
1144:                         rdCh();
1145:
1146:                         if (ch >= '0' && ch <= '9')
1147:                         {
1148:                                 fraction = rdNumber(10);
1149:                                 exponent = "0";
1150:                         } else
1151:                         {
1152:                                 // Somthing like rec.#1.field, so just return the integer
1153:                                 pop();
1154:                                 return new LexIntegerToken(value, location(tokline, tokpos, tokOffset, offset));
1155:                         }
1156:                 }
1157:
1158:                 unpush();
1159:
1160:                 if (ch == 'e' || ch == 'E')
1161:                 {
1162:                         if (fraction == null)
1163:                         {
1164:                                 fraction = "0";
1165:                         }
1166:
1167:                         switch (rdCh())
1168:                         {
1169:                                 case '+':
1170:                                 {
1171:                                         rdCh();
1172:                                         exponent = rdNumber(10);
1173:                                         break;
1174:                                 }
1175:
1176:                                 case '-':
1177:                                 {
1178:                                         rdCh();
1179:                                         exponent = rdNumber(10);
1180:                                         negative = true;
1181:                                         break;
1182:                                 }
1183:
1184:                                 case '0':
1185:                                 case '1':
1186:                                 case '2':
1187:                                 case '3':
1188:                                 case '4':
1189:                                 case '5':
1190:                                 case '6':
1191:                                 case '7':
1192:                                 case '8':
1193:                                 case '9':
1194:                                 {
1195:                                         exponent = rdNumber(10);
1196:                                         break;
1197:                                 }
1198:
1199:                                 default:
1200:                                         throwMessage(1010, floatSyntax);
1201:                         }
1202:                 }
1203:
1204:                 if (fraction != null)
1205:                 {
1206:                         String real = "+" + value + "." + fraction + "e"
1207:                                         + (negative ? "-" : "+") + exponent;
1208:
1209:                         return new LexRealToken(real, location(tokline, tokpos, tokOffset, offset));
1210:                 }
1211:
1212:                 return new LexIntegerToken(value, location(tokline, tokpos, tokOffset, offset));
1213:         }
1214:
1215:         /**
1216:          * Get the last token returned from the reader.
1217:          *
1218:          * @return The last token, or the first token if none read yet.
1219:          * @throws LexException
1220:          */
1221:
1222:         public LexToken getLast() throws LexException
1223:         {
1224:                 if (last == null)
1225:                 {
1226:                         nextToken();
1227:                 }
1228:
1229:                 return last;
1230:         }
1231:
1232:         /**
1233:          * @return True if the character passed can be the start of a variable name.
1234:          */
1235:
1236:         private boolean startOfName(char c)
1237:         {
1238:                 if (c < 0x0100)
1239:                 {
1240:                         return Character.isLetter(c) || c == '$';
1241:                 } else
1242:                 {
1243:                         switch (Character.getType(c))
1244:                         {
1245:                                 case Character.CONTROL:
1246:                                 case Character.LINE_SEPARATOR:
1247:                                 case Character.PARAGRAPH_SEPARATOR:
1248:                                 case Character.SPACE_SEPARATOR:
1249:                                 case Character.SURROGATE:
1250:                                 case Character.UNASSIGNED:
1251:                                 case Character.DECIMAL_DIGIT_NUMBER:
1252:                                 case Character.CONNECTOR_PUNCTUATION:
1253:                                         return false;
1254:
1255:                                 default:
1256:                                         return true;
1257:                         }
1258:                 }
1259:         }
1260:
1261:         /**
1262:          * @return True if the character passed can be part of a variable name.
1263:          */
1264:
1265:         private boolean restOfName(char c)
1266:         {
1267:                 if (c < 0x0100)
1268:                 {
1269:                         return Character.isLetterOrDigit(c) || c == '$' || c == '_'
1270:                                         || c == '\'';
1271:                 } else
1272:                 {
1273:                         switch (Character.getType(c))
1274:                         {
1275:                                 case Character.CONTROL:
1276:                                 case Character.LINE_SEPARATOR:
1277:                                 case Character.PARAGRAPH_SEPARATOR:
1278:                                 case Character.SPACE_SEPARATOR:
1279:                                 case Character.SURROGATE:
1280:                                 case Character.UNASSIGNED:
1281:                                         return false;
1282:
1283:                                 default:
1284:                                         return true;
1285:                         }
1286:                 }
1287:         }
1288:
1289:         /**
1290:          * Read a fully qualified module`name.
1291:          *
1292:          * @return A list of one or two name parts.
1293:          */
1294:
1295:         private List<String> rdName()
1296:         {
1297:                 List<String> names = new Vector<String>();
1298:                 names.add(rdIdentifier());
1299:
1300:                 if (ch == '`')
1301:                 {
1302:                         if (startOfName(rdCh()))
1303:                         {
1304:                                 names.add(rdIdentifier());
1305:                         }
1306:                 }
1307:
1308:                 if (names.size() == 2)
1309:                 {
1310:                         // We have the strange mk_Mod`name case...
1311:
1312:                         String first = names.get(0);
1313:
1314:                         if (first.startsWith("mk_") || first.startsWith("is_"))
1315:                         {
1316:                                 List<String> one = new Vector<String>();
1317:                                 one.add(first + "`" + names.get(1));
1318:                                 names = one;
1319:                         }
1320:                 }
1321:
1322:                 return names;
1323:         }
1324:
1325:         /**
1326:          * Read a simple identifier without a module name prefix.
1327:          *
1328:          * @return a simple name.
1329:          */
1330:
1331:         private String rdIdentifier()
1332:         {
1333:                 StringBuilder id = new StringBuilder();
1334:                 id.append(ch);
1335:
1336:                 while (restOfName(rdCh()))
1337:                 {
1338:                         id.append(ch);
1339:                 }
1340:
1341:                 return id.toString();
1342:         }
1343: }