Merge pull request #1 from decaf-lang/dev

fix antlr error recovery

Author: paulzfm

src/main/scala/decaf/frontend/parsing/Lexer.scala

@@ -1,6 +1,7 @@
 package decaf.frontend.parsing
 
-import decaf.driver.error.UnrecogCharError
+import decaf.driver.error._
+import decaf.frontend.parsing.Util.getPos
 import decaf.frontend.parsing.antlr.DecafLexer
 import org.antlr.v4.runtime.{CharStream, Token}
 
@@ -10,19 +11,65 @@ import org.antlr.v4.runtime.{CharStream, Token}
   *
   * @param in input stream
   */
-class Lexer(in: CharStream) extends DecafLexer(in) {
+class Lexer(in: CharStream, errorIssuer: ErrorIssuer) extends DecafLexer(in) {
+
+  // Temporary variables for parsing a string literal
+  private val buffer: StringBuilder = new StringBuilder
+  private var startPos: Pos = NoPos
 
   /**
-    * Catch unrecognized character error (override the generated code).
-    * This error will '''immediately interrupt''' the parsing.
+    * Lexer error handler and string literal parser.
+    * This method will be invoked before a token is returned to the parser.
+    *
+    * A special case is that unrecognized character error will '''immediately interrupt''' the parsing.
+    * Other lexer errors can be recovered.
     *
-    * @return the emitted token if no error occurs
+    * @return the emitted token
     */
-  override def emit(): Token = getType match {
+  override def emit: Token = getType match {
+    // unrecognized char: immediately interrupt
     case DecafLexer.UNRECOG_Char =>
-      val token = super.emit()
-      throw new UnrecogCharError(token.getText.head, Util.getPos(token))
-    case _ => super.emit()
+      val token = super.emit
+      throw new UnrecogCharError(token.getText.head, getPos(token))
+    // integer too large
+    case DecafLexer.INT_LIT =>
+      val token = super.emit
+      var literal = "0"
+      try {
+        literal = token.getText.toInt.toString
+      } catch {
+        case _: NumberFormatException => // not a valid 32-bit integer
+          errorIssuer.issue(new IntTooLargeError(token.getText, getPos(token)))
+      }
+      setText(literal)
+      super.emit
+    // string literal
+    case DecafLexer.UNTERM_STRING =>
+      errorIssuer.issue(new UntermStrError(buffer.toString, startPos))
+      super.emit
+    case DecafLexer.OPEN_STRING =>
+      buffer.clear()
+      buffer += '"'
+      val token = super.emit
+      startPos = getPos(token)
+      token
+    case DecafLexer.ERROR_NEWLINE =>
+      val token = super.emit
+      errorIssuer.issue(new NewlineInStrError(buffer.toString, getPos(token)))
+      token
+    case DecafLexer.BAD_ESC =>
+      val token = super.emit
+      errorIssuer.issue(new BadEscCharError(getPos(token)))
+      token
+    case DecafLexer.ESC | DecafLexer.VALID_CHAR =>
+      val token = super.emit
+      buffer ++= token.getText
+      token
+    case DecafLexer.CLOSE_STRING =>
+      buffer += '"'
+      setText(buffer.toString)
+      super.emit
+    case _ => super.emit
   }
 
   /**
@@ -41,7 +88,7 @@ class Lexer(in: CharStream) extends DecafLexer(in) {
     if (_mode == DecafLexer.IN_STRING) {
       setType(DecafLexer.UNTERM_STRING)
       setText("UNTERM_STRING")
-      val t = super.emit()
+      val t = emit
       popMode()
       t
     } else {

src/main/scala/decaf/frontend/parsing/Parser.scala

@@ -61,7 +61,7 @@ class Parser(implicit config: Config) extends Phase[InputStream, Tree]("parser",
     */
   override def transform(in: InputStream): Tree = {
     val stream = CharStreams.fromStream(in)
-    val lexer = new decaf.frontend.parsing.Lexer(stream)
+    val lexer = new decaf.frontend.parsing.Lexer(stream, this)
     val tokens = new CommonTokenStream(lexer)
     val parser = new DecafParser(tokens)
     parser.addErrorListener(ErrorListener)
@@ -98,7 +98,7 @@ class Parser(implicit config: Config) extends Phase[InputStream, Tree]("parser",
 
     override def syntaxError(recognizer: Recognizer[_, _], offendingSymbol: Any, lineNumber: Int,
                              charPositionInLine: Int, msg: String, e: RecognitionException): Unit = {
-      issue(new SyntaxError(msg, new Pos(lineNumber, charPositionInLine + 1)))
+      throw new SyntaxError(msg, new Pos(lineNumber, charPositionInLine + 1))
     }
   }
 
@@ -108,20 +108,20 @@ class Parser(implicit config: Config) extends Phase[InputStream, Tree]("parser",
 
   object TopLevelVisitor extends DecafParserBaseVisitor[TopLevel] {
 
-    override def visitTopLevel(ctx: DecafParser.TopLevelContext): TopLevel = positioned(ctx) {
+    override def visitTopLevel(ctx: DecafParser.TopLevelContext): TopLevel = {
       val classes = ctx.classDef.map(_.accept(ClassDefVisitor))
-      TopLevel(classes)
+      TopLevel(classes).setPos(classes.head.pos)
     }
   }
 
   object ClassDefVisitor extends DecafParserBaseVisitor[ClassDef] {
 
-    override def visitClassDef(ctx: DecafParser.ClassDefContext): ClassDef = positioned(ctx) {
+    override def visitClassDef(ctx: DecafParser.ClassDefContext): ClassDef = {
       val id = ctx.id.accept(IdVisitor)
       // NOTE: if an optional symbol (like extendsClause) is undefined, its corresponding field is null.
       val parent = if (ctx.extendsClause != null) Some(ctx.extendsClause.id.accept(IdVisitor)) else None
       val fields = ctx.field.map(_.accept(FieldVisitor))
-      ClassDef(id, parent, fields)
+      ClassDef(id, parent, fields).setPos(getPos(ctx.CLASS.getSymbol))
     }
   }
 
@@ -275,43 +275,15 @@ class Parser(implicit config: Config) extends Phase[InputStream, Tree]("parser",
     override def visitLiteral(ctx: DecafParser.LiteralContext): Expr = ctx.lit.accept(this)
 
     override def visitIntLit(ctx: DecafParser.IntLitContext): Expr = positioned(ctx) {
-      val literal = ctx.getText
-      var value = -1
-      try {
-        value = literal.toInt
-      } catch {
-        case _: NumberFormatException => // not a valid 32-bit integer
-          issue(new IntTooLargeError(literal, getPos(ctx.INT_LIT.getSymbol)))
-      }
-
-      IntLit(value)
+      IntLit(ctx.getText.toInt)
     }
 
     override def visitBoolLit(ctx: DecafParser.BoolLitContext): Expr = positioned(ctx) {
       BoolLit(ctx.getText.toBoolean)
     }
 
-    override def visitStringLit(ctx: DecafParser.StringLitContext): Expr = {
-      val buffer = new StringBuilder
-      val startPos = getPos(ctx.OPEN_STRING.getSymbol)
-      buffer += '"'
-      ctx.stringChar.foreach { node =>
-        if (node.ERROR_NEWLINE != null) { // handle new line in string
-          issue(new NewlineInStrError(buffer.toString, getPos(node.ERROR_NEWLINE.getSymbol)))
-        }
-        if (node.BAD_ESC != null) { // handle bad escape character
-          issue(new BadEscCharError(getPos(node.BAD_ESC.getSymbol)))
-        }
-        buffer ++= node.getText
-      }
-
-      if (ctx.UNTERM_STRING != null) { // handle unterminated string
-        issue(new UntermStrError(buffer.toString, startPos))
-      }
-
-      buffer += '"'
-      StringLit(buffer.toString).setPos(startPos)
-    }
+    override def visitStringLit(ctx: DecafParser.StringLitContext): Expr =
+      StringLit(ctx.CLOSE_STRING.getText).setPos(getPos(ctx.OPEN_STRING.getSymbol))
 
     override def visitNullLit(ctx: DecafParser.NullLitContext): Expr = positioned(ctx) { NullLit() }