Compiler support for scala-async, with a state machine transform phase running after erasure [ci: last-only]

build.sbt

@@ -450,6 +450,7 @@ val mimaFilterSettings = Seq {
     ProblemFilters.exclude[DirectMissingMethodProblem]("scala.collection.immutable.HashMap#HashTrieMap.getOrElse0"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("scala.collection.immutable.HashMap#HashMap1.getOrElse0"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("scala.collection.immutable.HashMap#HashMapCollision1.getOrElse0"),
+    ProblemFilters.exclude[ReversedMissingMethodProblem]("scala.reflect.runtime.ReflectSetup.phaseWithId"),
 
     ProblemFilters.exclude[MissingClassProblem]("scala.math.Ordering$IterableOrdering"),
     ProblemFilters.exclude[MissingClassProblem]("scala.math.Ordering$Tuple4Ordering"),
@@ -461,6 +462,14 @@ val mimaFilterSettings = Seq {
     ProblemFilters.exclude[MissingClassProblem]("scala.math.Ordering$Tuple6Ordering"),
     ProblemFilters.exclude[MissingClassProblem]("scala.math.Ordering$Tuple8Ordering"),
     ProblemFilters.exclude[MissingClassProblem]("scala.math.Ordering$Tuple5Ordering"),
+
+    ProblemFilters.exclude[DirectMissingMethodProblem]("scala.reflect.macros.Attachments.removeElement"),
+    ProblemFilters.exclude[DirectMissingMethodProblem]("scala.reflect.macros.Attachments.addElement"),
+    ProblemFilters.exclude[DirectMissingMethodProblem]("scala.reflect.macros.Attachments.containsElement"),
+
+    ProblemFilters.exclude[DirectMissingMethodProblem]("scala.reflect.runtime.Settings.async"),
+
+    ProblemFilters.exclude[DirectMissingMethodProblem]("scala.reflect.api.Internals#InternalApi.markForAsyncTransform"),
   )
 }
 
@@ -516,6 +525,7 @@ lazy val commonSettings = instanceSettings ++ clearSourceAndResourceDirectories
   cleanFiles += (classDirectory in Compile).value,
   cleanFiles += (target in Compile in doc).value,
   fork in run := true,
+  connectInput in run := true,
   scalacOptions in Compile += "-Ywarn-unused:imports",
   scalacOptions in Compile in doc ++= Seq(
     "-doc-footer", "epfl",
@@ -963,7 +973,7 @@ lazy val partestExtras = Project("partest-extras", file(".") / "src" / "partest-
   .settings(
     name := "scala-partest-extras",
     description := "Scala Compiler Testing Tool (compiler-specific extras)",
-    libraryDependencies += partestDep,
+    libraryDependencies ++= Seq(partestDep, junitDep),
     unmanagedSourceDirectories in Compile := List(baseDirectory.value)
   )
 
@@ -1294,6 +1304,7 @@ lazy val root: Project = (project in file("."))
         (testOnly in IntegrationTest in testP).toTask(" -- res scalap specialized").result map (_ -> "partest res scalap specialized"),
         (testOnly in IntegrationTest in testP).toTask(" -- instrumented presentation").result map (_ -> "partest instrumented presentation"),
         (testOnly in IntegrationTest in testP).toTask(" -- --srcpath scaladoc").result map (_ -> "partest --srcpath scaladoc"),
+        (testOnly in IntegrationTest in testP).toTask(" -- --srcpath async").result map (_ -> "partest --srcpath async"),
         (Keys.test in Test in osgiTestFelix).result map (_ -> "osgiTestFelix/test"),
         (Keys.test in Test in osgiTestEclipse).result map (_ -> "osgiTestEclipse/test"),
         (mimaReportBinaryIssues in library).result map (_ -> "library/mimaReportBinaryIssues"),

project/PartestUtil.scala

@@ -87,7 +87,7 @@ object PartestUtil {
       token(grepOption <~ Space) ~> token(globOrPattern, tokenCompletion)
     }
 
-    val SrcPath = ((token(srcPathOption) <~ Space) ~ token(StringBasic.examples(Set("files", "scaladoc")))) map {
+    val SrcPath = ((token(srcPathOption) <~ Space) ~ token(StringBasic.examples(Set("files", "scaladoc", "async")))) map {
       case opt ~ path =>
         srcPath = path
         opt + " " + path

project/ScalaOptionParser.scala

@@ -82,7 +82,7 @@ object ScalaOptionParser {
   }
 
   // TODO retrieve this data programmatically, ala https://github.com/scala/scala-tool-support/blob/master/bash-completion/src/main/scala/BashCompletion.scala
-  private def booleanSettingNames = List("-X", "-Xcheckinit", "-Xdev", "-Xdisable-assertions", "-Xexperimental", "-Xfatal-warnings", "-Xfull-lubs", "-Xfuture", "-Xlog-free-terms", "-Xlog-free-types", "-Xlog-implicit-conversions", "-Xlog-implicits", "-Xlog-reflective-calls",
+  private def booleanSettingNames = List("-X", "-Xasync", "-Xcheckinit", "-Xdev", "-Xdisable-assertions", "-Xexperimental", "-Xfatal-warnings", "-Xfull-lubs", "-Xfuture", "-Xlog-free-terms", "-Xlog-free-types", "-Xlog-implicit-conversions", "-Xlog-implicits", "-Xlog-reflective-calls",
     "-Xno-forwarders", "-Xno-patmat-analysis", "-Xno-uescape", "-Xnojline", "-Xprint-pos", "-Xprint-types", "-Xprompt", "-Xresident", "-Xshow-phases", "-Xstrict-inference", "-Xverify", "-Y",
     "-Ybreak-cycles", "-Ydebug", "-Ycompact-trees", "-YdisableFlatCpCaching", "-Ydoc-debug",
     "-Yide-debug", "-Yinfer-argument-types",

src/compiler/scala/reflect/macros/contexts/Internals.scala

@@ -55,5 +55,8 @@ trait Internals extends scala.tools.nsc.transform.TypingTransformers {
       val trans = new HofTypingTransformer(transformer)
       trans.atOwner(owner)(trans.transform(tree))
     }
+    override def markForAsyncTransform(owner: Symbol, method: DefDef, awaitSymbol: Symbol, config: Map[String, AnyRef]): DefDef = {
+      global.async.markForAsyncTransform(owner, method, awaitSymbol, config)
+    }
   }
-}
+}

src/compiler/scala/tools/nsc/Global.scala

@@ -39,6 +39,7 @@ import scala.language.postfixOps
 import scala.tools.nsc.ast.{TreeGen => AstTreeGen}
 import scala.tools.nsc.classpath._
 import scala.tools.nsc.profile.Profiler
+import scala.tools.nsc.transform.async.AsyncPhase
 import java.io.Closeable
 
 class Global(var currentSettings: Settings, reporter0: Reporter)
@@ -373,6 +374,12 @@ class Global(var currentSettings: Settings, reporter0: Reporter)
     getSourceFile(f)
   }
 
+  override lazy val internal: Internal = new SymbolTableInternal {
+    override def markForAsyncTransform(owner: Symbol, method: DefDef, awaitSymbol: Symbol, config: Map[String, AnyRef]): DefDef = {
+      async.markForAsyncTransform(owner, method, awaitSymbol, config)
+    }
+  }
+
   lazy val loaders = new {
     val global: Global.this.type = Global.this
     val platform: Global.this.platform.type = Global.this.platform
@@ -568,11 +575,17 @@ class Global(var currentSettings: Settings, reporter0: Reporter)
     val runsRightAfter = Some("erasure")
   } with PostErasure
 
+  // phaseName = "async"
+  object async extends {
+    val global: Global.this.type = Global.this
+    val runsAfter = List("posterasure")
+    val runsRightAfter = None
+  } with AsyncPhase
 
   // phaseName = "lambdalift"
   object lambdaLift extends {
     val global: Global.this.type = Global.this
-    val runsAfter = List("posterasure")
+    val runsAfter = List("async")
     val runsRightAfter = None
   } with LambdaLift
 
@@ -672,6 +685,7 @@ class Global(var currentSettings: Settings, reporter0: Reporter)
       explicitOuter           -> "this refs to outer pointers",
       erasure                 -> "erase types, add interfaces for traits",
       postErasure             -> "clean up erased inline classes",
+      async                   -> "transform async/await into a state machine",
       lambdaLift              -> "move nested functions to top level",
       constructors            -> "move field definitions into constructors",
       mixer                   -> "mixin composition",

src/compiler/scala/tools/nsc/backend/jvm/BCodeBodyBuilder.scala

@@ -655,7 +655,8 @@ abstract class BCodeBodyBuilder extends BCodeSkelBuilder {
           val sym = fun.symbol
 
           if (sym.isLabel) { // jump to a label
-            genLoadLabelArguments(args, labelDef(sym), app.pos)
+            def notFound() = abort("Not found: " + sym + " in " + labelDef)
+            genLoadLabelArguments(args, labelDef.getOrElse(sym, notFound()), app.pos)
             bc goTo programPoint(sym)
           } else if (isPrimitive(sym)) { // primitive method call
             generatedType = genPrimitiveOp(app, expectedType)

src/compiler/scala/tools/nsc/backend/jvm/BCodeIdiomatic.scala

@@ -456,7 +456,7 @@ abstract class BCodeIdiomatic {
       i = 1
       while (i < keys.length) {
         if (keys(i-1) == keys(i)) {
-          abort("duplicate keys in SWITCH, can't pick arbitrarily one of them to evict, see scala/bug#6011.")
+          abort("duplicate keys in SWITCH, can't pick arbitrarily one of them to evict, see scala/bug#6011: " + keys.sorted.toList)
         }
         i += 1
       }

src/compiler/scala/tools/nsc/backend/jvm/opt/BoxUnbox.scala

@@ -21,7 +21,6 @@ import scala.tools.asm.Opcodes._
 import scala.tools.asm.Type
 import scala.tools.asm.tree._
 import scala.tools.nsc.backend.jvm.BTypes.InternalName
-import scala.tools.nsc.backend.jvm.analysis.BackendUtils
 import scala.tools.nsc.backend.jvm.opt.BytecodeUtils._
 
 abstract class BoxUnbox {

src/compiler/scala/tools/nsc/settings/ScalaSettings.scala

@@ -113,6 +113,7 @@ trait ScalaSettings extends AbsScalaSettings
    * -X "Advanced" settings
    */
   val Xhelp              = BooleanSetting      ("-X", "Print a synopsis of advanced options.")
+  val async              = BooleanSetting      ("-Xasync", "Enable the async phase for scala.async.Async.{async,await}.")
   val checkInit          = BooleanSetting      ("-Xcheckinit", "Wrap field accessors to throw an exception on uninitialized access.")
   val developer          = BooleanSetting      ("-Xdev", "Indicates user is a developer - issue warnings about anything which seems amiss")
   val noassertions       = BooleanSetting      ("-Xdisable-assertions", "Generate no assertions or assumptions.") andThen (flag =>

src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala

@@ -239,7 +239,7 @@ abstract class SymbolLoaders {
           val currentphase = phase
           doComplete(root)
           phase = currentphase
-          informTime("loaded " + description, start)
+          if (settings.verbose) informTime("loaded " + description, start)
           ok = true
           setSource(root)
           setSource(root.companionSymbol) // module -> class, class -> module

src/compiler/scala/tools/nsc/transform/ExplicitOuter.scala

@@ -209,7 +209,8 @@ abstract class ExplicitOuter extends InfoTransform
    *  values for outer parameters of constructors.
    *  The class provides methods for referencing via outer.
    */
-  abstract class OuterPathTransformer(unit: CompilationUnit) extends TypingTransformer(unit) with UnderConstructionTransformer {
+  abstract class OuterPathTransformer(initLocalTyper: analyzer.Typer) extends TypingTransformer(initLocalTyper) with UnderConstructionTransformer {
+    def this(unit: CompilationUnit) { this(newRootLocalTyper(unit)) }
     /** The directly enclosing outer parameter, if we are in a constructor */
     protected var outerParam: Symbol = NoSymbol
 

src/compiler/scala/tools/nsc/transform/TypeAdaptingTransformer.scala

@@ -134,7 +134,18 @@ trait TypeAdaptingTransformer { self: TreeDSL =>
         val needsExtraCast = isPrimitiveValueType(tree.tpe.typeArgs.head) && !isPrimitiveValueType(pt.typeArgs.head)
         val tree1 = if (needsExtraCast) gen.mkRuntimeCall(nme.toObjectArray, List(tree)) else tree
         gen.mkAttributedCast(tree1, pt)
-      } else gen.mkAttributedCast(tree, pt)
+      } else {
+        tree match {
+          case ld: LabelDef =>
+            // Push the cast into the RHS of matchEnd LabelDefs.
+            ld.symbol.modifyInfo {
+              case MethodType(params, _) => MethodType(params, pt)
+            }
+            deriveLabelDef(ld)(rhs => cast(rhs, pt)).setType(pt)
+          case _ =>
+            gen.mkAttributedCast(tree, pt)
+        }
+      }
     }
 
     /** Adapt `tree` to expected type `pt`.

src/compiler/scala/tools/nsc/transform/TypingTransformers.scala

@@ -13,6 +13,9 @@
 package scala.tools.nsc
 package transform
 
+import scala.collection.mutable
+
+
 /** A base class for transforms.
  *  A transform contains a compiler phase which applies a tree transformer.
  */
@@ -21,12 +24,15 @@ trait TypingTransformers {
   val global: Global
   import global._
 
-  abstract class TypingTransformer(unit: CompilationUnit) extends Transformer {
-    var localTyper: analyzer.Typer =
-      if (phase.erasedTypes)
-        erasure.newTyper(erasure.rootContextPostTyper(unit, EmptyTree)).asInstanceOf[analyzer.Typer]
-      else // TODO: AM: should some phases use a regular rootContext instead of a post-typer one??
-        analyzer.newTyper(analyzer.rootContextPostTyper(unit, EmptyTree))
+  protected def newRootLocalTyper(unit: CompilationUnit): analyzer.Typer = if (phase.erasedTypes)
+    erasure.newTyper(erasure.rootContextPostTyper(unit, EmptyTree)).asInstanceOf[analyzer.Typer]
+  else // TODO: AM: should some phases use a regular rootContext instead of a post-typer one??
+    analyzer.newTyper(analyzer.rootContextPostTyper(unit, EmptyTree))
+
+  abstract class TypingTransformer(initLocalTyper: global.analyzer.Typer) extends Transformer {
+    def this(unit: CompilationUnit) = this(newRootLocalTyper(unit))
+    var localTyper: analyzer.Typer = initLocalTyper
+    currentOwner = localTyper.context.owner
     protected var curTree: Tree = _
 
     override final def atOwner[A](owner: Symbol)(trans: => A): A = atOwner(curTree, owner)(trans)
@@ -51,6 +57,58 @@ trait TypingTransformers {
           super.transform(tree)
       }
     }
+    def transformAtOwner(owner: Symbol, tree: Tree): Tree = atOwner(tree, owner) { transform(tree) }
+  }
+
+  private object ThicketAttachment
+  /** A base class for typing transformers that need to perform "thicket expansion". A thicket is the output of a
+   *  transformation that is flattened into the enclosing block.
+   */
+  abstract class ThicketTransformer(initLocalTyper: analyzer.Typer) extends TypingTransformer(initLocalTyper) {
+    private def expandThicket(t: Tree): List[Tree] = t match {
+      case Block(stats, expr) if t.attachments.containsElement(ThicketAttachment) =>
+        stats :+ expr
+      case _ => t :: Nil
+    }
+
+    def apply(tree: Tree): List[Tree] = expandThicket(transform(tree))
+
+    protected def Thicket(stats: List[Tree], expr: Tree): Tree = {
+      Block(stats, expr).updateAttachment(ThicketAttachment)
+    }
+    protected def Thicket(block: Block): Tree = {
+      block.updateAttachment(ThicketAttachment)
+    }
+
+    override def transform(tree: Tree): Tree = tree match {
+      case Block(stats, expr) =>
+        val transformedStats = transformTrees(stats)
+        val transformedExpr = transform(expr)
+        if ((stats eq transformedStats) && (expr eq transformedExpr)) tree
+        else {
+          val expanded = new mutable.ListBuffer[Tree]
+          def expandStats(): Unit = transformedStats.foreach {
+            case EmptyTree =>
+            case blk @ Block(stats, expr) if blk.attachments.containsElement(ThicketAttachment) =>
+              stats.foreach { s => if (s != EmptyTree) expanded += s }
+              if (expr != EmptyTree) expanded += expr
+            case t =>
+              expanded += t
+          }
+          def expandExpr(): Tree = transformedExpr match {
+            case blk @ Block(stats, expr) if blk.attachments.containsElement(ThicketAttachment) =>
+              stats.foreach { s => if (s != EmptyTree) expanded += s }
+              expr
+            case t =>
+              t
+          }
+          expandStats()
+          val expr1 = expandExpr()
+          treeCopy.Block(tree, expanded.toList, expr1)
+        }
+      case _ =>
+        super.transform(tree)
+    }
   }
 }
 

src/compiler/scala/tools/nsc/transform/UnCurry.scala

@@ -15,12 +15,10 @@ package tools.nsc
 package transform
 
 import scala.annotation.tailrec
-
 import symtab.Flags._
 import scala.collection.mutable
 import scala.collection.mutable.ListBuffer
 import scala.reflect.internal.util.ListOfNil
-
 import PartialFunction.cond
 
 /*<export> */
@@ -149,7 +147,7 @@ abstract class UnCurry extends InfoTransform
     /** Return non-local return key for given method */
     private def nonLocalReturnKey(meth: Symbol) =
       nonLocalReturnKeys.getOrElseUpdate(meth,
-        meth.newValue(unit.freshTermName("nonLocalReturnKey"), meth.pos, SYNTHETIC) setInfo ObjectTpe
+        meth.newValue(unit.freshTermName(nme.NON_LOCAL_RETURN_KEY_STRING), meth.pos, SYNTHETIC) setInfo ObjectTpe
       )
 
     /** Generate a non-local return throw with given return expression from given method.
@@ -409,7 +407,7 @@ abstract class UnCurry extends InfoTransform
       /* Transform tree `t` to { def f = t; f } where `f` is a fresh name */
       def liftTree(tree: Tree) = {
         debuglog("lifting tree at: " + (tree.pos))
-        val sym = currentOwner.newMethod(unit.freshTermName("liftedTree"), tree.pos)
+        val sym = currentOwner.newMethod(unit.freshTermName(nme.LIFTED_TREE), tree.pos, Flag.ARTIFACT)
         sym.setInfo(MethodType(List(), tree.tpe))
         tree.changeOwner(currentOwner, sym)
         localTyper.typedPos(tree.pos)(Block(
@@ -754,7 +752,7 @@ abstract class UnCurry extends InfoTransform
                       tpe
                   }
                 val info = info0.normalize
-                val tempValName = unit freshTermName (p.name + "$")
+                val tempValName = unit freshTermName (p.name.toStringWithSuffix("$"))
                 val newSym = dd.symbol.newTermSymbol(tempValName, p.pos, SYNTHETIC).setInfo(info)
                 atPos(p.pos)(ValDef(newSym, gen.mkAttributedCast(Ident(p.symbol), info)))
               }