Fix #8634: Support -release option

* A port of https://github.com/scala/scala/pull/6362/files with some improvements
* When running scalac on JDK 9+ the -release option assures that code is compiled with classes specific to the release available on the classpath.
  This applies to classes from the JDK itself and from external jars.
  If the compilation succeeds, bytecode for the specified release is produced.
* -target option gets renamed to -Xtarget. Using -release instead is preferred since -Xtarget sets the bytecode version without any checks so this might lead to producing code that breaks at runime

Author: prolativ

compiler/src/dotty/tools/backend/jvm/BCodeIdiomatic.scala

@@ -24,12 +24,17 @@ trait BCodeIdiomatic {
   import bTypes._
   import coreBTypes._
 
-  lazy val classfileVersion: Int = ctx.settings.target.value match {
-    case "jvm-1.5"     => asm.Opcodes.V1_5
-    case "jvm-1.6"     => asm.Opcodes.V1_6
-    case "jvm-1.7"     => asm.Opcodes.V1_7
-    case "jvm-1.8"     => asm.Opcodes.V1_8
-    case "jvm-9"       => asm.Opcodes.V9
+  lazy val target = Option(ctx.settings.release.value).filter(_.nonEmpty).getOrElse(ctx.settings.Xtarget.value)
+
+  lazy val classfileVersion: Int = target match {
+    case "8"  => asm.Opcodes.V1_8
+    case "9"  => asm.Opcodes.V9
+    case "10" => asm.Opcodes.V10
+    case "11" => asm.Opcodes.V11
+    case "12" => asm.Opcodes.V12
+    case "13" => asm.Opcodes.V13
+    case "14" => asm.Opcodes.V14
+    case "15" => asm.Opcodes.V15
   }
 
   lazy val majorVersion: Int = (classfileVersion & 0xFF)

compiler/src/dotty/tools/dotc/classpath/DirectoryClassPath.scala

@@ -7,11 +7,14 @@ import java.io.{File => JFile}
 import java.net.URL
 import java.nio.file.{FileSystems, Files}
 
-import dotty.tools.io.{AbstractFile, PlainFile, ClassPath, ClassRepresentation, EfficientClassPath}
+import dotty.tools.dotc.classpath.PackageNameUtils.{packageContains, separatePkgAndClassNames}
+import dotty.tools.io.{AbstractFile, PlainFile, ClassPath, ClassRepresentation, EfficientClassPath, JDK9Reflectors}
 import FileUtils._
+import PlainFile.toPlainFile
 
 import scala.collection.JavaConverters._
 import scala.collection.immutable.ArraySeq
+import scala.util.control.NonFatal
 
 /**
  * A trait allowing to look for classpath entries in directories. It provides common logic for
@@ -111,7 +114,7 @@ trait JFileDirectoryLookup[FileEntryType <: ClassRepresentation] extends Directo
     else Array()
   }
   protected def getName(f: JFile): String = f.getName
-  protected def toAbstractFile(f: JFile): AbstractFile = new PlainFile(new dotty.tools.io.File(f.toPath))
+  protected def toAbstractFile(f: JFile): AbstractFile = f.toPath.toPlainFile
   protected def isPackage(f: JFile): Boolean = f.isPackage
 
   assert(dir != null, "Directory file in DirectoryFileLookup cannot be null")
@@ -122,15 +125,33 @@ trait JFileDirectoryLookup[FileEntryType <: ClassRepresentation] extends Directo
 
 object JrtClassPath {
   import java.nio.file._, java.net.URI
-  def apply(): Option[ClassPath] =
-    try {
-      val fs = FileSystems.getFileSystem(URI.create("jrt:/"))
-      Some(new JrtClassPath(fs))
-    }
-    catch {
-      case _: ProviderNotFoundException | _: FileSystemNotFoundException =>
-        None
+  def apply(release: Option[String]): Option[ClassPath] = {
+    import scala.util.Properties._
+    if (!isJavaAtLeast("9")) None
+    else {
+      // Longer term we'd like an official API for this in the JDK
+      // Discussion: http://mail.openjdk.java.net/pipermail/compiler-dev/2018-March/thread.html#11738
+
+      val currentMajorVersion: Int = JDK9Reflectors.runtimeVersionMajor(JDK9Reflectors.runtimeVersion()).intValue()
+      release match {
+        case Some(v) if v.toInt < currentMajorVersion =>
+          try {
+            val ctSym = Paths.get(javaHome).resolve("lib").resolve("ct.sym")
+            if (Files.notExists(ctSym)) None
+            else Some(new CtSymClassPath(ctSym, v.toInt))
+          } catch {
+            case NonFatal(_) => None
+          }
+        case _ =>
+          try {
+            val fs = FileSystems.getFileSystem(URI.create("jrt:/"))
+            Some(new JrtClassPath(fs))
+          } catch {
+            case _: ProviderNotFoundException | _: FileSystemNotFoundException => None
+          }
+      }
     }
+  }
 }
 
 /**
@@ -157,20 +178,15 @@ final class JrtClassPath(fs: java.nio.file.FileSystem) extends ClassPath with No
   /** Empty string represents root package */
   override private[dotty] def hasPackage(pkg: PackageName): Boolean = packageToModuleBases.contains(pkg.dottedString)
 
-  override private[dotty] def packages(inPackage: PackageName): Seq[PackageEntry] = {
-    def matches(packageDottedName: String) =
-      if (packageDottedName.contains("."))
-        packageOf(packageDottedName) == inPackage.dottedString
-      else inPackage.isRoot
-    packageToModuleBases.keysIterator.filter(matches).map(PackageEntryImpl(_)).toVector
-  }
+  override private[dotty] def packages(inPackage: PackageName): Seq[PackageEntry] =
+    packageToModuleBases.keysIterator.filter(pack => packageContains(inPackage.dottedString, pack)).map(PackageEntryImpl(_)).toVector
 
   private[dotty] def classes(inPackage: PackageName): Seq[ClassFileEntry] =
     if (inPackage.isRoot) Nil
     else
       packageToModuleBases.getOrElse(inPackage.dottedString, Nil).flatMap(x =>
         Files.list(x.resolve(inPackage.dirPathTrailingSlash)).iterator().asScala.filter(_.getFileName.toString.endsWith(".class"))).map(x =>
-        ClassFileEntryImpl(new PlainFile(new dotty.tools.io.File(x)))).toVector
+        ClassFileEntryImpl(x.toPlainFile)).toVector
 
   override private[dotty] def list(inPackage: PackageName): ClassPathEntries =
     if (inPackage.isRoot) ClassPathEntries(packages(inPackage), Nil)
@@ -184,14 +200,75 @@ final class JrtClassPath(fs: java.nio.file.FileSystem) extends ClassPath with No
   def findClassFile(className: String): Option[AbstractFile] =
     if (!className.contains(".")) None
     else {
-      val inPackage = packageOf(className)
-      packageToModuleBases.getOrElse(inPackage, Nil).iterator.flatMap{x =>
+      val (inPackage, _) = separatePkgAndClassNames(className)
+      packageToModuleBases.getOrElse(inPackage, Nil).iterator.flatMap{ x =>
         val file = x.resolve(FileUtils.dirPath(className) + ".class")
-        if (Files.exists(file)) new PlainFile(new dotty.tools.io.File(file)) :: Nil else Nil
+        if (Files.exists(file)) file.toPlainFile :: Nil else Nil
       }.take(1).toList.headOption
     }
-  private def packageOf(dottedClassName: String): String =
-    dottedClassName.substring(0, dottedClassName.lastIndexOf("."))
+}
+
+/**
+  * Implementation `ClassPath` based on the \$JAVA_HOME/lib/ct.sym backing http://openjdk.java.net/jeps/247
+  */
+final class CtSymClassPath(ctSym: java.nio.file.Path, release: Int) extends ClassPath with NoSourcePaths {
+  import java.nio.file.Path, java.nio.file._
+
+  private val fileSystem: FileSystem = FileSystems.newFileSystem(ctSym, null: ClassLoader)
+  private val root: Path = fileSystem.getRootDirectories.iterator.next
+  private val roots = Files.newDirectoryStream(root).iterator.asScala.toList
+
+  // http://mail.openjdk.java.net/pipermail/compiler-dev/2018-March/011737.html
+  private def codeFor(major: Int): String = if (major < 10) major.toString else ('A' + (major - 10)).toChar.toString
+
+  private val releaseCode: String = codeFor(release)
+  private def fileNameMatchesRelease(fileName: String) = !fileName.contains("-") && fileName.contains(releaseCode) // exclude `9-modules`
+  private val rootsForRelease: List[Path] = roots.filter(root => fileNameMatchesRelease(root.getFileName.toString))
+
+  // e.g. "java.lang" -> Seq(/876/java/lang, /87/java/lang, /8/java/lang))
+  private val packageIndex: scala.collection.Map[String, scala.collection.Seq[Path]] = {
+    val index = collection.mutable.AnyRefMap[String, collection.mutable.ListBuffer[Path]]()
+    val isJava12OrHigher = scala.util.Properties.isJavaAtLeast("12")
+    rootsForRelease.foreach(root => Files.walk(root).iterator().asScala.filter(Files.isDirectory(_)).foreach { p =>
+      val moduleNamePathElementCount = if (isJava12OrHigher) 1 else 0
+      if (p.getNameCount > root.getNameCount + moduleNamePathElementCount) {
+        val packageDotted = p.subpath(moduleNamePathElementCount + root.getNameCount, p.getNameCount).toString.replace('/', '.')
+        index.getOrElseUpdate(packageDotted, new collection.mutable.ListBuffer) += p
+      }
+    })
+    index
+  }
+
+  /** Empty string represents root package */
+  override private[dotty] def hasPackage(pkg: PackageName) = packageIndex.contains(pkg.dottedString)
+  override private[dotty] def packages(inPackage: PackageName): Seq[PackageEntry] = {
+    packageIndex.keysIterator.filter(pack => packageContains(inPackage.dottedString, pack)).map(PackageEntryImpl(_)).toVector
+  }
+  private[dotty] def classes(inPackage: PackageName): Seq[ClassFileEntry] = {
+    if (inPackage.isRoot) Nil
+    else {
+      val sigFiles = packageIndex.getOrElse(inPackage.dottedString, Nil).iterator.flatMap(p =>
+        Files.list(p).iterator.asScala.filter(_.getFileName.toString.endsWith(".sig")))
+      sigFiles.map(f => ClassFileEntryImpl(f.toPlainFile)).toVector
+    }
+  }
+
+  override private[dotty] def list(inPackage: PackageName): ClassPathEntries =
+    if (inPackage.isRoot) ClassPathEntries(packages(inPackage), Nil)
+    else ClassPathEntries(packages(inPackage), classes(inPackage))
+
+  def asURLs: Seq[URL] = Nil
+  def asClassPathStrings: Seq[String] = Nil
+  def findClassFile(className: String): Option[AbstractFile] = {
+    if (!className.contains(".")) None
+    else {
+      val (inPackage, classSimpleName) = separatePkgAndClassNames(className)
+      packageIndex.getOrElse(inPackage, Nil).iterator.flatMap { p =>
+        val path = p.resolve(classSimpleName + ".sig")
+        if (Files.exists(path)) path.toPlainFile :: Nil else Nil
+      }.take(1).toList.headOption
+    }
+  }
 }
 
 case class DirectoryClassPath(dir: JFile) extends JFileDirectoryLookup[ClassFileEntryImpl] with NoSourcePaths {
@@ -201,9 +278,7 @@ case class DirectoryClassPath(dir: JFile) extends JFileDirectoryLookup[ClassFile
     val relativePath = FileUtils.dirPath(className)
     val classFile = new JFile(dir, relativePath + ".class")
     if (classFile.exists) {
-      val wrappedClassFile = new dotty.tools.io.File(classFile.toPath)
-      val abstractClassFile = new PlainFile(wrappedClassFile)
-      Some(abstractClassFile)
+      Some(classFile.toPath.toPlainFile)
     }
     else None
   }
@@ -228,11 +303,7 @@ case class DirectorySourcePath(dir: JFile) extends JFileDirectoryLookup[SourceFi
       .map(ext => new JFile(dir, relativePath + "." + ext))
       .collectFirst { case file if file.exists() => file }
 
-    sourceFile.map { file =>
-      val wrappedSourceFile = new dotty.tools.io.File(file.toPath)
-      val abstractSourceFile = new PlainFile(wrappedSourceFile)
-      abstractSourceFile
-    }
+    sourceFile.map(_.toPath.toPlainFile)
   }
 
   private[dotty] def sources(inPackage: PackageName): Seq[SourceFileEntry] = files(inPackage)

compiler/src/dotty/tools/dotc/classpath/FileUtils.scala

@@ -37,6 +37,11 @@ object FileUtils {
       // FIXME: drop last condition when we stop being compatible with Scala 2.11
   }
 
+  private val SUFFIX_CLASS = ".class"
+  private val SUFFIX_SCALA = ".scala"
+  private val SUFFIX_JAVA = ".java"
+  private val SUFFIX_SIG = ".sig"
+
   def stripSourceExtension(fileName: String): String =
     if (endsScala(fileName)) stripClassExtension(fileName)
     else if (endsJava(fileName)) stripJavaExtension(fileName)
@@ -46,23 +51,25 @@ object FileUtils {
 
   def dirPathInJar(forPackage: String): String = forPackage.replace('.', '/')
 
+  inline private def ends (filename:String, suffix:String) = filename.endsWith(suffix) && filename.length > suffix.length
+
   def endsClass(fileName: String): Boolean =
-    fileName.length > 6 && fileName.substring(fileName.length - 6) == ".class"
+    ends (fileName, SUFFIX_CLASS) || fileName.endsWith(SUFFIX_SIG)
 
   def endsScalaOrJava(fileName: String): Boolean =
     endsScala(fileName) || endsJava(fileName)
 
   def endsJava(fileName: String): Boolean =
-    fileName.length > 5 && fileName.substring(fileName.length - 5) == ".java"
+    ends (fileName, SUFFIX_JAVA)
 
   def endsScala(fileName: String): Boolean =
-    fileName.length > 6 && fileName.substring(fileName.length - 6) == ".scala"
+    ends (fileName, SUFFIX_SCALA)
 
   def stripClassExtension(fileName: String): String =
-    fileName.substring(0, fileName.length - 6) // equivalent of fileName.length - ".class".length
+    fileName.substring(0, fileName.lastIndexOf('.'))
 
   def stripJavaExtension(fileName: String): String =
-    fileName.substring(0, fileName.length - 5)
+    fileName.substring(0, fileName.length - 5) // equivalent of fileName.length - SUFFIX_JAVA.length
 
   // probably it should match a pattern like [a-z_]{1}[a-z0-9_]* but it cannot be changed
   // because then some tests in partest don't pass

compiler/src/dotty/tools/dotc/classpath/PackageNameUtils.scala

@@ -14,7 +14,7 @@ object PackageNameUtils {
    * @param fullClassName full class name with package
    * @return (package, simple class name)
    */
-  def separatePkgAndClassNames(fullClassName: String): (String, String) = {
+  inline def separatePkgAndClassNames(fullClassName: String): (String, String) = {
     val lastDotIndex = fullClassName.lastIndexOf('.')
     if (lastDotIndex == -1)
       (RootPackage, fullClassName)
@@ -23,4 +23,15 @@ object PackageNameUtils {
   }
 
   def packagePrefix(inPackage: String): String = if (inPackage == RootPackage) "" else inPackage + "."
+
+  /**
+   * `true` if `packageDottedName` is a package directly nested in `inPackage`, for example:
+   *   - `packageContains("scala", "scala.collection")`
+   *   - `packageContains("", "scala")`
+   */
+  def packageContains(inPackage: String, packageDottedName: String) = {
+    if (packageDottedName.contains("."))
+      packageDottedName.startsWith(inPackage) && packageDottedName.lastIndexOf('.') == inPackage.length
+    else inPackage == ""
+  }
 }

compiler/src/dotty/tools/dotc/classpath/ZipAndJarFileLookupFactory.scala

@@ -24,21 +24,22 @@ sealed trait ZipAndJarFileLookupFactory {
   private val cache = new FileBasedCache[ClassPath]
 
   def create(zipFile: AbstractFile)(using Context): ClassPath =
-    if (ctx.settings.YdisableFlatCpCaching.value || zipFile.file == null) createForZipFile(zipFile)
-    else createUsingCache(zipFile)
+    val release = Option(ctx.settings.release.value).filter(_.nonEmpty)
+    if (ctx.settings.YdisableFlatCpCaching.value || zipFile.file == null) createForZipFile(zipFile, release)
+    else createUsingCache(zipFile, release)
 
-  protected def createForZipFile(zipFile: AbstractFile): ClassPath
+  protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath
 
-  private def createUsingCache(zipFile: AbstractFile): ClassPath =
-    cache.getOrCreate(zipFile.file.toPath, () => createForZipFile(zipFile))
+  private def createUsingCache(zipFile: AbstractFile, release: Option[String]): ClassPath =
+    cache.getOrCreate(zipFile.file.toPath, () => createForZipFile(zipFile, release))
 }
 
 /**
  * Manages creation of classpath for class files placed in zip and jar files.
  * It should be the only way of creating them as it provides caching.
  */
 object ZipAndJarClassPathFactory extends ZipAndJarFileLookupFactory {
-  private case class ZipArchiveClassPath(zipFile: File)
+  private case class ZipArchiveClassPath(zipFile: File, override val release: Option[String])
     extends ZipArchiveFileLookup[ClassFileEntryImpl]
     with NoSourcePaths {
 
@@ -141,9 +142,9 @@ object ZipAndJarClassPathFactory extends ZipAndJarFileLookupFactory {
     case class PackageInfo(packageName: String, subpackages: List[AbstractFile])
   }
 
-  override protected def createForZipFile(zipFile: AbstractFile): ClassPath =
+  override protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath =
     if (zipFile.file == null) createWithoutUnderlyingFile(zipFile)
-    else ZipArchiveClassPath(zipFile.file)
+    else ZipArchiveClassPath(zipFile.file, release)
 
   private def createWithoutUnderlyingFile(zipFile: AbstractFile) = zipFile match {
     case manifestRes: ManifestResources =>
@@ -162,6 +163,8 @@ object ZipAndJarSourcePathFactory extends ZipAndJarFileLookupFactory {
   private case class ZipArchiveSourcePath(zipFile: File)
     extends ZipArchiveFileLookup[SourceFileEntryImpl]
     with NoClassPaths {
+    
+    def release: Option[String] = None
 
     override def asSourcePathString: String = asClassPathString
 
@@ -171,7 +174,7 @@ object ZipAndJarSourcePathFactory extends ZipAndJarFileLookupFactory {
     override protected def isRequiredFileType(file: AbstractFile): Boolean = file.isScalaOrJavaSource
   }
 
-  override protected def createForZipFile(zipFile: AbstractFile): ClassPath = ZipArchiveSourcePath(zipFile.file)
+  override protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath = ZipArchiveSourcePath(zipFile.file)
 }
 
 final class FileBasedCache[T] {

compiler/src/dotty/tools/dotc/classpath/ZipArchiveFileLookup.scala

@@ -17,13 +17,14 @@ import dotty.tools.io.{EfficientClassPath, ClassRepresentation}
  */
 trait ZipArchiveFileLookup[FileEntryType <: ClassRepresentation] extends EfficientClassPath {
   val zipFile: File
+  def release: Option[String]
 
   assert(zipFile != null, "Zip file in ZipArchiveFileLookup cannot be null")
 
   override def asURLs: Seq[URL] = Seq(zipFile.toURI.toURL)
   override def asClassPathStrings: Seq[String] = Seq(zipFile.getPath)
 
-  private val archive = new FileZipArchive(zipFile.toPath)
+  private val archive = new FileZipArchive(zipFile.toPath, release)
 
   override private[dotty] def packages(inPackage: PackageName): Seq[PackageEntry] = {
     for {

compiler/src/dotty/tools/dotc/config/PathResolver.scala

@@ -206,16 +206,19 @@ class PathResolver(using c: Context) {
     import classPathFactory._
 
     // Assemble the elements!
-    def basis: List[Traversable[ClassPath]] = List(
-      JrtClassPath.apply(),                         // 1. The Java 9 classpath (backed by the jrt:/ virtual system, if available)
-      classesInPath(javaBootClassPath),             // 2. The Java bootstrap class path.
-      contentsOfDirsInPath(javaExtDirs),            // 3. The Java extension class path.
-      classesInExpandedPath(javaUserClassPath),     // 4. The Java application class path.
-      classesInPath(scalaBootClassPath),            // 5. The Scala boot class path.
-      contentsOfDirsInPath(scalaExtDirs),           // 6. The Scala extension class path.
-      classesInExpandedPath(userClassPath),         // 7. The Scala application class path.
-      sourcesInPath(sourcePath)                     // 8. The Scala source path.
-    )
+    def basis: List[Traversable[ClassPath]] =
+      val release = Option(ctx.settings.release.value).filter(_.nonEmpty)
+
+      List(
+        JrtClassPath(release),                        // 1. The Java 9+ classpath (backed by the jrt:/ virtual system, if available)
+        classesInPath(javaBootClassPath),             // 2. The Java bootstrap class path.
+        contentsOfDirsInPath(javaExtDirs),            // 3. The Java extension class path.
+        classesInExpandedPath(javaUserClassPath),     // 4. The Java application class path.
+        classesInPath(scalaBootClassPath),            // 5. The Scala boot class path.
+        contentsOfDirsInPath(scalaExtDirs),           // 6. The Scala extension class path.
+        classesInExpandedPath(userClassPath),         // 7. The Scala application class path.
+        sourcesInPath(sourcePath)                     // 8. The Scala source path.
+      )
 
     lazy val containers: List[ClassPath] = basis.flatten.distinct