Skip to content

WIP reduce file stats in classpath setup #130

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: 2.13.x
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,16 @@ sealed trait ZipAndJarFileLookupFactory {
private val cache = new FileBasedCache[ZipSettings, ClassPath with Closeable]

def create(zipFile: AbstractFile, settings: Settings, closeableRegistry: CloseableRegistry): ClassPath = {
val disabled = (settings.YdisableFlatCpCaching.value && !settings.YforceFlatCpCaching.value) || zipFile.file == null
val jfile = zipFile.file
val disabled = (settings.YdisableFlatCpCaching.value && !settings.YforceFlatCpCaching.value) || jfile == null
val zipSettings = ZipSettings(settings.releaseValue)
cache.checkCacheability(zipFile.toURL :: Nil, checkStamps = true, disableCache = disabled) match {
case Left(_) =>
val result: ClassPath with Closeable = createForZipFile(zipFile, zipSettings)
closeableRegistry.registerCloseable(result)
result
case Right(paths) =>
cache.getOrCreate(zipSettings, paths, () => createForZipFile(zipFile, zipSettings), closeableRegistry, checkStamps = true)

if (disabled) {
val result: ClassPath with Closeable = createForZipFile(zipFile, zipSettings)
closeableRegistry.registerCloseable(result)
result
} else {
cache.getOrCreate(zipSettings, Seq(jfile.toPath -> zipFile.basicFileAttributes), () => createForZipFile(zipFile, zipSettings), closeableRegistry, checkStamps = true)
}
}

Expand Down Expand Up @@ -258,14 +259,18 @@ final class FileBasedCache[K, T] {
}
}

def getOrCreate(k: K, paths: Seq[Path], create: () => T, closeableRegistry: CloseableRegistry, checkStamps: Boolean): T = cache.synchronized {
val stamps = if (!checkStamps) Nil else paths.map { path =>
def getOrCreate(k: K, paths: Seq[Path], create: () => T,
closeableRegistry: CloseableRegistry, checkStamps: Boolean): T = {
val pathsAndAttrs = paths.map(x => (x, Files.readAttributes(x, classOf[BasicFileAttributes])))
getOrCreate0(k, pathsAndAttrs, create, closeableRegistry, checkStamps)
}
def getOrCreate0(k: K, paths: Seq[(Path, BasicFileAttributes)], create: () => T, closeableRegistry: CloseableRegistry, checkStamps: Boolean): T = cache.synchronized {
val stamps = if (!checkStamps) Nil else paths.map { case (path, attrs) =>
try {
val attrs = Files.readAttributes(path, classOf[BasicFileAttributes])
val lastModified = attrs.lastModifiedTime()
// only null on some platforms, but that's okay, we just use the last modified timestamp as our stamp
val fileKey = attrs.fileKey()
Stamp(lastModified, attrs.size(), if (fileKey == null) NoFileKey else fileKey)
val lastModified = attrs.lastModifiedTime()
// only null on some platforms, but that's okay, we just use the last modified timestamp as our stamp
val fileKey = attrs.fileKey()
Stamp(lastModified, attrs.size(), if (fileKey == null) NoFileKey else fileKey)
} catch {
case _: java.nio.file.NoSuchFileException =>
// Dummy stamp for (currently) non-existent file.
Expand Down
3 changes: 2 additions & 1 deletion src/compiler/scala/tools/nsc/settings/PathFactory.scala
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

package scala.tools.nsc.settings

import java.io.File
import scala.reflect.io.{AbstractFile, PlainFile}

/** Converts paths provided in compiler options (e.g elements of `-classpath` or the target directory of `-d`) to
Expand All @@ -37,6 +38,6 @@ trait PathFactory {
}

object DefaultPathFactory extends PathFactory {
override def getDirectory(path: String): AbstractFile = AbstractFile.getDirectory(path)
override def getDirectory(path: String): AbstractFile = AbstractFile.getDirectory(new File(path))
override def getFile(path: String): AbstractFile = new PlainFile(path)
}
33 changes: 29 additions & 4 deletions src/reflect/scala/reflect/io/AbstractFile.scala
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ import java.io.{BufferedOutputStream, ByteArrayOutputStream, IOException, InputS
import java.io.{File => JFile}
import java.net.URL
import java.nio.ByteBuffer

import java.nio.file.attribute.BasicFileAttributes
import java.nio.file.Files
import scala.collection.AbstractIterable

/**
Expand Down Expand Up @@ -46,10 +47,22 @@ object AbstractFile {
* readable zip or jar archive, returns an abstract directory
* backed by it. Otherwise, returns `null`.
*/
def getDirectory(file: File): AbstractFile =
if (file.isDirectory) new PlainFile(file)
else if (file.isFile && Path.isExtensionJarOrZip(file.jfile)) ZipArchive.fromFile(file)
def getDirectory(file: File): AbstractFile = {
val attrs = try {
Files.readAttributes(file.jfile.toPath, classOf[BasicFileAttributes])
} catch {
case _: IOException =>
return null
}

if (attrs.isDirectory) new PlainFile(file)
else if (attrs.isRegularFile && Path.isExtensionJarOrZip(file.jfile)) {
val za = ZipArchive.fromFile(file)
za.setBasicFileAttributesCached(attrs)
za
}
else null
}

/**
* If the specified URL exists and is a regular file or a directory, returns an
Expand Down Expand Up @@ -115,6 +128,18 @@ abstract class AbstractFile extends AbstractIterable[AbstractFile] {

/** Returns the underlying File if any and null otherwise. */
def file: JFile
def basicFileAttributes: BasicFileAttributes = {
if (basicFileAttributesCached != null)
basicFileAttributesCached
else file match {
case null => null
case f => Files.readAttributes(f.toPath, classOf[BasicFileAttributes])
}
}
private var basicFileAttributesCached: BasicFileAttributes = null
final def setBasicFileAttributesCached(attrs: BasicFileAttributes): Unit = {
basicFileAttributesCached = attrs
}

/** An underlying source, if known. Mostly, a zip/jar file. */
def underlyingSource: Option[AbstractFile] = None
Expand Down
Loading