[Caching] Reduce the number of cas ID passed on frontend commandline

Using IncludeTree::FileList to concat the include tree file systems that
are passed on the command-line. This significantly reduce the
command-line size, and also makes the cache key computation a lot
faster.

rdar://148752988

Author: cachemeifyoucan

include/swift/AST/DiagnosticsFrontend.def

@@ -532,6 +532,7 @@ ERROR(error_cache_key_creation, none, "cannot create cache key for compilation %
 ERROR(error_cas_file_ref, none, "cannot load file %0 from CAS filesystem", (StringRef))
 ERROR(error_cas_conflict_options, none, "cannot setup CAS due to conflicting '-cas-*' options", ())
 ERROR(error_cas_initialization, none, "CAS cannot be initialized from the specified '-cas-*' options: %0", (StringRef))
+ERROR(error_cas_malformed_input, none, "CAS input '%0' is malformed: %1", (StringRef, StringRef))
 WARNING(cache_replay_failed, none, "cache replay failed: %0", (StringRef))
 
 ERROR(error_failed_cached_diag, none, "failed to serialize cached diagnostics: %0", (StringRef))

include/swift/Basic/CASOptions.h

@@ -41,10 +41,10 @@ class CASOptions final {
   std::vector<std::string> CASFSRootIDs;
 
   /// Clang Include Trees.
-  std::vector<std::string> ClangIncludeTrees;
+  std::string ClangIncludeTree;
 
   /// Clang Include Tree FileList.
-  std::vector<std::string> ClangIncludeTreeFileList;
+  std::string ClangIncludeTreeFileList;
 
   /// CacheKey for input file.
   std::string InputFileKey;
@@ -62,7 +62,7 @@ class CASOptions final {
   /// Check to see if a CASFileSystem is required.
   bool requireCASFS() const {
     return EnableCaching &&
-           (!CASFSRootIDs.empty() || !ClangIncludeTrees.empty() ||
+           (!CASFSRootIDs.empty() || !ClangIncludeTree.empty() ||
             !ClangIncludeTreeFileList.empty() || !InputFileKey.empty() ||
             !BridgingHeaderPCHCacheKey.empty());
   }

include/swift/Frontend/CachingUtils.h

@@ -71,8 +71,8 @@ std::unique_ptr<llvm::MemoryBuffer> loadCachedCompileResultFromCacheKey(
 
 llvm::Expected<llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>>
 createCASFileSystem(llvm::cas::ObjectStore &CAS, ArrayRef<std::string> FSRoots,
-                    ArrayRef<std::string> IncludeTreeRoots,
-                    ArrayRef<std::string> IncludeTreeFileList);
+                    const std::string &IncludeTreeRoot,
+                    const std::string &IncludeTreeFileList);
 
 std::vector<std::string> remapPathsFromCommandLine(
     ArrayRef<std::string> Args,

lib/AST/PluginLoader.cpp

@@ -64,9 +64,9 @@ static StringRef pluginModuleNameStringFromPath(StringRef path) {
 
 static llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
 getPluginLoadingFS(ASTContext &Ctx) {
-  // If there is a clang include tree FS, using real file system to load plugin
+  // If there is an immutable file system, using real file system to load plugin
   // as the FS in SourceMgr doesn't support directory iterator.
-  if (Ctx.ClangImporterOpts.HasClangIncludeTreeRoot)
+  if (Ctx.CASOpts.HasImmutableFileSystem)
     return llvm::vfs::getRealFileSystem();
   return Ctx.SourceMgr.getFileSystem();
 }

lib/ClangImporter/ClangImporter.cpp

@@ -1186,8 +1186,7 @@ std::optional<std::vector<std::string>> ClangImporter::getClangCC1Arguments(
            CI->getFrontendOpts().ProgramAction ==
                clang::frontend::ActionKind::GeneratePCH) &&
           ctx.ClangImporterOpts.HasClangIncludeTreeRoot) {
-        CI->getFrontendOpts().CASIncludeTreeID =
-            ctx.CASOpts.ClangIncludeTrees.back();
+        CI->getFrontendOpts().CASIncludeTreeID = ctx.CASOpts.ClangIncludeTree;
         CI->getFrontendOpts().Inputs.clear();
       }
     }
@@ -1246,7 +1245,7 @@ std::optional<std::vector<std::string>> ClangImporter::getClangCC1Arguments(
 
   std::vector<std::string> FilteredModuleMapFiles;
   for (auto ModuleMapFile : CI->getFrontendOpts().ModuleMapFiles) {
-    if (ctx.ClangImporterOpts.HasClangIncludeTreeRoot) {
+    if (ctx.CASOpts.HasImmutableFileSystem) {
       // There is no need to add any module map file here. Issue a warning and
       // drop the option.
       Impl.diagnose(SourceLoc(), diag::module_map_ignored, ModuleMapFile);
@@ -1326,7 +1325,7 @@ ClangImporter::create(ASTContext &ctx,
       fileMapping.requiresBuiltinHeadersInSystemModules;
 
   // Avoid creating indirect file system when using include tree.
-  if (!ctx.ClangImporterOpts.HasClangIncludeTreeRoot) {
+  if (!ctx.CASOpts.HasImmutableFileSystem) {
     // Wrap Swift's FS to allow Clang to override the working directory
     VFS = llvm::vfs::RedirectingFileSystem::create(
         fileMapping.redirectedFiles, true, *ctx.SourceMgr.getFileSystem());

lib/DependencyScan/ScanDependencies.cpp

@@ -45,7 +45,7 @@
 #include "swift/Frontend/FrontendOptions.h"
 #include "swift/Frontend/ModuleInterfaceLoader.h"
 #include "swift/Strings.h"
-#include "clang/Basic/Module.h"
+#include "clang/CAS/IncludeTree.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetOperations.h"
 #include "llvm/ADT/SetVector.h"
@@ -69,7 +69,6 @@
 #include <sstream>
 #include <stack>
 #include <string>
-#include <algorithm>
 
 using namespace swift;
 using namespace swift::dependencies;
@@ -101,9 +100,6 @@ class ExplicitModuleDependencyResolver {
     if (resolvingDepInfo.isFinalized())
       return false;
 
-    if (auto ID = resolvingDepInfo.getClangIncludeTree())
-      includeTrees.push_back(*ID);
-
     for (const auto &depModuleID : dependencies) {
       const auto &depInfo = cache.findKnownDependency(depModuleID);
       switch (depModuleID.Kind) {
@@ -322,8 +318,10 @@ class ExplicitModuleDependencyResolver {
     // Collect CAS deppendencies from clang modules.
     if (!clangDepDetails.CASFileSystemRootID.empty())
       rootIDs.push_back(clangDepDetails.CASFileSystemRootID);
-    if (!clangDepDetails.CASClangIncludeTreeRootID.empty())
-      includeTrees.push_back(clangDepDetails.CASClangIncludeTreeRootID);
+    if (!clangDepDetails.CASClangIncludeTreeRootID.empty()) {
+      if (addIncludeTree(clangDepDetails.CASClangIncludeTreeRootID))
+        return true;
+    }
 
     collectUsedVFSOverlay(clangDepDetails);
 
@@ -358,12 +356,14 @@ class ExplicitModuleDependencyResolver {
           auto bridgeRoot = tracker->createTreeFromDependencies();
           if (!bridgeRoot)
             return diagnoseCASFSCreationError(bridgeRoot.takeError());
-          fileListIDs.push_back(bridgeRoot->getID().toString());
+
+          fileListRefs.push_back(bridgeRoot->getRef());
         }
       }
-    } else
-      includeTrees.push_back(sourceDepDetails.textualModuleDetails
-                                 .CASBridgingHeaderIncludeTreeRootID);
+    } else if (addIncludeTree(sourceDepDetails.textualModuleDetails
+                                  .CASBridgingHeaderIncludeTreeRootID))
+      return true;
+
     return false;
   };
 
@@ -499,9 +499,7 @@ class ExplicitModuleDependencyResolver {
       auto root = tracker->createTreeFromDependencies();
       if (!root)
         return diagnoseCASFSCreationError(root.takeError());
-      auto rootID = root->getID().toString();
-      dependencyInfoCopy.updateCASFileSystemRootID(rootID);
-      fileListIDs.push_back(rootID);
+      fileListRefs.push_back(root->getRef());
     } else if (auto *textualDep =
                    resolvingDepInfo.getAsSwiftInterfaceModule()) {
       tracker->startTracking();
@@ -516,9 +514,7 @@ class ExplicitModuleDependencyResolver {
       auto root = tracker->createTreeFromDependencies();
       if (!root)
         return diagnoseCASFSCreationError(root.takeError());
-      auto rootID = root->getID().toString();
-      dependencyInfoCopy.updateCASFileSystemRootID(rootID);
-      fileListIDs.push_back(rootID);
+      fileListRefs.push_back(root->getRef());
     }
 
     // Update build command line.
@@ -530,15 +526,8 @@ class ExplicitModuleDependencyResolver {
         commandline.push_back(rootID);
       }
 
-      for (auto tree : includeTrees) {
-        commandline.push_back("-clang-include-tree-root");
-        commandline.push_back(tree);
-      }
-
-      for (auto list : fileListIDs) {
-        commandline.push_back("-clang-include-tree-filelist");
-        commandline.push_back(list);
-      }
+      if (computeCASFileSystem(dependencyInfoCopy))
+        return true;
     }
 
     // Compute and update module cache key.
@@ -636,6 +625,53 @@ class ExplicitModuleDependencyResolver {
       cmd.push_back("-cache-disable-replay");
   }
 
+  bool addIncludeTree(StringRef includeTree) {
+    auto &db = cache.getScanService().getCAS();
+    auto casID = db.parseID(includeTree);
+    if (!casID) {
+      instance.getDiags().diagnose(SourceLoc(), diag::error_invalid_cas_id,
+                                   includeTree, toString(casID.takeError()));
+      return true;
+    }
+    auto ref = db.getReference(*casID);
+    if (!ref) {
+      instance.getDiags().diagnose(SourceLoc(), diag::error_load_input_from_cas,
+                                   includeTree);
+      return true;
+    }
+
+    auto root = clang::cas::IncludeTreeRoot::get(db, *ref);
+    if (!root) {
+      instance.getDiags().diagnose(SourceLoc(), diag::error_cas_malformed_input,
+                                   includeTree, toString(root.takeError()));
+      return true;
+    }
+
+    fileListRefs.push_back(root->getFileListRef());
+    return false;
+  }
+
+  bool computeCASFileSystem(ModuleDependencyInfo &dependencyInfoCopy) {
+    if (fileListRefs.empty())
+      return false;
+
+    auto &db = cache.getScanService().getCAS();
+    auto casFS =
+        clang::cas::IncludeTree::FileList::create(db, {}, fileListRefs);
+    if (!casFS) {
+      instance.getDiags().diagnose(SourceLoc(), diag::error_cas,
+                                   "CAS IncludeTree FileList creation",
+                                   toString(casFS.takeError()));
+      return true;
+    }
+
+    auto casID = casFS->getID().toString();
+    dependencyInfoCopy.updateCASFileSystemRootID(casID);
+    commandline.push_back("-clang-include-tree-filelist");
+    commandline.push_back(casID);
+    return false;
+  }
+
 private:
   const ModuleDependencyID &moduleID;
   ModuleDependenciesCache &cache;
@@ -644,8 +680,7 @@ class ExplicitModuleDependencyResolver {
 
   std::optional<SwiftDependencyTracker> tracker;
   std::vector<std::string> rootIDs;
-  std::vector<std::string> includeTrees;
-  std::vector<std::string> fileListIDs;
+  std::vector<llvm::cas::ObjectRef> fileListRefs;
   std::vector<std::string> commandline;
   std::vector<std::string> bridgingHeaderBuildCmd;
   llvm::StringMap<MacroPluginDependency> macros;

lib/Frontend/CachingUtils.cpp

@@ -472,16 +472,10 @@ static Expected<ObjectRef> mergeCASFileSystem(ObjectStore &CAS,
 
 Expected<IntrusiveRefCntPtr<vfs::FileSystem>>
 createCASFileSystem(ObjectStore &CAS, ArrayRef<std::string> FSRoots,
-                    ArrayRef<std::string> IncludeTrees,
-                    ArrayRef<std::string> IncludeTreeFileList) {
-  assert(!FSRoots.empty() || !IncludeTrees.empty() ||
+                    const std::string &IncludeTree,
+                    const std::string &IncludeTreeFileList) {
+  assert(!FSRoots.empty() || !IncludeTree.empty() ||
          !IncludeTreeFileList.empty() && "no root ID provided");
-  if (FSRoots.size() == 1 && IncludeTrees.empty()) {
-    auto ID = CAS.parseID(FSRoots.front());
-    if (!ID)
-      return ID.takeError();
-    return createCASFileSystem(CAS, *ID);
-  }
 
   auto NewRoot = mergeCASFileSystem(CAS, FSRoots);
   if (!NewRoot)
@@ -492,10 +486,9 @@ createCASFileSystem(ObjectStore &CAS, ArrayRef<std::string> FSRoots,
     return FS.takeError();
 
   auto CASFS = makeIntrusiveRefCnt<vfs::OverlayFileSystem>(std::move(*FS));
-  std::vector<clang::cas::IncludeTree::FileList::FileEntry> Files;
-  // Push all Include File System onto overlay.
-  for (auto &Tree : IncludeTrees) {
-    auto ID = CAS.parseID(Tree);
+
+  if (!IncludeTree.empty()) {
+    auto ID = CAS.parseID(IncludeTree);
     if (!ID)
       return ID.takeError();
 
@@ -510,46 +503,32 @@ createCASFileSystem(ObjectStore &CAS, ArrayRef<std::string> FSRoots,
     if (!ITF)
       return ITF.takeError();
 
-    auto Err = ITF->forEachFile(
-        [&](clang::cas::IncludeTree::File File,
-            clang::cas::IncludeTree::FileList::FileSizeTy Size) -> llvm::Error {
-          Files.push_back({File.getRef(), Size});
-          return llvm::Error::success();
-        });
+    auto ITFS = clang::cas::createIncludeTreeFileSystem(*ITF);
+    if (!ITFS)
+      return ITFS.takeError();
 
-    if (Err)
-      return std::move(Err);
+    CASFS->pushOverlay(*ITFS);
   }
 
-  for (auto &List: IncludeTreeFileList) {
-    auto ID = CAS.parseID(List);
+  if (!IncludeTreeFileList.empty()) {
+    auto ID = CAS.parseID(IncludeTreeFileList);
     if (!ID)
       return ID.takeError();
 
     auto Ref = CAS.getReference(*ID);
     if (!Ref)
       return createCASObjectNotFoundError(*ID);
-    auto IT = clang::cas::IncludeTree::FileList::get(CAS, *Ref);
-    if (!IT)
-      return IT.takeError();
+    auto ITF = clang::cas::IncludeTree::FileList::get(CAS, *Ref);
+    if (!ITF)
+      return ITF.takeError();
 
-    auto Err = IT->forEachFile(
-        [&](clang::cas::IncludeTree::File File,
-            clang::cas::IncludeTree::FileList::FileSizeTy Size) -> llvm::Error {
-          Files.push_back({File.getRef(), Size});
-          return llvm::Error::success();
-        });
+    auto ITFS = clang::cas::createIncludeTreeFileSystem(*ITF);
+    if (!ITFS)
+      return ITFS.takeError();
 
-    if (Err)
-      return std::move(Err);
+    CASFS->pushOverlay(std::move(*ITFS));
   }
 
-  auto ITFS = clang::cas::createIncludeTreeFileSystem(CAS, Files);
-  if (!ITFS)
-    return ITFS.takeError();
-
-  CASFS->pushOverlay(std::move(*ITFS));
-
   return CASFS;
 }
 

lib/Frontend/CompilerInvocation.cpp

@@ -763,18 +763,18 @@ static bool ParseCASArgs(CASOptions &Opts, ArgList &Args,
 
   for (const auto &A : Args.getAllArgValues(OPT_cas_fs))
     Opts.CASFSRootIDs.emplace_back(A);
-  for (const auto &A : Args.getAllArgValues(OPT_clang_include_tree_root))
-    Opts.ClangIncludeTrees.emplace_back(A);
-  for (const auto &A : Args.getAllArgValues(OPT_clang_include_tree_filelist))
-    Opts.ClangIncludeTreeFileList.emplace_back(A);
+  if (auto *A = Args.getLastArg(OPT_clang_include_tree_root))
+    Opts.ClangIncludeTree = A->getValue();
+  if (auto *A = Args.getLastArg(OPT_clang_include_tree_filelist))
+    Opts.ClangIncludeTreeFileList = A->getValue();
 
   if (const Arg *A = Args.getLastArg(OPT_input_file_key))
     Opts.InputFileKey = A->getValue();
 
   if (const Arg*A = Args.getLastArg(OPT_bridging_header_pch_key))
     Opts.BridgingHeaderPCHCacheKey = A->getValue();
 
-  if (!Opts.CASFSRootIDs.empty() || !Opts.ClangIncludeTrees.empty() ||
+  if (!Opts.CASFSRootIDs.empty() || !Opts.ClangIncludeTree.empty() ||
       !Opts.ClangIncludeTreeFileList.empty())
     Opts.HasImmutableFileSystem = true;
 

lib/Frontend/Frontend.cpp

@@ -630,11 +630,11 @@ bool CompilerInstance::setUpVirtualFileSystemOverlays() {
   }
 
   if (Invocation.getCASOptions().requireCASFS()) {
-    if (!CASOpts.CASFSRootIDs.empty() || !CASOpts.ClangIncludeTrees.empty() ||
+    if (!CASOpts.CASFSRootIDs.empty() || !CASOpts.ClangIncludeTree.empty() ||
         !CASOpts.ClangIncludeTreeFileList.empty()) {
       // Set up CASFS as BaseFS.
       auto FS = createCASFileSystem(*CAS, CASOpts.CASFSRootIDs,
-                                    CASOpts.ClangIncludeTrees,
+                                    CASOpts.ClangIncludeTree,
                                     CASOpts.ClangIncludeTreeFileList);
       if (!FS) {
         Diagnostics.diagnose(SourceLoc(), diag::error_cas_fs_creation,

lib/Sema/TypeCheckMacros.cpp

@@ -285,7 +285,7 @@ initializePlugin(ASTContext &ctx, CompilerPlugin *plugin, StringRef libraryPath,
   if (!libraryPath.empty()) {
 #if SWIFT_BUILD_SWIFT_SYNTAX
     llvm::SmallString<128> resolvedLibraryPath;
-    auto fs = ctx.ClangImporterOpts.HasClangIncludeTreeRoot
+    auto fs = ctx.CASOpts.HasImmutableFileSystem
                   ? llvm::vfs::getRealFileSystem()
                   : ctx.SourceMgr.getFileSystem();
     if (auto err = fs->getRealPath(libraryPath, resolvedLibraryPath)) {

test/CAS/module_deps_include_tree.swift

@@ -41,7 +41,6 @@
 // INCLUDE_TREE_F-NEXT: CHeaders/F.h
 
 // MAIN_CMD: -direct-clang-cc1-module-build
-// MAIN_CMD: -clang-include-tree-root
 // MAIN_CMD: -clang-include-tree-filelist
 
 import C