From 201e4faea720bfaaa39a2f5bc6d25c3c84efa788 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Wed, 23 Apr 2025 13:45:55 -0700 Subject: [PATCH] [Caching] Reduce the number of cas ID passed on frontend commandline Using IncludeTree::FileList to concat the include tree file systems that are passed on the command-line. This significantly reduce the command-line size, and also makes the cache key computation a lot faster. rdar://148752988 --- include/swift/AST/DiagnosticsFrontend.def | 1 + include/swift/Basic/CASOptions.h | 6 +- include/swift/Frontend/CachingUtils.h | 4 +- lib/AST/PluginLoader.cpp | 4 +- lib/ClangImporter/ClangImporter.cpp | 7 +- lib/DependencyScan/ScanDependencies.cpp | 91 ++++++++++++++++------- lib/Frontend/CachingUtils.cpp | 59 +++++---------- lib/Frontend/CompilerInvocation.cpp | 10 +-- lib/Frontend/Frontend.cpp | 4 +- lib/Sema/TypeCheckMacros.cpp | 2 +- test/CAS/module_deps_include_tree.swift | 1 - 11 files changed, 101 insertions(+), 88 deletions(-) diff --git a/include/swift/AST/DiagnosticsFrontend.def b/include/swift/AST/DiagnosticsFrontend.def index db4f2e792a1f9..20384a8f2a2d7 100644 --- a/include/swift/AST/DiagnosticsFrontend.def +++ b/include/swift/AST/DiagnosticsFrontend.def @@ -534,6 +534,7 @@ ERROR(error_cache_key_creation, none, "cannot create cache key for compilation % ERROR(error_cas_file_ref, none, "cannot load file %0 from CAS filesystem", (StringRef)) ERROR(error_cas_conflict_options, none, "cannot setup CAS due to conflicting '-cas-*' options", ()) ERROR(error_cas_initialization, none, "CAS cannot be initialized from the specified '-cas-*' options: %0", (StringRef)) +ERROR(error_cas_malformed_input, none, "CAS input '%0' is malformed: %1", (StringRef, StringRef)) WARNING(cache_replay_failed, none, "cache replay failed: %0", (StringRef)) ERROR(error_failed_cached_diag, none, "failed to serialize cached diagnostics: %0", (StringRef)) diff --git a/include/swift/Basic/CASOptions.h b/include/swift/Basic/CASOptions.h index fb63938e1f3fb..4f01df225ace7 100644 --- a/include/swift/Basic/CASOptions.h +++ b/include/swift/Basic/CASOptions.h @@ -41,10 +41,10 @@ class CASOptions final { std::vector CASFSRootIDs; /// Clang Include Trees. - std::vector ClangIncludeTrees; + std::string ClangIncludeTree; /// Clang Include Tree FileList. - std::vector ClangIncludeTreeFileList; + std::string ClangIncludeTreeFileList; /// CacheKey for input file. std::string InputFileKey; @@ -62,7 +62,7 @@ class CASOptions final { /// Check to see if a CASFileSystem is required. bool requireCASFS() const { return EnableCaching && - (!CASFSRootIDs.empty() || !ClangIncludeTrees.empty() || + (!CASFSRootIDs.empty() || !ClangIncludeTree.empty() || !ClangIncludeTreeFileList.empty() || !InputFileKey.empty() || !BridgingHeaderPCHCacheKey.empty()); } diff --git a/include/swift/Frontend/CachingUtils.h b/include/swift/Frontend/CachingUtils.h index e478c22daf56c..8f249cbd4159b 100644 --- a/include/swift/Frontend/CachingUtils.h +++ b/include/swift/Frontend/CachingUtils.h @@ -71,8 +71,8 @@ std::unique_ptr loadCachedCompileResultFromCacheKey( llvm::Expected> createCASFileSystem(llvm::cas::ObjectStore &CAS, ArrayRef FSRoots, - ArrayRef IncludeTreeRoots, - ArrayRef IncludeTreeFileList); + const std::string &IncludeTreeRoot, + const std::string &IncludeTreeFileList); std::vector remapPathsFromCommandLine( ArrayRef Args, diff --git a/lib/AST/PluginLoader.cpp b/lib/AST/PluginLoader.cpp index 271c484fc99b8..3fb2f6a5ec04d 100644 --- a/lib/AST/PluginLoader.cpp +++ b/lib/AST/PluginLoader.cpp @@ -64,9 +64,9 @@ static StringRef pluginModuleNameStringFromPath(StringRef path) { static llvm::IntrusiveRefCntPtr getPluginLoadingFS(ASTContext &Ctx) { - // If there is a clang include tree FS, using real file system to load plugin + // If there is an immutable file system, using real file system to load plugin // as the FS in SourceMgr doesn't support directory iterator. - if (Ctx.ClangImporterOpts.HasClangIncludeTreeRoot) + if (Ctx.CASOpts.HasImmutableFileSystem) return llvm::vfs::getRealFileSystem(); return Ctx.SourceMgr.getFileSystem(); } diff --git a/lib/ClangImporter/ClangImporter.cpp b/lib/ClangImporter/ClangImporter.cpp index 4b7e17d65a970..2d95fea602fb0 100644 --- a/lib/ClangImporter/ClangImporter.cpp +++ b/lib/ClangImporter/ClangImporter.cpp @@ -1187,8 +1187,7 @@ std::optional> ClangImporter::getClangCC1Arguments( CI->getFrontendOpts().ProgramAction == clang::frontend::ActionKind::GeneratePCH) && ctx.ClangImporterOpts.HasClangIncludeTreeRoot) { - CI->getFrontendOpts().CASIncludeTreeID = - ctx.CASOpts.ClangIncludeTrees.back(); + CI->getFrontendOpts().CASIncludeTreeID = ctx.CASOpts.ClangIncludeTree; CI->getFrontendOpts().Inputs.clear(); } } @@ -1247,7 +1246,7 @@ std::optional> ClangImporter::getClangCC1Arguments( std::vector FilteredModuleMapFiles; for (auto ModuleMapFile : CI->getFrontendOpts().ModuleMapFiles) { - if (ctx.ClangImporterOpts.HasClangIncludeTreeRoot) { + if (ctx.ClangImporterOpts.UseClangIncludeTree) { // There is no need to add any module map file here. Issue a warning and // drop the option. Impl.diagnose(SourceLoc(), diag::module_map_ignored, ModuleMapFile); @@ -1327,7 +1326,7 @@ ClangImporter::create(ASTContext &ctx, fileMapping.requiresBuiltinHeadersInSystemModules; // Avoid creating indirect file system when using include tree. - if (!ctx.ClangImporterOpts.HasClangIncludeTreeRoot) { + if (!ctx.CASOpts.HasImmutableFileSystem) { // Wrap Swift's FS to allow Clang to override the working directory VFS = llvm::vfs::RedirectingFileSystem::create( fileMapping.redirectedFiles, true, *ctx.SourceMgr.getFileSystem()); diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index 8547191c93df1..615b0e79265fe 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -45,7 +45,7 @@ #include "swift/Frontend/FrontendOptions.h" #include "swift/Frontend/ModuleInterfaceLoader.h" #include "swift/Strings.h" -#include "clang/Basic/Module.h" +#include "clang/CAS/IncludeTree.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetOperations.h" #include "llvm/ADT/SetVector.h" @@ -69,7 +69,6 @@ #include #include #include -#include using namespace swift; using namespace swift::dependencies; @@ -101,9 +100,6 @@ class ExplicitModuleDependencyResolver { if (resolvingDepInfo.isFinalized()) return false; - if (auto ID = resolvingDepInfo.getClangIncludeTree()) - includeTrees.push_back(*ID); - for (const auto &depModuleID : dependencies) { const auto &depInfo = cache.findKnownDependency(depModuleID); switch (depModuleID.Kind) { @@ -322,8 +318,10 @@ class ExplicitModuleDependencyResolver { // Collect CAS deppendencies from clang modules. if (!clangDepDetails.CASFileSystemRootID.empty()) rootIDs.push_back(clangDepDetails.CASFileSystemRootID); - if (!clangDepDetails.CASClangIncludeTreeRootID.empty()) - includeTrees.push_back(clangDepDetails.CASClangIncludeTreeRootID); + if (!clangDepDetails.CASClangIncludeTreeRootID.empty()) { + if (addIncludeTree(clangDepDetails.CASClangIncludeTreeRootID)) + return true; + } collectUsedVFSOverlay(clangDepDetails); @@ -358,12 +356,14 @@ class ExplicitModuleDependencyResolver { auto bridgeRoot = tracker->createTreeFromDependencies(); if (!bridgeRoot) return diagnoseCASFSCreationError(bridgeRoot.takeError()); - fileListIDs.push_back(bridgeRoot->getID().toString()); + + fileListRefs.push_back(bridgeRoot->getRef()); } } - } else - includeTrees.push_back(sourceDepDetails.textualModuleDetails - .CASBridgingHeaderIncludeTreeRootID); + } else if (addIncludeTree(sourceDepDetails.textualModuleDetails + .CASBridgingHeaderIncludeTreeRootID)) + return true; + return false; }; @@ -499,9 +499,7 @@ class ExplicitModuleDependencyResolver { auto root = tracker->createTreeFromDependencies(); if (!root) return diagnoseCASFSCreationError(root.takeError()); - auto rootID = root->getID().toString(); - dependencyInfoCopy.updateCASFileSystemRootID(rootID); - fileListIDs.push_back(rootID); + fileListRefs.push_back(root->getRef()); } else if (auto *textualDep = resolvingDepInfo.getAsSwiftInterfaceModule()) { tracker->startTracking(); @@ -516,9 +514,7 @@ class ExplicitModuleDependencyResolver { auto root = tracker->createTreeFromDependencies(); if (!root) return diagnoseCASFSCreationError(root.takeError()); - auto rootID = root->getID().toString(); - dependencyInfoCopy.updateCASFileSystemRootID(rootID); - fileListIDs.push_back(rootID); + fileListRefs.push_back(root->getRef()); } // Update build command line. @@ -530,15 +526,8 @@ class ExplicitModuleDependencyResolver { commandline.push_back(rootID); } - for (auto tree : includeTrees) { - commandline.push_back("-clang-include-tree-root"); - commandline.push_back(tree); - } - - for (auto list : fileListIDs) { - commandline.push_back("-clang-include-tree-filelist"); - commandline.push_back(list); - } + if (computeCASFileSystem(dependencyInfoCopy)) + return true; } // Compute and update module cache key. @@ -636,6 +625,53 @@ class ExplicitModuleDependencyResolver { cmd.push_back("-cache-disable-replay"); } + bool addIncludeTree(StringRef includeTree) { + auto &db = cache.getScanService().getCAS(); + auto casID = db.parseID(includeTree); + if (!casID) { + instance.getDiags().diagnose(SourceLoc(), diag::error_invalid_cas_id, + includeTree, toString(casID.takeError())); + return true; + } + auto ref = db.getReference(*casID); + if (!ref) { + instance.getDiags().diagnose(SourceLoc(), diag::error_load_input_from_cas, + includeTree); + return true; + } + + auto root = clang::cas::IncludeTreeRoot::get(db, *ref); + if (!root) { + instance.getDiags().diagnose(SourceLoc(), diag::error_cas_malformed_input, + includeTree, toString(root.takeError())); + return true; + } + + fileListRefs.push_back(root->getFileListRef()); + return false; + } + + bool computeCASFileSystem(ModuleDependencyInfo &dependencyInfoCopy) { + if (fileListRefs.empty()) + return false; + + auto &db = cache.getScanService().getCAS(); + auto casFS = + clang::cas::IncludeTree::FileList::create(db, {}, fileListRefs); + if (!casFS) { + instance.getDiags().diagnose(SourceLoc(), diag::error_cas, + "CAS IncludeTree FileList creation", + toString(casFS.takeError())); + return true; + } + + auto casID = casFS->getID().toString(); + dependencyInfoCopy.updateCASFileSystemRootID(casID); + commandline.push_back("-clang-include-tree-filelist"); + commandline.push_back(casID); + return false; + } + private: const ModuleDependencyID &moduleID; ModuleDependenciesCache &cache; @@ -644,8 +680,7 @@ class ExplicitModuleDependencyResolver { std::optional tracker; std::vector rootIDs; - std::vector includeTrees; - std::vector fileListIDs; + std::vector fileListRefs; std::vector commandline; std::vector bridgingHeaderBuildCmd; llvm::StringMap macros; diff --git a/lib/Frontend/CachingUtils.cpp b/lib/Frontend/CachingUtils.cpp index 7bc73326332b5..c181854f7708d 100644 --- a/lib/Frontend/CachingUtils.cpp +++ b/lib/Frontend/CachingUtils.cpp @@ -472,16 +472,10 @@ static Expected mergeCASFileSystem(ObjectStore &CAS, Expected> createCASFileSystem(ObjectStore &CAS, ArrayRef FSRoots, - ArrayRef IncludeTrees, - ArrayRef IncludeTreeFileList) { - assert(!FSRoots.empty() || !IncludeTrees.empty() || + const std::string &IncludeTree, + const std::string &IncludeTreeFileList) { + assert(!FSRoots.empty() || !IncludeTree.empty() || !IncludeTreeFileList.empty() && "no root ID provided"); - if (FSRoots.size() == 1 && IncludeTrees.empty()) { - auto ID = CAS.parseID(FSRoots.front()); - if (!ID) - return ID.takeError(); - return createCASFileSystem(CAS, *ID); - } auto NewRoot = mergeCASFileSystem(CAS, FSRoots); if (!NewRoot) @@ -492,10 +486,9 @@ createCASFileSystem(ObjectStore &CAS, ArrayRef FSRoots, return FS.takeError(); auto CASFS = makeIntrusiveRefCnt(std::move(*FS)); - std::vector Files; - // Push all Include File System onto overlay. - for (auto &Tree : IncludeTrees) { - auto ID = CAS.parseID(Tree); + + if (!IncludeTree.empty()) { + auto ID = CAS.parseID(IncludeTree); if (!ID) return ID.takeError(); @@ -510,46 +503,32 @@ createCASFileSystem(ObjectStore &CAS, ArrayRef FSRoots, if (!ITF) return ITF.takeError(); - auto Err = ITF->forEachFile( - [&](clang::cas::IncludeTree::File File, - clang::cas::IncludeTree::FileList::FileSizeTy Size) -> llvm::Error { - Files.push_back({File.getRef(), Size}); - return llvm::Error::success(); - }); + auto ITFS = clang::cas::createIncludeTreeFileSystem(*ITF); + if (!ITFS) + return ITFS.takeError(); - if (Err) - return std::move(Err); + CASFS->pushOverlay(*ITFS); } - for (auto &List: IncludeTreeFileList) { - auto ID = CAS.parseID(List); + if (!IncludeTreeFileList.empty()) { + auto ID = CAS.parseID(IncludeTreeFileList); if (!ID) return ID.takeError(); auto Ref = CAS.getReference(*ID); if (!Ref) return createCASObjectNotFoundError(*ID); - auto IT = clang::cas::IncludeTree::FileList::get(CAS, *Ref); - if (!IT) - return IT.takeError(); + auto ITF = clang::cas::IncludeTree::FileList::get(CAS, *Ref); + if (!ITF) + return ITF.takeError(); - auto Err = IT->forEachFile( - [&](clang::cas::IncludeTree::File File, - clang::cas::IncludeTree::FileList::FileSizeTy Size) -> llvm::Error { - Files.push_back({File.getRef(), Size}); - return llvm::Error::success(); - }); + auto ITFS = clang::cas::createIncludeTreeFileSystem(*ITF); + if (!ITFS) + return ITFS.takeError(); - if (Err) - return std::move(Err); + CASFS->pushOverlay(std::move(*ITFS)); } - auto ITFS = clang::cas::createIncludeTreeFileSystem(CAS, Files); - if (!ITFS) - return ITFS.takeError(); - - CASFS->pushOverlay(std::move(*ITFS)); - return CASFS; } diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index f3f151197c8eb..5e652513acf63 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -763,10 +763,10 @@ static bool ParseCASArgs(CASOptions &Opts, ArgList &Args, for (const auto &A : Args.getAllArgValues(OPT_cas_fs)) Opts.CASFSRootIDs.emplace_back(A); - for (const auto &A : Args.getAllArgValues(OPT_clang_include_tree_root)) - Opts.ClangIncludeTrees.emplace_back(A); - for (const auto &A : Args.getAllArgValues(OPT_clang_include_tree_filelist)) - Opts.ClangIncludeTreeFileList.emplace_back(A); + if (auto *A = Args.getLastArg(OPT_clang_include_tree_root)) + Opts.ClangIncludeTree = A->getValue(); + if (auto *A = Args.getLastArg(OPT_clang_include_tree_filelist)) + Opts.ClangIncludeTreeFileList = A->getValue(); if (const Arg *A = Args.getLastArg(OPT_input_file_key)) Opts.InputFileKey = A->getValue(); @@ -774,7 +774,7 @@ static bool ParseCASArgs(CASOptions &Opts, ArgList &Args, if (const Arg*A = Args.getLastArg(OPT_bridging_header_pch_key)) Opts.BridgingHeaderPCHCacheKey = A->getValue(); - if (!Opts.CASFSRootIDs.empty() || !Opts.ClangIncludeTrees.empty() || + if (!Opts.CASFSRootIDs.empty() || !Opts.ClangIncludeTree.empty() || !Opts.ClangIncludeTreeFileList.empty()) Opts.HasImmutableFileSystem = true; diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp index f95612e723286..894c321c968f9 100644 --- a/lib/Frontend/Frontend.cpp +++ b/lib/Frontend/Frontend.cpp @@ -630,11 +630,11 @@ bool CompilerInstance::setUpVirtualFileSystemOverlays() { } if (Invocation.getCASOptions().requireCASFS()) { - if (!CASOpts.CASFSRootIDs.empty() || !CASOpts.ClangIncludeTrees.empty() || + if (!CASOpts.CASFSRootIDs.empty() || !CASOpts.ClangIncludeTree.empty() || !CASOpts.ClangIncludeTreeFileList.empty()) { // Set up CASFS as BaseFS. auto FS = createCASFileSystem(*CAS, CASOpts.CASFSRootIDs, - CASOpts.ClangIncludeTrees, + CASOpts.ClangIncludeTree, CASOpts.ClangIncludeTreeFileList); if (!FS) { Diagnostics.diagnose(SourceLoc(), diag::error_cas_fs_creation, diff --git a/lib/Sema/TypeCheckMacros.cpp b/lib/Sema/TypeCheckMacros.cpp index 4428cf052f4ae..b2ea98fe99a18 100644 --- a/lib/Sema/TypeCheckMacros.cpp +++ b/lib/Sema/TypeCheckMacros.cpp @@ -282,7 +282,7 @@ initializePlugin(ASTContext &ctx, CompilerPlugin *plugin, StringRef libraryPath, if (!libraryPath.empty()) { #if SWIFT_BUILD_SWIFT_SYNTAX llvm::SmallString<128> resolvedLibraryPath; - auto fs = ctx.ClangImporterOpts.HasClangIncludeTreeRoot + auto fs = ctx.CASOpts.HasImmutableFileSystem ? llvm::vfs::getRealFileSystem() : ctx.SourceMgr.getFileSystem(); if (auto err = fs->getRealPath(libraryPath, resolvedLibraryPath)) { diff --git a/test/CAS/module_deps_include_tree.swift b/test/CAS/module_deps_include_tree.swift index 711d84a680864..5ae9d06981603 100644 --- a/test/CAS/module_deps_include_tree.swift +++ b/test/CAS/module_deps_include_tree.swift @@ -41,7 +41,6 @@ // INCLUDE_TREE_F-NEXT: CHeaders/F.h // MAIN_CMD: -direct-clang-cc1-module-build -// MAIN_CMD: -clang-include-tree-root // MAIN_CMD: -clang-include-tree-filelist import C