Hack lldb to load CAS content

This is a hack and very rough.

Author: cachemeifyoucan

clang/test/ClangScanDeps/gmodules.c

@@ -218,6 +218,7 @@ int main(void) {
   struct Right _right;
   struct Top _top;
   struct Prefix _prefix;
+  return 0;
 }
 
 //--- cas-config.template

lldb/include/lldb/Host/FileSystem.h

@@ -16,6 +16,7 @@
 #include "lldb/Utility/Status.h"
 #include "lldb/Utility/TildeExpressionResolver.h"
 
+#include "llvm/CAS/AutoCASFileSystem.h"
 #include "llvm/Support/Chrono.h"
 #include "llvm/Support/VirtualFileSystem.h"
 
@@ -33,13 +34,15 @@ class FileSystem {
   static const char *PATH_CONVERSION_ERROR;
 
   FileSystem()
-      : m_fs(llvm::vfs::getRealFileSystem()),
+      : m_fs(
+            llvm::cas::createAutoCASFileSystem(llvm::vfs::getRealFileSystem())),
         m_tilde_resolver(std::make_unique<StandardTildeExpressionResolver>()) {}
   FileSystem(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs)
-      : m_fs(std::move(fs)),
+      : m_fs(llvm::cas::createAutoCASFileSystem(std::move(fs))),
         m_tilde_resolver(std::make_unique<StandardTildeExpressionResolver>()) {}
   FileSystem(std::unique_ptr<TildeExpressionResolver> tilde_resolver)
-      : m_fs(llvm::vfs::getRealFileSystem()),
+      : m_fs(
+            llvm::cas::createAutoCASFileSystem(llvm::vfs::getRealFileSystem())),
         m_tilde_resolver(std::move(tilde_resolver)) {}
 
   FileSystem(const FileSystem &fs) = delete;

lldb/source/Host/common/FileSystem.cpp

@@ -295,12 +295,18 @@ FileSystem::CreateWritableDataBuffer(const llvm::Twine &path, uint64_t size,
 std::shared_ptr<DataBuffer>
 FileSystem::CreateDataBuffer(const llvm::Twine &path, uint64_t size,
                              uint64_t offset) {
-  const bool is_volatile = !IsLocal(path);
-  auto buffer =
-      GetMemoryBuffer<llvm::MemoryBuffer>(path, size, offset, is_volatile);
+  auto buffer = m_fs->getBufferForFile(path);
   if (!buffer)
     return {};
-  return std::make_shared<DataBufferLLVM>(std::move(buffer));
+
+  if (size == 0)
+    return std::make_shared<DataBufferLLVM>(std::move(*buffer));
+
+  auto slice = (*buffer)->getBuffer().slice(offset, offset + size);
+  auto sliced_buffer = llvm::MemoryBuffer::getMemBufferCopy(
+      slice, (*buffer)->getBufferIdentifier());
+
+  return std::make_shared<DataBufferLLVM>(std::move(sliced_buffer));
 }
 
 std::shared_ptr<WritableDataBuffer>

lldb/source/Utility/FileSpec.cpp

@@ -395,6 +395,9 @@ void FileSpec::GetPath(llvm::SmallVectorImpl<char> &path,
                        bool denormalize) const {
   path.append(m_directory.GetStringRef().begin(),
               m_directory.GetStringRef().end());
+  // FIXME: HACK
+  if (llvm::toStringRef(path).starts_with("llvmcas:"))
+    path.insert(path.end(), '/');
   // Since the path was normalized and all paths use '/' when stored in these
   // objects, we don't need to look for the actual syntax specific path
   // separator, we just look for and insert '/'.

llvm/include/llvm/CAS/AutoCASFileSystem.h

@@ -0,0 +1,48 @@
+//===- llvm/CAS/AutoCASFileSystem.h -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CAS_AUTOCASDISKFILESYSTEM_H
+#define LLVM_CAS_AUTOCASDISKFILESYSTEM_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CAS/CASConfiguration.h"
+#include "llvm/CAS/ObjectStore.h"
+#include "llvm/Support/VirtualFileSystem.h"
+
+namespace llvm::cas {
+
+/// A file system that will discover CASConfiguration when accessing new files,
+/// and able to load CASIDs after new CASes are loaded from discovered
+/// CASConfiguration.
+class AutoCASFileSystem : public vfs::ProxyFileSystem {
+  virtual void anchor() override;
+
+public:
+  AutoCASFileSystem(IntrusiveRefCntPtr<FileSystem> Base)
+      : vfs::ProxyFileSystem(Base) {}
+
+  llvm::ErrorOr<std::unique_ptr<vfs::File>>
+  openFileForRead(const Twine &Path) override;
+
+  ErrorOr<vfs::Status> status(const Twine &Path) override;
+
+private:
+  llvm::Error discoverNewCASConfiguration(const llvm::Twine &Path);
+  std::optional<std::unique_ptr<vfs::File>> lookupMaybeCASID(StringRef Path);
+
+  SmallVector<std::pair<CASConfiguration, std::shared_ptr<ObjectStore>>, 4>
+      LoadedCAS;
+  std::mutex DiscoverLock;
+};
+
+llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
+createAutoCASFileSystem(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> Base);
+
+} // namespace llvm::cas
+
+#endif

llvm/include/llvm/CAS/CASConfiguration.h

@@ -71,7 +71,7 @@ class CASConfiguration {
   static std::optional<std::pair<std::string, CASConfiguration>>
   createFromSearchConfigFile(
       StringRef Path,
-      llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS = nullptr);
+      llvm::vfs::FileSystem &VFS = *llvm::vfs::getRealFileSystem());
 };
 
 } // namespace llvm::cas

llvm/lib/CAS/AutoCASFileSystem.cpp

@@ -0,0 +1,111 @@
+//===- AutoCASFileSystem.cpp ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CAS/AutoCASFileSystem.h"
+#include "llvm/ADT/STLExtras.h"
+
+using namespace llvm;
+using namespace llvm::cas;
+
+void AutoCASFileSystem::anchor() {}
+
+namespace {
+class CASFile final : public vfs::File {
+  ObjectProxy Proxy;
+
+public:
+  CASFile(ObjectProxy Proxy) : Proxy(Proxy) {}
+
+  ErrorOr<vfs::Status> status() override {
+    // Return a mostly default constructed status but get the file size correct.
+    return vfs::Status(/*Name=*/{}, /*UID=*/{}, /*MTime=*/{}, /*User=*/{},
+                       /*Group=*/{}, /*Size=*/Proxy.getData().size(),
+                       /*Type=*/{}, /*Perms=*/{});
+  }
+
+  ErrorOr<std::unique_ptr<MemoryBuffer>> getBuffer(const Twine &Name,
+                                                   int64_t FileSize,
+                                                   bool RequiresNullTerminator,
+                                                   bool IsVolatile) override {
+    return Proxy.getMemoryBuffer();
+  }
+
+  ErrorOr<std::optional<cas::ObjectRef>> getObjectRefForContent() override {
+    return Proxy.getRef();
+  }
+
+  std::error_code close() override { return std::error_code(); }
+};
+} // namespace
+
+ErrorOr<std::unique_ptr<vfs::File>>
+AutoCASFileSystem::openFileForRead(const Twine &Path) {
+  if (auto E = discoverNewCASConfiguration(Path))
+    return errorToErrorCode(std::move(E));
+  if (auto File = lookupMaybeCASID(Path.str()))
+    return std::move(*File);
+  return vfs::ProxyFileSystem::openFileForRead(Path);
+}
+
+ErrorOr<vfs::Status> AutoCASFileSystem::status(const Twine &Path) {
+  if (auto E = discoverNewCASConfiguration(Path))
+    return errorToErrorCode(std::move(E));
+  if (auto File = lookupMaybeCASID(Path.str()))
+    return (*File)->status();
+  return vfs::ProxyFileSystem::status(Path);
+}
+
+Error AutoCASFileSystem::discoverNewCASConfiguration(const Twine &Path) {
+  auto NewConfig = CASConfiguration::createFromSearchConfigFile(
+      Path.str(), getUnderlyingFS());
+  if (!NewConfig)
+    return Error::success();
+
+  std::scoped_lock<std::mutex> Lock(DiscoverLock);
+
+  if (llvm::find_if(LoadedCAS, [&](const auto &Entry) {
+        return Entry.first == NewConfig->second;
+      }) != LoadedCAS.end())
+    return Error::success();
+
+  auto DB = NewConfig->second.createDatabases();
+  if (!DB)
+    return DB.takeError();
+
+  LoadedCAS.emplace_back(std::move(NewConfig->second), std::move(DB->first));
+  return Error::success();
+}
+
+std::optional<std::unique_ptr<vfs::File>>
+AutoCASFileSystem::lookupMaybeCASID(StringRef Path) {
+  if (LoadedCAS.empty())
+    return std::nullopt;
+
+  std::scoped_lock<std::mutex> Lock(DiscoverLock);
+  for (auto &CAS : llvm::reverse(LoadedCAS)) {
+    auto ID = CAS.second->parseID(Path);
+    if (!ID) {
+      consumeError(ID.takeError());
+      return std::nullopt;
+    }
+    auto Proxy = CAS.second->getProxy(*ID);
+    if (!Proxy) {
+      consumeError(Proxy.takeError());
+      return std::nullopt;
+    }
+    return std::make_unique<CASFile>(*Proxy);
+  }
+  return std::nullopt;
+}
+
+llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
+llvm::cas::createAutoCASFileSystem(
+    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> Base) {
+  return llvm::makeIntrusiveRefCnt<llvm::cas::AutoCASFileSystem>(
+      std::move(Base));
+}

llvm/lib/CAS/CASConfiguration.cpp

@@ -90,15 +90,12 @@ CASConfiguration::createFromConfig(StringRef Content) {
 }
 
 std::optional<std::pair<std::string, CASConfiguration>>
-CASConfiguration::createFromSearchConfigFile(
-    StringRef Path, IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
-  if (!VFS)
-    VFS = vfs::getRealFileSystem();
-
+CASConfiguration::createFromSearchConfigFile(StringRef Path,
+                                             vfs::FileSystem &VFS) {
   while (!Path.empty()) {
     SmallString<256> ConfigPath(Path);
     sys::path::append(ConfigPath, ".cas-config");
-    auto File = VFS->openFileForRead(ConfigPath);
+    auto File = VFS.openFileForRead(ConfigPath);
     if (!File || !*File) {
       Path = sys::path::parent_path(Path);
       continue;

llvm/lib/CAS/CMakeLists.txt

@@ -5,6 +5,7 @@ endif()
 add_llvm_component_library(LLVMCAS
   ActionCache.cpp
   ActionCaches.cpp
+  AutoCASFileSystem.cpp
   BuiltinCAS.cpp
   BuiltinUnifiedCASDatabases.cpp
   CASConfiguration.cpp

llvm/tools/dsymutil/dsymutil.cpp

@@ -24,6 +24,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/AutoCASFileSystem.h"
 #include "llvm/CAS/BuiltinUnifiedCASDatabases.h"
 #include "llvm/CAS/ObjectStore.h"
 #include "llvm/DebugInfo/DIContext.h"