harmonia-file: async file-tree libraries backed by cap-std

Add harmonia-file-io-pure and harmonia-file-fd implementing a generic
async file-tree abstraction mirroring nix's SourceAccessor and
FileSystemObjectSink, slim harmonia-file-core to the shared data types,
and route NAR dump/restore plus harmonia-cache/narlist through the new
FileSystemSource/FileSystemSink traits. The filesystem source wraps the
released cap-std in spawn_blocking rather than depending on the
unreleased cap-tokio PR, which is only that same sync crate behind
spawn_blocking anyway. read_link uses read_link_contents so absolute
symlink targets come back verbatim, and the dump strips the case-hack
suffix on case-insensitive filesystems to match nix.

Co-authored-by: Amaan Qureshi <git@amaanq.com>

Author: Ericson2314

Cargo.lock

@@ -17,6 +17,8 @@ members = [
     "harmonia-store-remote",
     "harmonia-store-aterm",
     "harmonia-file-core",
+    "harmonia-file-fd",
+    "harmonia-file-io-pure",
     "harmonia-file-nar",
     "harmonia-protocol",
     "harmonia-protocol-derive",
@@ -45,6 +47,7 @@ async-stream = "0.3"
 blake3 = "1.8"
 bstr = "1.11"
 bytes = "1.11"
+cap-std = "3"
 data-encoding = "2.11"
 derive_more = { version = "2.1", features = [ "display" ] }
 ed25519-dalek = "2"
@@ -98,6 +101,8 @@ tokio-test      = "0.4"
 
 # Internal crates
 harmonia-file-core             = { path = "harmonia-file-core" }
+harmonia-file-fd               = { path = "harmonia-file-fd" }
+harmonia-file-io-pure          = { path = "harmonia-file-io-pure" }
 harmonia-file-nar              = { path = "harmonia-file-nar" }
 harmonia-protocol              = { path = "harmonia-protocol" }
 harmonia-protocol-derive       = { path = "harmonia-protocol-derive" }

Cargo.toml

@@ -22,8 +22,8 @@ composed.
 ┌────────────────────────────┬─────────────────────────┐
 │  Format / File             │  Database               │
 │  harmonia-file-nar         │  harmonia-store-db      │
-│  harmonia-file-core        │                         │
-│  NAR pack/unpack, types    │  SQLite store metadata  │
+│  harmonia-file-core/io/fd  │                         │
+│  NAR pack/unpack, file I/O │  SQLite store metadata  │
 └────────────────────────────┴─────────────────────────┘
                          ↓
 ┌──────────────────────────────────────────────────────┐
@@ -51,7 +51,9 @@ composed.
 | [harmonia-store-aterm](../../harmonia-store-aterm/) | ATerm derivation parser |
 | [harmonia-store-path-info](../../harmonia-store-path-info/) | ValidPathInfo types (pure) |
 | [harmonia-store-db](../../harmonia-store-db/README.md) | SQLite store metadata |
-| [harmonia-file-core](../../harmonia-file-core/) | File tree types and serde (pure) |
+| [harmonia-file-core](../../harmonia-file-core/) | File tree data types and serde |
+| [harmonia-file-io-pure](../../harmonia-file-io-pure/) | Async IO traits, in-memory impls, listing |
+| [harmonia-file-fd](../../harmonia-file-fd/) | Filesystem source/sink via cap-std |
 | [harmonia-file-nar](../../harmonia-file-nar/README.md) | NAR archive format |
 | [harmonia-protocol](../../harmonia-protocol/README.md) | Daemon wire protocol |
 | [harmonia-protocol-derive](../../harmonia-protocol-derive/README.md) | Derive macros for protocol types |
@@ -100,16 +102,20 @@ graph BT
     end
     subgraph "File"
         file-core
+        file-fd
+        file-io-pure
         file-nar
     end
     bench
     client
     ssh-store
-    file-nar --> file-core
-    file-nar --> utils-io
+    file-io-pure --> file-core
     utils-hash --> utils-base-encoding
     utils-signature --> utils-base-encoding
+    file-fd --> file-io-pure
     store-path --> utils-hash
+    file-nar --> file-fd
+    file-nar --> utils-io
     store-content-address --> store-path
     store-ref-scan --> store-path
     store-derivation --> store-content-address
@@ -155,9 +161,11 @@ intra-workspace dependencies.
 - `harmonia-store-derivation`: derivations, derived paths, placeholders, realisations — depends on `harmonia-store-content-address`.
 - `harmonia-store-aterm`, `harmonia-store-path-info`, `harmonia-store-build-result`, `harmonia-store-nar-info`: format/metadata crates kept separate to avoid coupling.
 
-**File** (`harmonia-file-core`, `harmonia-file-nar`)
-- `harmonia-file-core`: pure file tree types and serde matching nix's JSON format.
-- `harmonia-file-nar`: NAR pack/unpack against generic `AsyncRead`/`AsyncWrite`. Streaming; never requires the full input in memory. Knows nothing about derivations or signatures.
+**File** (`harmonia-file-core`, `harmonia-file-io-pure`, `harmonia-file-fd`, `harmonia-file-nar`)
+- `harmonia-file-core`: pure file tree data types (`FileTree`, `FileSystemObject`), serde matching nix's JSON format.
+- `harmonia-file-io-pure`: async IO traits (`FileSystemSource`/`FileSystemSink`), in-memory implementations, listing functions.
+- `harmonia-file-fd`: real filesystem source/sink via `cap-std` (openat, no symlink following).
+- `harmonia-file-nar`: NAR pack/unpack against generic `FileSystemSource`/`FileSystemSink` traits. Streaming; never requires the full input in memory. Knows nothing about derivations or signatures.
 
 **Database** (`harmonia-store-db`)
 - Schema matches Nix's `db.sqlite` exactly.

docs/architecture/harmonia-store-structure.md

@@ -6,6 +6,7 @@ edition.workspace    = true
 license.workspace    = true
 homepage.workspace   = true
 repository.workspace = true
+readme               = "README.md"
 
 [[bin]]
 name = "harmonia-cache"
@@ -42,6 +43,8 @@ bytes                        = { workspace = true }
 futures-core                 = { workspace = true }
 futures-util                 = { workspace = true }
 harmonia-file-core           = { workspace = true }
+harmonia-file-fd             = { workspace = true }
+harmonia-file-io-pure        = { workspace = true }
 harmonia-file-nar            = { workspace = true }
 harmonia-store-db            = { workspace = true }
 harmonia-store-derivation    = { workspace = true }

harmonia-cache/Cargo.toml

@@ -71,6 +71,7 @@ pub enum ServeError {
     },
 
     #[error("Access denied: {path}")]
+    #[allow(dead_code)]
     AccessDenied { path: String },
 }