Skip to content

Commit f199440

Browse files
authored
Add list of layers file to spfs run command (#1136)
Adds support for list of layer references file (a runspec) to spfs EnvSpec items There are now 2 kinds of .spfs.yaml file that can be given to spfs run as environment spec items. This adds parsing and enums to support both, and moves LiveLayers to their own source code file. Signed-off-by: David Gilligan-Cook <[email protected]>
1 parent 2e722ec commit f199440

File tree

24 files changed

+663
-402
lines changed

24 files changed

+663
-402
lines changed

Cargo.lock

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

crates/spfs-cli/main/src/cmd_ls.rs

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,7 @@ impl CmdLs {
5757
self.username = tag.username_without_org().to_string();
5858
self.last_modified = tag.time.format("%b %e %H:%M").to_string();
5959
}
60-
EnvSpecItem::PartialDigest(_)
61-
| EnvSpecItem::Digest(_)
62-
| EnvSpecItem::LiveLayerFile(_) => (),
60+
EnvSpecItem::PartialDigest(_) | EnvSpecItem::Digest(_) | EnvSpecItem::SpecFile(_) => (),
6361
}
6462

6563
let item = repo.read_ref(&self.reference.to_string()).await?;

crates/spfs-cli/main/src/cmd_run.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -210,7 +210,7 @@ impl CmdRun {
210210

211211
self.exec_runtime_command(&mut runtime, &start_time).await
212212
} else if let Some(reference) = &self.reference {
213-
let live_layers = reference.load_live_layers()?;
213+
let live_layers = reference.load_live_layers();
214214
if !live_layers.is_empty() {
215215
tracing::debug!("with live layers: {live_layers:?}");
216216
};
@@ -305,7 +305,7 @@ impl CmdRun {
305305
let repo = spfs::storage::ProxyRepository::from_config(proxy_config)
306306
.await
307307
.wrap_err("Failed to build proxy repository for environment resolution")?;
308-
for item in reference.iter().filter(|i| !i.is_livelayerfile()) {
308+
for item in reference.iter().filter(|i| !i.is_livelayer()) {
309309
let digest = item.resolve_digest(&repo).await?;
310310
runtime.push_digest(digest);
311311
}

crates/spfs-encoding/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ workspace = true
1515
[dependencies]
1616
data-encoding = { workspace = true }
1717
ring = { workspace = true }
18+
serde = { workspace = true, features = ["derive"] }
1819
spfs-proto = { path = "../spfs-proto" }
1920
tokio = { version = "1.20", features = ["io-util", "io-std"] }
2021
thiserror = { workspace = true }

crates/spfs-encoding/src/hash.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ use std::task::Poll;
1010

1111
use data_encoding::BASE32;
1212
use ring::digest::{Context, SHA256};
13+
use serde::Deserialize;
1314
use tokio::io::{AsyncRead, AsyncWrite};
1415

1516
use super::{binary, Digest};
@@ -259,7 +260,7 @@ impl Decodable for String {
259260
}
260261

261262
/// The first N bytes of a digest that may still be unambiguous as a reference
262-
#[derive(Debug, Hash, Eq, PartialEq, Ord, PartialOrd, Clone)]
263+
#[derive(Deserialize, Debug, Hash, Eq, PartialEq, Ord, PartialOrd, Clone)]
263264
pub struct PartialDigest(Vec<u8>);
264265

265266
impl PartialDigest {

crates/spfs/src/check.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@ where
161161
.check_tag_spec(tag_spec)
162162
.await
163163
.map(CheckEnvItemResult::Tag)?,
164-
tracking::EnvSpecItem::LiveLayerFile(_) => {
164+
tracking::EnvSpecItem::SpecFile(_) => {
165165
// These items do not need checking, they can be ignored.
166166
// They do no represent spfs objects in a repo.
167167
CheckEnvItemResult::Object(CheckObjectResult::Ignorable)

crates/spfs/src/error.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
// https://github.com/spkenv/spk
44

55
use std::io;
6+
use std::path::PathBuf;
67
use std::str::Utf8Error;
78

89
use miette::Diagnostic;
@@ -165,6 +166,9 @@ pub enum Error {
165166
#[error("OverlayFS mount backend is not supported on windows.")]
166167
OverlayFsUnsupportedOnWindows,
167168

169+
#[error("Found duplicate spec file ({0}). Spec files can only be given once and must not contain circular references.")]
170+
DuplicateSpecFileReference(PathBuf),
171+
168172
#[error("{context}")]
169173
Wrapped {
170174
context: String,

crates/spfs/src/resolve.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -146,9 +146,9 @@ pub async fn compute_environment_manifest(
146146
tracking::EnvSpecItem::TagSpec(t) => {
147147
Some(repo.resolve_tag(t).map_ok(|t| t.target).boxed())
148148
}
149-
// LiveLayers are stored in the runtime, not as spfs
149+
// SpfsSpecFiles are stored in the runtime, not as spfs
150150
// objects/layers, so this filters them out.
151-
tracking::EnvSpecItem::LiveLayerFile(_) => None,
151+
tracking::EnvSpecItem::SpecFile(_) => None,
152152
})
153153
.collect();
154154
let stack = stack_futures.try_collect().await?;
Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
// Copyright (c) Contributors to the SPK project.
2+
// SPDX-License-Identifier: Apache-2.0
3+
// https://github.com/spkenv/spk
4+
5+
//! Definition and persistent storage of runtimes.
6+
7+
use std::fmt::Display;
8+
use std::path::PathBuf;
9+
10+
use serde::{Deserialize, Serialize};
11+
12+
use super::spec_api_version::SpecApiVersion;
13+
use crate::{Error, Result};
14+
15+
#[cfg(test)]
16+
#[path = "./live_layer_test.rs"]
17+
mod live_layer_test;
18+
19+
/// Data needed to bind mount a path onto an /spfs backend that uses
20+
/// overlayfs.
21+
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
22+
pub struct BindMount {
23+
/// Path to the source dir, or file, to bind mount into /spfs at
24+
/// the destination.
25+
#[serde(alias = "bind")]
26+
pub src: PathBuf,
27+
/// Where to attach the dir, or file, inside /spfs
28+
pub dest: String,
29+
}
30+
31+
impl BindMount {
32+
/// Checks the bind mount is valid for use in /spfs with the given parent directory
33+
pub(crate) fn validate(&self, parent: PathBuf) -> Result<()> {
34+
if !self.src.starts_with(parent.clone()) {
35+
return Err(Error::String(format!(
36+
"Bind mount is not valid: {} is not under the live layer's directory: {}",
37+
self.src.display(),
38+
parent.display()
39+
)));
40+
}
41+
42+
if !self.src.exists() {
43+
return Err(Error::String(format!(
44+
"Bind mount is not valid: {} does not exist",
45+
self.src.display()
46+
)));
47+
}
48+
49+
Ok(())
50+
}
51+
}
52+
53+
impl Display for BindMount {
54+
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
55+
write!(f, "{}:{}", self.src.display(), self.dest)
56+
}
57+
}
58+
59+
/// The kinds of contents that can be part of a live layer
60+
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
61+
#[serde(untagged)]
62+
pub enum LiveLayerContents {
63+
/// A directory or file that will be bind mounted over /spfs
64+
BindMount(BindMount),
65+
}
66+
67+
/// Data needed to add a live layer onto an /spfs overlayfs.
68+
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
69+
pub struct LiveLayer {
70+
/// The api format version of the live layer data
71+
pub api: SpecApiVersion,
72+
/// The contents that the live layer will put into /spfs
73+
pub contents: Vec<LiveLayerContents>,
74+
}
75+
76+
impl LiveLayer {
77+
/// Returns a list of the BindMounts in this LiveLayer
78+
pub fn bind_mounts(&self) -> Vec<BindMount> {
79+
self.contents
80+
.iter()
81+
.map(|c| match c {
82+
LiveLayerContents::BindMount(bm) => bm.clone(),
83+
})
84+
.collect::<Vec<_>>()
85+
}
86+
87+
/// Updates the live layer's contents entries using given parent
88+
/// directory. This will error if the resulting paths do not exist.
89+
///
90+
/// This should be called before validate()
91+
fn set_parent(&mut self, parent: PathBuf) -> Result<()> {
92+
let mut new_contents = Vec::new();
93+
94+
for entry in self.contents.iter() {
95+
let new_entry = match entry {
96+
LiveLayerContents::BindMount(bm) => {
97+
let full_path = match parent.join(bm.src.clone()).canonicalize() {
98+
Ok(abs_path) => abs_path.clone(),
99+
Err(err) => {
100+
return Err(Error::InvalidPath(parent.join(bm.src.clone()), err))
101+
}
102+
};
103+
104+
LiveLayerContents::BindMount(BindMount {
105+
src: full_path,
106+
dest: bm.dest.clone(),
107+
})
108+
}
109+
};
110+
111+
new_contents.push(new_entry);
112+
}
113+
self.contents = new_contents;
114+
115+
Ok(())
116+
}
117+
118+
/// Validates the live layer's contents are under the given parent
119+
/// directory and accessible by the current user.
120+
///
121+
/// This should be called after set_parent()
122+
fn validate(&self, parent: PathBuf) -> Result<()> {
123+
for entry in self.contents.iter() {
124+
match entry {
125+
LiveLayerContents::BindMount(bm) => bm.validate(parent.clone())?,
126+
}
127+
}
128+
Ok(())
129+
}
130+
131+
/// Sets the live layer's parent directory, which updates its
132+
/// contents, and then validates its contents.
133+
pub fn set_parent_and_validate(&mut self, parent: PathBuf) -> Result<()> {
134+
let abs_parent = match parent.canonicalize() {
135+
Ok(abs_path) => abs_path.clone(),
136+
Err(err) => return Err(Error::InvalidPath(parent.clone(), err)),
137+
};
138+
139+
self.set_parent(parent.clone())?;
140+
self.validate(abs_parent)
141+
}
142+
}
143+
144+
impl Display for LiveLayer {
145+
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
146+
write!(f, "{}:{:?}", self.api, self.contents)
147+
}
148+
}
Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
// Copyright (c) Contributors to the SPK project.
2+
// SPDX-License-Identifier: Apache-2.0
3+
// https://github.com/spkenv/spk
4+
5+
use std::fs::File;
6+
use std::io::Write;
7+
use std::path::PathBuf;
8+
use std::str::FromStr;
9+
10+
use rstest::rstest;
11+
12+
use crate::fixtures::*;
13+
use crate::runtime::{BindMount, LiveLayer, SpecApiVersion};
14+
use crate::tracking::SpecFile;
15+
16+
#[rstest]
17+
fn test_bindmount_creation() {
18+
let dir = "/some/dir/some/where";
19+
let mountpoint = "tests/tests/tests".to_string();
20+
let expected = format!("{dir}:{mountpoint}");
21+
22+
let mount = BindMount {
23+
src: PathBuf::from(dir),
24+
dest: mountpoint,
25+
};
26+
27+
assert_eq!(mount.to_string(), expected);
28+
}
29+
30+
#[rstest]
31+
fn test_bindmount_validate(tmpdir: tempfile::TempDir) {
32+
let path = tmpdir.path();
33+
let subdir = path.join("somedir");
34+
std::fs::create_dir(subdir.clone()).unwrap();
35+
36+
let mountpoint = "tests/tests/tests".to_string();
37+
38+
let mount = BindMount {
39+
src: subdir,
40+
dest: mountpoint,
41+
};
42+
43+
assert!(mount.validate(path.to_path_buf()).is_ok());
44+
}
45+
46+
#[rstest]
47+
fn test_bindmount_validate_fail_not_under_parent(tmpdir: tempfile::TempDir) {
48+
let path = tmpdir.path();
49+
let subdir = path.join("somedir");
50+
std::fs::create_dir(subdir.clone()).unwrap();
51+
52+
let mountpoint = "tests/tests/tests".to_string();
53+
54+
let mount = BindMount {
55+
src: subdir,
56+
dest: mountpoint,
57+
};
58+
59+
assert!(mount
60+
.validate(PathBuf::from_str("/tmp/no/its/parent/").unwrap())
61+
.is_err());
62+
}
63+
64+
#[rstest]
65+
fn test_bindmount_validate_fail_not_exists(tmpdir: tempfile::TempDir) {
66+
let path = tmpdir.path();
67+
let subdir = path.join("somedir");
68+
std::fs::create_dir(subdir.clone()).unwrap();
69+
70+
let mountpoint = "tests/tests/tests".to_string();
71+
72+
let missing_subdir = subdir.join("not_made");
73+
74+
let mount = BindMount {
75+
src: missing_subdir,
76+
dest: mountpoint,
77+
};
78+
79+
assert!(mount.validate(path.to_path_buf()).is_err());
80+
}
81+
82+
#[rstest]
83+
fn test_live_layer_file_load(tmpdir: tempfile::TempDir) {
84+
let dir = tmpdir.path();
85+
86+
let subdir = dir.join("testing");
87+
std::fs::create_dir(subdir.clone()).unwrap();
88+
89+
let yaml = format!(
90+
"# test live layer\napi: v0/livelayer\ncontents:\n - bind: {}\n dest: /spfs/test\n",
91+
subdir.display()
92+
);
93+
94+
let file_path = dir.join("layer.spfs.yaml");
95+
let mut tmp_file = File::create(file_path).unwrap();
96+
writeln!(tmp_file, "{}", yaml).unwrap();
97+
98+
let ll = SpecFile::parse(&dir.display().to_string()).unwrap();
99+
100+
if let SpecFile::LiveLayer(live_layer) = ll {
101+
assert!(live_layer.api == SpecApiVersion::V0Layer);
102+
assert!(!live_layer.contents.is_empty());
103+
assert!(live_layer.contents.len() == 1);
104+
} else {
105+
panic!("The test yaml should have parsed as a live layer, but it didn't")
106+
}
107+
}
108+
109+
#[rstest]
110+
fn test_live_layer_minimal_deserialize() {
111+
// Test a minimal yaml string that represents a LiveLayer. Note:
112+
// if more LiveLayer fields are added in future, they should have
113+
// #[serde(default)] set or be optional, so they are backwards
114+
// compatible with existing live layer configurations.
115+
let yaml: &str = "api: v0/livelayer\ncontents:\n";
116+
117+
let layer: LiveLayer = serde_yaml::from_str(yaml).unwrap();
118+
119+
assert!(layer.api == SpecApiVersion::V0Layer);
120+
}
121+
122+
#[rstest]
123+
#[should_panic]
124+
fn test_live_layer_deserialize_fail_no_contents_field() {
125+
let yaml: &str = "api: v0/livelayer\n";
126+
127+
// This should panic because the contents: field is missing
128+
let _layer: LiveLayer = serde_yaml::from_str(yaml).unwrap();
129+
}
130+
131+
#[rstest]
132+
#[should_panic]
133+
fn test_live_layer_deserialize_unknown_version() {
134+
let yaml: &str = "api: v9999999999999/invalidapi\ncontents:\n";
135+
136+
// This should panic because the api value is invalid
137+
let _layer: LiveLayer = serde_yaml::from_str(yaml).unwrap();
138+
}

0 commit comments

Comments
 (0)