diff --git a/Cargo.lock b/Cargo.lock index 81e6addc554..30fea417489 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1505,6 +1505,18 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "filetime" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.3.5", + "windows-sys 0.48.0", +] + [[package]] name = "finl_unicode" version = "1.2.0" @@ -2053,6 +2065,18 @@ dependencies = [ "serde", ] +[[package]] +name = "indicatif" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b" +dependencies = [ + "console", + "lazy_static", + "number_prefix", + "regex", +] + [[package]] name = "insta" version = "1.34.0" @@ -2626,6 +2650,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.32.1" @@ -4176,12 +4206,15 @@ dependencies = [ "dirs", "duct", "email_address", + "flate2", "futures", + "indicatif", "insta", "is-terminal", "itertools 0.11.0", "jsonwebtoken", "mimalloc", + "regex", "reqwest", "rustyline", "serde", @@ -4192,6 +4225,7 @@ dependencies = [ "spacetimedb-standalone", "syntect", "tabled", + "tar", "tempfile", "termcolor", "tokio", @@ -4749,6 +4783,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.11" @@ -6191,6 +6236,15 @@ dependencies = [ "tap", ] +[[package]] +name = "xattr" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4686009f71ff3e5c4dbcf1a282d0a44db3f021ba69350cd42086b3e5f1c6985" +dependencies = [ + "libc", +] + [[package]] name = "xi-unicode" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index b84c7f586f3..1487685cef1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -107,6 +107,7 @@ hyper = "0.14.18" im = "15.1" imara-diff = "0.1.3" indexmap = "2.0.0" +indicatif = "0.16" insta = { version = "1.21.0", features = ["toml"] } is-terminal = "0.4" itertools = "0.11.0" @@ -155,6 +156,8 @@ strum = { version = "0.25.0", features = ["derive"] } syn = { version = "2", features = ["full", "extra-traits"] } syntect = { version = "5.0.0", default-features = false, features = ["default-fancy"] } tabled = "0.14.0" +tar = "0.4" +tempdir = "0.3.7" tempfile = "3.8" termcolor = "1.2.0" thiserror = "1.0.37" diff --git a/crates/bindings-csharp/Runtime/Runtime.cs b/crates/bindings-csharp/Runtime/Runtime.cs index e0640d32bc4..c434fbdcaae 100644 --- a/crates/bindings-csharp/Runtime/Runtime.cs +++ b/crates/bindings-csharp/Runtime/Runtime.cs @@ -140,18 +140,20 @@ public void Reset() public class RawTableIter : IEnumerable { - private readonly uint tableId; - private readonly byte[]? filterBytes; + public readonly byte[] Schema; + + private readonly IEnumerator iter; public RawTableIter(uint tableId, byte[]? filterBytes = null) { - this.tableId = tableId; - this.filterBytes = filterBytes; + iter = new BufferIter(tableId, filterBytes); + iter.MoveNext(); + Schema = iter.Current; } public IEnumerator GetEnumerator() { - return new BufferIter(tableId, filterBytes); + return iter; } IEnumerator IEnumerable.GetEnumerator() diff --git a/crates/bindings-csharp/Runtime/bindings.c b/crates/bindings-csharp/Runtime/bindings.c index 00091e6a6ae..9b5a6ba102f 100644 --- a/crates/bindings-csharp/Runtime/bindings.c +++ b/crates/bindings-csharp/Runtime/bindings.c @@ -100,10 +100,7 @@ static MonoArray* stdb_buffer_consume(Buffer buf); // return out; // } -#define STDB_IMPORT_MODULE_MINOR(minor) "spacetime_7." #minor -#define STDB_IMPORT_MODULE STDB_IMPORT_MODULE_MINOR(0) - -__attribute__((import_module(STDB_IMPORT_MODULE), +__attribute__((import_module("spacetime"), import_name("_get_table_id"))) extern uint16_t _get_table_id(const char* name, size_t name_len, uint32_t* out); @@ -120,7 +117,7 @@ static uint32_t stdb_get_table_id(MonoString* name_) { return out; } -__attribute__((import_module(STDB_IMPORT_MODULE), +__attribute__((import_module("spacetime"), import_name("_create_index"))) extern uint16_t _create_index(const char* index_name, size_t index_name_len, @@ -144,7 +141,7 @@ static void stdb_create_index(MonoString* index_name_, check_result(result); } -__attribute__((import_module(STDB_IMPORT_MODULE), +__attribute__((import_module("spacetime"), import_name("_iter_by_col_eq"))) extern uint16_t _iter_by_col_eq(uint32_t table_id, uint32_t col_id, @@ -166,7 +163,7 @@ static MonoArray* stdb_iter_by_col_eq(uint32_t table_id, return stdb_buffer_consume(out); } -__attribute__((import_module(STDB_IMPORT_MODULE), +__attribute__((import_module("spacetime"), import_name("_insert"))) extern uint16_t _insert(uint32_t table_id, uint8_t* row, size_t row_len); @@ -178,7 +175,7 @@ static void stdb_insert(uint32_t table_id, MonoArray* row_) { check_result(result); } -// __attribute__((import_module(STDB_IMPORT_MODULE), +// __attribute__((import_module("spacetime"), // import_name("_delete_pk"))) extern uint16_t // _delete_pk(uint32_t table_id, const uint8_t* pk, size_t pk_len); @@ -190,7 +187,7 @@ static void stdb_insert(uint32_t table_id, MonoArray* row_) { // check_result(result); // } -// __attribute__((import_module(STDB_IMPORT_MODULE), +// __attribute__((import_module("spacetime"), // import_name("_delete_value"))) extern uint16_t // _delete_value(uint32_t table_id, const uint8_t* row, size_t row_len); @@ -202,7 +199,7 @@ static void stdb_insert(uint32_t table_id, MonoArray* row_) { // check_result(result); // } -__attribute__((import_module(STDB_IMPORT_MODULE), +__attribute__((import_module("spacetime"), import_name("_delete_by_col_eq"))) extern uint16_t _delete_by_col_eq(uint32_t table_id, uint32_t col_id, @@ -224,7 +221,7 @@ static uint32_t stdb_delete_by_col_eq(uint32_t table_id, return out; } -// __attribute__((import_module(STDB_IMPORT_MODULE), +// __attribute__((import_module("spacetime"), // import_name("_delete_range"))) extern uint16_t // _delete_range(uint32_t table_id, // uint32_t col_id, @@ -251,7 +248,7 @@ static uint32_t stdb_delete_by_col_eq(uint32_t table_id, // return out; // } -__attribute__((import_module(STDB_IMPORT_MODULE), +__attribute__((import_module("spacetime"), import_name("_iter_start"))) extern uint16_t _iter_start(uint32_t table_id, BufferIter* out); @@ -261,7 +258,7 @@ static void stdb_iter_start(uint32_t table_id, BufferIter* iter) { check_result(result); } -__attribute__((import_module(STDB_IMPORT_MODULE), +__attribute__((import_module("spacetime"), import_name("_iter_start_filtered"))) extern uint16_t _iter_start_filtered(uint32_t table_id, const uint8_t* filter, @@ -279,7 +276,7 @@ static void stdb_iter_start_filtered(uint32_t table_id, check_result(result); } -__attribute__((import_module(STDB_IMPORT_MODULE), +__attribute__((import_module("spacetime"), import_name("_iter_next"))) extern uint16_t _iter_next(BufferIter iter, Buffer* out); @@ -292,7 +289,7 @@ static MonoArray* stdb_iter_next(BufferIter iter) { return stdb_buffer_consume(out); } -__attribute__((import_module(STDB_IMPORT_MODULE), +__attribute__((import_module("spacetime"), import_name("_iter_drop"))) extern uint16_t _iter_drop(BufferIter iter); @@ -310,7 +307,7 @@ static void stdb_iter_drop(BufferIter* iter) { check_result(result); } -__attribute__((import_module(STDB_IMPORT_MODULE), +__attribute__((import_module("spacetime"), import_name("_console_log"))) extern void _console_log(uint8_t level, const char* target, @@ -338,7 +335,7 @@ static void stdb_console_log(MonoString* text_, free_string(filename); } -__attribute__((import_module(STDB_IMPORT_MODULE), +__attribute__((import_module("spacetime"), import_name("_schedule_reducer"))) extern void _schedule_reducer(const char* name, size_t name_len, @@ -363,7 +360,7 @@ static void stdb_schedule_reducer( free_string(name); } -__attribute__((import_module(STDB_IMPORT_MODULE), +__attribute__((import_module("spacetime"), import_name("_cancel_reducer"))) extern void _cancel_reducer(ScheduleToken token); @@ -371,11 +368,11 @@ static void stdb_cancel_reducer(ScheduleToken* token) { _cancel_reducer(*token); } -__attribute__((import_module(STDB_IMPORT_MODULE), +__attribute__((import_module("spacetime"), import_name("_buffer_len"))) extern size_t _buffer_len(Buffer buf); -__attribute__((import_module(STDB_IMPORT_MODULE), +__attribute__((import_module("spacetime"), import_name("_buffer_consume"))) extern void _buffer_consume(Buffer buf, uint8_t* into, size_t len); @@ -390,7 +387,7 @@ static MonoArray* stdb_buffer_consume(Buffer buf) { return result; } -__attribute__((import_module(STDB_IMPORT_MODULE), +__attribute__((import_module("spacetime"), import_name("_buffer_alloc"))) extern Buffer _buffer_alloc(const uint8_t* data, size_t data_len); @@ -779,3 +776,8 @@ __attribute__((export_name("__preinit__10_init_csharp"))) void __preinit__10_init_csharp() { _start(); } + +// __attribute__((export_name("SPACETIME_ABI_VERSION"))) - +// doesn't work on non-functions, must specify on command line +const uint32_t SPACETIME_ABI_VERSION = /* 5.0 */ (5 << 16) | 0; +const uint8_t SPACETIME_ABI_VERSION_IS_ADDR = 1; diff --git a/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets b/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets index c43794cba69..e2618dd297c 100644 --- a/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets +++ b/crates/bindings-csharp/Runtime/build/SpacetimeDB.Runtime.targets @@ -2,6 +2,7 @@ + diff --git a/crates/bindings-sys/src/lib.rs b/crates/bindings-sys/src/lib.rs index a09cee67f98..cc3c120ee4b 100644 --- a/crates/bindings-sys/src/lib.rs +++ b/crates/bindings-sys/src/lib.rs @@ -13,18 +13,35 @@ use std::ptr; use alloc::boxed::Box; -use spacetimedb_primitives::{ColId, TableId}; +/// The current version of the ABI. +/// +/// Exported as `SPACETIME_ABI_VERSION`, a `u32` WASM global. +/// If this global contains an address into linear memory at which the version is stored, +/// then a WASM global named `SPACETIME_ABI_VERSION_IS_ADDR` is also be exported. +/// +/// In rust this looks like: +/// ```rust,ignore +/// #[no_mangle] +/// static SPACETIME_ABI_VERSION: u32 = _; // right now, rust `static`s always export as an address. +/// #[no_mangle] +/// static SPACETIME_ABI_VERSION_IS_ADDR: () = (); +/// ``` +/// +/// The (big-endian) first 2 bytes constitute the major version (`A`) of the ABI, +/// and the last 2 bytes constitute the minor version (`B`). +/// +/// The semantics of a version number `A.B` is that a host implementing version `A.B` +/// can run a module declaring `X.Y` if and only if `X == A && Y <= B`. +/// So, the minor version is intended for backwards-compatible changes, e.g. adding a new function, +/// and the major version is for fully breaking changes. +pub const ABI_VERSION: u32 = 0x0005_0000; /// Provides a raw set of sys calls which abstractions can be built atop of. pub mod raw { use core::mem::ManuallyDrop; use spacetimedb_primitives::{ColId, TableId}; - // this module identifier determines the abi version that modules built with this crate depend - // on. Any non-breaking additions to the abi surface should be put in a new `extern {}` block - // with a module identifier with a minor version 1 above the previous highest minor version. - // For breaking changes, all functions should be moved into one new `spacetime_X.0` block. - #[link(wasm_import_module = "spacetime_7.0")] + #[link(wasm_import_module = "spacetime")] extern "C" { /* /// Create a table with `name`, a UTF-8 slice in WASM memory lasting `name_len` bytes, @@ -671,6 +688,7 @@ pub fn cancel_reducer(id: u64) { } pub use raw::{Buffer, BufferIter}; +use spacetimedb_primitives::{ColId, TableId}; impl Buffer { /// Returns the number of bytes of the data stored in the buffer. diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index eb295fbcac2..7c808bfe430 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -38,6 +38,14 @@ pub use log; pub type Result = core::result::Result; +#[no_mangle] +static SPACETIME_ABI_VERSION: u32 = { + assert!(spacetimedb_lib::MODULE_ABI_VERSION.to_u32() == sys::ABI_VERSION); + sys::ABI_VERSION +}; +#[no_mangle] +static SPACETIME_ABI_VERSION_IS_ADDR: () = (); + /// A context that any reducer is provided with. #[non_exhaustive] #[derive(Copy, Clone)] @@ -198,11 +206,52 @@ pub fn delete_by_col_eq(table_id: TableId, col_id: u8, value: &impl Serialize) - }) } -/// A table iterator which yields values of the `TableType` corresponding to the table. -type TableTypeTableIter = RawTableIter>; +/* +pub fn delete_pk(table_id: u32, primary_key: &PrimaryKey) -> Result<()> { + with_row_buf(|bytes| { + primary_key.encode(bytes); + sys::delete_pk(table_id, bytes) + }) +} -// Get the iterator for this table with an optional filter, -fn table_iter(table_id: TableId, filter: Option) -> Result> { +pub fn delete_filter bool>(table_id: u32, f: F) -> Result { + with_row_buf(|bytes| { + let mut count = 0; + for tuple_value in pv_table_iter(table_id, None)? { + if f(&tuple_value) { + count += 1; + bytes.clear(); + tuple_value.encode(bytes); + sys::delete_value(table_id, bytes)?; + } + } + Ok(count) + }) +} + +pub fn delete_range(table_id: u32, col_id: u8, range: Range) -> Result { + with_row_buf(|bytes| { + range.start.encode(bytes); + let mid = bytes.len(); + range.end.encode(bytes); + let (range_start, range_end) = bytes.split_at(mid); + sys::delete_range(table_id, col_id.into(), range_start, range_end) + }) +} +*/ + +// +// fn page_table(table_id : u32, pager_token : u32, read_entries : u32) { +// +// } + +// Get the buffer iterator for this table, +// with an optional filter, +// and return it and its decoded `ProductType` schema. +fn buffer_table_iter( + table_id: u32, + filter: Option, +) -> Result<(BufferIter, ProductType)> { // Decode the filter, if any. let filter = filter .as_ref() @@ -211,8 +260,35 @@ fn table_iter(table_id: TableId, filter: Option; + +// fn pv_table_iter(table_id: u32, filter: Option) -> Result { +// let (iter, schema) = buffer_table_iter(table_id, filter)?; +// let deserializer = ProductValueBufferDeserialize::new(schema); +// Ok(RawTableIter::new(iter, deserializer)) +// } + +/// A table iterator which yields values of the `TableType` corresponding to the table. +type TableTypeTableIter = RawTableIter>; +fn table_iter(table_id: u32, filter: Option) -> Result> { + // The TableType deserializer doesn't need the schema, as we have type-directed + // dispatch to deserialize any given `TableType`. + let (iter, _schema) = buffer_table_iter(table_id, filter)?; let deserializer = TableTypeBufferDeserialize::new(); Ok(RawTableIter::new(iter, deserializer).into()) } @@ -332,7 +408,7 @@ pub trait TableType: SpacetimeType + DeserializeOwned + Serialize { /// Returns an iterator over the rows in this table. fn iter() -> TableIter { - table_iter(Self::table_id(), None).unwrap() + table_iter(Self::table_id().0, None).unwrap() } /// Returns an iterator filtered by `filter` over the rows in this table. @@ -340,7 +416,7 @@ pub trait TableType: SpacetimeType + DeserializeOwned + Serialize { /// **NOTE:** Do not use directly. This is exposed as `query!(...)`. #[doc(hidden)] fn iter_filtered(filter: spacetimedb_lib::filter::Expr) -> TableIter { - table_iter(Self::table_id(), Some(filter)).unwrap() + table_iter(Self::table_id().0, Some(filter)).unwrap() } } diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index df19401ff57..2768ac12214 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -31,10 +31,13 @@ dirs.workspace = true duct.workspace = true email_address.workspace = true futures.workspace = true +flate2.workspace = true is-terminal.workspace = true itertools.workspace = true +indicatif.workspace = true jsonwebtoken.workspace = true mimalloc.workspace = true +regex.workspace = true reqwest.workspace = true rustyline.workspace = true serde = { workspace = true, features = ["derive"] } @@ -42,6 +45,7 @@ serde_json = { workspace = true, features = ["raw_value", "preserve_order"] } slab.workspace = true syntect.workspace = true tabled.workspace = true +tar.workspace = true tempfile.workspace = true termcolor.workspace = true tokio.workspace = true diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 2eb45b4ae78..e3144b3cb6b 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -31,6 +31,7 @@ pub fn get_subcommands() -> Vec { init::cli(), build::cli(), server::cli(), + upgrade::cli(), #[cfg(feature = "standalone")] start::cli(ProgramMode::CLI), ] @@ -55,6 +56,7 @@ pub async fn exec_subcommand(config: Config, cmd: &str, args: &ArgMatches) -> Re "server" => server::exec(config, args).await, #[cfg(feature = "standalone")] "start" => start::exec(args).await, + "upgrade" => upgrade::exec(args).await, unknown => Err(anyhow::anyhow!("Invalid subcommand: {}", unknown)), } } diff --git a/crates/cli/src/subcommands/generate/mod.rs b/crates/cli/src/subcommands/generate/mod.rs index 3a2eb7ada2e..3195af89188 100644 --- a/crates/cli/src/subcommands/generate/mod.rs +++ b/crates/cli/src/subcommands/generate/mod.rs @@ -6,8 +6,8 @@ use clap::ArgAction::SetTrue; use convert_case::{Case, Casing}; use duct::cmd; use spacetimedb_lib::sats::{AlgebraicType, Typespace}; -use spacetimedb_lib::{bsatn, MiscModuleExport, ModuleDef, ReducerDef, TableDef, TypeAlias, MODULE_ABI_MAJOR_VERSION}; -use wasmtime::{AsContext, Caller}; +use spacetimedb_lib::{bsatn, MiscModuleExport, ModuleDef, ReducerDef, TableDef, TypeAlias}; +use wasmtime::{AsContext, Caller, ExternType}; mod code_indenter; pub mod csharp; @@ -343,10 +343,18 @@ fn extract_descriptions(wasm_file: &Path) -> anyhow::Result { }; let mut store = wasmtime::Store::new(&engine, ctx); let mut linker = wasmtime::Linker::new(&engine); - linker.allow_shadowing(true).define_unknown_imports_as_traps(&module)?; - let module_name = &*format!("spacetime_{MODULE_ABI_MAJOR_VERSION}.0"); + linker.allow_shadowing(true); + for imp in module.imports() { + if let ExternType::Func(func_type) = imp.ty() { + linker + .func_new(imp.module(), imp.name(), func_type, |_, _, _| { + anyhow::bail!("don't call me!!") + }) + .unwrap(); + } + } linker.func_wrap( - module_name, + "spacetime", "_console_log", |caller: Caller<'_, WasmCtx>, _level: u32, @@ -366,7 +374,7 @@ fn extract_descriptions(wasm_file: &Path) -> anyhow::Result { } }, )?; - linker.func_wrap(module_name, "_buffer_alloc", WasmCtx::buffer_alloc)?; + linker.func_wrap("spacetime", "_buffer_alloc", WasmCtx::buffer_alloc)?; let instance = linker.instantiate(&mut store, &module)?; let memory = Memory { mem: instance.get_memory(&mut store, "memory").unwrap(), diff --git a/crates/cli/src/subcommands/mod.rs b/crates/cli/src/subcommands/mod.rs index a170383b043..41b665012f4 100644 --- a/crates/cli/src/subcommands/mod.rs +++ b/crates/cli/src/subcommands/mod.rs @@ -13,4 +13,5 @@ pub mod publish; pub mod repl; pub mod server; pub mod sql; +pub mod upgrade; pub mod version; diff --git a/crates/cli/src/subcommands/upgrade.rs b/crates/cli/src/subcommands/upgrade.rs new file mode 100644 index 00000000000..aaea75b775f --- /dev/null +++ b/crates/cli/src/subcommands/upgrade.rs @@ -0,0 +1,220 @@ +use std::io::Write; +use std::{env, fs}; + +extern crate regex; + +use crate::version; +use clap::{Arg, ArgMatches}; +use flate2::read::GzDecoder; +use futures::stream::StreamExt; +use indicatif::{ProgressBar, ProgressStyle}; +use regex::Regex; +use serde::Deserialize; +use serde_json::Value; +use std::path::Path; +use tar::Archive; + +pub fn cli() -> clap::Command { + clap::Command::new("upgrade") + .about("Checks for updates for the currently running spacetime CLI tool") + .arg(Arg::new("version").help("The specific version to upgrade to")) + .after_help("Run `spacetime help upgrade` for more detailed information.\n") +} + +#[derive(Deserialize)] +struct ReleaseAsset { + name: String, + browser_download_url: String, +} + +#[derive(Deserialize)] +struct Release { + tag_name: String, + assets: Vec, +} + +fn get_download_name() -> String { + let os = env::consts::OS; + let arch = env::consts::ARCH; + + let os_str = match os { + "macos" => "darwin", + "windows" => return "spacetime.exe".to_string(), + "linux" => "linux", + _ => panic!("Unsupported OS"), + }; + + let arch_str = match arch { + "x86_64" => "amd64", + "aarch64" => "arm64", + _ => panic!("Unsupported architecture"), + }; + + format!("spacetime.{}-{}.tar.gz", os_str, arch_str) +} + +fn clean_version(version: &str) -> Option { + let re = Regex::new(r"v?(\d+\.\d+\.\d+)").unwrap(); + re.captures(version) + .and_then(|cap| cap.get(1)) + .map(|match_| match_.as_str().to_string()) +} + +async fn get_release_tag_from_version(release_version: &str) -> Result, reqwest::Error> { + let release_version = format!("v{}-beta", release_version); + let url = "https://api.github.com/repos/clockworklabs/SpacetimeDB/releases"; + let client = reqwest::Client::builder() + .user_agent(format!("SpacetimeDB CLI/{}", version::CLI_VERSION)) + .build()?; + let releases: Vec = client + .get(url) + .header( + reqwest::header::USER_AGENT, + format!("SpacetimeDB CLI/{}", version::CLI_VERSION).as_str(), + ) + .send() + .await? + .json() + .await?; + + for release in releases.iter() { + if let Some(release_tag) = release["tag_name"].as_str() { + if release_tag.starts_with(&release_version) { + return Ok(Some(release_tag.to_string())); + } + } + } + Ok(None) +} + +async fn download_with_progress(client: &reqwest::Client, url: &str, temp_path: &Path) -> Result<(), anyhow::Error> { + let response = client.get(url).send().await?; + let total_size = match response.headers().get(reqwest::header::CONTENT_LENGTH) { + Some(size) => size.to_str().unwrap().parse::().unwrap(), + None => 0, + }; + + let pb = ProgressBar::new(total_size); + pb.set_style( + ProgressStyle::default_bar().template("{spinner} Downloading update... {bytes}/{total_bytes} ({eta})"), + ); + + let mut file = fs::File::create(temp_path)?; + let mut downloaded_bytes = 0; + + let mut response_stream = response.bytes_stream(); + while let Some(chunk) = response_stream.next().await { + let chunk = chunk?; + downloaded_bytes += chunk.len(); + pb.set_position(downloaded_bytes as u64); + file.write_all(&chunk)?; + } + + pb.finish_with_message("Download complete."); + Ok(()) +} + +pub async fn exec(args: &ArgMatches) -> Result<(), anyhow::Error> { + let version = args.get_one::("version"); + let current_exe_path = env::current_exe()?; + + let url = match version { + None => "https://api.github.com/repos/clockworklabs/SpacetimeDB/releases/latest".to_string(), + Some(release_version) => { + let release_tag = get_release_tag_from_version(release_version).await?; + if release_tag.is_none() { + return Err(anyhow::anyhow!("No release found for version {}", release_version)); + } + format!( + "https://api.github.com/repos/clockworklabs/SpacetimeDB/releases/tags/{}", + release_tag.unwrap() + ) + } + }; + + let client = reqwest::Client::builder() + .user_agent(format!("SpacetimeDB CLI/{}", version::CLI_VERSION)) + .build()?; + + print!("Finding version..."); + std::io::stdout().flush()?; + let release: Release = client.get(url).send().await?.json().await?; + let release_version = clean_version(&release.tag_name).unwrap(); + println!("done."); + + if release_version == version::CLI_VERSION { + println!("You're already running the latest version: {}", version::CLI_VERSION); + return Ok(()); + } + + let download_name = get_download_name(); + let asset = release.assets.iter().find(|&asset| asset.name == download_name); + + if asset.is_none() { + return Err(anyhow::anyhow!( + "No assets available for the detected OS and architecture." + )); + } + + println!( + "You are currently running version {} of spacetime. The version you're upgrading to is {}.", + version::CLI_VERSION, + release_version, + ); + println!( + "This will replace the current executable at {}.", + current_exe_path.display() + ); + print!("Do you want to continue? [y/N] "); + std::io::stdout().flush()?; + let mut input = String::new(); + std::io::stdin().read_line(&mut input)?; + if input.trim().to_lowercase() != "y" && input.trim().to_lowercase() != "yes" { + println!("Aborting upgrade."); + return Ok(()); + } + + let temp_dir = tempfile::tempdir()?.into_path(); + let temp_path = &temp_dir.join(download_name.clone()); + download_with_progress(&client, &asset.unwrap().browser_download_url, temp_path).await?; + + if download_name.to_lowercase().ends_with(".tar.gz") || download_name.to_lowercase().ends_with("tgz") { + let tar_gz = fs::File::open(temp_path)?; + let tar = GzDecoder::new(tar_gz); + let mut archive = Archive::new(tar); + let mut spacetime_found = false; + for mut file in archive.entries()?.filter_map(|e| e.ok()) { + if let Ok(path) = file.path() { + if path.ends_with("spacetime") { + spacetime_found = true; + file.unpack(temp_dir.join("spacetime"))?; + } + } + } + + if !spacetime_found { + fs::remove_dir_all(&temp_dir)?; + return Err(anyhow::anyhow!("Spacetime executable not found in archive")); + } + } + + let new_exe_path = if temp_path.to_str().unwrap().ends_with(".exe") { + temp_path.clone() + } else if download_name.ends_with(".tar.gz") { + temp_dir.join("spacetime") + } else { + fs::remove_dir_all(&temp_dir)?; + return Err(anyhow::anyhow!("Unsupported download type")); + }; + + // Move the current executable into a temporary directory, which will later be deleted by the OS + let current_exe_temp_dir = env::temp_dir(); + let current_exe_to_temp = current_exe_temp_dir.join("spacetime_old"); + fs::rename(¤t_exe_path, current_exe_to_temp)?; + fs::rename(new_exe_path, ¤t_exe_path)?; + fs::remove_dir_all(&temp_dir)?; + + println!("spacetime has been updated to version {}", release_version); + + Ok(()) +} diff --git a/crates/cli/src/subcommands/version.rs b/crates/cli/src/subcommands/version.rs index db45f155a00..646cdda85a4 100644 --- a/crates/cli/src/subcommands/version.rs +++ b/crates/cli/src/subcommands/version.rs @@ -1,6 +1,6 @@ use clap::{Arg, ArgAction::SetTrue, ArgMatches}; -const CLI_VERSION: &str = env!("CARGO_PKG_VERSION"); +pub const CLI_VERSION: &str = env!("CARGO_PKG_VERSION"); use crate::config::Config; diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index b4e273792ad..e41d32fb7f1 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -50,13 +50,9 @@ impl BufWriter for ChunkedWriter { } impl ChunkedWriter { - /// Flushes the data collected in the scratch space if it's larger than our - /// chunking threshold. - pub fn flush(&mut self) { - // For now, just send buffers over a certain fixed size. - const ITER_CHUNK_SIZE: usize = 64 * 1024; - - if self.scratch_space.len() > ITER_CHUNK_SIZE { + /// Flushes the currently populated part of the scratch space as a new chunk. + pub fn force_flush(&mut self) { + if !self.scratch_space.is_empty() { // We intentionally clone here so that our scratch space is not // recreated with zero capacity (via `Vec::new`), but instead can // be `.clear()`ed in-place and reused. @@ -70,11 +66,22 @@ impl ChunkedWriter { } } + /// Similar to [`Self::force_flush`], but only flushes if the data in the + /// scratch space is larger than our chunking threshold. + pub fn flush(&mut self) { + // For now, just send buffers over a certain fixed size. + const ITER_CHUNK_SIZE: usize = 64 * 1024; + + if self.scratch_space.len() > ITER_CHUNK_SIZE { + self.force_flush(); + } + } + /// Finalises the writer and returns all the chunks. pub fn into_chunks(mut self) -> Vec> { if !self.scratch_space.is_empty() { - // Avoid extra clone by just shrinking and pushing the scratch space - // in-place. + // This is equivalent to calling `force_flush`, but we avoid extra + // clone by just shrinking and pushing the scratch space in-place. self.chunks.push(self.scratch_space.into()); } self.chunks @@ -274,6 +281,10 @@ impl InstanceEnv { let stdb = &*self.dbic.relational_db; let tx = &mut *self.tx.get()?; + stdb.row_schema_for_table(tx, table_id)?.encode(&mut chunked_writer); + // initial chunk is expected to be schema itself, so force-flush it as a separate chunk + chunked_writer.force_flush(); + for row in stdb.iter(ctx, tx, table_id)? { row.view().encode(&mut chunked_writer); // Flush at row boundaries. @@ -319,12 +330,18 @@ impl InstanceEnv { } } + let mut chunked_writer = ChunkedWriter::default(); + let stdb = &self.dbic.relational_db; let tx = &mut *self.tx.get()?; let schema = stdb.schema_for_table(tx, table_id)?; let row_type = ProductType::from(&*schema); + // write and force flush schema as it's expected to be the first individual chunk + row_type.encode(&mut chunked_writer); + chunked_writer.force_flush(); + let filter = filter::Expr::from_bytes( // TODO: looks like module typespace is currently not hooked up to instances; // use empty typespace for now which should be enough for primitives @@ -342,8 +359,6 @@ impl InstanceEnv { _ => unreachable!("query should always return a table"), }; - let mut chunked_writer = ChunkedWriter::default(); - // write all rows and flush at row boundaries for row in results.data { row.data.encode(&mut chunked_writer); diff --git a/crates/core/src/host/wasm_common.rs b/crates/core/src/host/wasm_common.rs index 5aef3ae8c6b..a364fa1f7b3 100644 --- a/crates/core/src/host/wasm_common.rs +++ b/crates/core/src/host/wasm_common.rs @@ -1,4 +1,3 @@ -pub mod abi; pub mod instrumentation; pub mod module_host_actor; @@ -19,6 +18,9 @@ pub const INIT_DUNDER: &str = "__init__"; /// the reducer with this name is invoked when updating the database pub const UPDATE_DUNDER: &str = "__update__"; +pub const STDB_ABI_SYM: &str = "SPACETIME_ABI_VERSION"; +pub const STDB_ABI_IS_ADDR_SYM: &str = "SPACETIME_ABI_VERSION_IS_ADDR"; + #[derive(Debug, Copy, Clone, PartialEq, Eq)] #[allow(unused)] pub enum WasmType { @@ -209,7 +211,6 @@ impl FuncNames { pub enum ModuleCreationError { WasmCompileError(anyhow::Error), Init(#[from] module_host_actor::InitializationError), - Abi(#[from] abi::AbiVersionError), } pub trait ResourceIndex { diff --git a/crates/core/src/host/wasm_common/abi.rs b/crates/core/src/host/wasm_common/abi.rs deleted file mode 100644 index 20feb8140da..00000000000 --- a/crates/core/src/host/wasm_common/abi.rs +++ /dev/null @@ -1,57 +0,0 @@ -pub use spacetimedb_lib::VersionTuple; - -const MODULE_PREFIX: &str = "spacetime_"; - -pub fn determine_spacetime_abi( - imports: impl IntoIterator, - get_module: impl Fn(&I) -> &str, -) -> Result, AbiVersionError> { - let it = imports.into_iter().filter_map(|imp| { - let s = get_module(&imp); - let err = || AbiVersionError::Parse { module: s.to_owned() }; - s.strip_prefix(MODULE_PREFIX).map(|ver| { - let (major, minor) = ver.split_once('.').ok_or_else(err)?; - let (major, minor) = Option::zip(major.parse().ok(), minor.parse().ok()).ok_or_else(err)?; - Ok(VersionTuple { major, minor }) - }) - }); - itertools::process_results(it, |mut it| try_reduce(&mut it, refine_ver_req))? -} - -// TODO: replace with Iterator::try_reduce once stabilized -fn try_reduce( - it: &mut I, - f: impl FnMut(I::Item, I::Item) -> Result, -) -> Result, E> { - let Some(first) = it.next() else { return Ok(None) }; - it.try_fold(first, f).map(Some) -} - -fn refine_ver_req(ver: VersionTuple, new: VersionTuple) -> Result { - if ver.major != new.major { - Err(AbiVersionError::MultipleMajor(ver.major, new.major)) - } else { - Ok(Ord::max(ver, new)) - } -} - -pub fn verify_supported(implements: VersionTuple, got: VersionTuple) -> Result<(), AbiVersionError> { - if implements.supports(got) { - Ok(()) - } else { - Err(AbiVersionError::UnsupportedVersion { implements, got }) - } -} - -#[derive(thiserror::Error, Debug)] -pub enum AbiVersionError { - #[error("import module {module:?} has malformed version string")] - Parse { module: String }, - #[error("module cannot depend on both major version {0} and major version {1}")] - MultipleMajor(u16, u16), - #[error("abi version {got} is not supported (host implements {implements})")] - UnsupportedVersion { - got: VersionTuple, - implements: VersionTuple, - }, -} diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 532d8e8e588..763defc08be 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -10,7 +10,7 @@ use anyhow::{anyhow, Context}; use bytes::Bytes; use spacetimedb_lib::buffer::DecodeError; use spacetimedb_lib::identity::AuthCtx; -use spacetimedb_lib::{bsatn, Address, ModuleDef}; +use spacetimedb_lib::{bsatn, Address, ModuleDef, VersionTuple}; use spacetimedb_vm::expr::CrudExpr; use crate::database_instance_context::DatabaseInstanceContext; @@ -88,8 +88,20 @@ pub(crate) struct WasmModuleHostActor { energy_monitor: Arc, } +#[derive(thiserror::Error, Debug)] +pub enum AbiVersionError { + #[error("module doesn't indicate spacetime ABI version")] + NoVersion, + #[error("abi version is malformed somehow (out-of-bounds, etc)")] + Malformed, + #[error("abi version {got} is not supported (host implements {implement})")] + UnsupportedVersion { got: VersionTuple, implement: VersionTuple }, +} + #[derive(thiserror::Error, Debug)] pub enum InitializationError { + #[error(transparent)] + Abi(#[from] AbiVersionError), #[error(transparent)] Validation(#[from] ValidationError), #[error("setup function returned an error: {0}")] diff --git a/crates/core/src/host/wasmer/mod.rs b/crates/core/src/host/wasmer/mod.rs index 7e3740cccc8..96e9282c1dc 100644 --- a/crates/core/src/host/wasmer/mod.rs +++ b/crates/core/src/host/wasmer/mod.rs @@ -15,7 +15,7 @@ mod wasmer_module; use wasmer_module::WasmerModule; use super::scheduler::Scheduler; -use super::wasm_common::{abi, module_host_actor::WasmModuleHostActor, ModuleCreationError}; +use super::wasm_common::{module_host_actor::WasmModuleHostActor, ModuleCreationError}; use super::{EnergyMonitor, EnergyQuanta}; pub fn make_actor( @@ -47,12 +47,6 @@ pub fn make_actor( let module = Module::new(&engine, program_bytes).map_err(|e| ModuleCreationError::WasmCompileError(e.into()))?; - let abi = abi::determine_spacetime_abi(module.imports().functions(), wasmer::ImportType::module)?; - - if let Some(abi) = abi { - abi::verify_supported(WasmerModule::IMPLEMENTED_ABI, abi)?; - } - let module = WasmerModule::new(module, engine); WasmModuleHostActor::new(dbic, module_hash, module, scheduler, energy_monitor).map_err(Into::into) diff --git a/crates/core/src/host/wasmer/wasmer_module.rs b/crates/core/src/host/wasmer/wasmer_module.rs index c579f49d11b..cb755fdc3c5 100644 --- a/crates/core/src/host/wasmer/wasmer_module.rs +++ b/crates/core/src/host/wasmer/wasmer_module.rs @@ -1,13 +1,14 @@ use super::wasm_instance_env::WasmInstanceEnv; use super::Mem; use crate::host::instance_env::InstanceEnv; -use crate::host::wasm_common::module_host_actor::{DescribeError, InitializationError, ReducerOp}; +use crate::host::wasm_common::module_host_actor::{AbiVersionError, DescribeError, InitializationError, ReducerOp}; use crate::host::wasm_common::*; use crate::host::EnergyQuanta; use bytes::Bytes; +use spacetimedb_lib::VersionTuple; use wasmer::{ imports, AsStoreMut, Engine, ExternType, Function, FunctionEnv, Imports, Instance, Module, RuntimeError, Store, - TypedFunction, + TypedFunction, WasmPtr, }; use wasmer_middlewares::metering as wasmer_metering; @@ -45,13 +46,12 @@ impl WasmerModule { WasmerModule { module, engine } } - pub const IMPLEMENTED_ABI: abi::VersionTuple = abi::VersionTuple::new(7, 0); + pub const IMPLEMENTED_ABI: VersionTuple = VersionTuple::new(5, 0); fn imports(&self, store: &mut Store, env: &FunctionEnv) -> Imports { - #[allow(clippy::assertions_on_constants)] - const _: () = assert!(WasmerModule::IMPLEMENTED_ABI.major == spacetimedb_lib::MODULE_ABI_MAJOR_VERSION); + const _: () = assert!(WasmerModule::IMPLEMENTED_ABI.eq(spacetimedb_lib::MODULE_ABI_VERSION)); imports! { - "spacetime_7.0" => { + "spacetime" => { "_schedule_reducer" => Function::new_typed_with_env(store, env, WasmInstanceEnv::schedule_reducer), "_cancel_reducer" => Function::new_typed_with_env(store, env, WasmInstanceEnv::cancel_reducer), "_delete_by_col_eq" => Function::new_typed_with_env( @@ -149,6 +149,42 @@ impl module_host_actor::WasmInstancePre for WasmerModule { let mem = Mem::extract(&instance.exports).unwrap(); + // We could (and did in the past) parse the ABI version manually before the instantiation, + // but it gets complicated in presence of wasm-opt optimisations which might split encoded + // versions like `[...other data...]\00\00\03\00[...other data...]` by zeroes + // into several segments, so there is no single data segment containing the entire version. + // Instead, it's more reliable to extract the version from an instantiated module + // when all the data segments are loaded into the flat memory at correct offsets. + let abi_version = instance + .exports + .get_global(STDB_ABI_SYM) + .map_err(|_| AbiVersionError::NoVersion)?; + + let mut abi_version = match abi_version.get(&mut store) { + wasmer::Value::I32(x) => x as u32, + _ => return Err(AbiVersionError::Malformed.into()), + }; + + let abi_is_addr = instance.exports.get_global(STDB_ABI_IS_ADDR_SYM).is_ok(); + if abi_is_addr { + abi_version = u32::from_le_bytes( + mem.read_bytes(&store, WasmPtr::new(abi_version), 4) + .ok() + .and_then(|bytes| bytes.try_into().ok()) + .ok_or(AbiVersionError::Malformed)?, + ); + } + + let abi_version = VersionTuple::from_u32(abi_version); + + if !WasmerModule::IMPLEMENTED_ABI.supports(abi_version) { + return Err(AbiVersionError::UnsupportedVersion { + implement: WasmerModule::IMPLEMENTED_ABI, + got: abi_version, + } + .into()); + } + env.as_mut(&mut store).instantiate(mem); // Note: this budget is just for initializers diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index e97b10f3cfa..27c9fd0042e 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -41,10 +41,10 @@ pub use type_value::{AlgebraicValue, ProductValue}; pub use spacetimedb_sats as sats; -pub const MODULE_ABI_MAJOR_VERSION: u16 = 7; +pub const MODULE_ABI_VERSION: VersionTuple = VersionTuple::new(5, 0); // if it ends up we need more fields in the future, we can split one of them in two -#[derive(PartialEq, Eq, PartialOrd, Ord, Copy, Clone, Debug)] +#[derive(PartialEq, Eq, Copy, Clone, Debug)] pub struct VersionTuple { /// Breaking change; different major versions are not at all compatible with each other. pub major: u16,