diff --git a/cli/src/commands/sdk.rs b/cli/src/commands/sdk.rs index 8f3f7a01..1eb0abd0 100644 --- a/cli/src/commands/sdk.rs +++ b/cli/src/commands/sdk.rs @@ -246,13 +246,9 @@ fn load_stack_spec( let ast_json = fs::read_to_string(&ast.path) .with_context(|| format!("Failed to read stack file: {}", ast.path.display()))?; - let stack_spec: hyperstack_interpreter::ast::SerializableStackSpec = - serde_json::from_str(&ast_json).with_context(|| { - format!( - "Failed to deserialize stack AST from {}", - ast.path.display() - ) - })?; + // Use versioned loader for automatic version detection and migration + let stack_spec = hyperstack_interpreter::versioned::load_stack_spec(&ast_json) + .with_context(|| format!("Failed to load stack AST from {}", ast.path.display()))?; if stack_spec.entities.is_empty() { return Err(anyhow::anyhow!( diff --git a/docs/ast-versioning-guide.md b/docs/ast-versioning-guide.md new file mode 100644 index 00000000..1d00d24b --- /dev/null +++ b/docs/ast-versioning-guide.md @@ -0,0 +1,433 @@ +# AST Versioning Guide + +This guide explains how to add breaking changes to the AST and bump the version while maintaining backward compatibility. + +## Overview + +HyperStack uses **semantic versioning** for AST schemas (major.minor.patch): + +- **Major (X.0.0)**: Breaking structural changes (renamed fields, removed fields) +- **Minor (0.X.0)**: New optional fields, additions that don't break old code +- **Patch (0.0.X)**: Bug fixes, documentation changes + +## Quick Reference + +| Change Type | Version Bump | Migration Required? | +|------------|--------------|-------------------| +| Add new optional field | Minor (0.0.1 → 0.1.0) | No | +| Rename field | Major (0.0.1 → 1.0.0) | Yes | +| Remove field | Major (0.0.1 → 1.0.0) | Yes | +| Change field type | Major (0.0.1 → 1.0.0) | Yes | +| Restructure enum | Major (0.0.1 → 1.0.0) | Yes | + +## Step-by-Step: Adding a Breaking Change + +### Step 1: Define the New AST Version + +**⚠️ CRITICAL: You must update the version constant in BOTH crates.** + +The AST types are duplicated between `hyperstack-macros` (for compile-time code generation) and `interpreter` (for runtime) due to circular dependency constraints (proc-macro crates cannot depend on their output crates). Both crates must have the same `CURRENT_AST_VERSION` constant. + +**`hyperstack-macros/src/ast/types.rs`** +```rust +// Change this +pub const CURRENT_AST_VERSION: &str = "0.0.1"; + +// To this (for a minor bump) +pub const CURRENT_AST_VERSION: &str = "0.1.0"; + +// Or this (for a major bump) +pub const CURRENT_AST_VERSION: &str = "1.0.0"; +``` + +**`interpreter/src/ast.rs`** +```rust +// Mirror the EXACT same change here +pub const CURRENT_AST_VERSION: &str = "1.0.0"; +``` + +**Why two places?** The AST types exist in both crates: +- `hyperstack-macros`: Used at compile time when processing `#[hyperstack]` attributes +- `interpreter`: Used at runtime and for CLI tools (SDK generation, etc.) + +**Don't worry about forgetting:** There's a test (`test_ast_version_sync_*`) in both crates that will fail if the constants get out of sync. You'll see an error like: +``` +AST version mismatch! hyperstack-macros has '0.0.1', interpreter has '1.0.0'. +Both crates must have the same CURRENT_AST_VERSION. +Update both files when bumping the version. +``` + +### Step 2: Create the New AST Structure + +Define the new version of your types. You have two options: + +#### Option A: In-Place Changes (Recommended for Minor Bumps) + +For minor bumps (adding optional fields), just modify the existing struct: + +```rust +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SerializableStreamSpec { + #[serde(default = "default_ast_version")] + pub ast_version: String, + pub state_name: String, + // ... existing fields ... + + // NEW in v0.1.0 + #[serde(default)] + pub new_field: Option, +} +``` + +#### Option B: Separate Types (Required for Major Bumps) + +For major changes, create new struct definitions: + +**`hyperstack-macros/src/ast/types.rs`** +```rust +// Keep old version for migration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SerializableStreamSpecV1 { + pub state_name: String, + pub old_field: String, // This will be removed in v2 + // ... other v1 fields ... +} + +// New v2 structure +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SerializableStreamSpecV2 { + #[serde(default = "default_ast_version")] + pub ast_version: String, + pub state_name: String, + pub new_field: String, // Renamed from old_field + // ... other v2 fields ... +} + +// Keep the main type as latest +pub type SerializableStreamSpec = SerializableStreamSpecV2; +``` + +### Step 3: Add Migration Logic + +Update the versioned loader in **both** crates: + +**`hyperstack-macros/src/ast/versioned.rs`** + +```rust +pub fn load_stream_spec(json: &str) -> Result { + let raw: Value = serde_json::from_str(json) + .map_err(|e| VersionedLoadError::InvalidJson(e.to_string()))?; + + let version = raw + .get("ast_version") + .and_then(|v| v.as_str()) + .unwrap_or("0.0.1"); + + match version { + v if v == CURRENT_AST_VERSION => { + // Current version - deserialize directly + serde_json::from_value::(raw) + .map_err(|e| VersionedLoadError::InvalidStructure(e.to_string())) + } + "1.0.0" => { + // OLD: Load v1 and migrate to current + let v1: SerializableStreamSpecV1 = serde_json::from_value(raw) + .map_err(|e| VersionedLoadError::InvalidStructure(e.to_string()))?; + Ok(migrate_stream_v1_to_v2(v1)) + } + "0.0.1" => { + // OLD: Load v0.0.1 and migrate to current + let v0: SerializableStreamSpecV0 = serde_json::from_value(raw) + .map_err(|e| VersionedLoadError::InvalidStructure(e.to_string()))?; + Ok(migrate_v0_to_v2(v0)) + } + _ => Err(VersionedLoadError::UnsupportedVersion(version.to_string())), + } +} + +// Migration function +fn migrate_stream_v1_to_v2(v1: SerializableStreamSpecV1) -> SerializableStreamSpec { + SerializableStreamSpec { + ast_version: CURRENT_AST_VERSION.to_string(), + state_name: v1.state_name, + new_field: transform_old_field(v1.old_field), // Transform data + // ... migrate other fields ... + } +} +``` + +Do the same for `interpreter/src/versioned.rs`. + +### Step 4: Update Versioned Enums (Optional) + +If you're using the `VersionedStreamSpec` enum for explicit version handling: + +```rust +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "ast_version")] +pub enum VersionedStreamSpec { + #[serde(rename = "0.0.1")] + V1(SerializableStreamSpecV1), + #[serde(rename = "2.0.0")] + V2(SerializableStreamSpec), // Current version +} + +impl VersionedStreamSpec { + pub fn into_latest(self) -> SerializableStreamSpec { + match self { + VersionedStreamSpec::V1(v1) => migrate_stream_v1_to_v2(v1), + VersionedStreamSpec::V2(v2) => v2, + } + } +} +``` + +### Step 5: Update All Constructors + +Find all places that construct the spec and add the new `ast_version` field: + +```rust +// Old +let spec = SerializableStreamSpec { + state_name: "MyEntity".to_string(), + old_field: "value".to_string(), + // ... +}; + +// New +let spec = SerializableStreamSpec { + ast_version: CURRENT_AST_VERSION.to_string(), // ADD THIS + state_name: "MyEntity".to_string(), + new_field: transform_value("value"), // Updated field + // ... +}; +``` + +Common locations to check: +- `hyperstack-macros/src/stream_spec/ast_writer.rs` +- `hyperstack-macros/src/stream_spec/module.rs` +- `hyperstack-macros/src/stream_spec/idl_spec.rs` +- `interpreter/src/ast.rs` (to_serializable methods) +- `interpreter/src/typescript.rs` (test specs) + +### Step 6: Write Tests + +Add tests to verify migration works correctly: + +```rust +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_migrate_v1_to_v2() { + let v1_json = r#" + { + "ast_version": "0.0.1", + "state_name": "TestEntity", + "old_field": "old_value" + } + "#; + + let spec = load_stream_spec(v1_json).unwrap(); + assert_eq!(spec.ast_version, CURRENT_AST_VERSION); + assert_eq!(spec.state_name, "TestEntity"); + assert_eq!(spec.new_field, "transformed_value"); + } + + #[test] + fn test_load_v2_directly() { + let v2_json = r#" + { + "ast_version": "2.0.0", + "state_name": "TestEntity", + "new_field": "new_value" + } + "#; + + let spec = load_stream_spec(v2_json).unwrap(); + assert_eq!(spec.ast_version, CURRENT_AST_VERSION); + assert_eq!(spec.new_field, "new_value"); + } +} +``` + +### Step 7: Deprecation Window (Optional) + +If you want to deprecate old versions after a certain period: + +```rust +pub fn load_stream_spec(json: &str) -> Result { + // ... version detection ... + + match version { + "0.0.1" => { + // Log deprecation warning + eprintln!("WARNING: Loading deprecated AST v0.0.1. Please upgrade your AST files."); + + let v1: SerializableStreamSpecV1 = serde_json::from_value(raw) + .map_err(|e| VersionedLoadError::InvalidStructure(e.to_string()))?; + Ok(migrate_stream_v1_to_v2(v1)) + } + // ... + } +} +``` + +After your deprecation period, you can remove support: + +```rust +"0.0.1" => { + Err(VersionedLoadError::UnsupportedVersion( + "0.0.1 (deprecated, please upgrade your AST files)".to_string() + )) +} +``` + +## Complete Example: Field Rename + +Let's say we want to rename `old_name` to `new_name` in v2.0.0: + +### 1. Define v1 structure (in versioned.rs) + +```rust +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SerializableStreamSpecV1 { + pub state_name: String, + pub old_name: String, +} +``` + +### 2. Update main type + +```rust +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SerializableStreamSpec { + #[serde(default = "default_ast_version")] + pub ast_version: String, + pub state_name: String, + pub new_name: String, // Renamed from old_name +} +``` + +### 3. Add migration + +```rust +fn migrate_stream_v1_to_v2(v1: SerializableStreamSpecV1) -> SerializableStreamSpec { + SerializableStreamSpec { + ast_version: CURRENT_AST_VERSION.to_string(), + state_name: v1.state_name, + new_name: v1.old_name, // Direct mapping + } +} + +pub fn load_stream_spec(json: &str) -> Result { + let raw: Value = serde_json::from_str(json) + .map_err(|e| VersionedLoadError::InvalidJson(e.to_string()))?; + let version = raw.get("ast_version").and_then(|v| v.as_str()).unwrap_or("0.0.1"); + + match version { + v if v == CURRENT_AST_VERSION => { + // Current version - deserialize directly + serde_json::from_value::(raw) + .map_err(|e| VersionedLoadError::InvalidStructure(e.to_string())) + } + "0.0.1" => { + // OLD: Load v0.0.1 and migrate to current + let v1: SerializableStreamSpecV1 = serde_json::from_value(raw) + .map_err(|e| VersionedLoadError::InvalidStructure(e.to_string()))?; + Ok(migrate_stream_v1_to_v2(v1)) + } + _ => Err(VersionedLoadError::UnsupportedVersion(version.to_string())), + } +} +``` + +### 4. Test both directions + +```rust +#[test] +fn test_v1_migration() { + let json = r#"{"ast_version":"0.0.1","state_name":"Test","old_name":"Value"}"#; + let spec = load_stream_spec(json).unwrap(); + assert_eq!(spec.new_name, "Value"); + assert_eq!(spec.ast_version, CURRENT_AST_VERSION); +} + +#[test] +fn test_v2_native() { + let json = r#"{"ast_version":"2.0.0","state_name":"Test","new_name":"Value"}"#; + let spec = load_stream_spec(json).unwrap(); + assert_eq!(spec.new_name, "Value"); +} +``` + +## Best Practices + +1. **Always bump both crates** - Keep versions in sync between `hyperstack-macros` and `interpreter` + +2. **Keep old versions for 6+ months** - Give users time to upgrade their pipelines + +3. **Log migration warnings** - Let users know when their AST is being migrated + +4. **Test edge cases** - Missing fields, null values, malformed data + +5. **Document in CHANGELOG** - Note AST version changes in release notes + +6. **Use serde defaults** - For minor bumps, use `#[serde(default)]` to avoid breaking old ASTs + +7. **Validate after migration** - Ensure migrated ASTs pass validation + +## Migration Checklist + +Before releasing a new AST version: + +- [ ] Updated `CURRENT_AST_VERSION` in both type files +- [ ] Added migration logic in both versioned.rs files +- [ ] Updated all spec constructors +- [ ] Added tests for migration +- [ ] Tested loading old ASTs +- [ ] Tested loading new ASTs +- [ ] Updated documentation +- [ ] Added CHANGELOG entry +- [ ] Considered deprecation timeline + +## FAQ + +**Q: Can I skip versions? (e.g., 0.0.1 → 3.0.0)** + +A: Yes. You only need migration arms for versions that were actually released, not every intermediate version number. If someone has v0.0.1 and you jump to v3.0.0, you only need: +- 0.0.1 → 3.0.0 migration + +The numeric distance doesn't matter—only which versions actually exist in the wild. + +**Q: What if I need to rollback?** + +A: AST versions are additive. Keep old migration code and tests. Users with new ASTs can't go back, but old ASTs continue working. + +**Q: How do I deprecate a version?** + +A: After your deprecation window, change the migration to return an error: + +```rust +"0.0.1" => Err(VersionedLoadError::UnsupportedVersion( + "0.0.1 deprecated, run: hyperstack migrate-ast".to_string() +)) +``` + +**Q: Can I automate AST upgrades?** + +A: Yes! Create a CLI command that: +1. Loads old AST +2. Migrates to latest +3. Writes back to file + +```rust +pub fn upgrade_ast_file(path: &Path) -> Result<()> { + let json = fs::read_to_string(path)?; + let spec = load_stream_spec(&json)?; // Auto-migrates + let upgraded = serde_json::to_string_pretty(&spec)?; + fs::write(path, upgraded)?; + Ok(()) +} +``` \ No newline at end of file diff --git a/hyperstack-macros/src/ast/mod.rs b/hyperstack-macros/src/ast/mod.rs index a2c78206..24a5f950 100644 --- a/hyperstack-macros/src/ast/mod.rs +++ b/hyperstack-macros/src/ast/mod.rs @@ -49,6 +49,7 @@ mod reader; mod types; +pub mod versioned; pub(crate) mod writer; // Re-export all types for easy access diff --git a/hyperstack-macros/src/ast/types.rs b/hyperstack-macros/src/ast/types.rs index b01f437d..0024c11e 100644 --- a/hyperstack-macros/src/ast/types.rs +++ b/hyperstack-macros/src/ast/types.rs @@ -304,8 +304,20 @@ pub enum UnaryOp { // Stream Specification Types // ============================================================================ +/// Current AST version for SerializableStreamSpec and SerializableStackSpec +/// +/// ⚠️ IMPORTANT: This constant is duplicated in interpreter/src/ast.rs due to +/// circular dependency between proc-macro crates and their output crates. +/// When bumping this version, you MUST also update the constant in the +/// interpreter crate. A test in versioned.rs verifies they stay in sync. +pub const CURRENT_AST_VERSION: &str = "0.0.1"; + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SerializableStreamSpec { + /// AST schema version for backward compatibility + /// Uses semver format (e.g., "0.0.1") + #[serde(default = "default_ast_version")] + pub ast_version: String, pub state_name: String, #[serde(default)] pub program_id: Option, @@ -331,6 +343,10 @@ pub struct SerializableStreamSpec { pub views: Vec, } +fn default_ast_version() -> String { + CURRENT_AST_VERSION.to_string() +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct IdentitySpec { pub primary_keys: Vec, @@ -872,6 +888,10 @@ pub struct InstructionDef { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SerializableStackSpec { + /// AST schema version for backward compatibility + /// Uses semver format (e.g., "0.0.1") + #[serde(default = "default_ast_version")] + pub ast_version: String, pub stack_name: String, #[serde(default)] pub program_ids: Vec, diff --git a/hyperstack-macros/src/ast/versioned.rs b/hyperstack-macros/src/ast/versioned.rs new file mode 100644 index 00000000..5fee68ed --- /dev/null +++ b/hyperstack-macros/src/ast/versioned.rs @@ -0,0 +1,454 @@ +//! Versioned AST loader with automatic migration support. +//! +//! This module provides: +//! - Version detection from raw JSON +//! - Deserialization routing to the correct version +//! - Automatic migration to the latest AST format +//! +//! # Usage +//! +//! ```rust,ignore +//! use hyperstack_macros::ast::versioned::{load_stack_spec, load_stream_spec}; +//! +//! let stack = load_stack_spec(&json_string)?; +//! let stream = load_stream_spec(&json_string)?; +//! ``` + +// This module provides a public API for loading versioned ASTs. +// While not all items are currently used within this crate, they are +// part of the public API available for external use and future features +// like the `#[ast_spec]` macro or CLI tooling. + +use serde::Deserialize; +use serde_json::Value; +use std::fmt; + +use super::types::{SerializableStackSpec, SerializableStreamSpec, CURRENT_AST_VERSION}; + +/// Error type for versioned AST loading failures. +// Not yet used within this crate, but part of public API for future use +#[allow(dead_code)] +#[derive(Debug, Clone)] +pub enum VersionedLoadError { + /// The JSON could not be parsed + InvalidJson(String), + /// The AST version is not supported + UnsupportedVersion(String), + /// The AST structure is invalid for the detected version + InvalidStructure(String), +} + +impl fmt::Display for VersionedLoadError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + VersionedLoadError::InvalidJson(msg) => { + write!(f, "Invalid JSON: {}", msg) + } + VersionedLoadError::UnsupportedVersion(version) => { + write!( + f, + "Unsupported AST version: {}. Latest supported version: {}. \ + Older versions are supported via automatic migration.", + version, CURRENT_AST_VERSION + ) + } + VersionedLoadError::InvalidStructure(msg) => { + write!(f, "Invalid AST structure: {}", msg) + } + } + } +} + +impl std::error::Error for VersionedLoadError {} + +/// Load a stack spec from JSON with automatic version detection and migration. +/// +/// This function: +/// 1. Detects the AST version from the JSON +/// 2. Deserializes the appropriate version +/// 3. Migrates to the latest format if needed +/// +/// # Arguments +/// +/// * `json` - The JSON string containing the AST +/// +/// # Returns +/// +/// The deserialized and migrated `SerializableStackSpec` +/// +/// # Example +/// +/// ```rust,ignore +/// let json = std::fs::read_to_string("MyStack.stack.json")?; +/// let spec = load_stack_spec(&json)?; +/// ``` +// Not yet used within this crate, but part of public API for future use +#[allow(dead_code)] +pub fn load_stack_spec(json: &str) -> Result { + // Parse raw JSON to detect version + let raw: Value = + serde_json::from_str(json).map_err(|e| VersionedLoadError::InvalidJson(e.to_string()))?; + + // Extract version - default to "0.0.1" if not present (backwards compatibility) + let version = raw + .get("ast_version") + .and_then(|v| v.as_str()) + .unwrap_or("0.0.1"); + + // Route to appropriate deserializer based on version + match version { + v if v == CURRENT_AST_VERSION => { + // Current version - deserialize directly + serde_json::from_value::(raw) + .map_err(|e| VersionedLoadError::InvalidStructure(e.to_string())) + } + // Add migration arms for old versions here, e.g.: + // "0.0.1" => { migrate_v1_to_latest(raw) } + _ => { + // Unknown version + Err(VersionedLoadError::UnsupportedVersion(version.to_string())) + } + } +} + +/// Load a stream spec from JSON with automatic version detection and migration. +/// +/// Similar to `load_stack_spec` but for entity/stream specs. +/// +/// # Arguments +/// +/// * `json` - The JSON string containing the AST +/// +/// # Returns +/// +/// The deserialized and migrated `SerializableStreamSpec` +// Not yet used within this crate, but part of public API for future use +#[allow(dead_code)] +pub fn load_stream_spec(json: &str) -> Result { + // Parse raw JSON to detect version + let raw: Value = + serde_json::from_str(json).map_err(|e| VersionedLoadError::InvalidJson(e.to_string()))?; + + // Extract version - default to "0.0.1" if not present (backwards compatibility) + let version = raw + .get("ast_version") + .and_then(|v| v.as_str()) + .unwrap_or("0.0.1"); + + // Route to appropriate deserializer based on version + match version { + v if v == CURRENT_AST_VERSION => { + // Current version - deserialize directly + serde_json::from_value::(raw) + .map_err(|e| VersionedLoadError::InvalidStructure(e.to_string())) + } + // Add migration arms for old versions here, e.g.: + // "0.0.1" => { migrate_v1_to_latest(raw) } + _ => { + // Unknown version + Err(VersionedLoadError::UnsupportedVersion(version.to_string())) + } + } +} + +/// Versioned wrapper for SerializableStackSpec. +/// +/// This enum allows deserializing multiple AST versions and then +/// converting them to the latest format via `into_latest()`. +/// +/// ⚠️ IMPORTANT: This enum requires the `ast_version` field to be present in JSON. +/// It does NOT handle version-less (legacy) JSON files. For loading real-world ASTs +/// that may lack the `ast_version` field, use `load_stack_spec()` instead. +/// +// Not yet used within this crate, but part of public API for future use. +// Only Deserialize is derived to avoid duplicate `ast_version` keys +// (the inner struct already has this field, and we only use this for loading). +#[allow(dead_code)] +#[derive(Debug, Clone, Deserialize)] +#[serde(tag = "ast_version")] +pub enum VersionedStackSpec { + #[serde(rename = "0.0.1")] + V1(SerializableStackSpec), +} + +impl VersionedStackSpec { + /// Convert the versioned spec to the latest format. + /// + /// ⚠️ WARNING: This returns the spec with its original `ast_version` field unchanged. + /// If you need round-trip safety (e.g., serialize then deserialize), use `load_stack_spec` + /// instead, which properly sets `ast_version` to `CURRENT_AST_VERSION`. + // Not yet used within this crate, but part of public API for future use + #[allow(dead_code)] + pub fn into_latest(self) -> SerializableStackSpec { + match self { + VersionedStackSpec::V1(spec) => spec, + } + } +} + +/// Versioned wrapper for SerializableStreamSpec. +/// +/// This enum allows deserializing multiple AST versions and then +/// converting them to the latest format via `into_latest()`. +/// +/// ⚠️ IMPORTANT: This enum requires the `ast_version` field to be present in JSON. +/// It does NOT handle version-less (legacy) JSON files. For loading real-world ASTs +/// that may lack the `ast_version` field, use `load_stream_spec()` instead. +/// +// Not yet used within this crate, but part of public API for future use. +// Only Deserialize is derived to avoid duplicate `ast_version` keys +// (the inner struct already has this field, and we only use this for loading). +#[allow(dead_code)] +#[derive(Debug, Clone, Deserialize)] +#[serde(tag = "ast_version")] +pub enum VersionedStreamSpec { + #[serde(rename = "0.0.1")] + V1(SerializableStreamSpec), +} + +impl VersionedStreamSpec { + /// Convert the versioned spec to the latest format. + /// + /// ⚠️ WARNING: This returns the spec with its original `ast_version` field unchanged. + /// If you need round-trip safety (e.g., serialize then deserialize), use `load_stream_spec` + /// instead, which properly sets `ast_version` to `CURRENT_AST_VERSION`. + // Not yet used within this crate, but part of public API for future use + #[allow(dead_code)] + pub fn into_latest(self) -> SerializableStreamSpec { + match self { + VersionedStreamSpec::V1(spec) => spec, + } + } +} + +/// Detect the AST version from a JSON string without full deserialization. +/// +/// This is useful for logging, debugging, or routing decisions. +/// +/// # Arguments +/// +/// * `json` - The JSON string containing the AST +/// +/// # Returns +/// +/// The detected version string, or `"0.0.1"` if the field is absent (backwards compatibility default). +/// +/// # Example +/// +/// ```rust,ignore +/// let version = detect_ast_version(&json)?; +/// println!("AST version: {}", version); +/// ``` +// Not yet used within this crate, but part of public API for future use +#[allow(dead_code)] +pub fn detect_ast_version(json: &str) -> Result { + let raw: Value = + serde_json::from_str(json).map_err(|e| VersionedLoadError::InvalidJson(e.to_string()))?; + + Ok(raw + .get("ast_version") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + .unwrap_or_else(|| "0.0.1".to_string())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_load_stack_spec_v1() { + let json = r#" + { + "ast_version": "0.0.1", + "stack_name": "TestStack", + "program_ids": [], + "idls": [], + "entities": [], + "pdas": {}, + "instructions": [] + } + "#; + + let result = load_stack_spec(json); + assert!(result.is_ok()); + let spec = result.unwrap(); + assert_eq!(spec.stack_name, "TestStack"); + assert_eq!(spec.ast_version, CURRENT_AST_VERSION); + } + + #[test] + fn test_load_stack_spec_no_version_defaults_to_v1() { + // Test backwards compatibility - no ast_version field should default to 0.0.1 + let json = r#" + { + "stack_name": "TestStack", + "program_ids": [], + "idls": [], + "entities": [], + "pdas": {}, + "instructions": [] + } + "#; + + let result = load_stack_spec(json); + assert!(result.is_ok()); + let spec = result.unwrap(); + assert_eq!(spec.stack_name, "TestStack"); + assert_eq!(spec.ast_version, CURRENT_AST_VERSION); + } + + #[test] + fn test_load_stack_spec_unsupported_version() { + let json = r#" + { + "ast_version": "99.0.0", + "stack_name": "TestStack", + "program_ids": [], + "idls": [], + "entities": [], + "pdas": {}, + "instructions": [] + } + "#; + + let result = load_stack_spec(json); + assert!(result.is_err()); + match result.unwrap_err() { + VersionedLoadError::UnsupportedVersion(v) => assert_eq!(v, "99.0.0"), + _ => panic!("Expected UnsupportedVersion error"), + } + } + + #[test] + fn test_load_stream_spec_v1() { + let json = r#" + { + "ast_version": "0.0.1", + "state_name": "TestEntity", + "identity": {"primary_keys": ["id"], "lookup_indexes": []}, + "handlers": [], + "sections": [], + "field_mappings": {}, + "resolver_hooks": [], + "instruction_hooks": [], + "resolver_specs": [], + "computed_fields": [], + "computed_field_specs": [], + "views": [] + } + "#; + + let result = load_stream_spec(json); + assert!(result.is_ok()); + let spec = result.unwrap(); + assert_eq!(spec.state_name, "TestEntity"); + assert_eq!(spec.ast_version, CURRENT_AST_VERSION); + } + + #[test] + fn test_load_stream_spec_no_version_defaults_to_v1() { + // Test backwards compatibility - no ast_version field should default to 0.0.1 + let json = r#" + { + "state_name": "TestEntity", + "identity": {"primary_keys": ["id"], "lookup_indexes": []}, + "handlers": [], + "sections": [], + "field_mappings": {}, + "resolver_hooks": [], + "instruction_hooks": [], + "resolver_specs": [], + "computed_fields": [], + "computed_field_specs": [], + "views": [] + } + "#; + + let result = load_stream_spec(json); + assert!(result.is_ok()); + let spec = result.unwrap(); + assert_eq!(spec.state_name, "TestEntity"); + assert_eq!(spec.ast_version, CURRENT_AST_VERSION); + } + + #[test] + fn test_load_stream_spec_unsupported_version() { + let json = r#" + { + "ast_version": "99.0.0", + "state_name": "TestEntity", + "identity": {"primary_keys": ["id"], "lookup_indexes": []}, + "handlers": [], + "sections": [], + "field_mappings": {}, + "resolver_hooks": [], + "instruction_hooks": [], + "resolver_specs": [], + "computed_fields": [], + "computed_field_specs": [], + "views": [] + } + "#; + + let result = load_stream_spec(json); + assert!(result.is_err()); + match result.unwrap_err() { + VersionedLoadError::UnsupportedVersion(v) => assert_eq!(v, "99.0.0"), + _ => panic!("Expected UnsupportedVersion error"), + } + } + + #[test] + fn test_detect_ast_version() { + let json = r#"{"ast_version": "0.0.1", "stack_name": "Test"}"#; + assert_eq!(detect_ast_version(json).unwrap(), "0.0.1"); + + let json_no_version = r#"{"stack_name": "Test"}"#; + assert_eq!(detect_ast_version(json_no_version).unwrap(), "0.0.1"); + } + + /// Verifies that the AST version constant matches the interpreter crate. + /// This test ensures both crates stay in sync. + #[test] + fn test_ast_version_sync_with_interpreter() { + // Read the interpreter's ast.rs file + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); + let interpreter_ast_path = std::path::Path::new(&manifest_dir) + .join("..") // Go up to workspace root + .join("interpreter") + .join("src") + .join("ast.rs"); + + // Verify the file exists before attempting to read + assert!( + interpreter_ast_path.exists(), + "Cannot find interpreter source file at {:?}. \ + This test requires the source tree to be available.", + interpreter_ast_path + ); + + let content = std::fs::read_to_string(&interpreter_ast_path) + .expect("Failed to read interpreter/src/ast.rs"); + + // Parse the CURRENT_AST_VERSION constant + let version_line = content + .lines() + .find(|line| line.contains("pub const CURRENT_AST_VERSION")) + .expect("CURRENT_AST_VERSION not found in interpreter"); + + let version_str = version_line + .split('=') + .nth(1) + .and_then(|rhs| rhs.split('"').nth(1)) + .expect("Failed to parse version string"); + + assert_eq!( + version_str, CURRENT_AST_VERSION, + "AST version mismatch! hyperstack-macros has '{}', interpreter has '{}'. \ + Both crates must have the same CURRENT_AST_VERSION. \ + Update both files when bumping the version.", + CURRENT_AST_VERSION, version_str + ); + } +} diff --git a/hyperstack-macros/src/stream_spec/ast_writer.rs b/hyperstack-macros/src/stream_spec/ast_writer.rs index eb50026c..44a94366 100644 --- a/hyperstack-macros/src/stream_spec/ast_writer.rs +++ b/hyperstack-macros/src/stream_spec/ast_writer.rs @@ -201,6 +201,7 @@ pub fn build_ast( } let mut spec = SerializableStreamSpec { + ast_version: crate::ast::CURRENT_AST_VERSION.to_string(), state_name: entity_name.to_string(), program_id, idl: idl_snapshot, diff --git a/hyperstack-macros/src/stream_spec/idl_spec.rs b/hyperstack-macros/src/stream_spec/idl_spec.rs index 6e2595c6..81053091 100644 --- a/hyperstack-macros/src/stream_spec/idl_spec.rs +++ b/hyperstack-macros/src/stream_spec/idl_spec.rs @@ -456,6 +456,7 @@ pub fn process_idl_spec(mut module: ItemMod, idl_paths: &[String]) -> TokenStrea } let stack_spec = SerializableStackSpec { + ast_version: crate::ast::CURRENT_AST_VERSION.to_string(), stack_name: stack_name.clone(), program_ids: all_program_ids, idls: all_idl_snapshots, diff --git a/hyperstack-macros/src/stream_spec/module.rs b/hyperstack-macros/src/stream_spec/module.rs index 980351dd..68f3bc9b 100644 --- a/hyperstack-macros/src/stream_spec/module.rs +++ b/hyperstack-macros/src/stream_spec/module.rs @@ -130,6 +130,7 @@ pub fn process_module(mut module: ItemMod, attr: TokenStream) -> TokenStream { .collect(); let stack_spec = SerializableStackSpec { + ast_version: crate::ast::CURRENT_AST_VERSION.to_string(), stack_name: stack_name.clone(), program_ids: vec![], idls: vec![], diff --git a/interpreter/src/ast.rs b/interpreter/src/ast.rs index 74e1293e..5d800dd0 100644 --- a/interpreter/src/ast.rs +++ b/interpreter/src/ast.rs @@ -5,6 +5,18 @@ use std::marker::PhantomData; pub use hyperstack_idl::snapshot::*; +/// Current AST version for SerializableStreamSpec and SerializableStackSpec +/// +/// ⚠️ IMPORTANT: This constant is duplicated in hyperstack-macros/src/ast/types.rs due to +/// circular dependency between proc-macro crates and their output crates. +/// When bumping this version, you MUST also update the constant in the +/// hyperstack-macros crate. A test in versioned.rs verifies they stay in sync. +pub const CURRENT_AST_VERSION: &str = "0.0.1"; + +fn default_ast_version() -> String { + CURRENT_AST_VERSION.to_string() +} + pub fn idl_type_snapshot_to_rust_string(ty: &IdlTypeSnapshot) -> String { match ty { IdlTypeSnapshot::Simple(s) => map_simple_idl_type(s), @@ -377,6 +389,10 @@ pub enum UnaryOp { /// Serializable version of StreamSpec without phantom types #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SerializableStreamSpec { + /// AST schema version for backward compatibility + /// Uses semver format (e.g., "0.0.1") + #[serde(default = "default_ast_version")] + pub ast_version: String, pub state_name: String, /// Program ID (Solana address) - extracted from IDL #[serde(default)] @@ -489,6 +505,7 @@ impl TypedStreamSpec { /// Convert to serializable format pub fn to_serializable(&self) -> SerializableStreamSpec { let mut spec = SerializableStreamSpec { + ast_version: CURRENT_AST_VERSION.to_string(), state_name: self.state_name.clone(), program_id: None, idl: None, @@ -1314,6 +1331,10 @@ pub struct InstructionDef { /// Written to `.hyperstack/{StackName}.stack.json`. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SerializableStackSpec { + /// AST schema version for backward compatibility + /// Uses semver format (e.g., "0.0.1") + #[serde(default = "default_ast_version")] + pub ast_version: String, /// Stack name (PascalCase, derived from module ident) pub stack_name: String, /// Program IDs (one per IDL, in order) diff --git a/interpreter/src/lib.rs b/interpreter/src/lib.rs index a29cfec3..668ab172 100644 --- a/interpreter/src/lib.rs +++ b/interpreter/src/lib.rs @@ -36,6 +36,7 @@ pub mod scheduler; pub mod slot_hash_cache; pub mod spec_trait; pub mod typescript; +pub mod versioned; pub mod vm; pub mod vm_metrics; diff --git a/interpreter/src/typescript.rs b/interpreter/src/typescript.rs index 079f6591..32ea52ad 100644 --- a/interpreter/src/typescript.rs +++ b/interpreter/src/typescript.rs @@ -2340,6 +2340,7 @@ mod tests { #[test] fn test_derived_view_codegen() { let spec = SerializableStreamSpec { + ast_version: CURRENT_AST_VERSION.to_string(), state_name: "OreRound".to_string(), program_id: None, idl: None, diff --git a/interpreter/src/versioned.rs b/interpreter/src/versioned.rs new file mode 100644 index 00000000..650d7195 --- /dev/null +++ b/interpreter/src/versioned.rs @@ -0,0 +1,434 @@ +//! Versioned AST loader with automatic migration support. +//! +//! This module provides: +//! - Version detection from raw JSON +//! - Deserialization routing to the correct version +//! - Automatic migration to the latest AST format +//! +//! # Usage +//! +//! ```rust,ignore +//! use hyperstack_interpreter::versioned::{load_stack_spec, load_stream_spec}; +//! +//! let stack = load_stack_spec(&json_string)?; +//! let stream = load_stream_spec(&json_string)?; +//! ``` + +use serde::Deserialize; +use serde_json::Value; +use std::fmt; + +use crate::ast::{SerializableStackSpec, SerializableStreamSpec, CURRENT_AST_VERSION}; + +/// Error type for versioned AST loading failures. +#[derive(Debug, Clone)] +pub enum VersionedLoadError { + /// The JSON could not be parsed + InvalidJson(String), + /// The AST version is not supported + UnsupportedVersion(String), + /// The AST structure is invalid for the detected version + InvalidStructure(String), +} + +impl fmt::Display for VersionedLoadError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + VersionedLoadError::InvalidJson(msg) => { + write!(f, "Invalid JSON: {}", msg) + } + VersionedLoadError::UnsupportedVersion(version) => { + write!( + f, + "Unsupported AST version: {}. Latest supported version: {}. \ + Older versions are supported via automatic migration.", + version, CURRENT_AST_VERSION + ) + } + VersionedLoadError::InvalidStructure(msg) => { + write!(f, "Invalid AST structure: {}", msg) + } + } + } +} + +impl std::error::Error for VersionedLoadError {} + +/// Load a stack spec from JSON with automatic version detection and migration. +/// +/// This function: +/// 1. Detects the AST version from the JSON +/// 2. Deserializes the appropriate version +/// 3. Migrates to the latest format if needed +/// +/// # Arguments +/// +/// * `json` - The JSON string containing the AST +/// +/// # Returns +/// +/// The deserialized and migrated `SerializableStackSpec` +/// +/// # Example +/// +/// ```rust,ignore +/// let json = std::fs::read_to_string("MyStack.stack.json")?; +/// let spec = load_stack_spec(&json)?; +/// ``` +pub fn load_stack_spec(json: &str) -> Result { + // Parse raw JSON to detect version + let raw: Value = + serde_json::from_str(json).map_err(|e| VersionedLoadError::InvalidJson(e.to_string()))?; + + // Extract version - default to "0.0.1" if not present (backwards compatibility) + let version = raw + .get("ast_version") + .and_then(|v| v.as_str()) + .unwrap_or("0.0.1"); + + // Route to appropriate deserializer based on version + match version { + v if v == CURRENT_AST_VERSION => { + // Current version - deserialize directly + serde_json::from_value::(raw) + .map_err(|e| VersionedLoadError::InvalidStructure(e.to_string())) + } + // Add migration arms for old versions here, e.g.: + // "0.0.1" => { migrate_v1_to_latest(raw) } + _ => { + // Unknown version + Err(VersionedLoadError::UnsupportedVersion(version.to_string())) + } + } +} + +/// Load a stream spec from JSON with automatic version detection and migration. +/// +/// Similar to `load_stack_spec` but for entity/stream specs. +/// +/// # Arguments +/// +/// * `json` - The JSON string containing the AST +/// +/// # Returns +/// +/// The deserialized and migrated `SerializableStreamSpec` +pub fn load_stream_spec(json: &str) -> Result { + // Parse raw JSON to detect version + let raw: Value = + serde_json::from_str(json).map_err(|e| VersionedLoadError::InvalidJson(e.to_string()))?; + + // Extract version - default to "0.0.1" if not present (backwards compatibility) + let version = raw + .get("ast_version") + .and_then(|v| v.as_str()) + .unwrap_or("0.0.1"); + + // Route to appropriate deserializer based on version + match version { + v if v == CURRENT_AST_VERSION => { + // Current version - deserialize directly + serde_json::from_value::(raw) + .map_err(|e| VersionedLoadError::InvalidStructure(e.to_string())) + } + // Add migration arms for old versions here, e.g.: + // "0.0.1" => { migrate_v1_to_latest(raw) } + _ => { + // Unknown version + Err(VersionedLoadError::UnsupportedVersion(version.to_string())) + } + } +} + +/// Versioned wrapper for SerializableStackSpec. +/// +/// This enum allows deserializing multiple AST versions and then +/// converting them to the latest format via `into_latest()`. +/// +/// ⚠️ IMPORTANT: This enum requires the `ast_version` field to be present in JSON. +/// It does NOT handle version-less (legacy) JSON files. For loading real-world ASTs +/// that may lack the `ast_version` field, use `load_stack_spec()` instead. +/// +/// Note: Only Deserialize is derived to avoid duplicate `ast_version` keys +/// (the inner struct already has this field, and we only use this for loading). +#[derive(Debug, Clone, Deserialize)] +#[serde(tag = "ast_version")] +pub enum VersionedStackSpec { + #[serde(rename = "0.0.1")] + V1(SerializableStackSpec), +} + +impl VersionedStackSpec { + /// Convert the versioned spec to the latest format. + /// + /// ⚠️ WARNING: This returns the spec with its original `ast_version` field unchanged. + /// If you need round-trip safety (e.g., serialize then deserialize), use `load_stack_spec` + /// instead, which properly sets `ast_version` to `CURRENT_AST_VERSION`. + pub fn into_latest(self) -> SerializableStackSpec { + match self { + VersionedStackSpec::V1(spec) => spec, + } + } +} + +/// Versioned wrapper for SerializableStreamSpec. +/// +/// This enum allows deserializing multiple AST versions and then +/// converting them to the latest format via `into_latest()`. +/// +/// ⚠️ IMPORTANT: This enum requires the `ast_version` field to be present in JSON. +/// It does NOT handle version-less (legacy) JSON files. For loading real-world ASTs +/// that may lack the `ast_version` field, use `load_stream_spec()` instead. +/// +/// Note: Only Deserialize is derived to avoid duplicate `ast_version` keys +/// (the inner struct already has this field, and we only use this for loading). +#[derive(Debug, Clone, Deserialize)] +#[serde(tag = "ast_version")] +pub enum VersionedStreamSpec { + #[serde(rename = "0.0.1")] + V1(SerializableStreamSpec), +} + +impl VersionedStreamSpec { + /// Convert the versioned spec to the latest format. + /// + /// ⚠️ WARNING: This returns the spec with its original `ast_version` field unchanged. + /// If you need round-trip safety (e.g., serialize then deserialize), use `load_stream_spec` + /// instead, which properly sets `ast_version` to `CURRENT_AST_VERSION`. + pub fn into_latest(self) -> SerializableStreamSpec { + match self { + VersionedStreamSpec::V1(spec) => spec, + } + } +} + +/// Detect the AST version from a JSON string without full deserialization. +/// +/// This is useful for logging, debugging, or routing decisions. +/// +/// # Arguments +/// +/// * `json` - The JSON string containing the AST +/// +/// # Returns +/// +/// The detected version string, or `"0.0.1"` if the field is absent (backwards compatibility default). +/// +/// # Example +/// +/// ```rust,ignore +/// let version = detect_ast_version(&json)?; +/// println!("AST version: {}", version); +/// ``` +pub fn detect_ast_version(json: &str) -> Result { + let raw: Value = + serde_json::from_str(json).map_err(|e| VersionedLoadError::InvalidJson(e.to_string()))?; + + Ok(raw + .get("ast_version") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + .unwrap_or_else(|| "0.0.1".to_string())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_load_stack_spec_v1() { + let json = r#" + { + "ast_version": "0.0.1", + "stack_name": "TestStack", + "program_ids": [], + "idls": [], + "entities": [], + "pdas": {}, + "instructions": [] + } + "#; + + let result = load_stack_spec(json); + assert!(result.is_ok()); + let spec = result.unwrap(); + assert_eq!(spec.stack_name, "TestStack"); + assert_eq!(spec.ast_version, CURRENT_AST_VERSION); + } + + #[test] + fn test_load_stack_spec_no_version_defaults_to_v1() { + // Test backwards compatibility - no ast_version field should default to 0.0.1 + let json = r#" + { + "stack_name": "TestStack", + "program_ids": [], + "idls": [], + "entities": [], + "pdas": {}, + "instructions": [] + } + "#; + + let result = load_stack_spec(json); + assert!(result.is_ok()); + let spec = result.unwrap(); + assert_eq!(spec.stack_name, "TestStack"); + assert_eq!(spec.ast_version, CURRENT_AST_VERSION); + } + + #[test] + fn test_load_stack_spec_unsupported_version() { + let json = r#" + { + "ast_version": "99.0.0", + "stack_name": "TestStack", + "program_ids": [], + "idls": [], + "entities": [], + "pdas": {}, + "instructions": [] + } + "#; + + let result = load_stack_spec(json); + assert!(result.is_err()); + match result.unwrap_err() { + VersionedLoadError::UnsupportedVersion(v) => assert_eq!(v, "99.0.0"), + _ => panic!("Expected UnsupportedVersion error"), + } + } + + #[test] + fn test_load_stream_spec_v1() { + let json = r#" + { + "ast_version": "0.0.1", + "state_name": "TestEntity", + "identity": {"primary_keys": ["id"], "lookup_indexes": []}, + "handlers": [], + "sections": [], + "field_mappings": {}, + "resolver_hooks": [], + "instruction_hooks": [], + "resolver_specs": [], + "computed_fields": [], + "computed_field_specs": [], + "views": [] + } + "#; + + let result = load_stream_spec(json); + assert!(result.is_ok()); + let spec = result.unwrap(); + assert_eq!(spec.state_name, "TestEntity"); + assert_eq!(spec.ast_version, CURRENT_AST_VERSION); + } + + #[test] + fn test_load_stream_spec_no_version_defaults_to_v1() { + // Test backwards compatibility - no ast_version field should default to 0.0.1 + let json = r#" + { + "state_name": "TestEntity", + "identity": {"primary_keys": ["id"], "lookup_indexes": []}, + "handlers": [], + "sections": [], + "field_mappings": {}, + "resolver_hooks": [], + "instruction_hooks": [], + "resolver_specs": [], + "computed_fields": [], + "computed_field_specs": [], + "views": [] + } + "#; + + let result = load_stream_spec(json); + assert!(result.is_ok()); + let spec = result.unwrap(); + assert_eq!(spec.state_name, "TestEntity"); + assert_eq!(spec.ast_version, CURRENT_AST_VERSION); + } + + #[test] + fn test_load_stream_spec_unsupported_version() { + let json = r#" + { + "ast_version": "99.0.0", + "state_name": "TestEntity", + "identity": {"primary_keys": ["id"], "lookup_indexes": []}, + "handlers": [], + "sections": [], + "field_mappings": {}, + "resolver_hooks": [], + "instruction_hooks": [], + "resolver_specs": [], + "computed_fields": [], + "computed_field_specs": [], + "views": [] + } + "#; + + let result = load_stream_spec(json); + assert!(result.is_err()); + match result.unwrap_err() { + VersionedLoadError::UnsupportedVersion(v) => assert_eq!(v, "99.0.0"), + _ => panic!("Expected UnsupportedVersion error"), + } + } + + #[test] + fn test_detect_ast_version() { + let json = r#"{"ast_version": "0.0.1", "stack_name": "Test"}"#; + assert_eq!(detect_ast_version(json).unwrap(), "0.0.1"); + + let json_no_version = r#"{"stack_name": "Test"}"#; + assert_eq!(detect_ast_version(json_no_version).unwrap(), "0.0.1"); + } + + /// Verifies that the AST version constant matches the hyperstack-macros crate. + /// This test ensures both crates stay in sync. + #[test] + fn test_ast_version_sync_with_macros() { + // Read the hyperstack-macros' types.rs file + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); + let macros_types_path = std::path::Path::new(&manifest_dir) + .join("..") // Go up to workspace root + .join("hyperstack-macros") + .join("src") + .join("ast") + .join("types.rs"); + + // Verify the file exists before attempting to read + assert!( + macros_types_path.exists(), + "Cannot find hyperstack-macros source file at {:?}. \ + This test requires the source tree to be available.", + macros_types_path + ); + + let content = std::fs::read_to_string(¯os_types_path) + .expect("Failed to read hyperstack-macros/src/ast/types.rs"); + + // Parse the CURRENT_AST_VERSION constant + let version_line = content + .lines() + .find(|line| line.contains("pub const CURRENT_AST_VERSION")) + .expect("CURRENT_AST_VERSION not found in hyperstack-macros"); + + let version_str = version_line + .split('=') + .nth(1) + .and_then(|rhs| rhs.split('"').nth(1)) + .expect("Failed to parse version string"); + + assert_eq!( + version_str, CURRENT_AST_VERSION, + "AST version mismatch! interpreter has '{}', hyperstack-macros has '{}'. \ + Both crates must have the same CURRENT_AST_VERSION. \ + Update both files when bumping the version.", + CURRENT_AST_VERSION, version_str + ); + } +}