feat(storage): at-rest encryption for node content via key providers

plans/PHASE1-SQLITE-HARDENING.md

@@ -1,6 +1,6 @@
 # Feature Specification: Phase 1 SQLite Hardening
 
-**Status:** Implemented — Phase 1 merged via PRs #47–#51; Phase 2 items 1–3 merged via PRs #53–#55
+**Status:** Implemented — Phase 1 merged via PRs #47–#51; Phase 2 complete (PRs #53–#59: HNSW incremental/restore, backup rotation/scheduler, lock fix, encryption)
 **Author:** Patel230
 **Date:** 2026-08-15
 **Repos affected:** `GrayCodeAI/yaad` (via hawk submodule `external/yaad`)
@@ -185,11 +185,12 @@ does not exist on Windows):**
 - [x] `feat/storage-hnsw` — PR #50 (`6c03254`)
 - [x] `feat/storage-backup` — PR #51 (`dfedbce`)
 
-### Phase 2 (items 1-3 shipped)
+### Phase 2 (items 1-4 shipped)
 - ~~Incremental HNSW updates on `SaveEmbedding`/`DeleteEmbedding`~~ — **done** (PR #53: `HNSWIndex.Upsert`/`Remove` with link pruning + entry-point repair; `Store` write paths apply them; transactional writes invalidate the cache after commit)
 - ~~Persist-and-restore of the HNSW graph across restarts~~ — **done** (PR #54: versioned payload in `embeddings_hnsw` carrying `version`/`m`/`efConstruction`; `HNSWIndex.Restore` requires parameter + membership parity, `BuildHNSWIndex` is the force-rebuild path)
-- ~~Backup rotation~~ — **done** (PR #55: `Store.RotateBackups(dir, keep, maxAge)` with count/age retention and stale-`.tmp` cleanup; a daemon-side scheduler is not yet implemented)
-- Not yet implemented: encryption columns (migration v5) wired to a real key provider
+- ~~Backup rotation~~ — **done** (PR #55: `Store.RotateBackups(dir, keep, maxAge)` with count/age retention and stale-`.tmp` cleanup)
+- ~~Backup scheduler~~ — **done** (PR #57: `Store.ScheduleBackups(dir, interval, keep, maxAge)` returns a `BackupScheduler` — immediate first snapshot, then per-interval best-effort `VACUUM INTO` + rotation; hawk starts it from session startup via `YaadBridge.EnsureBackups()`, hourly, keep 7 / 30d)
+- ~~Encryption columns (migration v5) wired to a real key provider~~ — **done** (PR #59: app-layer AES-256-GCM — `KeyProvider` interface + `EnvKeyProvider` (`YAAD_ENCRYPTION_KEY`), version-bound ciphertexts `yaad.aes256gcm.v{n}`, transparent legacy-plaintext reads with progressive re-encrypt on update, `SearchNodes` in-memory fallback over decrypted content; hawk opts in via the same env var)
 
 ## Testing Strategy
 

storage/crypto.go

@@ -0,0 +1,223 @@
+package storage
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+
+	"golang.org/x/crypto/hkdf"
+)
+
+// CipherScheme identifies the at-rest encryption format used for node
+// content. The scheme is embedded in every ciphertext so future formats can
+// coexist with current ones on reads.
+const CipherScheme = "yaad.aes256gcm.v"
+
+// CiphertextPrefix is the full marker stored at the head of encrypted node
+// content. Content without this prefix is treated as legacy plaintext.
+//
+// The prefix is deliberately namespaced and self-describing: FTS5 and LIKE
+// queries against encrypted rows simply never match plaintext queries, and a
+// stored value can always be told apart from content a user typed.
+const CiphertextPrefix = CipherScheme
+
+// hkdfSaltDomain binds derived keys to this column domain. Rotating keys or
+// adding encrypted columns later should change this domain.
+const hkdfSaltDomain = "yaad node content at rest"
+
+// KeyProvider supplies master key material for at-rest encryption. It is the
+// security boundary between the storage layer and wherever the real secret
+// lives — environment, keychain, KMS. Implementations return raw key bytes
+// per version; CurrentVersion names the version used for new writes and
+// older versions remain available so rotation can re-encrypt progressively.
+type KeyProvider interface {
+	// KeyBytes returns raw master key material for the requested version.
+	// Unknown versions return an error; empty material is rejected.
+	KeyBytes(version int) ([]byte, error)
+	// CurrentVersion is the key version used to encrypt new data.
+	CurrentVersion() int
+}
+
+// EnvKeyProvider is a KeyProvider backed by an environment variable. It is
+// the wired "real key provider" for yaad's encryption columns: the secret
+// never touches disk, only the caller's process environment. Only version 0
+// is supported — hosts that need rotation should layer their own provider.
+type EnvKeyProvider struct {
+	envVar string
+}
+
+// NewEnvKeyProvider returns a provider reading the master key from envVar.
+// The variable is read lazily on each KeyBytes call so tests can set it via
+// t.Setenv after construction.
+func NewEnvKeyProvider(envVar string) *EnvKeyProvider {
+	return &EnvKeyProvider{envVar: envVar}
+}
+
+// CurrentVersion implements KeyProvider.
+func (p *EnvKeyProvider) CurrentVersion() int { return 0 }
+
+// KeyBytes implements KeyProvider.
+func (p *EnvKeyProvider) KeyBytes(version int) ([]byte, error) {
+	if version != 0 {
+		return nil, fmt.Errorf("%w: %d", ErrUnknownKeyVersion, version)
+	}
+	raw := strings.TrimSpace(os.Getenv(p.envVar))
+	if raw == "" {
+		return nil, fmt.Errorf("%w: %s is unset or empty", ErrKeyMaterial, p.envVar)
+	}
+	return []byte(raw), nil
+}
+
+// NodeCipher encrypts and decrypts single string values with
+// AES-256-GCM. Per-version encryption keys are derived from provider master
+// material via HKDF-SHA256 (deterministic per version, never stored); each
+// encrypted value carries a random 12-byte nonce and authenticates the key
+// version as associated data, so ciphertexts cannot be relabeled across
+// versions. Encrypted output is ASCII (prefix + base64url) so it round-trips
+// through TEXT columns, backups, and JSON without encoding changes.
+//
+// A NodeCipher is safe for concurrent use.
+type NodeCipher struct {
+	provider KeyProvider
+	current  int // key version for new writes, captured at construction
+
+	mu      sync.RWMutex
+	derived map[int]cipher.AEAD // lazily derived per version
+}
+
+// NewNodeCipher validates the provider eagerly: the current key version is
+// derived now so misconfiguration (missing env var, bad material) fails at
+// setup instead of on the first write.
+func NewNodeCipher(p KeyProvider) (*NodeCipher, error) {
+	if p == nil {
+		return nil, fmt.Errorf("nil key provider")
+	}
+	c := &NodeCipher{
+		provider: p,
+		current:  p.CurrentVersion(),
+		derived:  make(map[int]cipher.AEAD),
+	}
+	if _, err := c.aeadFor(c.current); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+// KeyVersion returns the key version used for new encrypted values.
+func (c *NodeCipher) KeyVersion() int { return c.current }
+
+// Encrypt seals plaintext with the current key version. Empty strings pass
+// through unchanged so nullable columns stay empty rather than becoming a
+// decryptable "empty" blob.
+func (c *NodeCipher) Encrypt(plaintext string) (string, error) {
+	if plaintext == "" {
+		return "", nil
+	}
+	aead, err := c.aeadFor(c.current)
+	if err != nil {
+		return "", err
+	}
+	nonce := make([]byte, aead.NonceSize())
+	if _, err := rand.Read(nonce); err != nil {
+		return "", fmt.Errorf("generate nonce: %w", err)
+	}
+	sealed := aead.Seal(nil, nonce, []byte(plaintext), versionAAD(c.current))
+
+	payload := make([]byte, 0, len(nonce)+len(sealed))
+	payload = append(payload, nonce...)
+	payload = append(payload, sealed...)
+	return CipherScheme + strconv.Itoa(c.current) + "." + base64.RawURLEncoding.EncodeToString(payload), nil
+}
+
+// Decrypt opens a ciphertext produced by Encrypt. Values without the cipher
+// prefix are returned unchanged — this is how a store transparently reads
+// legacy plaintext rows written before encryption was enabled.
+func (c *NodeCipher) Decrypt(stored string) (string, error) {
+	if !IsEncryptedValue(stored) {
+		return stored, nil
+	}
+	rest := stored[len(CipherScheme):]
+	dot := strings.IndexByte(rest, '.')
+	if dot < 1 {
+		return "", fmt.Errorf("%w: missing version separator", ErrMalformedCiphertext)
+	}
+	version, err := strconv.Atoi(rest[:dot])
+	if err != nil {
+		return "", fmt.Errorf("%w: bad key version %q", ErrMalformedCiphertext, rest[:dot])
+	}
+	payload, err := base64.RawURLEncoding.DecodeString(rest[dot+1:])
+	if err != nil {
+		return "", fmt.Errorf("%w: bad base64 payload", ErrMalformedCiphertext)
+	}
+	aead, err := c.aeadFor(version)
+	if err != nil {
+		return "", err
+	}
+	ns := aead.NonceSize()
+	if len(payload) < ns+aead.Overhead() {
+		return "", fmt.Errorf("%w: payload shorter than nonce+tag", ErrMalformedCiphertext)
+	}
+	plaintext, err := aead.Open(nil, payload[:ns], payload[ns:], versionAAD(version))
+	if err != nil {
+		return "", fmt.Errorf("%w", ErrDecryptionFailed)
+	}
+	return string(plaintext), nil
+}
+
+// IsEncryptedValue reports whether a stored string carries the cipher prefix.
+func IsEncryptedValue(s string) bool {
+	return strings.HasPrefix(s, CiphertextPrefix)
+}
+
+// aeadFor returns (and caches) the AES-256-GCM AEAD derived from version's
+// master material. Double-checked under the mutex so hot reads avoid the
+// derivation path entirely.
+func (c *NodeCipher) aeadFor(version int) (cipher.AEAD, error) {
+	c.mu.RLock()
+	a := c.derived[version]
+	c.mu.RUnlock()
+	if a != nil {
+		return a, nil
+	}
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if a := c.derived[version]; a != nil {
+		return a, nil
+	}
+	master, err := c.provider.KeyBytes(version)
+	if err != nil {
+		return nil, err
+	}
+	if len(master) == 0 {
+		return nil, fmt.Errorf("%w: provider returned empty material for version %d", ErrKeyMaterial, version)
+	}
+	hk := hkdf.New(sha256.New, master, []byte(hkdfSaltDomain), []byte("aes256gcm key v"+strconv.Itoa(version)))
+	key := make([]byte, 32)
+	if _, err := io.ReadFull(hk, key); err != nil {
+		return nil, fmt.Errorf("derive key: %w", err)
+	}
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, fmt.Errorf("aes cipher: %w", err)
+	}
+	aead, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, fmt.Errorf("gcm: %w", err)
+	}
+	c.derived[version] = aead
+	return aead, nil
+}
+
+// versionAAD binds each ciphertext to the key version that sealed it.
+func versionAAD(version int) []byte {
+	return []byte("yaad-node-v" + strconv.Itoa(version))
+}