fix: DeleteNode cascade, go.mod build, DSN pragmas, chunking, backup visibility, key policy

CHANGELOG.md

@@ -7,6 +7,76 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+- **Standalone module resolution**: `go.mod` required
+  `github.com/GrayCodeAI/hawk-mcpkit v0.0.0` (a non-existent version, only
+  satisfiable through the monorepo `replace`), which made yaad unbuildable
+  whenever it was consumed as a dependency module. The requirement is now the
+  tagged `v0.1.5` (which ships `ServeHTTPWithShutdown`, the newest mcpkit API
+  yaad uses) and `go.sum` carries its hashes; the local `replace` remains for
+  monorepo development. The dead `ServeSSE`/`ServeSSEWithShutdown` wrappers in
+  `internal/server` were removed — they referenced mcpkit APIs added after
+  v0.1.5 and had no callers in yaad.
+- **Per-connection PRAGMAs now cover the whole connection pool**:
+  `synchronous`, `wal_autocheckpoint`, `temp_store`, `mmap_size`,
+  `cache_size`, and `recursive_triggers` were issued once through
+  `db.ExecContext`, which only configured the single pooled connection that
+  served the call — the other four connections (pool size 5) silently ran on
+  SQLite defaults. They are now `_pragma=` DSN parameters, which the
+  modernc.org/sqlite driver applies to every new connection. `page_size`
+  (a no-op after the database file exists) was dropped, and `PRAGMA optimize`
+  moved from startup to `Store.Close()`, matching SQLite's guidance to run it
+  near connection close.
+- **Batch metadata and edge-count queries chunk large ID sets**:
+  `LoadNodeMetadata`/`FillNodeMetadata` and `CountEdgesBatch` built a single
+  `IN (...)` with every requested ID, ignoring the `maxSQLVariables` (900)
+  host-parameter budget the other batch helpers (`GetNodesBatch`,
+  `GetAllEdgesFor`, `GetEdgesBetween`) already chunk at — large ID sets could
+  exceed SQLite's per-statement parameter limit. Both now fetch in
+  `maxSQLVariables`-sized chunks and merge results, with regression tests
+  covering >900 IDs.
+- **Backup failures are now visible and backups are fsynced**:
+  - `BackupScheduler.run` discarded every snapshot error (`_ = b.RunNow`), so
+    a dead backup pipeline looked healthy. Failures are now logged via
+    `log/slog` and tracked in the new `BackupScheduler.Status()` accessor
+    (last success/error timestamps, snapshot/failure counters), with
+    `RunNow` recording its outcome too.
+  - `RotateBackups` silently ignored `os.Remove` errors; unexpected ones are
+    now logged (a vanished file is still fine).
+  - `Store.Backup` now fsyncs the snapshot file before renaming it into
+    place and best-effort fsyncs the backup directory after the rename, so a
+    crash cannot leave a renamed-but-not-durable (or truncated) backup.
+- **`EnvKeyProvider` enforces a minimum key policy**: the raw environment
+  value was used as master key material regardless of length, so a short
+  passphrase (whose entropy HKDF cannot fix) was silently accepted.
+  `KeyBytes` now accepts a raw string of at least 32 bytes, or a base64- or
+  hex-encoded value that decodes to exactly 32 bytes (raw wins when a value
+  qualifies both ways), and otherwise returns an error naming the
+  environment variable.
+  **Upgrade note:** deployments using a shorter key are rejected after this
+  change (in hawk this disables the yaad memory bridge with a logged
+  warning — it never falls back to plaintext). Rows already encrypted under
+  a now-rejected short key cannot be decrypted by `EnvKeyProvider` anymore;
+  set a compliant key and re-create or migrate that data.
+- **Process lock paths are normalized**: `AcquireProcessLock` keys its
+  in-process lock registry by the raw `dbPath + ".lock"` string, so
+  lexically equivalent paths (`dir/db`, `dir/./db`, `dir/x/../db`) opened two
+  lock-file handles in one process and the second open falsely reported
+  "database locked by another yaad process". Lock paths are now normalized
+  with `filepath.Abs`.
+- **`storage.DeleteNode` foreign-key violations**: `deleteNodeQ` only deleted
+  `edges` + `nodes`, so deleting a node with child rows in `embeddings`,
+  `embeddings_hnsw`, `file_watch`, or `node_signatures` failed with
+  `FOREIGN KEY constraint failed` (no `ON DELETE CASCADE` in the schema), and
+  `node_versions` / `node_metadata` rows leaked as orphans. All child rows are
+  now deleted explicitly, leaf tables first, in the same transaction.
+- **Swallowed delete errors in engine passes**: `GarbageCollect`
+  (`engine/decay.go`), the sparsifier passes (`engine/sparsify.go`),
+  `consolidateDuplicates` (`engine/improve.go`), and the LLM consolidator
+  (`engine/llm_consolidation.go`) silently ignored `DeleteNode` failures and
+  mis-reported removed/merged/pruned counts. Failures are now logged via
+  `log/slog` and only successful deletions are counted.
+
 ## [0.2.0] — 2026-07-14
 
 ### Changed

engine/decay.go

@@ -2,6 +2,7 @@ package engine
 
 import (
 	"context"
+	"log/slog"
 	"math"
 	"time"
 
@@ -142,13 +143,20 @@ func GarbageCollect(ctx context.Context, store storage.Storage, cfg DecayConfig)
 
 	// Phase 2: delete all collected IDs
 	removed := 0
+	failed := 0
 	for _, id := range toDelete {
 		if err := ctx.Err(); err != nil {
 			return removed, err
 		}
-		if err := store.DeleteNode(ctx, id); err == nil {
-			removed++
+		if err := store.DeleteNode(ctx, id); err != nil {
+			failed++
+			slog.Warn("gc: delete node failed", "node_id", id, "error", err)
+			continue
 		}
+		removed++
+	}
+	if failed > 0 {
+		slog.Error("gc: failed to delete nodes", "failed", failed, "removed", removed)
 	}
 	return removed, nil
 }

engine/improve.go

@@ -16,6 +16,7 @@ package engine
 import (
 	"context"
 	"fmt"
+	"log/slog"
 	"strings"
 	"sync/atomic"
 	"time"
@@ -241,6 +242,7 @@ func (e *Engine) consolidateDuplicates(ctx context.Context, nodes []*storage.Nod
 				continue
 			}
 			if err := e.store.DeleteNode(ctx, n.ID); err != nil {
+				slog.Warn("improve: failed to delete duplicate node", "node_id", n.ID, "error", err)
 				continue
 			}
 			merged++

engine/llm_consolidation.go

@@ -3,6 +3,7 @@ package engine
 import (
 	"context"
 	"fmt"
+	"log/slog"
 	"sort"
 	"strings"
 	"unicode"
@@ -249,7 +250,7 @@ func (lc *LLMConsolidator) applyMergePlan(ctx context.Context, plan *MergePlan)
 	for _, nid := range plan.NodeIDs {
 		if err := lc.store.DeleteNode(ctx, nid); err != nil {
 			// Best-effort delete; log but don't fail.
-			continue
+			slog.Warn("llm-consolidation: failed to delete merged-away node", "node_id", nid, "error", err)
 		}
 	}
 

engine/sparsify.go

@@ -3,6 +3,7 @@ package engine
 import (
 	"context"
 	"fmt"
+	"log/slog"
 	"sort"
 	"strings"
 
@@ -128,7 +129,9 @@ func (s *Sparsifier) mergeNearDuplicates(ctx context.Context) (int, error) {
 					_ = s.store.UpdateNodeContent(ctx, primary.ID, combined)
 				}
 				// Archive the duplicate.
-				_ = s.store.DeleteNode(ctx, duplicate.ID)
+				if err := s.store.DeleteNode(ctx, duplicate.ID); err != nil {
+					slog.Warn("sparsify: failed to delete merged duplicate", "node_id", duplicate.ID, "error", err)
+				}
 				processed[duplicate.ID] = true
 				merged++
 			}
@@ -200,7 +203,10 @@ func (s *Sparsifier) compressLowValueClusters(ctx context.Context) (int, error)
 
 		// Delete compressed nodes
 		for _, n := range toCompress {
-			_ = s.store.DeleteNode(ctx, n.ID)
+			if err := s.store.DeleteNode(ctx, n.ID); err != nil {
+				slog.Warn("sparsify: failed to delete compressed node", "node_id", n.ID, "error", err)
+				continue
+			}
 			compressed++
 		}
 	}
@@ -226,7 +232,10 @@ func (s *Sparsifier) pruneOrphans(ctx context.Context) (int, error) {
 			continue
 		}
 		if inbound+outbound == 0 && n.AccessCount <= 1 {
-			_ = s.store.DeleteNode(ctx, n.ID)
+			if err := s.store.DeleteNode(ctx, n.ID); err != nil {
+				slog.Warn("sparsify: failed to prune orphan node", "node_id", n.ID, "error", err)
+				continue
+			}
 			pruned++
 		}
 	}

internal/server/mcp_core.go

@@ -56,19 +56,6 @@ func (s *MCPServer) ServeHTTPWithShutdown(addr string) (*mcpserver.StreamableHTT
 	return s.srv.ServeHTTPWithShutdown(addr)
 }
 
-// ServeSSE starts the MCP server on the SSE (Server-Sent Events) transport.
-// This is the classic MCP transport used by Claude Desktop.
-func (s *MCPServer) ServeSSE(addr string) error {
-	return s.srv.ServeSSE(addr)
-}
-
-// ServeSSEWithShutdown starts the MCP server on the SSE transport and
-// returns the underlying server so the caller can invoke Shutdown for
-// graceful teardown.
-func (s *MCPServer) ServeSSEWithShutdown(addr string) (*mcpserver.SSEServer, error) {
-	return s.srv.ServeSSEWithShutdown(addr)
-}
-
 // mcp returns the underlying mcp-go server so the progress notification
 // helper can reach mcp-go's send path directly.
 func (s *MCPServer) mcp() *mcpserver.MCPServer {

storage/backup.go

@@ -37,9 +37,40 @@ func (s *Store) Backup(ctx context.Context, backupPath string) error {
 		_ = os.Remove(tmp)
 		return fmt.Errorf("restrict backup file permissions: %w", err)
 	}
+	// fsync the snapshot before the rename: once renamed, the file reads as
+	// a complete backup, so its contents must already be on durable storage.
+	if err := syncFile(tmp); err != nil {
+		_ = os.Remove(tmp)
+		return fmt.Errorf("sync backup file: %w", err)
+	}
 	if err := os.Rename(tmp, backupPath); err != nil {
 		_ = os.Remove(tmp)
 		return fmt.Errorf("finalize backup: %w", err)
 	}
+	// Best-effort fsync of the directory so the rename itself survives a
+	// crash. Directory Sync is a no-op / unsupported on some platforms
+	// (Windows rejects fsync on directory handles), hence the ignore.
+	_ = syncDir(dir)
 	return nil
 }
+
+// syncFile flushes a file's contents to durable storage.
+func syncFile(path string) error {
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = f.Close() }()
+	return f.Sync()
+}
+
+// syncDir fsyncs a directory entry so a rename inside it is durable. Errors
+// are ignored by callers: several platforms cannot sync directory handles.
+func syncDir(path string) error {
+	d, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = d.Close() }()
+	return d.Sync()
+}

storage/backup_rotate.go

@@ -2,14 +2,26 @@ package storage
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"io/fs"
+	"log/slog"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
 	"time"
 )
 
+// removeBackupFile deletes a pruned backup (or stale temp file), logging any
+// error other than "already gone" so rotation failures are visible instead of
+// silently leaving the directory over retention.
+func removeBackupFile(path string) {
+	if err := os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) {
+		slog.Warn("backup rotation: failed to remove backup file", "path", path, "error", err)
+	}
+}
+
 // defaultTmpMaxAge bounds how long aborted Backup temp files may linger
 // before RotateBackups sweeps them.
 const defaultTmpMaxAge = 24 * time.Hour
@@ -49,7 +61,7 @@ func (s *Store) RotateBackups(ctx context.Context, dir string, keep int, maxAge
 		// Sweep abandoned VACUUM INTO temp files from failed backups.
 		if strings.HasSuffix(name, ".tmp") {
 			if time.Since(info.ModTime()) > defaultTmpMaxAge {
-				_ = os.Remove(path)
+				removeBackupFile(path)
 			}
 			continue
 		}
@@ -70,7 +82,7 @@ func (s *Store) RotateBackups(ctx context.Context, dir string, keep int, maxAge
 		tooMany := keep > 0 && i >= keep
 		tooOld := maxAge > 0 && time.Since(b.modTime) > maxAge && i > 0
 		if tooMany || tooOld {
-			_ = os.Remove(b.path)
+			removeBackupFile(b.path)
 		}
 	}
 	return nil