From c25fda53a8e7ecb0b216273999a114d4158e8ced Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 11:29:06 +0000 Subject: [PATCH 01/35] fix(entities): reject an unknown type on MODIFY ATTRIBUTE, add DROP DEFAULT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mendixlabs/mxcli#910. `ALTER ENTITY … MODIFY ATTRIBUTE UserID SET DEFAULT NULL` silently rewrote an Integer attribute to `Enumeration(set)`. MODIFY ATTRIBUTE always parses a type, and the grammar's last dataType alternative is a bare qualifiedName (an entity reference), so `SET` was read as the type and `DEFAULT NULL` as a constraint. The executor then wrote an EnumerationAttributeType with no enumeration behind it. Two things make this worse than reported. It is not specific to SET DEFAULT NULL — an ordinary typo (`MODIFY ATTRIBUTE A Integr`) corrupts identically. And the result is not merely wrong, it is unloadable: mx check dies before running any validation with System.ArgumentNullException: Value cannot be null. (Parameter 'value') at EnumerationAttributeType.set_EnumerationId CREATE ENTITY has rejected exactly this since #552, but the guard was inline in the create handler and never reached MODIFY. It is extracted to validateAttributeTypeRef and applied on the modify path before the attribute is touched. Refusing alone would leave the reporter's actual goal unreachable, since restating a type is the only way to keep a default and there was no way to clear one. So DROP DEFAULT ON ATTRIBUTE is added, and the refusal names it. A calculated attribute is refused rather than silently converted to a plain stored one. mxcli's own `syntax` topic documented `MODIFY ATTRIBUTE AttrName SET DEFAULT val` — the corrupting form — which is the likeliest way the reporter arrived at it. Verified against a pre-fix binary: that documented statement produced `Enumeration(set) default set.5`. The topic now shows the type as mandatory, names the trap, and lists DROP DEFAULT. Verified end to end on Mendix 11.13.0: before, mx check could not load the project; after the same flow it reports 0 errors. Each test was checked to fail against a stubbed guard. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018hifgRSawfaRWXS44YKtSJ --- .claude/skills/fix-issue.md | 1 + .../skills/mendix/generate-domain-model.md | 9 + cmd/mxcli/syntax/features_domain_model.go | 4 +- docs-site/src/language/alter-entity.md | 29 ++ .../910-modify-attribute-unknown-type.mdl | 61 +++++ mdl/ast/ast_entity.go | 1 + mdl/executor/alter_entity_modify_type_test.go | 256 ++++++++++++++++++ mdl/executor/attribute_type_ref.go | 69 +++++ mdl/executor/cmd_entities.go | 57 ++-- mdl/grammar/domains/MDLDomainModel.g4 | 1 + mdl/visitor/visitor_entity.go | 11 + 11 files changed, 480 insertions(+), 19 deletions(-) create mode 100644 mdl-examples/bug-tests/910-modify-attribute-unknown-type.mdl create mode 100644 mdl/executor/alter_entity_modify_type_test.go create mode 100644 mdl/executor/attribute_type_ref.go diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index ca969a1f2..6ebd0d603 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -524,3 +524,4 @@ extracting `OffsetExpression`/`LimitExpression`. | Windows Defender flags the mxcli **Windows** release binary as `Trojan:Script/Sabsik.EN.A!ml`; enterprise EDR (Defender for Endpoint, CrowdStrike, SentinelOne) blocks it harder. Not the generic unsigned-Go-binary false positive of #185 | The binary genuinely embedded **chisel**, a dual-use tunnelling/pivoting tool (SSH over WebSocket), on every platform — although the tunnel only ever runs inside a Linux container. `run --hub` linked `chisel/client`, `tunnel-hub` linked `chisel/server`, so windows/darwin carried 32 packages incl. the whole `x/crypto/ssh` stack for a feature they cannot use | `cmd/mxcli/docker/tunnel_linux.go` + `tunnel_other.go` (client seam), `cmd/mxcli/tunnelhub/control_linux.go` + `control_other.go` (server seam), `scripts/check-tunnel-deps.sh` (guard) | **Never obfuscate, pack or rename to dodge the scanner** — attacker tradecraft, and it makes the binary less trustworthy, not more. **Code signing does not fix this class**: a signed binary containing chisel is still flagged behaviourally; signing only addresses #185's generic false positive. The fix is to stop shipping the capability where it is unused: one interface per seam, `_linux.go` impl + `!linux` stub, commands still registered everywhere but failing with an actionable message. **Prove absence three ways, and know that `go tool nm` is not one of them** — release ldflags `-s -w` strip the symbol table, so nm reports "no symbols" whether or not the code is linked and would give a false pass; use `go list -deps`, `go version -m`, and `strings` (nm only on a deliberately unstripped build). **Guard against the transitive path, not the name**: match the module list (`x/crypto/ssh`, `gorilla/websocket`, `armon/go-socks5`, `jpillora/*`) so re-entry without the word "chisel" still trips it, and assert a **positive control** (chisel IS in the linux graph) so the check cannot pass vacuously. Verified by re-adding the import and watching the guard fail on all four windows/darwin targets. Result: -13.5 MB (-14.7%) on windows+darwin, linux unchanged. See ADR-0009 | | CE7375 "must be published and be the key when associations are exposed as an associated object id" on a service publishing no associations | `PublishAssociations` is the representation, not a yes/no — `No` selects "as an associated object id", which needs the system ID as key | `mdl/executor/validate_odata_service_shape.go` | Set `PublishAssociations: Yes` ("as a link", and the default when omitted). MDL-ODATA06 warns at check time | | `mx check` exits with `System.ArgumentOutOfRangeException: length ('-1')` and no error code | an OData service `Path` with no slash in it; mxbuild throws out of its own validator | `mdl/executor/validate_odata_service_shape.go` | Path must not start with `/` and must end with one (CE6550/CE6552). MDL-ODATA05 catches all three | +| `ALTER ENTITY … MODIFY ATTRIBUTE X SET DEFAULT ` (or any unrecognised word in the type position, including a typo like `Integr`) silently retypes the attribute to `Enumeration()`, and the project then **cannot be loaded at all** — `mx check` dies before validating with `System.ArgumentNullException: Value cannot be null. (Parameter 'value') at EnumerationAttributeType.set_EnumerationId` | `MODIFY ATTRIBUTE` requires a `dataType`, whose last grammar alternative is a bare `qualifiedName` (entity reference), so any word matches and the visitor maps it to `TypeEnumeration`. The executor wrote it with no enumeration behind it. CREATE ENTITY had guarded this since #552; the guard was **inline in the create handler** and never applied to MODIFY | `mdl/executor/attribute_type_ref.go` (new), `mdl/executor/cmd_entities.go` (`AlterEntityModifyAttribute` case) | Extract the create-path guard to `validateAttributeTypeRef` and call it from the modify path **before** mutating the attribute. Add `DROP DEFAULT ON ATTRIBUTE ` so clearing a default has a spelling that does not go through the type slot, and point the refusal at it. Check `mxcli syntax` too: the topic documented `MODIFY ATTRIBUTE AttrName SET DEFAULT val` — the corrupting form — which is how the reporter got there. When a guard is added inline to one handler, grep for sibling handlers that take the same input. mendixlabs/mxcli#910 | diff --git a/.claude/skills/mendix/generate-domain-model.md b/.claude/skills/mendix/generate-domain-model.md index b2907492b..84245609e 100644 --- a/.claude/skills/mendix/generate-domain-model.md +++ b/.claude/skills/mendix/generate-domain-model.md @@ -988,6 +988,15 @@ alter entity Module.Customer **Supported operations:** ADD ATTRIBUTE, RENAME ATTRIBUTE, MODIFY ATTRIBUTE (type + `NULLABLE`/`NOT NULL`/`UNIQUE`/`DEFAULT` constraints), DROP ATTRIBUTE, SET DOCUMENTATION, SET COMMENT, ADD INDEX, DROP INDEX, SET POSITION. +> **`MODIFY ATTRIBUTE` always takes a type** — restate it even when you only want +> to change a constraint. Its type slot accepts a bare qualified name, so a +> clause written in the type position is read as a type name: +> `MODIFY ATTRIBUTE X SET DEFAULT 0` treats `SET` as the type. mxcli now refuses +> that; before it did, the statement rewrote the attribute to an enumeration and +> produced a project Mendix could not open (#910). +> +> To clear a default value use **`DROP DEFAULT ON ATTRIBUTE `**. + ### Entity Positioning Guidelines When creating or repositioning entities, follow these layout rules for readable domain models: diff --git a/cmd/mxcli/syntax/features_domain_model.go b/cmd/mxcli/syntax/features_domain_model.go index ce3eaeca3..1bc38d2ba 100644 --- a/cmd/mxcli/syntax/features_domain_model.go +++ b/cmd/mxcli/syntax/features_domain_model.go @@ -51,8 +51,8 @@ func init() { "event handler", "documentation", "if not exists", "if exists", "idempotent", }, - Syntax: "ALTER ENTITY Module.Name ADD ATTRIBUTE [IF NOT EXISTS] AttrName: Type [constraints];\nALTER ENTITY Module.Name DROP ATTRIBUTE [IF EXISTS] AttrName;\nALTER ENTITY Module.Name RENAME ATTRIBUTE OldName TO NewName;\nALTER ENTITY Module.Name MODIFY ATTRIBUTE AttrName SET DEFAULT val;\nALTER ENTITY Module.Name ADD INDEX [name] [ON] (attr1, attr2);\nALTER ENTITY Module.Name SET DOCUMENTATION 'text';\nALTER ENTITY Module.Name ADD EVENT HANDLER ON BEFORE COMMIT CALL Module.MF RAISE ERROR;\n\nIF NOT EXISTS / IF EXISTS make the add/drop a no-op (skipped, not an error)\nwhen the attribute is already present / already gone — so a domain script\nre-runs cleanly. For a whole script, 'mxcli exec --continue-on-error' reports\neach failed statement and keeps going instead of halting at the first.", - Example: "ALTER ENTITY Shop.Customer ADD ATTRIBUTE Phone: String(20);\nALTER ENTITY Shop.Customer ADD ATTRIBUTE IF NOT EXISTS Phone: String(20); -- re-runnable\nALTER ENTITY Shop.Customer DROP ATTRIBUTE IF EXISTS OldField; -- re-runnable\nALTER ENTITY Shop.Customer RENAME ATTRIBUTE Email TO EmailAddress;\nALTER ENTITY Shop.Customer ADD INDEX ON (EmailAddress);\nALTER ENTITY Shop.Customer\n ADD EVENT HANDLER ON BEFORE COMMIT CALL Shop.Validate($currentObject) RAISE ERROR;", + Syntax: "ALTER ENTITY Module.Name ADD ATTRIBUTE [IF NOT EXISTS] AttrName: Type [constraints];\nALTER ENTITY Module.Name DROP ATTRIBUTE [IF EXISTS] AttrName;\nALTER ENTITY Module.Name RENAME ATTRIBUTE OldName TO NewName;\nALTER ENTITY Module.Name MODIFY ATTRIBUTE AttrName Type [DEFAULT val];\nALTER ENTITY Module.Name DROP DEFAULT ON ATTRIBUTE AttrName;\nALTER ENTITY Module.Name ADD INDEX [name] [ON] (attr1, attr2);\nALTER ENTITY Module.Name SET DOCUMENTATION 'text';\nALTER ENTITY Module.Name ADD EVENT HANDLER ON BEFORE COMMIT CALL Module.MF RAISE ERROR;\n\nMODIFY ATTRIBUTE always takes a type — restate it even when you are only\nchanging the default. There is no 'MODIFY ATTRIBUTE X SET DEFAULT v' form:\nSET would be read as the type name. Use DROP DEFAULT to clear one.\n\nIF NOT EXISTS / IF EXISTS make the add/drop a no-op (skipped, not an error)\nwhen the attribute is already present / already gone — so a domain script\nre-runs cleanly. For a whole script, 'mxcli exec --continue-on-error' reports\neach failed statement and keeps going instead of halting at the first.", + Example: "ALTER ENTITY Shop.Customer ADD ATTRIBUTE Phone: String(20);\nALTER ENTITY Shop.Customer ADD ATTRIBUTE IF NOT EXISTS Phone: String(20); -- re-runnable\nALTER ENTITY Shop.Customer DROP ATTRIBUTE IF EXISTS OldField; -- re-runnable\nALTER ENTITY Shop.Customer RENAME ATTRIBUTE Email TO EmailAddress;\nALTER ENTITY Shop.Customer MODIFY ATTRIBUTE Phone String(30) DEFAULT ''; -- type restated\nALTER ENTITY Shop.Customer DROP DEFAULT ON ATTRIBUTE Phone; -- clear a default\nALTER ENTITY Shop.Customer ADD INDEX ON (EmailAddress);\nALTER ENTITY Shop.Customer\n ADD EVENT HANDLER ON BEFORE COMMIT CALL Shop.Validate($currentObject) RAISE ERROR;", SeeAlso: []string{"domain-model.entity.create", "domain-model.entity.attributes"}, }) diff --git a/docs-site/src/language/alter-entity.md b/docs-site/src/language/alter-entity.md index 9d1da77bd..af92f2e9c 100644 --- a/docs-site/src/language/alter-entity.md +++ b/docs-site/src/language/alter-entity.md @@ -41,6 +41,34 @@ Change the type or constraints of an existing attribute with `MODIFY ATTRIBUTE`: ```sql ALTER ENTITY Sales.Customer MODIFY ATTRIBUTE Name: String(400) NOT NULL; + +**The type is not optional.** `MODIFY ATTRIBUTE` always parses a type, and its +type slot accepts a bare qualified name (an entity or enumeration reference), so +a clause written where the type belongs is read *as* the type: + +```mdl +-- WRONG: `SET` is read as the type name, not as a keyword +ALTER ENTITY Sales.Customer MODIFY ATTRIBUTE Discount SET DEFAULT 0; + +-- Right: restate the type +ALTER ENTITY Sales.Customer MODIFY ATTRIBUTE Discount Decimal DEFAULT 0; +``` + +mxcli refuses the first form and names the alternatives. Before it did, that +statement silently rewrote the attribute to an enumeration and produced a +project Mendix could not open ([#910](https://github.com/mendixlabs/mxcli/issues/910)). + +### Clearing a default + +`DROP DEFAULT` removes a default value without touching the type: + +```mdl +ALTER ENTITY Sales.Customer DROP DEFAULT ON ATTRIBUTE Discount; +``` + +Clearing a default that is already absent is a no-op, not an error. A +*calculated* attribute is refused rather than silently converted to a plain +stored one — that is a different change. ``` ## RENAME Attributes @@ -137,6 +165,7 @@ ALTER ENTITY . ADD ATTRIBUTE : [constraints]; ALTER ENTITY . DROP ATTRIBUTE ; ALTER ENTITY . MODIFY ATTRIBUTE : [constraints]; +ALTER ENTITY . DROP DEFAULT ON ATTRIBUTE ; ALTER ENTITY . RENAME ATTRIBUTE TO ; diff --git a/mdl-examples/bug-tests/910-modify-attribute-unknown-type.mdl b/mdl-examples/bug-tests/910-modify-attribute-unknown-type.mdl new file mode 100644 index 000000000..884248871 --- /dev/null +++ b/mdl-examples/bug-tests/910-modify-attribute-unknown-type.mdl @@ -0,0 +1,61 @@ +-- ============================================================================ +-- Bug 910: MODIFY ATTRIBUTE with an unrecognised type corrupted the project +-- ============================================================================ +-- https://github.com/mendixlabs/mxcli/issues/910 +-- +-- Reported as: `ALTER ENTITY … MODIFY ATTRIBUTE UserID SET DEFAULT NULL` +-- silently turned an Integer attribute into an Enumeration. +-- +-- MODIFY ATTRIBUTE always takes a type, and the grammar's last dataType +-- alternative is a bare qualified name (an entity reference), so `SET` was read +-- as the type and `DEFAULT NULL` as a constraint. The executor then wrote an +-- EnumerationAttributeType with no enumeration behind it — and the result was +-- not merely wrong, it was unloadable. Before the fix, `mx check` died before +-- running any validation: +-- +-- System.ArgumentNullException: Value cannot be null. (Parameter 'value') +-- at EnumerationAttributeType.set_EnumerationId +-- +-- CREATE ENTITY has rejected the same mistake since #552; MODIFY ATTRIBUTE +-- never got the guard. +-- +-- Run: mxcli exec 910-modify-attribute-unknown-type.mdl -p app.mpr +-- Then: mx check the project — it must load and report 0 errors. +-- ============================================================================ + +create module Bug910; + +create persistent entity Bug910.OdooSession ( + UserID: integer default 7, + Note: string(200) +); +/ + +-- ---------------------------------------------------------------------------- +-- The regression. Both of these must now be REFUSED, leaving UserID an Integer. +-- Uncomment either one to confirm; both used to succeed and corrupt the project. +-- ---------------------------------------------------------------------------- + +-- The reported spelling — `set` lands in the type position: +-- alter entity Bug910.OdooSession modify attribute UserID set default null; + +-- The same defect reached by an ordinary typo: +-- alter entity Bug910.OdooSession modify attribute UserID Integr; + +-- Expected error for either: +-- Error: attribute 'UserID': unknown type 'set' — not a primitive, enumeration, or entity +-- MODIFY ATTRIBUTE always takes a type, so a clause it does not recognise is read as one. +-- To clear a default value: ALTER ENTITY … DROP DEFAULT ON ATTRIBUTE UserID; +-- To change the type: ALTER ENTITY … MODIFY ATTRIBUTE UserID ; + +-- ---------------------------------------------------------------------------- +-- What the reporter actually wanted: clear the default, keep the type. +-- ---------------------------------------------------------------------------- +alter entity Bug910.OdooSession drop default on attribute UserID; + +-- UserID is now `Integer` with no default. A genuine retype still works: +alter entity Bug910.OdooSession modify attribute UserID long; + +-- Verify: +-- describe entity Bug910.OdooSession; +-- -> UserID: Long (no default, no enumeration) diff --git a/mdl/ast/ast_entity.go b/mdl/ast/ast_entity.go index eb638a452..489222a5f 100644 --- a/mdl/ast/ast_entity.go +++ b/mdl/ast/ast_entity.go @@ -70,6 +70,7 @@ const ( AlterEntityAddEventHandler // ADD EVENT HANDLER ON BEFORE/AFTER CREATE/COMMIT/DELETE/ROLLBACK CALL Mod.MF AlterEntityDropEventHandler // DROP EVENT HANDLER ON BEFORE/AFTER CREATE/COMMIT/DELETE/ROLLBACK AlterEntitySetAllowCreateChangeLocally // SET ALLOW_CREATE_CHANGE_LOCALLY = true/false + AlterEntityDropDefault // DROP DEFAULT ON ATTRIBUTE — clear a default value ) // EventHandlerDef represents an event handler in CREATE/ALTER ENTITY syntax. diff --git a/mdl/executor/alter_entity_modify_type_test.go b/mdl/executor/alter_entity_modify_type_test.go new file mode 100644 index 000000000..ddd2ee3bf --- /dev/null +++ b/mdl/executor/alter_entity_modify_type_test.go @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// modifyTypeTestCtx builds an ExecContext over one entity with one Integer +// attribute, and reports what MODIFY ATTRIBUTE wrote back. +func modifyTypeTestCtx(t *testing.T) (*ExecContext, func() *domainmodel.Attribute) { + t.Helper() + mod := mkModule("App") + ent := &domainmodel.Entity{ + BaseElement: model.BaseElement{ID: nextID("ent")}, + ContainerID: nextID("dm"), + Name: "OdooSession", + Persistable: true, + Attributes: []*domainmodel.Attribute{ + {Name: "UserID", Type: &domainmodel.IntegerAttributeType{}}, + }, + } + dm := &domainmodel.DomainModel{ + BaseElement: model.BaseElement{ID: nextID("dm")}, + ContainerID: mod.ID, + Entities: []*domainmodel.Entity{ent}, + } + h := mkHierarchy(mod) + withContainer(h, dm.ID, mod.ID) + withContainer(h, ent.ContainerID, dm.ID) + + var written *domainmodel.Attribute + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListModulesFunc: func() ([]*model.Module, error) { return []*model.Module{mod}, nil }, + ListDomainModelsFunc: func() ([]*domainmodel.DomainModel, error) { return []*domainmodel.DomainModel{dm}, nil }, + GetDomainModelFunc: func(id model.ID) (*domainmodel.DomainModel, error) { return dm, nil }, + UpdateEntityFunc: func(dmID model.ID, e *domainmodel.Entity) error { + for _, a := range e.Attributes { + if a.Name == "UserID" { + written = a + } + } + return nil + }, + } + ctx, _ := newMockCtx(t, withBackend(mb), withHierarchy(h)) + return ctx, func() *domainmodel.Attribute { return written } +} + +// modifyToUnknownType runs MODIFY ATTRIBUTE against a type name that resolves to +// neither a primitive, an enumeration, nor an entity. The visitor maps any bare +// qualified name to TypeEnumeration (the TypeEnumeration/TypeEntity ambiguity), +// which is how a stray word reaches the executor as a type at all. +func modifyToUnknownType(ctx *ExecContext, typeName string) error { + return execAlterEntity(ctx, &ast.AlterEntityStmt{ + Name: ast.QualifiedName{Module: "App", Name: "OdooSession"}, + Operation: ast.AlterEntityModifyAttribute, + AttributeName: "UserID", + DataType: ast.DataType{ + Kind: ast.TypeEnumeration, + EnumRef: &ast.QualifiedName{Name: typeName}, + }, + }) +} + +// TestModifyAttributeRejectsUnknownType is the regression test for +// mendixlabs/mxcli#910. +// +// `ALTER ENTITY … MODIFY ATTRIBUTE UserID SET DEFAULT NULL` parses as +// dataType=`SET` (the grammar's last dataType alternative is a bare +// qualifiedName, so any word is a candidate entity/enum reference) with +// `DEFAULT NULL` as a constraint. The executor then wrote an EnumerationAttribute +// whose enumeration does not exist — and the resulting .mpr cannot be loaded at +// all: `mx check` dies with +// +// System.ArgumentNullException: Value cannot be null. (Parameter 'value') +// at EnumerationAttributeType.set_EnumerationId +// +// CREATE ENTITY has rejected exactly this since #552; MODIFY ATTRIBUTE never got +// the same guard, which is the whole bug. +func TestModifyAttributeRejectsUnknownType(t *testing.T) { + ctx, written := modifyTypeTestCtx(t) + + err := modifyToUnknownType(ctx, "set") + if err == nil { + t.Fatal("MODIFY ATTRIBUTE accepted an unknown type — this writes an unloadable .mpr (#910)") + } + if !strings.Contains(err.Error(), "set") { + t.Errorf("error %q does not name the offending type", err) + } + if w := written(); w != nil { + t.Errorf("the attribute was written despite the error: type is now %v", w.Type) + } +} + +// TestModifyAttributeRejectsTypoedType covers the same defect reached by an +// ordinary typo rather than the SET DEFAULT NULL shape. It is the more likely +// way to hit it, and it corrupts the project identically. +func TestModifyAttributeRejectsTypoedType(t *testing.T) { + ctx, written := modifyTypeTestCtx(t) + + if err := modifyToUnknownType(ctx, "Integr"); err == nil { + t.Fatal("MODIFY ATTRIBUTE accepted a misspelled type name (#910)") + } + if w := written(); w != nil { + t.Errorf("the attribute was written despite the error: type is now %v", w.Type) + } +} + +// TestModifyAttributeErrorNamesTheAlternatives keeps the refusal actionable: a +// user who typed SET DEFAULT NULL needs to be told what to type instead, or the +// error just moves the confusion. +func TestModifyAttributeErrorNamesTheAlternatives(t *testing.T) { + ctx, _ := modifyTypeTestCtx(t) + err := modifyToUnknownType(ctx, "set") + if err == nil { + t.Fatal("expected an error") + } + if !strings.Contains(err.Error(), "DROP DEFAULT") { + t.Errorf("error %q does not point at DROP DEFAULT, the way to clear a default", err) + } +} + +// TestModifyAttributeAcceptsAPrimitive guards against the fix over-reaching: an +// ordinary retype must still work. +func TestModifyAttributeAcceptsAPrimitive(t *testing.T) { + ctx, written := modifyTypeTestCtx(t) + + err := execAlterEntity(ctx, &ast.AlterEntityStmt{ + Name: ast.QualifiedName{Module: "App", Name: "OdooSession"}, + Operation: ast.AlterEntityModifyAttribute, + AttributeName: "UserID", + DataType: ast.DataType{Kind: ast.TypeString, Length: 100}, + }) + if err != nil { + t.Fatalf("a legitimate retype was rejected: %v", err) + } + if written() == nil { + t.Fatal("a legitimate retype was not written") + } +} + +// dropDefaultTestCtx builds a context over one attribute carrying the given +// value, and reports what was written back. +func dropDefaultTestCtx(t *testing.T, val *domainmodel.AttributeValue) (*ExecContext, func() *domainmodel.Attribute) { + t.Helper() + mod := mkModule("App") + ent := &domainmodel.Entity{ + BaseElement: model.BaseElement{ID: nextID("ent")}, + ContainerID: nextID("dm"), + Name: "OdooSession", + Persistable: true, + Attributes: []*domainmodel.Attribute{ + {Name: "UserID", Type: &domainmodel.IntegerAttributeType{}, Value: val}, + }, + } + dm := &domainmodel.DomainModel{ + BaseElement: model.BaseElement{ID: nextID("dm")}, + ContainerID: mod.ID, + Entities: []*domainmodel.Entity{ent}, + } + h := mkHierarchy(mod) + withContainer(h, dm.ID, mod.ID) + withContainer(h, ent.ContainerID, dm.ID) + + var written *domainmodel.Attribute + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListModulesFunc: func() ([]*model.Module, error) { return []*model.Module{mod}, nil }, + ListDomainModelsFunc: func() ([]*domainmodel.DomainModel, error) { return []*domainmodel.DomainModel{dm}, nil }, + GetDomainModelFunc: func(id model.ID) (*domainmodel.DomainModel, error) { return dm, nil }, + UpdateEntityFunc: func(dmID model.ID, e *domainmodel.Entity) error { + for _, a := range e.Attributes { + if a.Name == "UserID" { + written = a + } + } + return nil + }, + } + ctx, _ := newMockCtx(t, withBackend(mb), withHierarchy(h)) + return ctx, func() *domainmodel.Attribute { return written } +} + +func dropDefault(ctx *ExecContext, attr string) error { + return execAlterEntity(ctx, &ast.AlterEntityStmt{ + Name: ast.QualifiedName{Module: "App", Name: "OdooSession"}, + Operation: ast.AlterEntityDropDefault, + AttributeName: attr, + }) +} + +// TestDropDefaultClearsTheValueAndKeepsTheType is the other half of #910: the +// reporter wanted to clear a default, and the only spelling that looked right +// destroyed the project. DROP DEFAULT does it without touching the type. +func TestDropDefaultClearsTheValueAndKeepsTheType(t *testing.T) { + ctx, written := dropDefaultTestCtx(t, &domainmodel.AttributeValue{DefaultValue: "7"}) + + if err := dropDefault(ctx, "UserID"); err != nil { + t.Fatalf("DROP DEFAULT: %v", err) + } + w := written() + if w == nil { + t.Fatal("nothing was written") + } + if w.Value != nil { + t.Errorf("the default survived: %+v", w.Value) + } + if _, ok := w.Type.(*domainmodel.IntegerAttributeType); !ok { + t.Errorf("the attribute type changed to %T — DROP DEFAULT must not retype", w.Type) + } +} + +// TestDropDefaultRefusesACalculatedAttribute pins that a CalculatedValue is not +// treated as a default. Clearing it would quietly convert a calculated attribute +// into a plain stored one — a different operation, and a lossy one. +func TestDropDefaultRefusesACalculatedAttribute(t *testing.T) { + ctx, written := dropDefaultTestCtx(t, &domainmodel.AttributeValue{Type: "CalculatedValue"}) + + err := dropDefault(ctx, "UserID") + if err == nil { + t.Fatal("DROP DEFAULT silently cleared a calculated attribute's value") + } + if !strings.Contains(err.Error(), "calculated") { + t.Errorf("error %q does not explain that the attribute is calculated", err) + } + if written() != nil { + t.Error("the attribute was written despite the refusal") + } +} + +func TestDropDefaultOnMissingAttribute(t *testing.T) { + ctx, _ := dropDefaultTestCtx(t, &domainmodel.AttributeValue{DefaultValue: "7"}) + if err := dropDefault(ctx, "NoSuchAttr"); err == nil { + t.Fatal("DROP DEFAULT on a non-existent attribute succeeded") + } +} + +// TestDropDefaultOnAnAttributeWithoutOne is a no-op, not an error: clearing a +// default that is already absent is the state the user asked for. +func TestDropDefaultOnAnAttributeWithoutOne(t *testing.T) { + ctx, written := dropDefaultTestCtx(t, nil) + if err := dropDefault(ctx, "UserID"); err != nil { + t.Fatalf("DROP DEFAULT on an attribute with no default: %v", err) + } + if w := written(); w == nil || w.Value != nil { + t.Errorf("expected a written attribute with no value, got %+v", w) + } +} diff --git a/mdl/executor/attribute_type_ref.go b/mdl/executor/attribute_type_ref.go new file mode 100644 index 000000000..8ad4eabc6 --- /dev/null +++ b/mdl/executor/attribute_type_ref.go @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "github.com/mendixlabs/mxcli/mdl/ast" + mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" +) + +// validateAttributeTypeRef rejects an attribute type that names neither a +// primitive, a known enumeration, nor a known entity. +// +// The visitor maps every bare qualified name to TypeEnumeration (the +// TypeEnumeration/TypeEntity ambiguity documented in CLAUDE.md), so an +// unrecognised word arrives here looking like an enumeration reference. Writing +// it produces an EnumerationAttributeType whose enumeration does not exist, and +// Mendix cannot load the result at all — `mx check` dies before any validation +// runs: +// +// System.ArgumentNullException: Value cannot be null. (Parameter 'value') +// at EnumerationAttributeType.set_EnumerationId +// +// This lived inline in CREATE ENTITY (added for #552) and was never applied to +// ALTER ENTITY … MODIFY ATTRIBUTE, so the same typo corrupted a project through +// the modify path while the create path rejected it cleanly. It is shared here +// so a third caller cannot repeat that. +func validateAttributeTypeRef(ctx *ExecContext, attrName string, dt ast.DataType) error { + if dt.Kind != ast.TypeEnumeration || dt.EnumRef == nil { + return nil + } + refModule := dt.EnumRef.Module + refName := dt.EnumRef.Name + if findEnumeration(ctx, refModule, refName) != nil { + return nil + } + if _, err := findEntity(ctx, refModule, refName); err == nil { + return nil + } + return mdlerrors.NewValidationf( + "attribute '%s': unknown type '%s' — not a primitive, enumeration, or entity", + attrName, dt.EnumRef.String()) +} + +// validateModifyAttributeTypeRef is validateAttributeTypeRef with the hint that +// only MODIFY ATTRIBUTE needs. +// +// MODIFY ATTRIBUTE requires a type — the grammar has no form without one — and +// its last dataType alternative is a bare qualifiedName. So a user reaching for +// something the syntax does not have writes a statement whose first word lands +// in the type position: +// +// ALTER ENTITY M.E MODIFY ATTRIBUTE UserID SET DEFAULT NULL +// ^^^ parsed as the type +// +// The bare "unknown type 'set'" is accurate but unhelpful there, because the +// user was not trying to name a type at all. Naming DROP DEFAULT turns the +// refusal into the answer (mendixlabs/mxcli#910). +func validateModifyAttributeTypeRef(ctx *ExecContext, attrName string, dt ast.DataType) error { + err := validateAttributeTypeRef(ctx, attrName, dt) + if err == nil { + return nil + } + return mdlerrors.NewValidationf( + "%s\n"+ + " MODIFY ATTRIBUTE always takes a type, so a clause it does not recognise is read as one.\n"+ + " To clear a default value: ALTER ENTITY … DROP DEFAULT ON ATTRIBUTE %s;\n"+ + " To change the type: ALTER ENTITY … MODIFY ATTRIBUTE %s ;", + err.Error(), attrName, attrName) +} diff --git a/mdl/executor/cmd_entities.go b/mdl/executor/cmd_entities.go index 33edaa03d..86b5f1a95 100644 --- a/mdl/executor/cmd_entities.go +++ b/mdl/executor/cmd_entities.go @@ -155,25 +155,10 @@ func execCreateEntity(ctx *ExecContext, s *ast.CreateEntityStmt) error { } // Validate TypeEnumeration attribute refs before writing anything. - // The visitor uses TypeEnumeration for both enum and entity type references - // (TypeEnumeration vs TypeEntity ambiguity). Accept the ref when it resolves - // to either a known enumeration or a known entity; reject unknown names fast - // so typos don't silently produce corrupt models. for _, a := range s.Attributes { - if a.Type.Kind != ast.TypeEnumeration || a.Type.EnumRef == nil { - continue - } - refModule := a.Type.EnumRef.Module - refName := a.Type.EnumRef.Name - if findEnumeration(ctx, refModule, refName) != nil { - continue - } - if _, err := findEntity(ctx, refModule, refName); err == nil { - continue + if err := validateAttributeTypeRef(ctx, a.Name, a.Type); err != nil { + return err } - return mdlerrors.NewValidationf( - "attribute '%s': unknown type '%s' — not a primitive, enumeration, or entity", - a.Name, a.Type.EnumRef.String()) } // Create attributes and build name-to-ID map for validation rules and indexes @@ -842,11 +827,49 @@ func execAlterEntity(ctx *ExecContext, s *ast.AlterEntityStmt) error { invalidateDomainModelsCache(ctx) fmt.Fprintf(ctx.Output, "Renamed attribute '%s' to '%s' on entity %s\n", s.AttributeName, s.NewName, s.Name) + case ast.AlterEntityDropDefault: + // Clearing a default is its own operation because MODIFY ATTRIBUTE cannot + // express it: that form always takes a type, and its type slot accepts a + // bare qualified name, so `MODIFY ATTRIBUTE X SET DEFAULT NULL` read SET + // as the type and wrote an unloadable project (#910). + found := false + for _, attr := range entity.Attributes { + if attr.Name != s.AttributeName { + continue + } + // Only the stored default goes. A CalculatedValue is not a default — + // dropping it would silently turn a calculated attribute into a plain + // one, which is a different operation the user did not ask for. + if attr.Value != nil && attr.Value.Type == "CalculatedValue" { + return mdlerrors.NewValidationf( + "attribute '%s' is calculated, not defaulted — DROP DEFAULT does not apply "+ + "(use MODIFY ATTRIBUTE to change how it is computed)", s.AttributeName) + } + attr.Value = nil + found = true + break + } + if !found { + return mdlerrors.NewNotFoundMsg("attribute", s.AttributeName, + fmt.Sprintf("attribute '%s' not found on entity %s", s.AttributeName, s.Name)) + } + if err := ctx.Backend.UpdateEntity(dm.ID, entity); err != nil { + return mdlerrors.NewBackend("drop default", err) + } + invalidateHierarchy(ctx) + invalidateDomainModelsCache(ctx) + fmt.Fprintf(ctx.Output, "Dropped default value on attribute '%s' of entity %s\n", s.AttributeName, s.Name) + case ast.AlterEntityModifyAttribute: // CALCULATED attributes are only supported on persistent entities if s.Calculated && !entity.Persistable { return mdlerrors.NewValidationf("attribute '%s': calculated attributes are only supported on persistent entities", s.AttributeName) } + // Reject a type that resolves to nothing BEFORE touching the attribute. + // Writing one produces a .mpr Mendix cannot load at all (#910). + if err := validateModifyAttributeTypeRef(ctx, s.AttributeName, s.DataType); err != nil { + return err + } found := false for _, attr := range entity.Attributes { if attr.Name == s.AttributeName { diff --git a/mdl/grammar/domains/MDLDomainModel.g4 b/mdl/grammar/domains/MDLDomainModel.g4 index 1f1a954f9..4dd9d1d9f 100644 --- a/mdl/grammar/domains/MDLDomainModel.g4 +++ b/mdl/grammar/domains/MDLDomainModel.g4 @@ -205,6 +205,7 @@ alterEntityAction | MODIFY COLUMN attributeName COLON? dataType attributeConstraint* | DROP ATTRIBUTE ifExists? attributeName | DROP COLUMN ifExists? attributeName + | DROP DEFAULT ON ATTRIBUTE attributeName // clear an attribute's default value | SET DOCUMENTATION STRING_LITERAL | SET COMMENT STRING_LITERAL | SET POSITION LPAREN NUMBER_LITERAL COMMA NUMBER_LITERAL RPAREN diff --git a/mdl/visitor/visitor_entity.go b/mdl/visitor/visitor_entity.go index 2880796b4..1949b06cd 100644 --- a/mdl/visitor/visitor_entity.go +++ b/mdl/visitor/visitor_entity.go @@ -614,6 +614,17 @@ func (b *Builder) ExitAlterEntityAction(ctx *parser.AlterEntityActionContext) { return } + // DROP DEFAULT ON ATTRIBUTE — checked before DROP ATTRIBUTE, which + // also matches DROP + ATTRIBUTE and would otherwise swallow it. + if ctx.DROP() != nil && ctx.DEFAULT() != nil && ctx.ATTRIBUTE() != nil && len(attrNames) >= 1 { + b.statements = append(b.statements, &ast.AlterEntityStmt{ + Name: name, + Operation: ast.AlterEntityDropDefault, + AttributeName: attributeNameText(attrNames[0]), + }) + return + } + // DROP ATTRIBUTE / DROP COLUMN if ctx.DROP() != nil && (ctx.ATTRIBUTE() != nil || ctx.COLUMN() != nil) && len(attrNames) >= 1 { b.statements = append(b.statements, &ast.AlterEntityStmt{ From e923be7115a8395cd52cedf11d595306380cece7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 12:05:54 +0000 Subject: [PATCH 02/35] docs: triage mxcli-banking FINDINGS.md against current main Verifies the 8-slice feedback report from the mxcli-banking app against this checkout. 13 of 14 mxcli-side claims reproduce; roots pinned for 12. Notable: - DROP ATTRIBUTE's validation-rule cleanup is dead code: the executor matches on element ID while parseValidationRule stores a qualified name in AttributeID. Same confusion affects MemberAccess. - Microflow ApplyEntityAccess is hardcoded false in both engines with no MDL syntax to set it, which is the reported datasource leak. - CONV010's ALLOWED_ACTIONS uses BSON storage names the linter never sees; QUAL004 counts only call/schedule edges. - Two new defects not in the report: dropping the last attribute of an entity is a silent no-op, and a synced skill documents a NON_PERSISTENT spelling that does not parse. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01KTdvGwVewkdgNQGDCxpQPZ --- .../2026-08-17-banking-app-findings-triage.md | 497 ++++++++++++++++++ 1 file changed, 497 insertions(+) create mode 100644 docs/12-bug-reports/2026-08-17-banking-app-findings-triage.md diff --git a/docs/12-bug-reports/2026-08-17-banking-app-findings-triage.md b/docs/12-bug-reports/2026-08-17-banking-app-findings-triage.md new file mode 100644 index 000000000..62fa17af8 --- /dev/null +++ b/docs/12-bug-reports/2026-08-17-banking-app-findings-triage.md @@ -0,0 +1,497 @@ +# Triage: mxcli-banking FINDINGS.md + +Source: (966 lines, 8 slices) +Reported against: mxcli `nightly-20260815-0dda3a76`, Mendix 11.13.0 +Triaged against: `5c70014` (main), 2026-08-17 + +The banking app is a full CRUD/security/transfer app built end-to-end with MDL +and verified in a real browser. Its findings file is the most detailed external +report mxcli has had. This document separates the **mxcli defects** from the +Mendix-platform behaviour the report also (correctly) records, and pins a root +cause for each defect that reproduced here. + +Every "reproduced" claim below was measured on this checkout with a real +project. Claims that could not be measured here say so, and say why. + +--- + +## Verdict summary + +| # | Finding | Verdict | Root cause pinned | +|---|---|---|---| +| 1 | `DROP ATTRIBUTE` orphans the validation rule (CE1613) | **Confirmed defect** | yes | +| 2 | Dropping the *last* attribute is a silent no-op | **New defect** (not in the report) | partially | +| 3 | Microflow "Apply entity access" not settable — datasource leak | **Confirmed gap** | yes | +| 4 | Lint CONV010 false-positives on every `ACT_` microflow | **Confirmed defect** | yes | +| 5 | Lint QUAL004 misses datasource/action/nav references | **Confirmed defect** | yes | +| 6 | `mxcli syntax` prints `NON_PERSISTENT`, parser wants `NON-PERSISTENT` | **Confirmed doc defect** | yes | +| 6b | Synced skill documents a *third*, non-parsing spelling | **New defect** (not in the report) | yes | +| 7 | ComboBox rejects `onChangeEvent` | **Confirmed gap** | yes | +| 8 | `mxcli exec` applies scripts `mxcli check` rejects | **Confirmed gap** | yes | +| 9 | `mxcli check` does not validate `ALTER SETTINGS MODEL` keys | **Confirmed** | yes | +| 10 | Optimistic locking not settable from MDL | **Confirmed gap** | yes (key identified) | +| 11 | `DROP USER ROLE` leaves demo users dangling | **Confirmed defect** | yes | +| 12 | `SHOW MESSAGE` clause-order error is misleading | **Confirmed UX defect** | yes | +| 13 | `ALTER MODULE/PAGE SET DOCUMENTATION` is a parse error | **Confirmed gap** | yes | +| 14 | OQL view entities are write-only in MDL | **Not reproduced here** — needs 11.x | no | +| — | Everything else (Mendix platform behaviour) | **Correct, not our bug** | n/a | + +--- + +## 1. `DROP ATTRIBUTE` orphans the validation rule — the cleanup is dead code + +**Reported:** slice 1. `ALTER ENTITY … DROP ATTRIBUTE FullName` removed the +attribute and left the validation rule its `NOT NULL` had created, pointing at +an attribute that no longer exists → `CE1613`. Unrecoverable with mxcli alone: +there is no `DROP VALIDATION RULE`, and `DESCRIBE ENTITY` does not emit +validation rules, so the orphan is invisible to a describe→edit→exec round trip. +The reporter had to tear down and rebuild the entity and everything referencing +it. + +**Status: confirmed, with a one-line root cause.** This is the highest-severity +item in the report and the cheapest to fix. + +Minimal reproduction (this checkout, `sdk/mpr/testdata/v1-project`): + +``` +create module B4 +create persistent entity B4.C ( Nm: String(100) not null error 'req', Age: Integer ) +alter entity B4.C drop attribute Nm +``` + +Resulting BSON: + +```json +"Attributes": ["Age"], +"ValidationRules": [ + { "$Type": "DomainModels$ValidationRule", + "Attribute": "B4.C.Nm", + "RuleInfo": { "$Type": "DomainModels$RequiredRuleInfo" } } +] +``` + +Note the executor prints no `Removed 1 validation rule(s)` line — the cleanup +it already has did not fire. + +### Root cause + +`mdl/executor/cmd_entities.go` (DROP ATTRIBUTE) removes rules by element ID: + +```go +droppedID := entity.Attributes[idx].ID +for _, vr := range entity.ValidationRules { + if vr.AttributeID != droppedID { keepRules = append(keepRules, vr) } +} +``` + +But `sdk/mpr/parser_domainmodel.go:641` (`parseValidationRule`) stores a +**qualified name** in that field, because Mendix stores the rule's attribute +reference as a `BY_NAME_REFERENCE` string, not an ID: + +```go +} else if attrName, ok := attrRef.(string); ok { + // Store qualified name as ID - will need to resolve later + rule.AttributeID = model.ID(attrName) +} +``` + +The comment says "will need to resolve later" — it never is. `"B4.C.Nm"` is +never equal to a binary element ID, so **the cleanup can never match a rule read +from disk**, which is every rule Studio Pro or a prior mxcli run wrote. The +cleanup code has been in place since `5dc4a46` (2026-07-29) — before the nightly +the banking app used — and has been dead the whole time. + +The writer already handles the dual meaning (`serializeValidationRule`, +`writer_domainmodel.go:1291` explicitly branches on `strings.Contains(attrIDStr, ".")`). +The executor does not. + +**The same bug hits `MemberAccess`.** BSON stores those as qualified-name +strings too (verified in the same dump: `"Attribute": "NanoflowCommons.Geolocation.Timestamp"`), +and the DROP ATTRIBUTE handler filters them with the identical +`ma.AttributeID != droppedID` comparison. That is a plausible source of the +report's `CE0066 "Entity access is out of date"` as well. + +### Fix + +Compare on both spellings in the executor — or better, resolve the qualified +name to an element ID once at parse time so the model layer has one +representation. Index attributes should be checked for the same confusion. +Add the missing `DESCRIBE ENTITY` output for validation rules so an orphan is at +least *visible*, and consider `ALTER ENTITY … DROP VALIDATION RULE` for cleanup +of orphans already in the wild — without it, every project that has already hit +this needs Studio Pro. + +--- + +## 2. Dropping the last attribute of an entity is a silent no-op — NEW + +Not in the report; found while reproducing #1. + +``` +create module B6 +create persistent entity B6.C ( Age: Integer, Nm: String(50) ) +alter entity B6.C drop attribute Age -> works +alter entity B6.C drop attribute Nm -> "Dropped attribute 'Nm' from entity B6.C" +``` + +After the second drop the file hash is **unchanged** and +`describe entity B6.C` still shows `Nm: String(50)`. No validation rules are +involved; `MXCLI_ALWAYS_WRITE=1` does not change the outcome, so this is not +write elision — the model mutation itself never lands. The command reports +success either way. + +Severity is lower than #1 (dropping every attribute is rare) but the failure +mode — reporting success while doing nothing — is the worst shape a write can +have, and it is the same shape as #11 and the `revoke` bug in +`revoke_orphaned_rules.md`. Worth a targeted test at the writer level. + +--- + +## 3. Microflow "Apply entity access" is not settable — the datasource leak + +**Reported:** slice 2, marked "the most important finding so far". A microflow +retrieve ignores entity access unless the microflow's *Apply entity access* +property is set. MDL has no syntax for it, so a datasource microflow that does +not spell out its own ownership constraint **retrieves every customer's rows**. +Access rules still blank the attributes at render time, so the page looks +correct while other people's objects have been loaded. Second-order damage +measured: `LIMIT 5` was applied *before* the access filter, so the user lost one +of their own rows to make room for rows they cannot read. + +The report also (correctly) corrects its own slice 1 conclusion because of this, +and derives the right test rule: assert row *counts*, because "the other +customer's ID is not in the DOM" cannot distinguish "not retrieved" from +"retrieved and blanked". + +**Status: confirmed.** Both engines hardcode the property to `false`: + +- `mdl/backend/modelsdk/microflow_write.go:187` — `out.SetApplyEntityAccess(false)` +- `sdk/mpr/writer_microflow.go:96` — `{Key: "ApplyEntityAccess", Value: false}` + +`microflowOption` in `mdl/grammar/domains/MDLMicroflow.g4:103` accepts only +`FOLDER` and `COMMENT`. The `APPLY` lexer token exists (`MDLLexer.g4:686`) but +is used only by settings rules. `MicroflowsMicroflow.ApplyEntityAccess` and +`MicroflowsRule.ApplyEntityAccess` are both present in `generated/metamodel`, +so nothing is missing at the metamodel layer. + +This is a security-relevant default. mxcli currently makes it impossible to +write a microflow datasource that is safe by construction — the only available +mitigation is the one the reporter used (hand-write the XPath constraint in +every retrieve), and nothing in mxcli tells you that. + +### Fix + +Add a microflow header option (`APPLY ENTITY ACCESS`), wire it through +AST/visitor/executor to both engines, emit it from `DESCRIBE MICROFLOW`, and — +separately — consider a lint rule for the actual defect: a microflow used as a +page datasource that retrieves a security-constrained entity with neither +`ApplyEntityAccess` nor an XPath constraint mentioning `[%CurrentUser%]`. That +rule would have caught this app's bug in slice 1 rather than slice 2. + +--- + +## 4. CONV010 flags exactly what it documents as allowed + +**Reported:** slice 3. 11 false positives out of 13 findings, burying 2 real +ones. + +**Status: confirmed, verbatim.** `.claude/lint-rules/conv010_act_microflow_content.star`: + +```python +ALLOWED_ACTIONS = ("ShowFormAction", "CloseFormAction", "ShowHomeFormAction", + "ShowMessageAction", "DownloadFileAction") +ALLOWED_ACTIVITY_TYPES = ("SubMicroflow", ...) +``` + +Those are the **BSON storage names** (per the storage-name table in CLAUDE.md). +The linter never sees them: `getMicroflowActionType` +(`mdl/catalog/builder_microflows.go:284`) derives `action_type` from the **Go +type name** — + +```go +return strings.TrimPrefix(fmt.Sprintf("%T", action), "*microflows.") +``` + +— giving `ShowPageAction`, `ClosePageAction`, `MicroflowCallAction`. So the +allowlist matches nothing, and every `ACT_` microflow that shows a page, closes +a page, or calls a sub-microflow is flagged. + +The reporter's fix (list both spellings) is right. Audited the other 28 bundled +rules for the same stale vocabulary: **CONV010 is the only one affected.** + +A test that asserts each rule's action-name constants exist in the catalog's +vocabulary would prevent the whole class. + +--- + +## 5. QUAL004 misses three kinds of reference, not one + +**Reported:** slice 1, as a false positive on page datasources. + +**Status: confirmed, and broader than reported.** +`.claude/lint-rules/orphaned_elements.star` counts a microflow as referenced +only when: + +```python +if ref.ref_kind == "call" or ref.ref_kind == "schedule": +``` + +The reference builder emits nine other kinds (`builder_references.go:18-36`). +For microflows the ones that mean "this is an entry point" and are ignored: + +- `datasource` — a page/widget microflow datasource (the reported case; the + edge **is** emitted, `extractDataSourceRefs` handles `*pages.MicroflowSource`) +- `action` — a widget button calling a microflow +- `calculate` — a calculated attribute's microflow + +The page half of the rule has the same shape: it counts only `show_page`, and +ignores `home_page`, `login_page` and `menu_item`. A page reachable only from +navigation is reported orphaned; the `ENTRY_PAGE_PATTERNS` list +(`Home`/`Login`/`Index`/`Dashboard`) masks this for exactly the pages most +likely to be navigation targets, which is why it has gone unnoticed. + +The `schedule` kind was added for precisely this reason (see the comment in +`builder_references.go:485`); the other five were not. + +--- + +## 6. `NON_PERSISTENT` in the syntax help does not parse + +**Reported:** slice 2. + +**Status: confirmed.** The lexer *token* is named `NON_PERSISTENT` but matches a +hyphen (`MDLLexer.g4:32`: `NON_PERSISTENT: N O N '-' P E R S I S T E N T;`). +`cmd/mxcli/syntax/features_domain_model.go` lines 27 and 40 print the token name +into user-facing syntax. Measured: + +``` +CREATE NON-PERSISTENT ENTITY Test.Foo ( Name: String(100) ); -> Syntax OK +CREATE NON_PERSISTENT ENTITY Test.Foo ( Name: String(100) ); -> no viable alternative +``` + +Two-line fix. Also present in `docs-site/src/language/lexical-structure.md`, +`docs/05-mdl-specification/01-language-reference.md` and +`docs/06-mdl-reference/grammar-reference.md`, where it is a token-name listing +and therefore arguably correct — but the syntax help is not. + +### 6b. A synced skill documents a third spelling that also does not parse — NEW + +Not in the report. `.claude/skills/mendix/rest-call-from-json.md` — which +`mxcli init` **ships into every user project** — uses a different form entirely, +five times: + +``` +create entity Module.MyRootObject (NON_PERSISTENT) +``` + +Measured: `mismatched input ')' expecting ':'`. So the skill mxcli hands to +agents in user projects teaches syntax that cannot parse. Worse than the syntax +help, because it is the thing an LLM reads first. + +--- + +## 7. ComboBox rejects `onChangeEvent`, and the template already carries it + +**Reported:** slice 2. `OnChange` works on `DATEPICKER`; every spelling is +MDL-WIDGET01 "has no property" on `COMBOBOX`, even though the generated widget +doc lists `onChangeEvent` and `onChangeDatabaseEvent`. The consequence is +silent: an unwired account picker does nothing, so the page had to be redesigned +around an explicit button. + +**Status: confirmed, and the machinery is already there.** + +- `sdk/widgets/definitions/combobox.def.json` declares four `knownProperties`, + all `optionsSourceAssociation*`. Neither change event is among them. +- `sdk/widgets/templates/mendix-11.6/combobox.json` **does** contain both + `onChangeEvent` and `onChangeDatabaseEvent`. + +So the widget can carry the property and the doc generator can see it; only the +`.def.json` allowlist blocks it. The doc/def mismatch is itself worth a test: +a property the generated doc advertises should be a property MDL accepts. + +--- + +## 8. `mxcli exec` has no pre-flight gate + +**Reported:** slice 2. "A page with an invalid widget property was written to +the model anyway. Always run `check` before `exec`." + +**Status: confirmed.** `cmd/mxcli/cmd_exec.go` has two flags (`--project`, +`--continue-on-error`) and no validation step. Parse errors do stop it — I +verified that — but the semantic checks `mxcli check` runs (MDL-WIDGET*, MDL0xx, +reference validation) are never invoked by `exec`. + +That makes the documented workflow "always run check first" a convention nothing +enforces, on a tool whose whole point is unattended agent use. `exec` should run +the same semantic checks and refuse on errors, with `--no-check`/`--force` to +opt out. + +--- + +## 9–10. Settings: unvalidated keys, and optimistic locking + +**Reported:** slice 4. + +**#9 confirmed:** `ALTER SETTINGS MODEL OptimisticLocking = true;` passes +`mxcli check` cleanly and fails only at `exec` with `unknown model setting`. The +grammar accepts any identifier there. The reporter's summary is the right one: +"check passed" means the text parses, not that the statement means anything. +`check` should validate settings keys against the same table `exec` uses. + +**#10 confirmed, and the key is identifiable.** The report's correction is +accurate — Mendix *does* ship optimistic locking as an app setting, and it is +exactly the mitigation for the read-then-write balance race the app documents. +The stored property is `EnableDataStorageOptimisticLocking` on +`Settings$ModelSettings` (read directly out of a project's BSON while triaging +this). Adding it to the accepted model settings is small, and it takes an item +off the "needs Studio Pro" list for a security-relevant setting. + +Same list, not investigated here: strict mode (SEC005), which the report also +flags as Studio-Pro-only and which weakens XPath constraint enforcement +(CVE-2023-23835). Worth checking whether it is equally reachable. + +--- + +## 11. `DROP USER ROLE` does not follow inbound references + +**Reported:** slice 1. `DROP USER ROLE User` succeeded and left the blank app's +`demo_user` referencing it → `CE1613`. + +**Status: confirmed.** `execDropUserRole` (`mdl/executor/cmd_security_write.go:288`) +checks the role exists, calls `RemoveUserRole`, prints success. There is no +demo-user check, no navigation-profile check, and — unlike DROP ATTRIBUTE — not +even a warning. + +Same family as #1: mxcli drops the thing you named without following what points +at it. The report's framing is right, and it is worth deciding this once as a +policy (cascade, refuse, or warn) rather than per-command. + +--- + +## 12. The `SHOW MESSAGE` clause-order error sends you to the wrong place + +**Reported:** slice 4. + +**Status: confirmed.** Measured: + +``` +SHOW MESSAGE 'Ref {1}.' OBJECTS [$x] TYPE Information; + -> mismatched input 'TYPE' expecting ';' + 'Type' is a reserved keyword in MDL. Use a different name like: + - Type_ (add underscore suffix) ... +``` + +The reserved-keyword hint fires on the token text regardless of context, so a +pure clause-ordering problem is reported as an identifier clash that does not +exist. Cheap fix: suppress the keyword hint when the offending token is a +keyword in a valid position for the statement, or add the ordering to the +message. + +--- + +## 13. Module and page documentation are unreachable + +**Reported:** slice 1 — `ALTER PAGE … SET DOCUMENTATION` and +`ALTER MODULE … SET DOCUMENTATION` are both parse errors, a `/** */` docblock +before `CREATE PAGE` is not picked up (entities and microflows do take one), so +the two QUAL002 lint findings this raises are not fixable with mxcli. + +**Status: confirmed** (`ALTER MODULE B6 SET DOCUMENTATION 'hello'` → +`no viable alternative at input 'SETDOCUMENTATION'`). Note the shape of the +complaint: mxcli's own linter raises a finding mxcli gives you no way to fix. + +--- + +## 14. OQL view entities — reported, not reproduced here + +**Reported:** slice 6, as the headline finding that killed the dashboard design. +mxcli can CREATE a view entity and Mendix accepts it (0 errors), after which MDL +cannot reference it at all: `GRANT` → `entity not found`, +`RETURNS LIST OF` → `entity not found for return type`, `SHOW ENTITIES` does not +list it. Plus three quieter traps: `--` comments inside the OQL body fail the +parse (and `exec` then does not run, which reads as success if you grep for a +success line); `CREATE OR MODIFY` does not prune members the OQL no longer +produces (CE6770 until DROP + recreate); pass-through columns inherit the source +length and `cast()` is not in the grammar. + +**Status: not reproduced.** View entities require Mendix 10.18+ and the only +projects in this checkout are 9.24; no mxbuild is cached in this environment, so +creating an 11.x project was out of scope for a triage pass. + +Reading the code does *not* obviously support the claim — the read path parses +`Source` (`parser_domainmodel.go:105`), `isViewEntity` is used in validation, +and several executor paths handle `DomainModels$OqlViewEntitySource` on entities +obtained from the reader — so a view entity ought to appear in `dm.Entities` and +therefore in `SHOW ENTITIES`. Something version- or engine-specific is going on. + +**This is the top item to reproduce next**, on a real 11.13 project. The report +is careful and its other claims all held up, so I would not discount it. + +The associated finding — that Mendix view entities *can* have associations (you +select the associated object's `.ID`) and MDL cannot declare one — is worth +recording as a feature gap independent of the above. + +--- + +## What is correct and is not our bug + +A large fraction of the report is Mendix platform behaviour, accurately +diagnosed. Recording it here so nobody re-opens it as an mxcli issue, and +because several items belong in mxcli's *skills* even though they are not +defects: + +- `mxcli check` passing does not mean `mx check` passes — the report's single + most emphasised operational lesson. Correct by design (MDL syntax + mxcli's + own rules vs. the Mendix model), but the skills should say it as plainly as + the report does. +- `Administration.Account` already defines `FullName`/`Email` (CE0069); a user + entity may not carry Required/Unique rules (CE7247); every user role needs a + System module role (CE0156); the stock `User` role is not inert and costs one + CE2729 per widget on the default home page. +- Database datasources **do** apply entity access; microflow datasources do + not. The report states this pair better than our docs do. +- Row-level security makes associated objects unreadable, not just unlisted → + you must denormalise across the security boundary. Four attributes in the app + exist for this reason, plus the counter-example (a plain String caption on + readable reference data needs nothing). +- `HEAD()`, `COUNT()`, `SUM()` are list activities, not expression functions + (CE0117). `id` is an XPath pseudo-attribute, not a member. +- `RETRIEVE … LIMIT 1` returns an object, not a list. +- A microflow is one transaction — the whole atomicity story, for free. +- Mendix commits an input to the model on **blur**; Mendix serves page URLs + under `/p/`; the trial licence caps concurrent sessions and Playwright pages + leak them (and `/logout` is a 404 — `mx.logout()` is the working call). +- `Show Message` is blocking; a microflow-datasource grid does not refresh + after a delete. +- No forward references, in either direction, between microflows and pages. + mxcli's hint is good; the report's complaint is that + `--continue-on-error` leaves the model partially updated, which is fair. + +The two test-methodology rules the report derives are worth lifting into +`.claude/skills/verify-in-runtime.md` more or less verbatim: + +> A check that asserts something is ABSENT needs a sibling check that the same +> thing is PRESENT for someone. + +> A literal about *seeded* data is fine; a literal about data that any other +> suite mutates is a time bomb. + +Both were earned the hard way: the first found the `/p/` prefix after four +security assertions had been passing for the wrong reason, and the second found +four stale tests during a regression run. + +--- + +## Suggested order of work + +1. **#1** — validation-rule / member-access cleanup keyed on the wrong field. + Data-corrupting, one-line root cause, already has (dead) code. +2. **#3** — `APPLY ENTITY ACCESS`. Security-relevant, currently unexpressible. +3. **#8** — `exec` pre-flight gate. Cheap, and it is the backstop that would + have softened several other findings. +4. **#4, #5** — the two lint rules. Cheap, and false positives are actively + harmful: CONV010's 11 false positives buried 2 real findings. +5. **#6/#6b** — three documented spellings of `NON-PERSISTENT`, two of which do + not parse, one of which ships to every user project. +6. **#14** — reproduce on 11.13 before designing anything. +7. **#2, #9, #10, #11, #12, #13** — individually small. From d895d64ebd74ddd88d982e42227ca376651acb43 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 13:03:49 +0000 Subject: [PATCH 03/35] fix(executor): RENAME ATTRIBUTE now updates cross-references (#910) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `alter entity Mod.Ent rename attribute Old to New;` renamed the attribute inside the domain model and stopped there. Everywhere else Mendix stores an attribute reference it stores the fully qualified name as a string, so microflow create/change members, page attribute widgets, and the entity's own validation and access rules all kept pointing at the old name. mxbuild reports each one as CE1613 "The selected attribute 'Mod.Ent.Old' no longer exists."; Studio Pro still opens the project, so nothing looks wrong until a build. `mxcli rename` already had the project-wide reference scanner for entities, enumerations, associations, modules and documents — attributes were the one kind never wired to it. The handler now calls RenameReferences and reports how many documents it touched. Order is load-bearing: the scan runs AFTER UpdateEntity, not before. It is a raw-BSON pass over every unit including the domain model, while UpdateEntity re-serializes the whole domain model from the parsed entity — scanning first would hand the model write the last word and undo the scan's edits inside the domain model, which is where a validation rule's Attribute string and a MemberAccess.Attribute live. execRenameEntity gets away with the opposite order only because entity refs within the domain model are binary pointers. Also adds the missing collision guard: renaming onto a name the entity already uses produced two attributes no reference could tell apart. Scope, stated rather than hidden: uses inside expressions ($obj/Attr) and XPath constraints ([Attr = ...]) are free text where a bare name is only resolvable from the type of what precedes it, so they are not rewritten. mxbuild reports those as CE0117 / CE0161 and the command now says so instead of reading as complete. Controls on Mendix 11.13.0 with the new bug-test script: a build from origin/main leaves 3 errors after the rename (2 microflow members + the validation rule), this build leaves 0. Each new unit test was shown to fail with the reported symptom against the stubbed-out fix. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018hifgRSawfaRWXS44YKtSJ --- .claude/skills/fix-issue.md | 1 + .../skills/mendix/generate-domain-model.md | 6 +- cmd/mxcli/syntax/features_domain_model.go | 2 +- docs-site/src/appendixes/quick-reference.md | 2 +- docs-site/src/language/alter-entity.md | 9 + .../reference/domain-model/alter-entity.md | 4 +- docs/01-project/MDL_QUICK_REFERENCE.md | 2 +- ...10-rename-attribute-updates-references.mdl | 66 ++++++ .../alter_entity_rename_attribute_test.go | 189 ++++++++++++++++++ mdl/executor/cmd_entities.go | 46 ++++- 10 files changed, 316 insertions(+), 11 deletions(-) create mode 100644 mdl-examples/bug-tests/910-rename-attribute-updates-references.mdl create mode 100644 mdl/executor/alter_entity_rename_attribute_test.go diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index ca969a1f2..5a2a146e0 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -524,3 +524,4 @@ extracting `OffsetExpression`/`LimitExpression`. | Windows Defender flags the mxcli **Windows** release binary as `Trojan:Script/Sabsik.EN.A!ml`; enterprise EDR (Defender for Endpoint, CrowdStrike, SentinelOne) blocks it harder. Not the generic unsigned-Go-binary false positive of #185 | The binary genuinely embedded **chisel**, a dual-use tunnelling/pivoting tool (SSH over WebSocket), on every platform — although the tunnel only ever runs inside a Linux container. `run --hub` linked `chisel/client`, `tunnel-hub` linked `chisel/server`, so windows/darwin carried 32 packages incl. the whole `x/crypto/ssh` stack for a feature they cannot use | `cmd/mxcli/docker/tunnel_linux.go` + `tunnel_other.go` (client seam), `cmd/mxcli/tunnelhub/control_linux.go` + `control_other.go` (server seam), `scripts/check-tunnel-deps.sh` (guard) | **Never obfuscate, pack or rename to dodge the scanner** — attacker tradecraft, and it makes the binary less trustworthy, not more. **Code signing does not fix this class**: a signed binary containing chisel is still flagged behaviourally; signing only addresses #185's generic false positive. The fix is to stop shipping the capability where it is unused: one interface per seam, `_linux.go` impl + `!linux` stub, commands still registered everywhere but failing with an actionable message. **Prove absence three ways, and know that `go tool nm` is not one of them** — release ldflags `-s -w` strip the symbol table, so nm reports "no symbols" whether or not the code is linked and would give a false pass; use `go list -deps`, `go version -m`, and `strings` (nm only on a deliberately unstripped build). **Guard against the transitive path, not the name**: match the module list (`x/crypto/ssh`, `gorilla/websocket`, `armon/go-socks5`, `jpillora/*`) so re-entry without the word "chisel" still trips it, and assert a **positive control** (chisel IS in the linux graph) so the check cannot pass vacuously. Verified by re-adding the import and watching the guard fail on all four windows/darwin targets. Result: -13.5 MB (-14.7%) on windows+darwin, linux unchanged. See ADR-0009 | | CE7375 "must be published and be the key when associations are exposed as an associated object id" on a service publishing no associations | `PublishAssociations` is the representation, not a yes/no — `No` selects "as an associated object id", which needs the system ID as key | `mdl/executor/validate_odata_service_shape.go` | Set `PublishAssociations: Yes` ("as a link", and the default when omitted). MDL-ODATA06 warns at check time | | `mx check` exits with `System.ArgumentOutOfRangeException: length ('-1')` and no error code | an OData service `Path` with no slash in it; mxbuild throws out of its own validator | `mdl/executor/validate_odata_service_shape.go` | Path must not start with `/` and must end with one (CE6550/CE6552). MDL-ODATA05 catches all three | +| `alter entity Mod.Ent rename attribute Old to New;` reports success, and `mx check` then fails the app with one **CE1613 "The selected attribute 'Mod.Ent.Old' no longer exists."** per create/change activity member, page attribute widget, validation rule and access rule that named it. Studio Pro still opens the project, so nothing looks wrong until a build | The handler renamed the attribute inside the domain model and stopped. Everywhere else Mendix stores an attribute reference it stores the **fully qualified name as a string**, so none of them followed. `mxcli rename` already had the project-wide scanner (entity, enumeration, association, module, microflow, page, constant, java action) — attributes were the one kind never wired to it | `mdl/executor/cmd_entities.go` (`ast.AlterEntityRenameAttribute`), `mdl/executor/alter_entity_rename_attribute_test.go` | Call `ctx.Backend.RenameReferences(Mod.Ent.Old → Mod.Ent.New, false)` and report the count. **Order is load-bearing: scan AFTER `UpdateEntity`, never before.** The scan is a raw-BSON pass over every unit *including the domain model*, while `UpdateEntity` re-serializes the whole domain model from the parsed entity — so scanning first hands the model write the last word and silently undoes the scan's edits inside the domain model, which is exactly where a validation rule's `Attribute` string and a `MemberAccess.Attribute` live. `execRenameEntity` gets away with the opposite order only because entity refs *within* the domain model are binary pointers, not strings. Also added the missing collision guard (renaming onto an existing attribute name made two attributes no reference could tell apart). **Scope, stated rather than hidden**: uses inside expressions (`$obj/Attr`) and XPath constraints (`[Attr = ...]`) are free text where a bare name is only resolvable from the type of what precedes it — they are NOT rewritten, mxbuild reports them as CE0117 / CE0161, and the command now says so instead of reading as complete. Controls on Mendix 11.13.0 with `mdl-examples/bug-tests/910-rename-attribute-updates-references.mdl`: pre-fix build 3 errors, post-fix 0. mendixlabs/mxcli#910 (problem 2) | diff --git a/.claude/skills/mendix/generate-domain-model.md b/.claude/skills/mendix/generate-domain-model.md index b2907492b..c9e32f581 100644 --- a/.claude/skills/mendix/generate-domain-model.md +++ b/.claude/skills/mendix/generate-domain-model.md @@ -950,7 +950,11 @@ alter entity Module.Order add attribute VATRate: decimal add attribute VATAmount: decimal; --- Rename an attribute (preserves data) +-- Rename an attribute (preserves data). Every reference Mendix stores as a +-- reference follows it: microflow create/change members, page attribute +-- widgets, validation rules, access rules. Expressions ($Order/CreatedDate) +-- and XPath constraints ([CreatedDate > ...]) are free text and are NOT +-- rewritten — mxbuild reports those as CE0117 / CE0161, so build afterwards. alter entity Module.Order rename attribute CreatedDate to OrderDate; diff --git a/cmd/mxcli/syntax/features_domain_model.go b/cmd/mxcli/syntax/features_domain_model.go index ce3eaeca3..2aa903330 100644 --- a/cmd/mxcli/syntax/features_domain_model.go +++ b/cmd/mxcli/syntax/features_domain_model.go @@ -51,7 +51,7 @@ func init() { "event handler", "documentation", "if not exists", "if exists", "idempotent", }, - Syntax: "ALTER ENTITY Module.Name ADD ATTRIBUTE [IF NOT EXISTS] AttrName: Type [constraints];\nALTER ENTITY Module.Name DROP ATTRIBUTE [IF EXISTS] AttrName;\nALTER ENTITY Module.Name RENAME ATTRIBUTE OldName TO NewName;\nALTER ENTITY Module.Name MODIFY ATTRIBUTE AttrName SET DEFAULT val;\nALTER ENTITY Module.Name ADD INDEX [name] [ON] (attr1, attr2);\nALTER ENTITY Module.Name SET DOCUMENTATION 'text';\nALTER ENTITY Module.Name ADD EVENT HANDLER ON BEFORE COMMIT CALL Module.MF RAISE ERROR;\n\nIF NOT EXISTS / IF EXISTS make the add/drop a no-op (skipped, not an error)\nwhen the attribute is already present / already gone — so a domain script\nre-runs cleanly. For a whole script, 'mxcli exec --continue-on-error' reports\neach failed statement and keeps going instead of halting at the first.", + Syntax: "ALTER ENTITY Module.Name ADD ATTRIBUTE [IF NOT EXISTS] AttrName: Type [constraints];\nALTER ENTITY Module.Name DROP ATTRIBUTE [IF EXISTS] AttrName;\nALTER ENTITY Module.Name RENAME ATTRIBUTE OldName TO NewName;\nALTER ENTITY Module.Name MODIFY ATTRIBUTE AttrName SET DEFAULT val;\nALTER ENTITY Module.Name ADD INDEX [name] [ON] (attr1, attr2);\nALTER ENTITY Module.Name SET DOCUMENTATION 'text';\nALTER ENTITY Module.Name ADD EVENT HANDLER ON BEFORE COMMIT CALL Module.MF RAISE ERROR;\n\nIF NOT EXISTS / IF EXISTS make the add/drop a no-op (skipped, not an error)\nwhen the attribute is already present / already gone — so a domain script\nre-runs cleanly. For a whole script, 'mxcli exec --continue-on-error' reports\neach failed statement and keeps going instead of halting at the first.\n\nRENAME ATTRIBUTE also rewrites every stored reference to the attribute —\nmicroflow create/change members, page widgets, and the entity's own validation\nand access rules — and reports how many it touched. Uses inside expressions\n($obj/Attr) and XPath constraints ([Attr = ...]) are free text and are NOT\nrewritten; mxbuild reports those as CE0117 / CE0161.", Example: "ALTER ENTITY Shop.Customer ADD ATTRIBUTE Phone: String(20);\nALTER ENTITY Shop.Customer ADD ATTRIBUTE IF NOT EXISTS Phone: String(20); -- re-runnable\nALTER ENTITY Shop.Customer DROP ATTRIBUTE IF EXISTS OldField; -- re-runnable\nALTER ENTITY Shop.Customer RENAME ATTRIBUTE Email TO EmailAddress;\nALTER ENTITY Shop.Customer ADD INDEX ON (EmailAddress);\nALTER ENTITY Shop.Customer\n ADD EVENT HANDLER ON BEFORE COMMIT CALL Shop.Validate($currentObject) RAISE ERROR;", SeeAlso: []string{"domain-model.entity.create", "domain-model.entity.attributes"}, }) diff --git a/docs-site/src/appendixes/quick-reference.md b/docs-site/src/appendixes/quick-reference.md index f2699ee64..0ee764959 100644 --- a/docs-site/src/appendixes/quick-reference.md +++ b/docs-site/src/appendixes/quick-reference.md @@ -44,7 +44,7 @@ Modifies an existing entity without full replacement. | Add attribute | `ALTER ENTITY Module.Name ADD ATTRIBUTE Attr: Type [constraints];` | One action per statement | | Drop attribute | `ALTER ENTITY Module.Name DROP ATTRIBUTE AttrName;` | | | Modify attribute | `ALTER ENTITY Module.Name MODIFY ATTRIBUTE Attr: NewType [constraints];` | Change type/constraints | -| Rename attribute | `ALTER ENTITY Module.Name RENAME ATTRIBUTE OldName TO NewName;` | | +| Rename attribute | `ALTER ENTITY Module.Name RENAME ATTRIBUTE OldName TO NewName;` | Also rewrites stored references; expressions and XPath constraints are not rewritten | | Add index | `ALTER ENTITY Module.Name ADD INDEX [IdxName] (Col1 [ASC\|DESC], ...);` | Name optional | | Drop index | `ALTER ENTITY Module.Name DROP INDEX IdxName;` | By index name | | Set documentation | `ALTER ENTITY Module.Name SET DOCUMENTATION 'text';` | | diff --git a/docs-site/src/language/alter-entity.md b/docs-site/src/language/alter-entity.md index 9d1da77bd..d372e1e5f 100644 --- a/docs-site/src/language/alter-entity.md +++ b/docs-site/src/language/alter-entity.md @@ -51,6 +51,15 @@ Rename an attribute with `RENAME ATTRIBUTE`: ALTER ENTITY Sales.Customer RENAME ATTRIBUTE Phone TO PhoneNumber; ``` +Every reference stored *as* a reference follows the rename — microflow create and +change members, page attribute widgets, and the entity's own validation and +access rules — and the command reports how many documents it updated. + +Expressions (`$Customer/Phone`) and XPath constraints (`[Phone = '06-1234']`) +are stored as free text and are **not** rewritten: a bare name there is only +resolvable from the type of what precedes it. `mx check` reports those as CE0117 +and CE0161, so build after renaming an attribute used in either. + ## ADD INDEX Add an index to the entity (the index name is optional): diff --git a/docs-site/src/reference/domain-model/alter-entity.md b/docs-site/src/reference/domain-model/alter-entity.md index 489b6ffa8..0b639d150 100644 --- a/docs-site/src/reference/domain-model/alter-entity.md +++ b/docs-site/src/reference/domain-model/alter-entity.md @@ -28,7 +28,9 @@ The `DROP ATTRIBUTE` operation removes an attribute by name. Dropping an attribu The `MODIFY ATTRIBUTE` operation changes the type or constraints of an existing attribute. The attribute name must already exist in the entity. The full attribute definition (type and constraints) replaces the current one. -The `RENAME ATTRIBUTE` operation changes an attribute's name. This updates references within the entity but does not automatically update microflows, pages, or access rules that reference the old name. +The `RENAME ATTRIBUTE` operation changes an attribute's name and rewrites every reference Mendix stores as a reference — microflow create and change activity members, page attribute widgets, and the entity's own validation and access rules — reporting how many documents it touched. Renaming onto a name the entity already uses is refused. + +Two kinds of use are **not** rewritten, because Mendix stores them as free text in which a bare attribute name is only resolvable from the type of whatever precedes it: microflow expressions (`$obj/Phone`) and XPath constraints (`[Phone = '...']`). The command prints a note saying so; `mx check` reports those as CE0117 and CE0161 respectively, so run a build after renaming an attribute that appears in either. The `ADD INDEX` and `DROP INDEX` operations manage database indexes. `ADD INDEX` takes an optional index name followed by one or more columns, each with an optional `ASC` or `DESC` sort direction; `DROP INDEX` takes the index name. diff --git a/docs/01-project/MDL_QUICK_REFERENCE.md b/docs/01-project/MDL_QUICK_REFERENCE.md index fc8ffd027..3c8bc8ddc 100644 --- a/docs/01-project/MDL_QUICK_REFERENCE.md +++ b/docs/01-project/MDL_QUICK_REFERENCE.md @@ -88,7 +88,7 @@ Modifies an existing entity without full replacement. | Add attributes | `alter entity Module.Name add (attr: type [constraints]);` | One or more attributes | | Drop attributes | `alter entity Module.Name drop (AttrName, ...);` | | | Modify attributes | `alter entity Module.Name modify (attr: NewType [constraints]);` | Change type/constraints | -| Rename attribute | `alter entity Module.Name rename OldName to NewName;` | | +| Rename attribute | `alter entity Module.Name rename attribute OldName to NewName;` | Also rewrites stored references (microflow members, page widgets, validation/access rules). Expressions and XPath constraints are free text and are **not** rewritten | | Add index | `alter entity Module.Name add index [name] [on] (Col1 [asc\|desc], ...);` | `on` is optional (SQL-like) | | Drop index | `alter entity Module.Name drop index (Col1, ...);` | | | Add event handler | `alter entity Module.Name add event handler on before commit call Mod.MF($currentObject) [raise error];` | `($currentObject)` or `()`, RAISE ERROR only on BEFORE | diff --git a/mdl-examples/bug-tests/910-rename-attribute-updates-references.mdl b/mdl-examples/bug-tests/910-rename-attribute-updates-references.mdl new file mode 100644 index 000000000..d1cfbb1c8 --- /dev/null +++ b/mdl-examples/bug-tests/910-rename-attribute-updates-references.mdl @@ -0,0 +1,66 @@ +-- ============================================================================ +-- Bug #910 (problem 2): RENAME ATTRIBUTE left every reference dangling +-- ============================================================================ +-- +-- Symptom: +-- alter entity App.Person rename attribute LastName to Surname; +-- reported success, and then `mx check` failed the app: +-- [CE1613] "The selected attribute 'App.Person.LastName' no longer exists." +-- at Create object activity 'Create Person (FirstName, LastName)' +-- ...once per create/change member, page attribute widget, validation rule +-- and access rule that named the attribute. Measured on a Mendix 11.13.0 +-- project: 0 errors before the rename, 3 after running this script against the +-- pre-fix build (2 microflow members + the "required" validation rule), 0 with +-- the fix. Studio Pro could still open the app, which is what made it look fine. +-- +-- Root cause: +-- The ALTER ENTITY handler renamed the attribute inside the domain model and +-- stopped there. Every other place Mendix stores an attribute reference keeps +-- the fully qualified name as a STRING ("App.Person.LastName"), so none of +-- them followed. `mxcli rename` already had the project-wide reference +-- scanner for entities, enumerations, associations and modules — attributes +-- were simply never wired to it. +-- +-- Fix: +-- The handler now calls RenameReferences(Mod.Ent.Old → Mod.Ent.New) and +-- reports the count. The scan runs AFTER UpdateEntity on purpose: it is a +-- raw-BSON pass over every unit including the domain model, and writing the +-- model second would re-serialize it from the parsed entity and undo the +-- scan's edits there (which is where a validation rule's reference lives). +-- +-- Uses inside expressions ($obj/Attr) and XPath constraints ([Attr = ...]) +-- are free text and are still NOT rewritten — a bare name there is only +-- resolvable from the type of what precedes it. The rename says so, and +-- mxbuild reports them as CE0117 / CE0161. +-- +-- Verify: +-- mxcli exec mdl-examples/bug-tests/910-rename-attribute-updates-references.mdl -p app.mpr +-- mx check app.mpr # 0 errors — before the fix, 3x CE1613 +-- ============================================================================ + +create module RenameRefs; +/ +create persistent entity RenameRefs.Person ( + FirstName: String(100), + LastName: String(100) not null error 'required' +); +/ +-- A create and a change activity, each naming the attribute by qualified name. +create or replace microflow RenameRefs.ACT_MakePerson () +begin + $P = create RenameRefs.Person ( + FirstName = 'Ada', + LastName = 'Lovelace' + ); + commit $P; + change $P (LastName = 'King'); +end; +/ +-- The rename must carry all of it: both microflow members and the entity's own +-- "required" validation rule, which stores "RenameRefs.Person.LastName". +alter entity RenameRefs.Person rename attribute LastName to Surname; +/ +-- Renaming onto a name the entity already uses is refused rather than creating +-- two attributes that no reference can tell apart. +-- alter entity RenameRefs.Person rename attribute Surname to FirstName; +-- → attribute 'FirstName' already exists on entity RenameRefs.Person diff --git a/mdl/executor/alter_entity_rename_attribute_test.go b/mdl/executor/alter_entity_rename_attribute_test.go new file mode 100644 index 000000000..40de60ac1 --- /dev/null +++ b/mdl/executor/alter_entity_rename_attribute_test.go @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// renameAttrCall records one RenameReferences invocation. +type renameAttrCall struct { + old, new string + dryRun bool +} + +// renameAttrTestCtx builds an ExecContext over Sudoku.Game carrying attrs, and +// returns the call log so a test can assert what the rename asked the backend to +// do — and in which order. +func renameAttrTestCtx(t *testing.T, hits int, attrs ...string) (*ExecContext, *[]string, *[]renameAttrCall) { + t.Helper() + mod := mkModule("Sudoku") + game := &domainmodel.Entity{ + BaseElement: model.BaseElement{ID: nextID("ent")}, + ContainerID: nextID("dm"), + Name: "Game", + Persistable: true, + } + for _, a := range attrs { + game.Attributes = append(game.Attributes, &domainmodel.Attribute{Name: a}) + } + dm := &domainmodel.DomainModel{ + BaseElement: model.BaseElement{ID: nextID("dm")}, + ContainerID: mod.ID, + Entities: []*domainmodel.Entity{game}, + } + h := mkHierarchy(mod) + withContainer(h, dm.ID, mod.ID) + withContainer(h, game.ContainerID, dm.ID) + + var order []string + var calls []renameAttrCall + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListModulesFunc: func() ([]*model.Module, error) { return []*model.Module{mod}, nil }, + ListDomainModelsFunc: func() ([]*domainmodel.DomainModel, error) { return []*domainmodel.DomainModel{dm}, nil }, + GetDomainModelFunc: func(id model.ID) (*domainmodel.DomainModel, error) { return dm, nil }, + UpdateEntityFunc: func(dmID model.ID, e *domainmodel.Entity) error { + order = append(order, "UpdateEntity") + return nil + }, + RenameReferencesFunc: func(oldName, newName string, dryRun bool) ([]types.RenameHit, error) { + order = append(order, "RenameReferences") + calls = append(calls, renameAttrCall{oldName, newName, dryRun}) + if hits == 0 { + return nil, nil + } + return []types.RenameHit{{ + UnitID: "u1", UnitType: "Microflows$Microflow", Name: "ACT_Play", Count: hits, + }}, nil + }, + } + ctx, _ := newMockCtx(t, withBackend(mb), withHierarchy(h)) + return ctx, &order, &calls +} + +func renameAttrStmt(from, to string) *ast.AlterEntityStmt { + return &ast.AlterEntityStmt{ + Name: ast.QualifiedName{Module: "Sudoku", Name: "Game"}, + Operation: ast.AlterEntityRenameAttribute, + AttributeName: from, + NewName: to, + } +} + +// TestAlterEntityRenameAttributeUpdatesReferences is the regression test for +// issue #910 (problem 2). Renaming an attribute used to change only the domain +// model, leaving every microflow, page and validation rule pointing at the old +// qualified name — mxbuild reports each one as CE1613 "The selected attribute +// 'Mod.Entity.Old' no longer exists." +func TestAlterEntityRenameAttributeUpdatesReferences(t *testing.T) { + ctx, _, calls := renameAttrTestCtx(t, 4, "PuzzleNo", "Score") + + assertNoError(t, execAlterEntity(ctx, renameAttrStmt("PuzzleNo", "PuzzleNumber"))) + + if len(*calls) != 1 { + t.Fatalf("expected exactly one reference scan, got %d: %+v", len(*calls), *calls) + } + got := (*calls)[0] + if got.old != "Sudoku.Game.PuzzleNo" || got.new != "Sudoku.Game.PuzzleNumber" { + t.Errorf("scanned for %q → %q, want the fully qualified attribute names "+ + "(Sudoku.Game.PuzzleNo → Sudoku.Game.PuzzleNumber)", got.old, got.new) + } + if got.dryRun { + t.Error("the reference scan ran as a dry run, so nothing was written") + } +} + +// TestAlterEntityRenameAttributeReportsReferenceCount pins that the user is told +// how much the rename touched. A rename that silently rewrites four documents is +// as hard to review as one that rewrites none. +func TestAlterEntityRenameAttributeReportsReferenceCount(t *testing.T) { + ctx, _, _ := renameAttrTestCtx(t, 4, "PuzzleNo") + + assertNoError(t, execAlterEntity(ctx, renameAttrStmt("PuzzleNo", "PuzzleNumber"))) + + out := ctx.Output.(interface{ String() string }).String() + if !strings.Contains(out, "4 reference") { + t.Errorf("the rename did not report the references it updated, got: %q", out) + } +} + +// TestAlterEntityRenameAttributeWarnsAboutTextUses pins that the rename says +// what it did NOT do. Expressions and XPath constraints name an attribute in +// free text, where a bare name is only resolvable from the type of what precedes +// it, so the reference scan leaves them alone — and mxbuild then reports them as +// CE0117 / CE0161. A rename that reports only its successes reads as complete. +func TestAlterEntityRenameAttributeWarnsAboutTextUses(t *testing.T) { + ctx, _, _ := renameAttrTestCtx(t, 0, "PuzzleNo") + + assertNoError(t, execAlterEntity(ctx, renameAttrStmt("PuzzleNo", "PuzzleNumber"))) + + out := ctx.Output.(interface{ String() string }).String() + for _, want := range []string{"expressions", "XPath", "PuzzleNo"} { + if !strings.Contains(out, want) { + t.Errorf("the rename does not mention %q in its note about text uses, got: %q", want, out) + } + } +} + +// TestAlterEntityRenameAttributeReferencesAfterModel pins the ordering. +// +// The reference scan is a raw-BSON pass over every unit, the domain model +// included; UpdateEntity re-serializes the whole domain model from the parsed +// model. Scanning first and writing the model second would hand the model write +// the last word and undo the scan's edits inside the domain model — which is +// where a validation rule's attribute reference lives. +func TestAlterEntityRenameAttributeReferencesAfterModel(t *testing.T) { + ctx, order, _ := renameAttrTestCtx(t, 1, "PuzzleNo") + + assertNoError(t, execAlterEntity(ctx, renameAttrStmt("PuzzleNo", "PuzzleNumber"))) + + want := []string{"UpdateEntity", "RenameReferences"} + if len(*order) != len(want) { + t.Fatalf("got call order %v, want %v", *order, want) + } + for i := range want { + if (*order)[i] != want[i] { + t.Fatalf("got call order %v, want %v", *order, want) + } + } +} + +// TestAlterEntityRenameAttributeCollision pins that renaming onto a name the +// entity already uses is refused. Without the check the two attributes become +// indistinguishable by name, and every reference to either one is rewritten to +// point at whichever the model resolves first. +func TestAlterEntityRenameAttributeCollision(t *testing.T) { + ctx, _, calls := renameAttrTestCtx(t, 0, "PuzzleNo", "Score") + + err := execAlterEntity(ctx, renameAttrStmt("PuzzleNo", "Score")) + if err == nil { + t.Fatal("expected renaming onto an existing attribute name to be refused") + } + if !strings.Contains(err.Error(), "Score") { + t.Errorf("the error does not name the colliding attribute: %v", err) + } + if len(*calls) != 0 { + t.Errorf("references were rewritten despite the collision: %+v", *calls) + } +} + +// TestAlterEntityRenameAttributeMissingDoesNotScan pins that a rename that +// cannot find its attribute leaves the project alone. +func TestAlterEntityRenameAttributeMissingDoesNotScan(t *testing.T) { + ctx, _, calls := renameAttrTestCtx(t, 0, "PuzzleNo") + + if err := execAlterEntity(ctx, renameAttrStmt("Nope", "Whatever")); err == nil { + t.Fatal("expected a not-found error") + } + if len(*calls) != 0 { + t.Errorf("references were rewritten for an attribute that does not exist: %+v", *calls) + } +} diff --git a/mdl/executor/cmd_entities.go b/mdl/executor/cmd_entities.go index 33edaa03d..115ac9960 100644 --- a/mdl/executor/cmd_entities.go +++ b/mdl/executor/cmd_entities.go @@ -824,23 +824,57 @@ func execAlterEntity(ctx *ExecContext, s *ast.AlterEntityStmt) error { fmt.Fprintf(ctx.Output, "Added attribute '%s' to entity %s\n", a.Name, s.Name) case ast.AlterEntityRenameAttribute: - found := false + var target *domainmodel.Attribute for _, attr := range entity.Attributes { - if attr.Name == s.AttributeName { - attr.Name = s.NewName - found = true - break + switch attr.Name { + case s.AttributeName: + target = attr + case s.NewName: + return mdlerrors.NewValidationf("attribute '%s' already exists on entity %s", s.NewName, s.Name) } } - if !found { + if target == nil { return mdlerrors.NewNotFoundMsg("attribute", s.AttributeName, fmt.Sprintf("attribute '%s' not found on entity %s", s.AttributeName, s.Name)) } + target.Name = s.NewName if err := ctx.Backend.UpdateEntity(dm.ID, entity); err != nil { return mdlerrors.NewBackend("rename attribute", err) } + + // Everything that points at an attribute — a create/change activity's + // member, a page's attribute widget, the entity's own validation and + // access rules — stores the fully qualified name as a string. Renaming + // only the domain model leaves every one of them dangling, which mxbuild + // reports as CE1613 "The selected attribute 'Mod.Entity.Old' no longer + // exists." (#910). The scan runs *after* UpdateEntity on purpose: it is a + // raw-BSON pass over every unit including the domain model, and writing + // the model afterwards would re-serialize it from the parsed entity and + // undo the scan's edits there. + hits, err := ctx.Backend.RenameReferences( + s.Name.String()+"."+s.AttributeName, + s.Name.String()+"."+s.NewName, + false, + ) + if err != nil { + return mdlerrors.NewBackend("update attribute references", err) + } + invalidateHierarchy(ctx) invalidateDomainModelsCache(ctx) fmt.Fprintf(ctx.Output, "Renamed attribute '%s' to '%s' on entity %s\n", s.AttributeName, s.NewName, s.Name) + if n := totalRefCount(hits); n > 0 { + fmt.Fprintf(ctx.Output, "Updated %d reference(s) in %d document(s)\n", n, len(hits)) + } + // The scan above rewrites every reference Mendix stores *as a reference*. + // It cannot rewrite the two places an attribute is named in free text — + // microflow expressions ($obj/Attr) and XPath constraints ([Attr = …]) — + // because a bare name there is only resolvable with the type of what + // precedes it. mxbuild does report them (CE0117 / CE0161), so say so + // rather than let a half-done rename look finished. + fmt.Fprintf(ctx.Output, + "Note: uses in expressions ($obj/%s) and XPath constraints are stored as text "+ + "and were not rewritten — run 'mxcli docker check' to find them.\n", + s.AttributeName) case ast.AlterEntityModifyAttribute: // CALCULATED attributes are only supported on persistent entities From 1b2db9475be77b971707522a1d5c2c67a141e74e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 11:34:08 +0000 Subject: [PATCH 04/35] fix(widgets): serialize a required object-list TextTemplate with its shipped default (#891) Authoring an Accordion raised CE0463 "the definition of this widget has changed" -- on a widget created seconds earlier against the very package it was built from. An object-list item's REQUIRED TextTemplate that the author leaves unset was serialized as null. The empty-ClientTemplate convention lived in emptyClientTemplateRules, a hardcoded per-widget table covering only DataGrid columns, so every other object-list widget fell through it. In a stock blank app the Accordion's group (required headerText) failed, and thirteen shipped widgets declare object lists with texttemplate items. Required-ness is genuinely absent from the widget XML here -- and the widget schema defaults `required` to TRUE -- so headerText is mandatory though nothing says so. It now comes from the widget's own PropertyTypes, needing no per-widget table and unable to go stale against a package upgrade. Both weaker forms were measured and rejected. null is CE0463; an EMPTY Forms$ClientTemplate is CE4899 "Property 'Groups/1/Text' is required" -- so the intuitive "emit an empty template" fix only moves the error. Only the widget's shipped satisfy both, which is what mx update-widgets writes: mxcli and the reference now emit the same 'Header'/'Koptekst'. Scoped to REQUIRED properties. An optional unset TextTemplate keeps its null, matching Studio Pro; filling every one is the documented way to take CE0463 from 33 to 127. Measured against a baseline fixture (accordion + pop-up menu + a plain DataGrid2): 3 errors -> 2, no new errors of any code, DataGrid2 untouched. Control run with the call site removed fails the new test. PropertyTypeIDEntry exists in three packages, so the field is carried through convertPropTypeIDs; without that it silently never arrives. The pop-up menu's remaining CE0463 is a DIFFERENT defect -- its `caption` is required="false" in the XML, so this rule correctly does not fire on it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013uQvFDd5R4eNqqita59jM8 --- .claude/skills/fix-issue.md | 1 + CHANGELOG.md | 2 + .../modelsdk/widget_pluggable_write.go | 4 + mdl/backend/widgetobj/builder.go | 76 ++++++++++ .../objectlist_required_texttemplate_test.go | 135 ++++++++++++++++++ mdl/types/widget_property_type.go | 13 ++ modelsdk/widgets/loader.go | 40 +++++- sdk/pages/pages_widgets_advanced.go | 11 ++ 8 files changed, 277 insertions(+), 5 deletions(-) create mode 100644 mdl/backend/widgetobj/objectlist_required_texttemplate_test.go diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index ca969a1f2..2c44c7191 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -524,3 +524,4 @@ extracting `OffsetExpression`/`LimitExpression`. | Windows Defender flags the mxcli **Windows** release binary as `Trojan:Script/Sabsik.EN.A!ml`; enterprise EDR (Defender for Endpoint, CrowdStrike, SentinelOne) blocks it harder. Not the generic unsigned-Go-binary false positive of #185 | The binary genuinely embedded **chisel**, a dual-use tunnelling/pivoting tool (SSH over WebSocket), on every platform — although the tunnel only ever runs inside a Linux container. `run --hub` linked `chisel/client`, `tunnel-hub` linked `chisel/server`, so windows/darwin carried 32 packages incl. the whole `x/crypto/ssh` stack for a feature they cannot use | `cmd/mxcli/docker/tunnel_linux.go` + `tunnel_other.go` (client seam), `cmd/mxcli/tunnelhub/control_linux.go` + `control_other.go` (server seam), `scripts/check-tunnel-deps.sh` (guard) | **Never obfuscate, pack or rename to dodge the scanner** — attacker tradecraft, and it makes the binary less trustworthy, not more. **Code signing does not fix this class**: a signed binary containing chisel is still flagged behaviourally; signing only addresses #185's generic false positive. The fix is to stop shipping the capability where it is unused: one interface per seam, `_linux.go` impl + `!linux` stub, commands still registered everywhere but failing with an actionable message. **Prove absence three ways, and know that `go tool nm` is not one of them** — release ldflags `-s -w` strip the symbol table, so nm reports "no symbols" whether or not the code is linked and would give a false pass; use `go list -deps`, `go version -m`, and `strings` (nm only on a deliberately unstripped build). **Guard against the transitive path, not the name**: match the module list (`x/crypto/ssh`, `gorilla/websocket`, `armon/go-socks5`, `jpillora/*`) so re-entry without the word "chisel" still trips it, and assert a **positive control** (chisel IS in the linux graph) so the check cannot pass vacuously. Verified by re-adding the import and watching the guard fail on all four windows/darwin targets. Result: -13.5 MB (-14.7%) on windows+darwin, linux unchanged. See ADR-0009 | | CE7375 "must be published and be the key when associations are exposed as an associated object id" on a service publishing no associations | `PublishAssociations` is the representation, not a yes/no — `No` selects "as an associated object id", which needs the system ID as key | `mdl/executor/validate_odata_service_shape.go` | Set `PublishAssociations: Yes` ("as a link", and the default when omitted). MDL-ODATA06 warns at check time | | `mx check` exits with `System.ArgumentOutOfRangeException: length ('-1')` and no error code | an OData service `Path` with no slash in it; mxbuild throws out of its own validator | `mdl/executor/validate_odata_service_shape.go` | Path must not start with `/` and must end with one (CE6550/CE6552). MDL-ODATA05 catches all three | +| Authoring a pluggable widget with an **object list** (Accordion group, Pop-up menu item, chart series) raises **CE0463** "the definition of this widget has changed" on the widget itself, on a widget created seconds ago against the current package | An object-list item's **required TextTemplate** that the author left unset was serialized as `null`. `emptyClientTemplateRules` in `widgetobj/builder.go` is a hardcoded per-widget table covering only DataGrid columns, so every other object-list widget fell through it. Required-ness is genuinely absent from the widget XML for these properties — and the widget schema **defaults `required` to true** — so the Accordion's `headerText` is mandatory even though nothing says so | `mdl/backend/widgetobj/builder.go` (`isUnsetRequiredTextTemplate`, `buildDefaultTextClientTemplateProperty`), plus the entry plumbing in `mdl/types/widget_property_type.go`, `modelsdk/widgets/loader.go`, `sdk/pages/pages_widgets_advanced.go`, `mdl/backend/modelsdk/widget_pluggable_write.go` (`convertPropTypeIDs`) | Serialize a required unset TextTemplate with the widget's **shipped ``**, read off the template ValueType's `Translations` beside the `Required` flag. **Both weaker forms were measured and fail**: `null` is CE0463, and an *empty* `Forms$ClientTemplate` is **CE4899** "Property 'Groups/1/Text' is required" — so the intuitive "emit an empty template" fix only moves the error. Populating is what `mx update-widgets` itself writes (verified: mxcli and the reference now emit the same `'Header'`/`'Koptekst'`). Scope it to **required** properties: filling every TextTemplate is the documented way to take CE0463 from 33 to 127. Note `PropertyTypeIDEntry` exists in THREE places (`mdl/types` — canonical and aliased by `modelsdk/widgets`; `sdk/pages` — what the builder consumes; `sdk/widgets` — the legacy engine's own), so a field added for this must be carried through `convertPropTypeIDs` or it silently never arrives. Repro `mdl-examples/bug-tests/891-accordion-group-nested-widgets.mdl`. Issue #891 | diff --git a/CHANGELOG.md b/CHANGELOG.md index c49954257..439614566 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **Authoring a pluggable widget with an object list no longer raises CE0463** (#891) — an object-list item's *required* TextTemplate that the author left unset was written as `null`, so a freshly authored Accordion failed "the definition of this widget has changed" against the very package it was built from. The empty-ClientTemplate convention was a hardcoded table covering only DataGrid columns; required-ness now comes from the widget's own PropertyTypes, and the property is serialized with the widget's shipped translations — the same text `mx update-widgets` writes. Both weaker forms were measured and rejected: `null` is CE0463 and an empty template is CE4899. Optional TextTemplates keep their null, which is what Studio Pro stores. + - **`DESCRIBE PAGE` no longer renders an Accordion group empty** (#891) — an object-list item's child widgets (the group's `content` slot) were never read, and the emitter had no body to put them in, so a group holding a DataGrid2 described as a bare `group group1 (…)` and a describe→exec round-trip silently deleted the grid. Both halves are fixed, and the description now re-parses with the nested widgets intact. Applies to any pluggable widget's object-list items, not just Accordion. - **`ALTER PAGE INSERT`/`REPLACE` no longer corrupts a page when a DataGrid2 column is named without its grid** (#891) — `REPLACE NextRunAt WITH { COLUMN … }` reported success while writing a layout container into the grid's column list, leaving a project Studio Pro and mxbuild could not **load** (`InvalidCastException: DivContainer → WidgetObject`). `DESCRIBE PAGE` skipped the malformed node, so REPLACE looked like a clean deletion and INSERT like a harmless no-op; both were corruption, and neither required the grid to be nested in a pluggable widget. A bare name that resolves to an object-list item is now refused, naming the qualified `grid.column` form — which always worked and is unaffected. diff --git a/mdl/backend/modelsdk/widget_pluggable_write.go b/mdl/backend/modelsdk/widget_pluggable_write.go index 5acb6bb25..698c479ed 100644 --- a/mdl/backend/modelsdk/widget_pluggable_write.go +++ b/mdl/backend/modelsdk/widget_pluggable_write.go @@ -213,6 +213,10 @@ func convertPropTypeIDs(src map[string]types.PropertyTypeIDEntry) map[string]pag Required: v.Required, ObjectTypeID: v.ObjectTypeID, } + for _, t := range v.DefaultTranslations { + entry.DefaultTranslations = append(entry.DefaultTranslations, + pages.PropertyTranslation{LanguageCode: t.LanguageCode, Text: t.Text}) + } if len(v.NestedPropertyIDs) > 0 { entry.NestedPropertyIDs = convertPropTypeIDs(v.NestedPropertyIDs) entry.NestedKeyOrder = append([]string(nil), v.NestedKeyOrder...) diff --git a/mdl/backend/widgetobj/builder.go b/mdl/backend/widgetobj/builder.go index 27a00a25b..e4a40eb26 100644 --- a/mdl/backend/widgetobj/builder.go +++ b/mdl/backend/widgetobj/builder.go @@ -293,6 +293,8 @@ func buildObjectListItemBSON(widgetID, listPropertyKey string, parentEntry pages prop = buildItemChildWidgetsProperty(nestedEntry, childWidgets) case shouldEmitEmptyClientTemplate(widgetID, listPropertyKey, k, itemKind): prop = buildEmptyClientTemplateProperty(nestedEntry) + case isUnsetRequiredTextTemplate(nestedEntry): + prop = buildDefaultTextClientTemplateProperty(nestedEntry) default: prop = createDefaultWidgetProperty(nestedEntry) } @@ -376,6 +378,80 @@ var emptyClientTemplateRules = map[string]map[string]map[objectListItemKind]map[ }, } +// isUnsetRequiredTextTemplate reports whether an object-list item sub-property +// is a REQUIRED TextTemplate the author did not set (#891). +// +// This generalises emptyClientTemplateRules, which is a hardcoded table covering +// only DataGrid columns; every other object-list widget fell through it to a +// null. In a stock blank app an Accordion group (required `headerText`) and a +// Pop-up menu basic item (required `caption`) both raised CE0463, and thirteen +// shipped widgets declare object lists with texttemplate items. +// +// Required-ness comes from the widget's own PropertyTypes, so this needs no +// per-widget table and cannot go stale against a package upgrade. Note the +// widget XML schema defaults `required` to TRUE when the attribute is absent — +// exactly the Accordion's headerText — so a parser reading a missing attribute +// as false silently disables this (mpk.go encodes the default as +// `Required: p.Required != "false"`). +func isUnsetRequiredTextTemplate(e pages.PropertyTypeIDEntry) bool { + return e.Required && strings.EqualFold(e.ValueType, "TextTemplate") +} + +// buildDefaultTextClientTemplateProperty emits a required TextTemplate carrying +// the widget's shipped default text. +// +// Both weaker forms fail, which is why this is not simply the empty builder: +// TextTemplate=null is CE0463 "the definition of this widget has changed", and +// an EMPTY Forms$ClientTemplate is CE4899 "Property 'Groups/1/Text' is +// required". Populating from the ValueType's Translations is what +// `mx update-widgets` itself writes (the Accordion's headerText ships +// 'Header'/'Koptekst'). With no shipped translations there is nothing better to +// write than the empty template. +func buildDefaultTextClientTemplateProperty(entry pages.PropertyTypeIDEntry) bson.D { + if len(entry.DefaultTranslations) == 0 { + return buildEmptyClientTemplateProperty(entry) + } + value := createDefaultWidgetValue(entry) + value = setBSONField(value, "TextTemplate", buildClientTemplateWithTranslations(entry.DefaultTranslations)) + return bson.D{ + {Key: "$ID", Value: types.UUIDToBlob(types.GenerateID())}, + {Key: "$Type", Value: "CustomWidgets$WidgetProperty"}, + {Key: "TypePointer", Value: types.UUIDToBlob(entry.PropertyTypeID)}, + {Key: "Value", Value: value}, + } +} + +// buildClientTemplateWithTranslations mirrors BuildEmptyClientTemplate but fills +// Template.Items with one Texts$Translation per shipped language. The Fallback +// stays empty and Parameters keeps marker 2 — the shape `mx update-widgets` +// produces. +func buildClientTemplateWithTranslations(translations []pages.PropertyTranslation) bson.D { + items := bson.A{int32(3)} + for _, t := range translations { + items = append(items, bson.D{ + {Key: "$ID", Value: bsonutil.NewIDBsonBinary()}, + {Key: "$Type", Value: "Texts$Translation"}, + {Key: "LanguageCode", Value: t.LanguageCode}, + {Key: "Text", Value: t.Text}, + }) + } + return bson.D{ + {Key: "$ID", Value: bsonutil.NewIDBsonBinary()}, + {Key: "$Type", Value: "Forms$ClientTemplate"}, + {Key: "Fallback", Value: bson.D{ + {Key: "$ID", Value: bsonutil.NewIDBsonBinary()}, + {Key: "$Type", Value: "Texts$Text"}, + {Key: "Items", Value: bson.A{int32(3)}}, + }}, + {Key: "Parameters", Value: bson.A{int32(2)}}, + {Key: "Template", Value: bson.D{ + {Key: "$ID", Value: bsonutil.NewIDBsonBinary()}, + {Key: "$Type", Value: "Texts$Text"}, + {Key: "Items", Value: items}, + }}, + } +} + // shouldEmitEmptyClientTemplate returns true when an unset TextTemplate-typed // property should be serialized as an empty Forms$ClientTemplate (Items=[3] // in both Fallback and Template, no Translation entries) instead of diff --git a/mdl/backend/widgetobj/objectlist_required_texttemplate_test.go b/mdl/backend/widgetobj/objectlist_required_texttemplate_test.go new file mode 100644 index 000000000..162290c44 --- /dev/null +++ b/mdl/backend/widgetobj/objectlist_required_texttemplate_test.go @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Issue #891: authoring an Accordion raised CE0463 "the definition of this +// widget has changed". +// +// An object-list item's REQUIRED TextTemplate that the author leaves unset was +// serialized as null. emptyClientTemplateRules covers only DataGrid columns, so +// every other object-list widget fell through it — in a stock blank app the +// Accordion's group (required `headerText`) failed, and thirteen shipped widgets +// declare object lists with texttemplate items. +// +// Both weaker forms were measured and rejected: +// null -> CE0463 "the definition of this widget has changed" +// empty template -> CE4899 "Property 'Groups/1/Text' is required" +// +// Only the widget's own shipped translations satisfy both, which is what +// `mx update-widgets` writes. +package widgetobj + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/sdk/pages" + "go.mongodb.org/mongo-driver/bson" +) + +// accordionGroupEntry mirrors the Accordion's `groups` object list: a required +// `headerText` TextTemplate shipping 'Header'/'Koptekst', plus an optional one. +func accordionGroupEntry() pages.PropertyTypeIDEntry { + return pages.PropertyTypeIDEntry{ + PropertyTypeID: "00000000000000000000000000000001", + NestedKeyOrder: []string{"headerText", "optionalText"}, + NestedPropertyIDs: map[string]pages.PropertyTypeIDEntry{ + "headerText": { + PropertyTypeID: "00000000000000000000000000000002", + ValueType: "TextTemplate", + Required: true, + DefaultTranslations: []pages.PropertyTranslation{ + {LanguageCode: "en_US", Text: "Header"}, + {LanguageCode: "nl_NL", Text: "Koptekst"}, + }, + }, + "optionalText": { + PropertyTypeID: "00000000000000000000000000000003", + ValueType: "TextTemplate", + Required: false, + DefaultTranslations: []pages.PropertyTranslation{ + {LanguageCode: "en_US", Text: "Tooltip"}, + }, + }, + }, + } +} + +// collectTranslations gathers every Texts$Translation under a node. +func collectTranslations(v any, out *[][2]string) { + switch n := v.(type) { + case bson.D: + isTranslation := false + var lang, text string + for _, e := range n { + if e.Key == "$Type" && e.Value == "Texts$Translation" { + isTranslation = true + } + if e.Key == "LanguageCode" { + lang, _ = e.Value.(string) + } + if e.Key == "Text" { + text, _ = e.Value.(string) + } + } + if isTranslation { + *out = append(*out, [2]string{lang, text}) + } + for _, e := range n { + collectTranslations(e.Value, out) + } + case bson.A: + for _, e := range n { + collectTranslations(e, out) + } + } +} + +// Goes through buildObjectListItemBSON, not the helper, so removing the call +// site fails this test — a helper-level assertion would prove the helper works +// and nothing about the wiring. +func TestObjectListItem_RequiredTextTemplateGetsShippedDefault(t *testing.T) { + got := buildObjectListItemBSON( + "com.mendix.widget.web.accordion.Accordion", "groups", + accordionGroupEntry(), + backend.ObjectListItemSpec{}, // author set nothing + ) + + var found [][2]string + collectTranslations(got, &found) + + want := map[string]string{"en_US": "Header", "nl_NL": "Koptekst"} + seen := map[string]string{} + for _, p := range found { + seen[p[0]] = p[1] + } + for lang, text := range want { + if seen[lang] != text { + t.Errorf("required headerText missing its shipped %s default: got %q, want %q (null here is CE0463, empty is CE4899)", + lang, seen[lang], text) + } + } + + // The OPTIONAL TextTemplate must NOT be filled — Studio Pro leaves an unset + // optional one null, and filling every template took CE0463 from 33 to 127 + // in a previous attempt at this class of fix. + if seen["en_US"] == "Tooltip" || len(found) > len(want) { + t.Errorf("an optional TextTemplate was populated too; translations found: %v", found) + } +} + +func TestIsUnsetRequiredTextTemplate(t *testing.T) { + cases := []struct { + name string + in pages.PropertyTypeIDEntry + want bool + }{ + {"required texttemplate", pages.PropertyTypeIDEntry{ValueType: "TextTemplate", Required: true}, true}, + {"optional texttemplate", pages.PropertyTypeIDEntry{ValueType: "TextTemplate", Required: false}, false}, + {"required string", pages.PropertyTypeIDEntry{ValueType: "String", Required: true}, false}, + {"required attribute", pages.PropertyTypeIDEntry{ValueType: "Attribute", Required: true}, false}, + } + for _, tc := range cases { + if got := isUnsetRequiredTextTemplate(tc.in); got != tc.want { + t.Errorf("%s: got %v, want %v", tc.name, got, tc.want) + } + } +} diff --git a/mdl/types/widget_property_type.go b/mdl/types/widget_property_type.go index dcb221904..0bf23969f 100644 --- a/mdl/types/widget_property_type.go +++ b/mdl/types/widget_property_type.go @@ -2,6 +2,13 @@ package types +// PropertyTranslation is one widget-shipped translation of a property's default +// text, read from the template ValueType's Translations array. +type PropertyTranslation struct { + LanguageCode string + Text string +} + // PropertyTypeIDEntry holds the IDs for a property type from a cloned pluggable widget template. // This is an engine-internal struct used by WidgetObjectBuilder; it is not a BSON wire type. type PropertyTypeIDEntry struct { @@ -10,6 +17,12 @@ type PropertyTypeIDEntry struct { DefaultValue string // Default value from the template's ValueType ValueType string // Type of value (Boolean, Integer, String, DataSource, etc.) Required bool // Whether this property is required + // DefaultTranslations are the widget-shipped for this property. + // A REQUIRED TextTemplate the author leaves unset must be serialized WITH this + // text: a null there is CE0463 "the definition of this widget has changed", + // and an empty Forms$ClientTemplate is CE4899 "Property … is required" (#891). + // This is what `mx update-widgets` itself writes. + DefaultTranslations []PropertyTranslation DataSourceProperty string // Non-empty when this attribute is linked to another DataSource property // For object list properties (IsList=true with ObjectType), these hold nested IDs ObjectTypeID string // ID of the nested ObjectType (for object lists like columns) diff --git a/modelsdk/widgets/loader.go b/modelsdk/widgets/loader.go index 3aa1c7e02..f7075225a 100644 --- a/modelsdk/widgets/loader.go +++ b/modelsdk/widgets/loader.go @@ -571,6 +571,7 @@ func extractNestedPropertyTypes(val any, idMapping map[string]string, nestedProp // Extract DefaultValue, Type, and Required from nested ValueType var nestedDefaultValue, nestedValueType string var nestedRequired bool + var nestedTranslations []types.PropertyTranslation if vtVal, ok := propType["ValueType"]; ok { if vt, ok := vtVal.(map[string]any); ok { if dv, ok := vt["DefaultValue"].(string); ok { @@ -582,6 +583,9 @@ func extractNestedPropertyTypes(val any, idMapping map[string]string, nestedProp if r, ok := vt["Required"].(bool); ok { nestedRequired = r } + // Widget-shipped default text. A required TextTemplate + // must serialize with it, not empty and not null (#891). + nestedTranslations = readPropertyTranslations(vt["Translations"]) } } @@ -594,11 +598,12 @@ func extractNestedPropertyTypes(val any, idMapping map[string]string, nestedProp *nestedKeyOrder = append(*nestedKeyOrder, propKey) } nestedPropertyIDs[propKey] = PropertyTypeIDEntry{ - PropertyTypeID: propTypeID, - ValueTypeID: valueTypeID, - DefaultValue: nestedDefaultValue, - ValueType: nestedValueType, - Required: nestedRequired, + PropertyTypeID: propTypeID, + ValueTypeID: valueTypeID, + DefaultValue: nestedDefaultValue, + ValueType: nestedValueType, + Required: nestedRequired, + DefaultTranslations: nestedTranslations, } } } @@ -1072,3 +1077,28 @@ func ListAvailableTemplates() []string { } return result } + +// readPropertyTranslations reads a template ValueType's widget-shipped +// Translations array (marker-prefixed CustomWidgets$WidgetTranslation entries) +// into the property entry. Used to serialize a required TextTemplate the author +// left unset — see types.PropertyTypeIDEntry.DefaultTranslations (#891). +func readPropertyTranslations(v any) []types.PropertyTranslation { + arr, ok := v.([]any) + if !ok { + return nil + } + var out []types.PropertyTranslation + for _, e := range arr { + m, ok := e.(map[string]any) + if !ok { + continue + } + lang, _ := m["LanguageCode"].(string) + if lang == "" { + continue + } + text, _ := m["Text"].(string) + out = append(out, types.PropertyTranslation{LanguageCode: lang, Text: text}) + } + return out +} diff --git a/sdk/pages/pages_widgets_advanced.go b/sdk/pages/pages_widgets_advanced.go index 78ddacc7f..e56faa93b 100644 --- a/sdk/pages/pages_widgets_advanced.go +++ b/sdk/pages/pages_widgets_advanced.go @@ -170,12 +170,23 @@ type CustomWidget struct { } // PropertyTypeIDEntry holds the IDs for a property type from a cloned widget. +// PropertyTranslation is one widget-shipped translation of a property's +// default text. +type PropertyTranslation struct { + LanguageCode string + Text string +} + type PropertyTypeIDEntry struct { PropertyTypeID string ValueTypeID string DefaultValue string // Default value from the template's ValueType ValueType string // Type of value (Boolean, Integer, String, DataSource, etc.) Required bool // Whether this property is required + // DefaultTranslations are the widget-shipped for this + // property. A required TextTemplate the author leaves unset serializes with + // this text — null is CE0463, empty is CE4899 (#891). + DefaultTranslations []PropertyTranslation // For object list properties (IsList=true with ObjectType), these hold nested IDs ObjectTypeID string // ID of the nested ObjectType (for object lists like columns) NestedPropertyIDs map[string]PropertyTypeIDEntry // Property IDs within the nested ObjectType From 646ee29f52c6c8c89757f3cc589261bf68bafcff Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 13:30:48 +0000 Subject: [PATCH 05/35] docs(proposals): record attribute rename as a consumer of expression typing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The half of #910 the reference scanner cannot reach — attribute names inside microflow expressions ($Order/Status) and XPath constraints ([Status = 'Open']) — needs exactly the resolution machinery PROPOSAL_expression_type_checking already builds for its check and refs consumers. Adds it as a fourth consumer alongside those, and links the dependency from the rename proposal's scope exclusions (which did not mention the gap, and listed attribute rename as plainly "Works"). Three things worth having written down rather than rediscovered: - The XPath half needs no type system and is shippable before build-order step 1. A constraint's target entity is known structurally, the parse and re-render round trip already ships (ParseXPathConstraint / expressionToXPath / SplitXPathPredicateGroups), and enrichXPathConstraintForDescribe is a working precedent for resolving bare attribute names against a known entity. Only multi-hop association paths need the resolver. - The expression half is genuinely step 1, and is the one place the dormant exprcheck parser is in the hot path: stored expressions come back from BSON as raw strings, not as the mdl/ast nodes the visitor produces. - The proposal's failure mode inverts for a mutating consumer. Unresolved → KindUnknown → "catch less, never false-positive" is right for an advisory check; a rename must instead refuse to rewrite and report, because silently rewriting an occurrence whose type could not be established corrupts a model that was valid. That is a constraint on the resolver interface, not on its callers. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018hifgRSawfaRWXS44YKtSJ --- .../PROPOSAL_expression_type_checking.md | 55 +++++++++++++++++++ .../PROPOSAL_rename_refactoring.md | 3 +- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/docs/11-proposals/PROPOSAL_expression_type_checking.md b/docs/11-proposals/PROPOSAL_expression_type_checking.md index d79e2b78a..a5ed84991 100644 --- a/docs/11-proposals/PROPOSAL_expression_type_checking.md +++ b/docs/11-proposals/PROPOSAL_expression_type_checking.md @@ -116,6 +116,61 @@ already intended for the catalog `refs` consumer), not microflow-coupled. --- +## Fourth consumer: attribute rename + +`alter entity Mod.Ent rename attribute Old to New` rewrites every reference +Mendix stores *as a reference* — a create/change member, a page attribute +widget, a validation or access rule — because those hold the fully qualified +name as a string and a string scan finds them. It cannot touch the two places an +attribute is named in **free text**, and those are exactly the two this proposal +learns to read: + +| Shape | Example | Why a string scan cannot do it | +|-------|---------|-------------------------------| +| Expression | `$Order/Status` | The bare segment is only an attribute of `Mod.Order` if `$Order` is typed — the scope walker's job | +| XPath constraint | `[Status = 'Open']`, `[Mod.A_B/Mod.B/Status = …]` | The bare step belongs to the constraint's *target* entity, or to the entity the association hops reach | + +mxbuild reports the leftovers as **CE0117** and **CE0161** (measured on 11.13.0), +so the rename tells the user they exist rather than reading as complete — but a +rename that half-renames is the weakest form of the feature. This consumer is +what finishes it. Origin: mendixlabs/mxcli#910. + +**Two asymmetries worth knowing before scheduling it.** + +*The XPath half is nearly there and does not need the type system.* A +constraint's target entity is known **structurally** — a retrieve names its +entity, a page datasource names its entity, an access rule belongs to one — so a +top-level bare step needs no inference at all. The round trip already exists and +already ships: `visitor.ParseXPathConstraint` → typed `XPathPathExpr` → +`expressionToXPath`, with `visitor.SplitXPathPredicateGroups` for the sibling +groups Mendix concatenates and a verbatim pass-through on any group that will not +parse (#772). `enrichXPathConstraintForDescribe` is the working precedent: it +already walks a constraint resolving bare attribute names against a known entity, +for enum enrichment. Only the **multi-hop association path** needs this +proposal's resolver, to answer "which entity does this hop land on". + +*The expression half is genuinely step 1.* Stored expressions come back from BSON +as raw strings, so this is the one place the dormant `exprcheck` lexer/parser is +in the hot path rather than the `mdl/ast` nodes the visitor produces; then the +scope walker types `$Var`, and `ModelResolver` walks the path segments. + +**One thing that does not transfer: the failure mode inverts.** For checking, +unresolved → `KindUnknown` → *catch less, never false-positive*, and § 4 is right +to call that correct for an advisory gate. A **mutating** consumer cannot inherit +it: unresolved must mean **do not rewrite, and report the occurrence**. Silently +rewriting an occurrence whose type could not be established corrupts a model that +was valid — strictly worse than today's honest half-rename. So the resolver needs +to return *unresolved* distinguishably from *resolved to something else*, which is +a constraint on the interface, not on the callers. (This is the same +guard-don't-drop stance as [ADR-0005](../13-decisions/0005-semantic-model-interface-currency.md).) + +Sequencing follows from the asymmetry: the XPath half is shippable **before** +step 1 lands, against structural entity knowledge alone, with multi-hop paths +reported-not-rewritten until the resolver exists. The expression half waits for +step 1. + +--- + ## Background: Mendix Type System Mendix expressions use these types: diff --git a/docs/11-proposals/PROPOSAL_rename_refactoring.md b/docs/11-proposals/PROPOSAL_rename_refactoring.md index 98fcfbf76..81d76ef33 100644 --- a/docs/11-proposals/PROPOSAL_rename_refactoring.md +++ b/docs/11-proposals/PROPOSAL_rename_refactoring.md @@ -21,7 +21,7 @@ A safe RENAME command that automatically updates all references would be a signi | Operation | Status | |-----------|--------| -| `alter entity ... rename attribute Old to New` | Works | +| `alter entity ... rename attribute Old to New` | Works, and updates stored references. Free-text uses (expressions, XPath) are excluded — see Scope Exclusions | | `alter enumeration ... rename value Old to New` | Works | | `rename entity Module.Old to New` | Grammar only, not implemented | | `rename module Old to New` | Grammar only, not implemented | @@ -235,6 +235,7 @@ Renaming an entity changes the proxy class name in `javasource//proxies/ ## Scope Exclusions +- **Attribute names in expressions and XPath constraints**: `$Order/Status` and `[Status = 'Open']` name an attribute in **free text**, where the bare name is only resolvable from the type of what precedes it — so the qualified-name scanner this proposal is built on cannot see them. `alter entity … rename attribute` rewrites every *stored* reference and says plainly that it leaves these; mxbuild reports them as CE0117 / CE0161. Closing the gap needs the resolution machinery in [`PROPOSAL_expression_type_checking.md`](PROPOSAL_expression_type_checking.md) (§ Fourth consumer), which also records why a *mutating* consumer must fail differently from a checking one. Origin: mendixlabs/mxcli#910 - **Java source file updates**: Out of scope — rename produces correct MPR but Java files need manual update - **Widget property string references**: Pluggable widget properties may contain entity/attribute names as strings — these are not updatable without widget-specific knowledge - **Git history**: Rename doesn't create a git rename operation — it modifies files in place From 201fd6f3cd1da704316e4617e96a6a697a169321 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 13:57:34 +0000 Subject: [PATCH 06/35] feat(settings): allow enabling optimistic locking from MDL ALTER SETTINGS MODEL refused every spelling of Mendix's optimistic locking setting with a bare 'unknown model setting', which put a security-relevant runtime setting on the needs-Studio-Pro list. Reported by the mxcli-banking app as the unclosable half of a read-then-write balance race in a transfer microflow. alter settings model EnableDataStorageOptimisticLocking = true; Every layer but one was already in place: model.ModelSettings carried the field and both engines read and wrote it, so the property round-tripped faithfully on every write while being unreachable from MDL. Only the executor's assignment case was missing. The stored key is verified against Studio-Pro-created projects on both 9.24 and 11.13.0. Unlike JavaVersion -> JavaMajorVersion (renamed between 11.6 and 11.12) this one did not move, so a single spelling is correct for every supported version. The unknown-key error now lists the accepted model keys, guarded by a drift test. The report tried three wrong names in a row and was never pointed at the right one. Verified on an 11.13.0 project: correct BSON key written with JavaMajorVersion preserved beside it, mx check 0 errors, re-running is byte-identical (MXCLI_ALWAYS_WRITE=1 as the control that does churn), and a non-boolean value is refused by mxcli check (MDL-SET02) and the write. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01KTdvGwVewkdgNQGDCxpQPZ --- .claude/skills/mendix/project-settings.md | 22 ++- cmd/mxcli/syntax/features_misc.go | 13 +- .../2026-08-17-banking-app-findings-triage.md | 55 +++++-- .../14-project-settings-examples.mdl | 19 +++ mdl/executor/cmd_settings.go | 40 ++++- mdl/executor/cmd_settings_optimistic_test.go | 139 ++++++++++++++++++ mdl/executor/validate_settings.go | 5 +- 7 files changed, 277 insertions(+), 16 deletions(-) create mode 100644 mdl/executor/cmd_settings_optimistic_test.go diff --git a/.claude/skills/mendix/project-settings.md b/.claude/skills/mendix/project-settings.md index ce601688f..e5b2c14ca 100644 --- a/.claude/skills/mendix/project-settings.md +++ b/.claude/skills/mendix/project-settings.md @@ -34,8 +34,27 @@ alter settings model JavaVersion = 'Java21'; -- or '21'; see note below alter settings model RoundingMode = 'HalfUp'; alter settings model AllowUserMultipleSessions = true; alter settings model ScheduledEventTimeZoneCode = 'Etc/UTC'; +alter settings model EnableDataStorageOptimisticLocking = true; ``` +**Optimistic locking** is App Settings → Runtime → *Optimistic locking* in Studio +Pro. With it on, the runtime tracks an `MxObjectVersion` on every persistable +entity and a commit whose version no longer matches the database throws +`ConcurrentModificationRuntimeException`. + +Reach for it when a microflow reads a value, decides on it, and writes it back — +the classic "check the balance, then debit it" shape. A microflow is one +transaction, so each run is *atomic*, but that does not make two concurrent runs +*serialisable*: both can pass the check and the second overwrites the first. With +optimistic locking on, the second commit fails and its whole microflow rolls back +instead of silently overdrawing the account. + +It **detects, it does not retry.** Mendix's guidance is that the handler must +catch the exception, *reload* the object, re-apply and re-commit — "trying to +commit the same object without reloading always results in an optimistic locking +error." Without that the user sees a failure rather than a transfer that works. +The money is safe either way, which is the half that matters. + **JavaVersion spelling.** Mendix renamed this property between versions: up to 11.6 it stores `JavaVersion` = `'Java21'`, from 11.12 it stores `JavaMajorVersion` = `'21'`. Write either spelling — mxcli reads which one the project uses and stores @@ -64,7 +83,8 @@ alter settings configuration 'Default' ``` `HttpPortNumber`, `ServerPortNumber`, `BcryptCost`, `DefaultTaskParallelism` and -`WorkflowEngineParallelism` are Integer-typed, and `AllowUserMultipleSessions` is +`WorkflowEngineParallelism` are Integer-typed, and `AllowUserMultipleSessions` and +`EnableDataStorageOptimisticLocking` are Boolean. An unparseable value is rejected by `mxcli check` (MDL-SET01 / MDL-SET02) and by the write itself — it is no longer silently ignored. Quoted numbers are fine: `HttpPortNumber = '8080'` and `HttpPortNumber = 8080` are equivalent. diff --git a/cmd/mxcli/syntax/features_misc.go b/cmd/mxcli/syntax/features_misc.go index d6fa7d893..d558e74a3 100644 --- a/cmd/mxcli/syntax/features_misc.go +++ b/cmd/mxcli/syntax/features_misc.go @@ -180,6 +180,7 @@ DISCONNECT;`, "alter settings", "modify settings", "change settings", "after startup", "before shutdown", "hash algorithm", "database type", "constant override", "language", + "optimistic locking", "concurrency", "lost update", }, Syntax: `ALTER SETTINGS MODEL = ; ALTER SETTINGS CONFIGURATION '' = , ...; @@ -191,6 +192,7 @@ CREATE CONFIGURATION '' [ = , ...]; DROP CONFIGURATION '';`, Example: `ALTER SETTINGS MODEL AfterStartupMicroflow = 'Module.MF_Startup'; ALTER SETTINGS MODEL HashAlgorithm = 'BCrypt'; +ALTER SETTINGS MODEL EnableDataStorageOptimisticLocking = true; ALTER SETTINGS CONFIGURATION 'Default' DatabaseType = 'PostgreSql', DatabaseUrl = 'localhost:5432', @@ -203,7 +205,16 @@ CREATE CONFIGURATION 'Production' -- DatabaseType must be a Mendix database type: -- Db2, Hsqldb, MySql, Oracle, PostgreSql, SapHana, SqlServer --- (matched case-insensitively and stored in the spelling above).`, +-- (matched case-insensitively and stored in the spelling above). + +-- MODEL accepts these keys (an unknown one is refused and lists them): +-- AfterStartupMicroflow, BeforeShutdownMicroflow, HealthCheckMicroflow, +-- HashAlgorithm, BcryptCost, JavaVersion, RoundingMode, +-- AllowUserMultipleSessions, ScheduledEventTimeZoneCode, +-- EnableDataStorageOptimisticLocking +-- EnableDataStorageOptimisticLocking is Studio Pro's App Settings → Runtime → +-- "Optimistic locking": it makes a stale commit fail instead of silently +-- overwriting, which is the fix for a read-then-write race in a microflow.`, SeeAlso: []string{"settings.show"}, }) diff --git a/docs/12-bug-reports/2026-08-17-banking-app-findings-triage.md b/docs/12-bug-reports/2026-08-17-banking-app-findings-triage.md index 62fa17af8..46b562960 100644 --- a/docs/12-bug-reports/2026-08-17-banking-app-findings-triage.md +++ b/docs/12-bug-reports/2026-08-17-banking-app-findings-triage.md @@ -29,7 +29,7 @@ project. Claims that could not be measured here say so, and say why. | 7 | ComboBox rejects `onChangeEvent` | **Confirmed gap** | yes | | 8 | `mxcli exec` applies scripts `mxcli check` rejects | **Confirmed gap** | yes | | 9 | `mxcli check` does not validate `ALTER SETTINGS MODEL` keys | **Confirmed** | yes | -| 10 | Optimistic locking not settable from MDL | **Confirmed gap** | yes (key identified) | +| 10 | Optimistic locking not settable from MDL | **Fixed** — see below | yes (key identified) | | 11 | `DROP USER ROLE` leaves demo users dangling | **Confirmed defect** | yes | | 12 | `SHOW MESSAGE` clause-order error is misleading | **Confirmed UX defect** | yes | | 13 | `ALTER MODULE/PAGE SET DOCUMENTATION` is a parse error | **Confirmed gap** | yes | @@ -338,13 +338,44 @@ grammar accepts any identifier there. The reporter's summary is the right one: "check passed" means the text parses, not that the statement means anything. `check` should validate settings keys against the same table `exec` uses. -**#10 confirmed, and the key is identifiable.** The report's correction is -accurate — Mendix *does* ship optimistic locking as an app setting, and it is -exactly the mitigation for the read-then-write balance race the app documents. +**#10 fixed.** The report's correction is accurate — Mendix *does* ship optimistic +locking as an app setting, and it is exactly the mitigation for the read-then-write +balance race the app documents. + The stored property is `EnableDataStorageOptimisticLocking` on -`Settings$ModelSettings` (read directly out of a project's BSON while triaging -this). Adding it to the accepted model settings is small, and it takes an item -off the "needs Studio Pro" list for a security-relevant setting. +`Settings$ModelSettings`, verified against Studio-Pro-created projects on **both** +9.24 and **11.13.0**. Unlike `JavaVersion` → `JavaMajorVersion` (which did rename +between 11.6 and 11.12, exactly as CLAUDE.md warns) this key did not move, so one +spelling is correct for every version mxcli supports. + +Every layer except one was already in place — `model.ModelSettings` carried the +field, and both engines read and wrote it (`sdk/mpr/parser_settings.go:164`, +`writer_settings.go:102`, `mdl/backend/modelsdk/settings_{read,write}.go`). The +only thing missing was the executor's assignment case, so the property was +round-tripped faithfully on every write while being unreachable from MDL. + +``` +alter settings model EnableDataStorageOptimisticLocking = true; +``` + +Measured on the 11.13.0 project: the key lands in the BSON as +`"EnableDataStorageOptimisticLocking": true`, `JavaMajorVersion` is preserved +untouched beside it, `mx check` reports **0 errors**, re-running the statement +leaves the `.mpr` byte-identical (with `MXCLI_ALWAYS_WRITE=1` as the control run +that does churn), and a non-boolean value is refused by `mxcli check` (MDL-SET02) +and by the write. + +The unknown-key error now lists the accepted keys. The report tried +`OptimisticLocking`, `UseOptimisticLocking` and `EnableOptimisticLocking` in turn +and got a bare `unknown model setting` each time; the first of those now answers +with the real name. + +Still on the "needs Studio Pro" list and **not** addressed here: strict mode +(SEC005), which the report also flags and which weakens XPath constraint +enforcement (CVE-2023-23835). Worth checking whether it is equally reachable — +`Settings$ModelSettings` on 11.13 also carries `UseSystemContextForBackgroundTasks`, +`UseOQLVersion2`, `UseDatabaseForeignKeyConstraints`, `DecimalScale`, +`FirstDayOfWeek` and `SslCertificateAlgorithm`, none of which MDL exposes either. Same list, not investigated here: strict mode (SEC005), which the report also flags as Studio-Pro-only and which weakens XPath constraint enforcement @@ -414,9 +445,11 @@ success line); `CREATE OR MODIFY` does not prune members the OQL no longer produces (CE6770 until DROP + recreate); pass-through columns inherit the source length and `cast()` is not in the grammar. -**Status: not reproduced.** View entities require Mendix 10.18+ and the only -projects in this checkout are 9.24; no mxbuild is cached in this environment, so -creating an 11.x project was out of scope for a triage pass. +**Status: not yet reproduced.** View entities require Mendix 10.18+ and the only +projects in this checkout were 9.24 at triage time. An 11.13.0 project now exists +for this (`mxcli new --version 11.13.0`, mxbuild cached at +`/root/.mxcli/mxbuild/11.13.0`), so the reproduction is no longer blocked — it +just has not been run. Reading the code does *not* obviously support the claim — the read path parses `Source` (`parser_domainmodel.go:105`), `isViewEntity` is used in validation, @@ -494,4 +527,4 @@ four stale tests during a regression run. 5. **#6/#6b** — three documented spellings of `NON-PERSISTENT`, two of which do not parse, one of which ships to every user project. 6. **#14** — reproduce on 11.13 before designing anything. -7. **#2, #9, #10, #11, #12, #13** — individually small. +7. **#2, #9, #11, #12, #13** — individually small. (**#10 is done.**) diff --git a/mdl-examples/doctype-tests/14-project-settings-examples.mdl b/mdl-examples/doctype-tests/14-project-settings-examples.mdl index 5d59cbac0..a72f18b10 100644 --- a/mdl-examples/doctype-tests/14-project-settings-examples.mdl +++ b/mdl-examples/doctype-tests/14-project-settings-examples.mdl @@ -51,6 +51,25 @@ alter settings model BcryptCost = 12, AllowUserMultipleSessions = true; +/** + * Example 1.4: Enable optimistic locking + * + * Studio Pro calls this App Settings -> Runtime -> "Optimistic locking". The + * runtime then tracks an MxObjectVersion per persistable entity, and a commit + * whose version no longer matches the database throws + * ConcurrentModificationRuntimeException. + * + * Turn it on when a microflow reads a value, decides on it, and writes it back + * -- "check the balance, then debit it". A microflow is one transaction, so each + * run is atomic, but that does not make two concurrent runs serialisable: both + * can pass the check and the second overwrites the first. With this on, the + * second commit fails and its microflow rolls back instead of overdrawing. + * + * It detects rather than retries -- the handler must catch, reload the object, + * re-apply and re-commit. + */ +alter settings model EnableDataStorageOptimisticLocking = true; + -- MARK: Configuration Settings -- ============================================================================ diff --git a/mdl/executor/cmd_settings.go b/mdl/executor/cmd_settings.go index f7a990a0a..1a5afb9ae 100644 --- a/mdl/executor/cmd_settings.go +++ b/mdl/executor/cmd_settings.go @@ -131,6 +131,7 @@ func describeSettings(ctx *ExecContext, configName string) error { parts = append(parts, fmt.Sprintf(" JavaVersion = '%s'", ms.JavaVersion)) parts = append(parts, fmt.Sprintf(" RoundingMode = '%s'", ms.RoundingMode)) parts = append(parts, fmt.Sprintf(" AllowUserMultipleSessions = %t", ms.AllowUserMultipleSessions)) + parts = append(parts, fmt.Sprintf(" EnableDataStorageOptimisticLocking = %t", ms.EnableDataStorageOptimisticLocking)) if ms.ScheduledEventTimeZoneCode != "" { parts = append(parts, fmt.Sprintf(" ScheduledEventTimeZoneCode = '%s'", ms.ScheduledEventTimeZoneCode)) } @@ -170,6 +171,27 @@ func describeSettings(ctx *ExecContext, configName string) error { return nil } +// modelSettingKeys names every property the ALTER SETTINGS MODEL switch below +// assigns, so an unknown key can be rejected with a list of what would have +// worked. A bare "unknown model setting" sent the mxcli-banking app through +// three wrong guesses at the optimistic-locking property without ever naming the +// real one. +// +// Hand-maintained alongside the switch: add a case, add it here. +// TestModelSettingKeys_AllAccepted fails when a listed key is not accepted. +var modelSettingKeys = []string{ + "AfterStartupMicroflow", + "BeforeShutdownMicroflow", + "HealthCheckMicroflow", + "HashAlgorithm", + "BcryptCost", + "JavaVersion", + "RoundingMode", + "AllowUserMultipleSessions", + "ScheduledEventTimeZoneCode", + "EnableDataStorageOptimisticLocking", +} + // alterSettings modifies project settings based on ALTER SETTINGS statement. func alterSettings(ctx *ExecContext, stmt *ast.AlterSettingsStmt) error { if !ctx.ConnectedForWrite() { @@ -216,8 +238,24 @@ func alterSettings(ctx *ExecContext, stmt *ast.AlterSettingsStmt) error { ps.Model.AllowUserMultipleSessions = v case "ScheduledEventTimeZoneCode": ps.Model.ScheduledEventTimeZoneCode = valStr + case "EnableDataStorageOptimisticLocking": + // Mendix's App Settings → Runtime → "Optimistic locking". With it + // on, the runtime tracks an MxObjectVersion per persistable entity + // and a commit whose version no longer matches the database throws + // ConcurrentModificationRuntimeException — the mitigation for a + // read-then-write race (check balance, then write it) inside a + // microflow, which the transaction alone does not make serialisable. + // It detects rather than retries: the handler must catch, reload, + // re-apply and re-commit. + v, err := settingsBool(key, valStr) + if err != nil { + return err + } + ps.Model.EnableDataStorageOptimisticLocking = v default: - return mdlerrors.NewUnsupported("unknown model setting: " + key) + return mdlerrors.NewUnsupported(fmt.Sprintf( + "unknown model setting: %s\n valid keys: %s", + key, strings.Join(modelSettingKeys, ", "))) } } diff --git a/mdl/executor/cmd_settings_optimistic_test.go b/mdl/executor/cmd_settings_optimistic_test.go new file mode 100644 index 000000000..3869e4019 --- /dev/null +++ b/mdl/executor/cmd_settings_optimistic_test.go @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/model" +) + +// TestAlterSettingsModel_OptimisticLocking covers the gap reported by the +// mxcli-banking app: Mendix ships optimistic locking as an app setting (App +// Settings → Runtime), which is the mitigation for a read-then-write balance +// race in a transfer microflow, and ALTER SETTINGS MODEL refused every spelling +// of it with "unknown model setting". The property was already read and written +// by both engines — only the executor's assignment switch was missing. +// +// The stored key is EnableDataStorageOptimisticLocking, verified against a +// Studio-Pro-created project on both Mendix 9.24 and 11.13. Unlike JavaVersion +// (renamed to JavaMajorVersion between 11.6 and 11.12) this one did not move, so +// a single spelling is correct for every version mxcli supports. +func TestAlterSettingsModel_OptimisticLocking(t *testing.T) { + tests := []struct { + name string + given string + want bool + }{ + {name: "enable", given: "true", want: true}, + {name: "disable", given: "false", want: false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var written *model.ProjectSettings + ctx, _ := newMockCtx(t, withBackend(captureSettingsBackend(&written))) + + err := alterSettings(ctx, &ast.AlterSettingsStmt{ + Section: "model", + Properties: map[string]any{"EnableDataStorageOptimisticLocking": tc.given}, + }) + if err != nil { + t.Fatalf("alterSettings: %v", err) + } + if written == nil { + t.Fatal("no settings written") + } + if got := written.Model.EnableDataStorageOptimisticLocking; got != tc.want { + t.Errorf("EnableDataStorageOptimisticLocking = %v, want %v", got, tc.want) + } + }) + } +} + +// A non-boolean value must be refused rather than silently skipped — the same +// silent-no-op shape as mendixlabs/mxcli#805, where a bad Integer value skipped +// the assignment while the handler reported success. +func TestAlterSettingsModel_OptimisticLockingRejectsNonBool(t *testing.T) { + wrote := false + ctx, _ := newMockCtx(t, withBackend(settingsBackend(&wrote))) + + err := alterSettings(ctx, &ast.AlterSettingsStmt{ + Section: "model", + Properties: map[string]any{"EnableDataStorageOptimisticLocking": "yes-please"}, + }) + if err == nil { + t.Fatal("expected an error for a non-boolean value, got nil") + } + if !strings.Contains(err.Error(), "EnableDataStorageOptimisticLocking") { + t.Errorf("error should name the offending key, got: %v", err) + } + if wrote { + t.Error("a rejected statement must not write") + } +} + +// DESCRIBE must emit the property so a describe → edit → exec round trip can +// carry it. Without this the setting is writable but invisible, which is how a +// user discovers it does not exist. +func TestDescribeSettings_EmitsOptimisticLocking(t *testing.T) { + var written *model.ProjectSettings + ctx, buf := newMockCtx(t, withBackend(captureSettingsBackend(&written))) + + if err := describeSettings(ctx, ""); err != nil { + t.Fatalf("describeSettings: %v", err) + } + assertContainsStr(t, buf.String(), "EnableDataStorageOptimisticLocking") +} + +// TestModelSettingKeys_AllAccepted is the drift guard for the hand-maintained +// modelSettingKeys list: a name that appears in the error message's "valid keys" +// but is not actually assigned by the switch would send the next reader down the +// same dead end the bare message did. +func TestModelSettingKeys_AllAccepted(t *testing.T) { + for _, key := range modelSettingKeys { + t.Run(key, func(t *testing.T) { + wrote := false + ctx, _ := newMockCtx(t, withBackend(settingsBackend(&wrote))) + + // A value valid for every kind in the list: parses as a string, and + // the typed keys are covered by TestTypedSettingsKeys_MatchExecutor. + val := "1" + if kind, ok := typedSettingsKeys["model"][key]; ok && kind == settingsKindBool { + val = "true" + } + err := alterSettings(ctx, &ast.AlterSettingsStmt{ + Section: "model", + Properties: map[string]any{key: val}, + }) + if err != nil && strings.Contains(err.Error(), "unknown model setting") { + t.Errorf("%s is listed as valid but the executor rejects it: %v", key, err) + } + }) + } +} + +// The unknown-setting error must list what IS accepted. The banking app tried +// OptimisticLocking, UseOptimisticLocking and EnableOptimisticLocking in turn +// and got a bare "unknown model setting" each time, with nothing pointing at the +// real name. +func TestAlterSettingsModel_UnknownKeyListsValidKeys(t *testing.T) { + wrote := false + ctx, _ := newMockCtx(t, withBackend(settingsBackend(&wrote))) + + err := alterSettings(ctx, &ast.AlterSettingsStmt{ + Section: "model", + Properties: map[string]any{"OptimisticLocking": "true"}, + }) + if err == nil { + t.Fatal("expected an error for an unknown key, got nil") + } + if !strings.Contains(err.Error(), "EnableDataStorageOptimisticLocking") { + t.Errorf("error should point at the real key name, got: %v", err) + } + if wrote { + t.Error("a rejected statement must not write") + } +} diff --git a/mdl/executor/validate_settings.go b/mdl/executor/validate_settings.go index 692676b16..c5838ae8e 100644 --- a/mdl/executor/validate_settings.go +++ b/mdl/executor/validate_settings.go @@ -27,8 +27,9 @@ const ( // fails if the two drift apart. var typedSettingsKeys = map[string]map[string]settingsValueKind{ "model": { - "BcryptCost": settingsKindInt, - "AllowUserMultipleSessions": settingsKindBool, + "BcryptCost": settingsKindInt, + "AllowUserMultipleSessions": settingsKindBool, + "EnableDataStorageOptimisticLocking": settingsKindBool, }, "workflows": { "DefaultTaskParallelism": settingsKindInt, From 35efa0207201ddc386eff179e94ddb9c5e250f36 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 14:00:04 +0000 Subject: [PATCH 07/35] feat(rename): rewrite XPath constraints when an attribute is renamed (#910) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit XPath names an attribute as a bare step — [Status = 'Open'] — so the qualified-name scan that fixes "Mod.Entity.Attr" references cannot see it, and a scan for the bare name would corrupt string literals, function names and other entities' identically-named attributes. Renaming therefore left every constraint dangling, which mxbuild reports as CE0161. No type inference is needed for this half. A constraint's target entity is known structurally — a retrieve names its Entity, a widget data source its EntityRef.QualifiedName, an access rule the entity it sits inside — and every further hop is an association or entity named in the path, resolved against a flat index built from the domain models. Three design points, each of which cost a wrong first attempt: - The edit is textual, gated by the parse, not a re-render of the parsed tree. Re-rendering would churn spacing in constraints the rename has no business touching, and any parser/renderer disagreement would corrupt a working one. Only a single identifier token changes. - The load-bearing safety check is a count invariant. ParseXPathConstraint runs with ANTLR's error listeners removed, so it recovers and can hand back a tree that silently omits part of its input — [LastName = 'L' FirstName] parses and the trailing step is simply absent. Requiring the lexical occurrence count to equal the walked occurrence count is what makes leaning on that parse safe. - A mutating consumer needs three answers, not two: this entity's attribute (rewrite), a different entity's (leave alone, say nothing), and cannot tell (leave alone, report). Collapsing the last two made the pass silently skip constraints it should have questioned. Also fixes a second defect the same rename exposed. UpdateEntity re-derives an access rule's members from the attributes it can match, so renaming in the model orphaned the old member and the reference scan then renamed the orphan into a duplicate — three members for a two-attribute entity, reported as CE0066 "Entity access is out of date". The entity's own by-name references are now fixed in the model before the write. Trap worth recording: a domain model entity node is stored under DomainModels$EntityImpl, not DomainModels$Entity (that is the metamodel's name for a reference target). Matching the latter matched nothing and made every access rule look like a cautious refusal rather than a bug. Controls on Mendix 11.13.0 with the new bug-test script: the same script without the rename is 0 errors; with the rename it was 3x CE0161 and then 1x CE0066; it is now 0. On the earlier repro the progression is 5 errors (pre-fix) to 3 (qualified-name half) to 1 — and that one is the microflow expression, which is reported as out of scope and needs the resolver in PROPOSAL_expression_type_checking. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018hifgRSawfaRWXS44YKtSJ --- .claude/skills/fix-issue.md | 1 + .../skills/mendix/generate-domain-model.md | 11 +- cmd/mxcli/syntax/features_domain_model.go | 2 +- docs-site/src/appendixes/quick-reference.md | 2 +- docs-site/src/language/alter-entity.md | 13 +- .../reference/domain-model/alter-entity.md | 6 +- docs/01-project/MDL_QUICK_REFERENCE.md | 2 +- .../PROPOSAL_expression_type_checking.md | 39 ++- .../PROPOSAL_rename_refactoring.md | 4 +- ...10-rename-attribute-updates-references.mdl | 7 +- ...910-rename-attribute-xpath-constraints.mdl | 60 ++++ .../alter_entity_rename_attribute_test.go | 20 +- mdl/executor/cmd_entities.go | 35 +- mdl/executor/rename_attribute_entity_rules.go | 61 ++++ .../rename_attribute_entity_rules_test.go | 90 +++++ mdl/executor/xpath_rename.go | 86 +++++ mdl/executor/xpath_rename_model.go | 104 ++++++ mdl/xpathrefs/rewrite.go | 297 ++++++++++++++++ mdl/xpathrefs/rewrite_test.go | 250 ++++++++++++++ mdl/xpathrefs/scan.go | 263 +++++++++++++++ mdl/xpathrefs/scan_test.go | 318 ++++++++++++++++++ 21 files changed, 1629 insertions(+), 42 deletions(-) create mode 100644 mdl-examples/bug-tests/910-rename-attribute-xpath-constraints.mdl create mode 100644 mdl/executor/rename_attribute_entity_rules.go create mode 100644 mdl/executor/rename_attribute_entity_rules_test.go create mode 100644 mdl/executor/xpath_rename.go create mode 100644 mdl/executor/xpath_rename_model.go create mode 100644 mdl/xpathrefs/rewrite.go create mode 100644 mdl/xpathrefs/rewrite_test.go create mode 100644 mdl/xpathrefs/scan.go create mode 100644 mdl/xpathrefs/scan_test.go diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index 5a2a146e0..b8ae991d7 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -525,3 +525,4 @@ extracting `OffsetExpression`/`LimitExpression`. | CE7375 "must be published and be the key when associations are exposed as an associated object id" on a service publishing no associations | `PublishAssociations` is the representation, not a yes/no — `No` selects "as an associated object id", which needs the system ID as key | `mdl/executor/validate_odata_service_shape.go` | Set `PublishAssociations: Yes` ("as a link", and the default when omitted). MDL-ODATA06 warns at check time | | `mx check` exits with `System.ArgumentOutOfRangeException: length ('-1')` and no error code | an OData service `Path` with no slash in it; mxbuild throws out of its own validator | `mdl/executor/validate_odata_service_shape.go` | Path must not start with `/` and must end with one (CE6550/CE6552). MDL-ODATA05 catches all three | | `alter entity Mod.Ent rename attribute Old to New;` reports success, and `mx check` then fails the app with one **CE1613 "The selected attribute 'Mod.Ent.Old' no longer exists."** per create/change activity member, page attribute widget, validation rule and access rule that named it. Studio Pro still opens the project, so nothing looks wrong until a build | The handler renamed the attribute inside the domain model and stopped. Everywhere else Mendix stores an attribute reference it stores the **fully qualified name as a string**, so none of them followed. `mxcli rename` already had the project-wide scanner (entity, enumeration, association, module, microflow, page, constant, java action) — attributes were the one kind never wired to it | `mdl/executor/cmd_entities.go` (`ast.AlterEntityRenameAttribute`), `mdl/executor/alter_entity_rename_attribute_test.go` | Call `ctx.Backend.RenameReferences(Mod.Ent.Old → Mod.Ent.New, false)` and report the count. **Order is load-bearing: scan AFTER `UpdateEntity`, never before.** The scan is a raw-BSON pass over every unit *including the domain model*, while `UpdateEntity` re-serializes the whole domain model from the parsed entity — so scanning first hands the model write the last word and silently undoes the scan's edits inside the domain model, which is exactly where a validation rule's `Attribute` string and a `MemberAccess.Attribute` live. `execRenameEntity` gets away with the opposite order only because entity refs *within* the domain model are binary pointers, not strings. Also added the missing collision guard (renaming onto an existing attribute name made two attributes no reference could tell apart). **Scope, stated rather than hidden**: uses inside expressions (`$obj/Attr`) and XPath constraints (`[Attr = ...]`) are free text where a bare name is only resolvable from the type of what precedes it — they are NOT rewritten, mxbuild reports them as CE0117 / CE0161, and the command now says so instead of reading as complete. Controls on Mendix 11.13.0 with `mdl-examples/bug-tests/910-rename-attribute-updates-references.mdl`: pre-fix build 3 errors, post-fix 0. mendixlabs/mxcli#910 (problem 2) | +| After a `rename attribute`, `mx check` reports **CE0161 "Error(s) in XPath constraint."** at every retrieve, widget data source or entity access rule whose constraint named the attribute — and, if the attribute had a `READ *` access rule, **CE0066 "Entity access is out of date"** on the domain model | Two different causes with the same trigger. (a) XPath names an attribute as a **bare step** (`[Status = 'Open']`), which the qualified-name reference scanner cannot see — and which a scan for the bare name must not touch, since the same three letters are a string literal here, a function name there, and another entity's attribute somewhere else. (b) `UpdateEntity` re-derives an access rule's members from the attributes it can match, so renaming the attribute in the model left the old member orphaned; the reference scan then renamed the orphan into a **duplicate** of the new member | `mdl/xpathrefs/` (new: `rewrite.go`, `scan.go`), `mdl/executor/xpath_rename_model.go`, `mdl/executor/xpath_rename.go`, `mdl/executor/rename_attribute_entity_rules.go`, `mdl/executor/cmd_entities.go` | (a) **XPath needs no type inference** — a constraint's target entity is known *structurally* (a retrieve names its `Entity`, a widget data source its `EntityRef.QualifiedName`, an access rule the entity it sits inside), and every further hop is an association or entity named in the path. Resolve each bare step to its owning entity, then **edit one identifier token textually**; do NOT re-render the parsed tree, which would churn spacing in constraints the rename has no business touching and would corrupt any constraint the parser and renderer disagree about. **The load-bearing safety check is a count invariant**: `visitor.ParseXPathConstraint` runs with ANTLR's error listeners removed, so it recovers and can return a tree that silently omits part of its input (`[LastName = 'L' FirstName]` parses; the trailing step is simply absent) — require the lexical occurrence count and the walked occurrence count to agree, or refuse. Unresolvable → report and leave alone, never guess: for a *mutating* consumer the checking world's "unresolved → assume nothing → catch less" inverts into "unresolved → change nothing". (b) Fix the entity's own by-name references (`MemberAccess.AttributeName`, `ValidationRule.AttributeID`, range `Min/MaxAttributeQualifiedName`) **in the model before `UpdateEntity`**, not by leaving them to the scan afterwards. **Trap that cost an hour**: the stored `$Type` of a domain model entity node is `DomainModels$EntityImpl`, not `DomainModels$Entity` (that is the metamodel's name for a *reference target*) — checking the latter matched nothing and made every access rule look like a cautious refusal instead of a bug. Controls on Mendix 11.13.0 with `mdl-examples/bug-tests/910-rename-attribute-xpath-constraints.mdl`: same script without the rename = 0 errors, with the rename pre-fix = 3× CE0161 then 1× CE0066, post-fix = 0. mendixlabs/mxcli#910 (problem 2, XPath half) | diff --git a/.claude/skills/mendix/generate-domain-model.md b/.claude/skills/mendix/generate-domain-model.md index c9e32f581..b1ad267b8 100644 --- a/.claude/skills/mendix/generate-domain-model.md +++ b/.claude/skills/mendix/generate-domain-model.md @@ -950,11 +950,12 @@ alter entity Module.Order add attribute VATRate: decimal add attribute VATAmount: decimal; --- Rename an attribute (preserves data). Every reference Mendix stores as a --- reference follows it: microflow create/change members, page attribute --- widgets, validation rules, access rules. Expressions ($Order/CreatedDate) --- and XPath constraints ([CreatedDate > ...]) are free text and are NOT --- rewritten — mxbuild reports those as CE0117 / CE0161, so build afterwards. +-- Rename an attribute (preserves data). Every stored reference follows it: +-- microflow create/change members, page attribute widgets, validation rules, +-- access rules -- and XPath constraints too ([CreatedDate > ...]), including +-- ones that reach the entity through an association. Microflow expressions +-- ($Order/CreatedDate) are NOT rewritten -- mxbuild reports those as CE0117, +-- so build afterwards. alter entity Module.Order rename attribute CreatedDate to OrderDate; diff --git a/cmd/mxcli/syntax/features_domain_model.go b/cmd/mxcli/syntax/features_domain_model.go index 2aa903330..43c229afe 100644 --- a/cmd/mxcli/syntax/features_domain_model.go +++ b/cmd/mxcli/syntax/features_domain_model.go @@ -51,7 +51,7 @@ func init() { "event handler", "documentation", "if not exists", "if exists", "idempotent", }, - Syntax: "ALTER ENTITY Module.Name ADD ATTRIBUTE [IF NOT EXISTS] AttrName: Type [constraints];\nALTER ENTITY Module.Name DROP ATTRIBUTE [IF EXISTS] AttrName;\nALTER ENTITY Module.Name RENAME ATTRIBUTE OldName TO NewName;\nALTER ENTITY Module.Name MODIFY ATTRIBUTE AttrName SET DEFAULT val;\nALTER ENTITY Module.Name ADD INDEX [name] [ON] (attr1, attr2);\nALTER ENTITY Module.Name SET DOCUMENTATION 'text';\nALTER ENTITY Module.Name ADD EVENT HANDLER ON BEFORE COMMIT CALL Module.MF RAISE ERROR;\n\nIF NOT EXISTS / IF EXISTS make the add/drop a no-op (skipped, not an error)\nwhen the attribute is already present / already gone — so a domain script\nre-runs cleanly. For a whole script, 'mxcli exec --continue-on-error' reports\neach failed statement and keeps going instead of halting at the first.\n\nRENAME ATTRIBUTE also rewrites every stored reference to the attribute —\nmicroflow create/change members, page widgets, and the entity's own validation\nand access rules — and reports how many it touched. Uses inside expressions\n($obj/Attr) and XPath constraints ([Attr = ...]) are free text and are NOT\nrewritten; mxbuild reports those as CE0117 / CE0161.", + Syntax: "ALTER ENTITY Module.Name ADD ATTRIBUTE [IF NOT EXISTS] AttrName: Type [constraints];\nALTER ENTITY Module.Name DROP ATTRIBUTE [IF EXISTS] AttrName;\nALTER ENTITY Module.Name RENAME ATTRIBUTE OldName TO NewName;\nALTER ENTITY Module.Name MODIFY ATTRIBUTE AttrName SET DEFAULT val;\nALTER ENTITY Module.Name ADD INDEX [name] [ON] (attr1, attr2);\nALTER ENTITY Module.Name SET DOCUMENTATION 'text';\nALTER ENTITY Module.Name ADD EVENT HANDLER ON BEFORE COMMIT CALL Module.MF RAISE ERROR;\n\nIF NOT EXISTS / IF EXISTS make the add/drop a no-op (skipped, not an error)\nwhen the attribute is already present / already gone — so a domain script\nre-runs cleanly. For a whole script, 'mxcli exec --continue-on-error' reports\neach failed statement and keeps going instead of halting at the first.\n\nRENAME ATTRIBUTE also rewrites every reference to the attribute: the stored\nqualified names (microflow create/change members, page widgets, the entity's own\nvalidation and access rules) AND the bare steps inside XPath constraints, which\nare resolved to their owning entity first so another entity's identically-named\nattribute is left alone. A constraint that cannot be resolved is reported and\nleft unchanged, never guessed at. Uses inside microflow expressions ($obj/Attr)\nare free text and are NOT rewritten; mxbuild reports those as CE0117.", Example: "ALTER ENTITY Shop.Customer ADD ATTRIBUTE Phone: String(20);\nALTER ENTITY Shop.Customer ADD ATTRIBUTE IF NOT EXISTS Phone: String(20); -- re-runnable\nALTER ENTITY Shop.Customer DROP ATTRIBUTE IF EXISTS OldField; -- re-runnable\nALTER ENTITY Shop.Customer RENAME ATTRIBUTE Email TO EmailAddress;\nALTER ENTITY Shop.Customer ADD INDEX ON (EmailAddress);\nALTER ENTITY Shop.Customer\n ADD EVENT HANDLER ON BEFORE COMMIT CALL Shop.Validate($currentObject) RAISE ERROR;", SeeAlso: []string{"domain-model.entity.create", "domain-model.entity.attributes"}, }) diff --git a/docs-site/src/appendixes/quick-reference.md b/docs-site/src/appendixes/quick-reference.md index 0ee764959..72cf36b49 100644 --- a/docs-site/src/appendixes/quick-reference.md +++ b/docs-site/src/appendixes/quick-reference.md @@ -44,7 +44,7 @@ Modifies an existing entity without full replacement. | Add attribute | `ALTER ENTITY Module.Name ADD ATTRIBUTE Attr: Type [constraints];` | One action per statement | | Drop attribute | `ALTER ENTITY Module.Name DROP ATTRIBUTE AttrName;` | | | Modify attribute | `ALTER ENTITY Module.Name MODIFY ATTRIBUTE Attr: NewType [constraints];` | Change type/constraints | -| Rename attribute | `ALTER ENTITY Module.Name RENAME ATTRIBUTE OldName TO NewName;` | Also rewrites stored references; expressions and XPath constraints are not rewritten | +| Rename attribute | `ALTER ENTITY Module.Name RENAME ATTRIBUTE OldName TO NewName;` | Also rewrites stored references and XPath constraints; microflow expressions are not rewritten | | Add index | `ALTER ENTITY Module.Name ADD INDEX [IdxName] (Col1 [ASC\|DESC], ...);` | Name optional | | Drop index | `ALTER ENTITY Module.Name DROP INDEX IdxName;` | By index name | | Set documentation | `ALTER ENTITY Module.Name SET DOCUMENTATION 'text';` | | diff --git a/docs-site/src/language/alter-entity.md b/docs-site/src/language/alter-entity.md index d372e1e5f..8fb171ea5 100644 --- a/docs-site/src/language/alter-entity.md +++ b/docs-site/src/language/alter-entity.md @@ -55,10 +55,15 @@ Every reference stored *as* a reference follows the rename — microflow create change members, page attribute widgets, and the entity's own validation and access rules — and the command reports how many documents it updated. -Expressions (`$Customer/Phone`) and XPath constraints (`[Phone = '06-1234']`) -are stored as free text and are **not** rewritten: a bare name there is only -resolvable from the type of what precedes it. `mx check` reports those as CE0117 -and CE0161, so build after renaming an attribute used in either. +XPath constraints follow too — `[Phone = '06-1234']` and paths that reach the +entity through an association alike — because a constraint's target entity is +known from where it is stored. Another entity's attribute of the same name is +left alone, and any constraint that cannot be resolved is reported rather than +rewritten. + +Microflow expressions (`$Customer/Phone`) are **not** rewritten: a bare name +there is only resolvable from the type of what precedes it. `mx check` reports +those as CE0117, so build after renaming an attribute used in one. ## ADD INDEX diff --git a/docs-site/src/reference/domain-model/alter-entity.md b/docs-site/src/reference/domain-model/alter-entity.md index 0b639d150..97b354d33 100644 --- a/docs-site/src/reference/domain-model/alter-entity.md +++ b/docs-site/src/reference/domain-model/alter-entity.md @@ -30,7 +30,9 @@ The `MODIFY ATTRIBUTE` operation changes the type or constraints of an existing The `RENAME ATTRIBUTE` operation changes an attribute's name and rewrites every reference Mendix stores as a reference — microflow create and change activity members, page attribute widgets, and the entity's own validation and access rules — reporting how many documents it touched. Renaming onto a name the entity already uses is refused. -Two kinds of use are **not** rewritten, because Mendix stores them as free text in which a bare attribute name is only resolvable from the type of whatever precedes it: microflow expressions (`$obj/Phone`) and XPath constraints (`[Phone = '...']`). The command prints a note saying so; `mx check` reports those as CE0117 and CE0161 respectively, so run a build after renaming an attribute that appears in either. +XPath constraints are rewritten too, even though they name the attribute as a bare step. Each constraint's target entity is known from where it is stored — a retrieve names its entity, a widget data source names its entity, an access rule lives inside one — and every further hop is an association or entity named in the path, so `[Phone = '...']` and `[Sales.Order_Customer/Sales.Customer/Phone = '...']` both follow the rename while another entity's identically-named attribute does not. A constraint mxcli cannot resolve is listed in a warning and left exactly as it was. + +One kind of use is **not** rewritten: microflow expressions (`$obj/Phone`), where a bare name is only resolvable from the type of what precedes it. The command prints a note saying so, and `mx check` reports the leftovers as CE0117. The `ADD INDEX` and `DROP INDEX` operations manage database indexes. `ADD INDEX` takes an optional index name followed by one or more columns, each with an optional `ASC` or `DESC` sort direction; `DROP INDEX` takes the index name. @@ -128,7 +130,7 @@ ALTER ENTITY Sales.Customer ## Notes - Each `ALTER ENTITY` statement performs a single operation. Chain multiple statements for multiple changes. -- `RENAME` does not update references in microflows, pages, or access rules. Update those separately or use `SHOW IMPACT OF` to find affected elements. +- `RENAME ATTRIBUTE` updates references in microflows, pages and access rules, and rewrites XPath constraints. It does not rewrite microflow expressions (`$obj/Attr`); use `SHOW IMPACT OF` to find the documents to check, or build and read the CE0117s. - `DROP` removes the attribute's validation rules and index entries automatically. ## See Also diff --git a/docs/01-project/MDL_QUICK_REFERENCE.md b/docs/01-project/MDL_QUICK_REFERENCE.md index 3c8bc8ddc..354b9feb2 100644 --- a/docs/01-project/MDL_QUICK_REFERENCE.md +++ b/docs/01-project/MDL_QUICK_REFERENCE.md @@ -88,7 +88,7 @@ Modifies an existing entity without full replacement. | Add attributes | `alter entity Module.Name add (attr: type [constraints]);` | One or more attributes | | Drop attributes | `alter entity Module.Name drop (AttrName, ...);` | | | Modify attributes | `alter entity Module.Name modify (attr: NewType [constraints]);` | Change type/constraints | -| Rename attribute | `alter entity Module.Name rename attribute OldName to NewName;` | Also rewrites stored references (microflow members, page widgets, validation/access rules). Expressions and XPath constraints are free text and are **not** rewritten | +| Rename attribute | `alter entity Module.Name rename attribute OldName to NewName;` | Also rewrites stored references (microflow members, page widgets, validation/access rules) and XPath constraints. Microflow expressions are free text and are **not** rewritten | | Add index | `alter entity Module.Name add index [name] [on] (Col1 [asc\|desc], ...);` | `on` is optional (SQL-like) | | Drop index | `alter entity Module.Name drop index (Col1, ...);` | | | Add event handler | `alter entity Module.Name add event handler on before commit call Mod.MF($currentObject) [raise error];` | `($currentObject)` or `()`, RAISE ERROR only on BEFORE | diff --git a/docs/11-proposals/PROPOSAL_expression_type_checking.md b/docs/11-proposals/PROPOSAL_expression_type_checking.md index a5ed84991..817f3acf4 100644 --- a/docs/11-proposals/PROPOSAL_expression_type_checking.md +++ b/docs/11-proposals/PROPOSAL_expression_type_checking.md @@ -123,17 +123,18 @@ Mendix stores *as a reference* — a create/change member, a page attribute widget, a validation or access rule — because those hold the fully qualified name as a string and a string scan finds them. It cannot touch the two places an attribute is named in **free text**, and those are exactly the two this proposal -learns to read: +learns to read (one of which has since been closed without it — see the status +note at the end of this section): | Shape | Example | Why a string scan cannot do it | |-------|---------|-------------------------------| | Expression | `$Order/Status` | The bare segment is only an attribute of `Mod.Order` if `$Order` is typed — the scope walker's job | | XPath constraint | `[Status = 'Open']`, `[Mod.A_B/Mod.B/Status = …]` | The bare step belongs to the constraint's *target* entity, or to the entity the association hops reach | -mxbuild reports the leftovers as **CE0117** and **CE0161** (measured on 11.13.0), -so the rename tells the user they exist rather than reading as complete — but a -rename that half-renames is the weakest form of the feature. This consumer is -what finishes it. Origin: mendixlabs/mxcli#910. +mxbuild reports the leftovers as **CE0117** (expressions) and **CE0161** (XPath), +measured on 11.13.0, so the rename tells the user they exist rather than reading +as complete — but a rename that half-renames is the weakest form of the feature. +This consumer is what finishes it. Origin: mendixlabs/mxcli#910. **Two asymmetries worth knowing before scheduling it.** @@ -165,9 +166,31 @@ a constraint on the interface, not on the callers. (This is the same guard-don't-drop stance as [ADR-0005](../13-decisions/0005-semantic-model-interface-currency.md).) Sequencing follows from the asymmetry: the XPath half is shippable **before** -step 1 lands, against structural entity knowledge alone, with multi-hop paths -reported-not-rewritten until the resolver exists. The expression half waits for -step 1. +step 1 lands, against structural entity knowledge alone. The expression half +waits for step 1. + +**Status: the XPath half has shipped** (`mdl/xpathrefs`), and multi-hop paths did +not need to wait for the resolver after all — Mendix spells the intermediate +entity out in a stored path, so a hop resolves against a flat entity/association +index built from the domain models rather than against a type system. Three +findings worth carrying into step 1: + +- **A mutating consumer needs a third answer, not two.** The walk distinguishes + *this entity's attribute*, *a different entity's attribute* (a definite answer — + leave it alone, say nothing) and *could not tell* (refuse and report). Collapsing + the last two into one "not ours" made the rewrite silently skip constraints it + should have questioned. `KindUnknown` as § 4 defines it is the second and third + merged, which is fine for a checker and not for this. +- **A lenient parser can still gate a rewrite, if the edit is counted against it.** + `visitor.ParseXPathConstraint` runs with ANTLR's error listeners removed and can + return a tree that omits part of its input. Requiring the lexical occurrence + count to equal the walked occurrence count is what makes leaning on that parse + safe; the expression half will need the same discipline, because `exprcheck`'s + parser recovers by design. +- **Edit tokens, do not re-render.** Re-rendering the parsed tree would rewrite + spacing in constraints the rename has no business touching, and any + parser/renderer disagreement would corrupt a working one. The rewrite replaces a + single identifier and leaves every other byte alone. --- diff --git a/docs/11-proposals/PROPOSAL_rename_refactoring.md b/docs/11-proposals/PROPOSAL_rename_refactoring.md index 81d76ef33..1a0dccddd 100644 --- a/docs/11-proposals/PROPOSAL_rename_refactoring.md +++ b/docs/11-proposals/PROPOSAL_rename_refactoring.md @@ -21,7 +21,7 @@ A safe RENAME command that automatically updates all references would be a signi | Operation | Status | |-----------|--------| -| `alter entity ... rename attribute Old to New` | Works, and updates stored references. Free-text uses (expressions, XPath) are excluded — see Scope Exclusions | +| `alter entity ... rename attribute Old to New` | Works: updates stored references and XPath constraints. Microflow expressions are excluded — see Scope Exclusions | | `alter enumeration ... rename value Old to New` | Works | | `rename entity Module.Old to New` | Grammar only, not implemented | | `rename module Old to New` | Grammar only, not implemented | @@ -235,7 +235,7 @@ Renaming an entity changes the proxy class name in `javasource//proxies/ ## Scope Exclusions -- **Attribute names in expressions and XPath constraints**: `$Order/Status` and `[Status = 'Open']` name an attribute in **free text**, where the bare name is only resolvable from the type of what precedes it — so the qualified-name scanner this proposal is built on cannot see them. `alter entity … rename attribute` rewrites every *stored* reference and says plainly that it leaves these; mxbuild reports them as CE0117 / CE0161. Closing the gap needs the resolution machinery in [`PROPOSAL_expression_type_checking.md`](PROPOSAL_expression_type_checking.md) (§ Fourth consumer), which also records why a *mutating* consumer must fail differently from a checking one. Origin: mendixlabs/mxcli#910 +- **Attribute names in microflow expressions**: `$Order/Status` names an attribute in **free text**, where the bare name is only resolvable from the type of what precedes it — so the qualified-name scanner this proposal is built on cannot see it. `alter entity … rename attribute` rewrites every *stored* reference, rewrites XPath constraints (`mdl/xpathrefs` — a constraint's target entity is known structurally, so it needed no type system), and says plainly that it leaves expressions; mxbuild reports those as CE0117. Closing the last gap needs the resolution machinery in [`PROPOSAL_expression_type_checking.md`](PROPOSAL_expression_type_checking.md) (§ Fourth consumer), which also records why a *mutating* consumer must fail differently from a checking one. Origin: mendixlabs/mxcli#910 - **Java source file updates**: Out of scope — rename produces correct MPR but Java files need manual update - **Widget property string references**: Pluggable widget properties may contain entity/attribute names as strings — these are not updatable without widget-specific knowledge - **Git history**: Rename doesn't create a git rename operation — it modifies files in place diff --git a/mdl-examples/bug-tests/910-rename-attribute-updates-references.mdl b/mdl-examples/bug-tests/910-rename-attribute-updates-references.mdl index d1cfbb1c8..042d7f329 100644 --- a/mdl-examples/bug-tests/910-rename-attribute-updates-references.mdl +++ b/mdl-examples/bug-tests/910-rename-attribute-updates-references.mdl @@ -28,10 +28,11 @@ -- model second would re-serialize it from the parsed entity and undo the -- scan's edits there (which is where a validation rule's reference lives). -- --- Uses inside expressions ($obj/Attr) and XPath constraints ([Attr = ...]) --- are free text and are still NOT rewritten — a bare name there is only +-- XPath constraints are handled separately, by mdl/xpathrefs — see +-- 910-rename-attribute-xpath-constraints.mdl. Uses inside microflow +-- expressions ($obj/Attr) are still NOT rewritten: a bare name there is only -- resolvable from the type of what precedes it. The rename says so, and --- mxbuild reports them as CE0117 / CE0161. +-- mxbuild reports them as CE0117. -- -- Verify: -- mxcli exec mdl-examples/bug-tests/910-rename-attribute-updates-references.mdl -p app.mpr diff --git a/mdl-examples/bug-tests/910-rename-attribute-xpath-constraints.mdl b/mdl-examples/bug-tests/910-rename-attribute-xpath-constraints.mdl new file mode 100644 index 000000000..7aaf82568 --- /dev/null +++ b/mdl-examples/bug-tests/910-rename-attribute-xpath-constraints.mdl @@ -0,0 +1,60 @@ +-- ============================================================================ +-- Bug #910 (problem 2, XPath half): RENAME ATTRIBUTE left XPath constraints +-- ============================================================================ +-- +-- Symptom: +-- After the qualified-name half of the fix, a rename still broke every XPath +-- constraint naming the attribute: +-- [error] [CE0161] "Error(s) in XPath constraint." +-- at Retrieve object(s) activity 'Retrieve list of Person from database' +-- XPath names an attribute as a BARE step — `[FirstName = 'Ada']` — so the +-- string scan that fixes "Mod.Entity.Attr" references cannot see it, and a +-- scan for the bare name would rewrite string literals, function names, and +-- other entities' identically-named attributes. +-- +-- Fix (mdl/xpathrefs): +-- Each constraint's target entity is known STRUCTURALLY — a retrieve names +-- its entity, a widget data source names its entity, an access rule lives +-- inside one — so no type inference is needed. Every further hop is an +-- association or entity named in the path itself. The rewrite resolves each +-- bare step to its owning entity, then edits ONE identifier token, leaving +-- every other byte alone (re-rendering the parsed tree would churn spacing +-- in constraints the rename has no business touching). +-- +-- Anything it cannot resolve is reported and left untouched, never guessed. +-- +-- Verify: +-- mxcli exec mdl-examples/bug-tests/910-rename-attribute-xpath-constraints.mdl -p app.mpr +-- mx check app.mpr # 0 errors — before the fix, 3x CE0161 +-- ============================================================================ + +create module XPathRn; +/ +create persistent entity XPathRn.Person ( FirstName: String(100), LastName: String(100) ); +-- Order carries an attribute of the SAME name. It must not move. +create persistent entity XPathRn.Order ( FirstName: String(100), Total: Decimal ); +/ +create association XPathRn.Order_Person from XPathRn.Order to XPathRn.Person; +/ +create or replace microflow XPathRn.ACT_FindPeople () +begin + -- (1) bare step on the renamed entity itself + retrieve $People from XPathRn.Person where [FirstName = 'Ada']; + -- (2) the same name on a DIFFERENT entity — must survive the rename + retrieve $Orders from XPathRn.Order where [FirstName = 'Ada']; + -- (3) reached through an association hop: the constraint belongs to Order, + -- and only the hop to Person makes the bare step ours + retrieve $Theirs from XPathRn.Order + where [XPathRn.Order_Person/XPathRn.Person/FirstName = 'Ada']; +end; +/ +create module role XPathRn.User; +grant XPathRn.User on XPathRn.Person (read *) where '[FirstName != '''']'; +/ +alter entity XPathRn.Person rename attribute FirstName to GivenName; +/ +-- Expected afterwards (describe entity / describe microflow): +-- (1) [GivenName = 'Ada'] +-- (2) [FirstName = 'Ada'] <- untouched, it is Order's +-- (3) [XPathRn.Order_Person/XPathRn.Person/GivenName = 'Ada'] +-- access rule: [GivenName != ''] diff --git a/mdl/executor/alter_entity_rename_attribute_test.go b/mdl/executor/alter_entity_rename_attribute_test.go index 40de60ac1..14688eafe 100644 --- a/mdl/executor/alter_entity_rename_attribute_test.go +++ b/mdl/executor/alter_entity_rename_attribute_test.go @@ -115,22 +115,28 @@ func TestAlterEntityRenameAttributeReportsReferenceCount(t *testing.T) { } } -// TestAlterEntityRenameAttributeWarnsAboutTextUses pins that the rename says -// what it did NOT do. Expressions and XPath constraints name an attribute in -// free text, where a bare name is only resolvable from the type of what precedes -// it, so the reference scan leaves them alone — and mxbuild then reports them as -// CE0117 / CE0161. A rename that reports only its successes reads as complete. -func TestAlterEntityRenameAttributeWarnsAboutTextUses(t *testing.T) { +// TestAlterEntityRenameAttributeWarnsAboutExpressions pins that the rename says +// what it did NOT do. Microflow expressions name an attribute in free text, +// where a bare name is only resolvable from the type of what precedes it, so the +// reference scan leaves them alone and mxbuild reports them as CE0117. A rename +// that reports only its successes reads as complete. +// +// It must not also claim that about XPath constraints, which are rewritten — a +// warning that names something already handled trains the reader to ignore it. +func TestAlterEntityRenameAttributeWarnsAboutExpressions(t *testing.T) { ctx, _, _ := renameAttrTestCtx(t, 0, "PuzzleNo") assertNoError(t, execAlterEntity(ctx, renameAttrStmt("PuzzleNo", "PuzzleNumber"))) out := ctx.Output.(interface{ String() string }).String() - for _, want := range []string{"expressions", "XPath", "PuzzleNo"} { + for _, want := range []string{"expressions", "PuzzleNo"} { if !strings.Contains(out, want) { t.Errorf("the rename does not mention %q in its note about text uses, got: %q", want, out) } } + if strings.Contains(out, "XPath constraints are stored as text") { + t.Errorf("the note still claims XPath constraints are not rewritten, got: %q", out) + } } // TestAlterEntityRenameAttributeReferencesAfterModel pins the ordering. diff --git a/mdl/executor/cmd_entities.go b/mdl/executor/cmd_entities.go index 115ac9960..f2e664075 100644 --- a/mdl/executor/cmd_entities.go +++ b/mdl/executor/cmd_entities.go @@ -837,6 +837,14 @@ func execAlterEntity(ctx *ExecContext, s *ast.AlterEntityStmt) error { return mdlerrors.NewNotFoundMsg("attribute", s.AttributeName, fmt.Sprintf("attribute '%s' not found on entity %s", s.AttributeName, s.Name)) } target.Name = s.NewName + // The entity's own access and validation rules hold the attribute's + // qualified name as a string, so they have to move with it *in the model* + // — not merely in the BSON the reference scan rewrites afterwards. + // Leaving them to the scan produced a duplicate member: UpdateEntity saw + // an attribute with no matching MemberAccess and added one, and the scan + // then renamed the stale entry into a second copy of it. mxbuild caught + // that as CE0066 "Entity access is out of date". + renameAttributeInEntityRules(entity, s.Name.String(), s.AttributeName, s.NewName) if err := ctx.Backend.UpdateEntity(dm.ID, entity); err != nil { return mdlerrors.NewBackend("rename attribute", err) } @@ -859,21 +867,32 @@ func execAlterEntity(ctx *ExecContext, s *ast.AlterEntityStmt) error { return mdlerrors.NewBackend("update attribute references", err) } + // XPath constraints name the attribute as a bare step, so the scan above + // cannot see them. They are resolvable without any type inference — a + // constraint's target entity is known structurally and every further hop + // is named in the path — so they are rewritten here rather than left for + // the user. See mdl/xpathrefs for why the edit is textual and what it + // refuses to touch. + xres, err := renameAttributeInXPath(ctx, s.Name.String(), string(dm.ID), s.Name.Name, s.AttributeName, s.NewName) + if err != nil { + return mdlerrors.NewBackend("update attribute references in XPath constraints", err) + } + invalidateHierarchy(ctx) invalidateDomainModelsCache(ctx) fmt.Fprintf(ctx.Output, "Renamed attribute '%s' to '%s' on entity %s\n", s.AttributeName, s.NewName, s.Name) if n := totalRefCount(hits); n > 0 { fmt.Fprintf(ctx.Output, "Updated %d reference(s) in %d document(s)\n", n, len(hits)) } - // The scan above rewrites every reference Mendix stores *as a reference*. - // It cannot rewrite the two places an attribute is named in free text — - // microflow expressions ($obj/Attr) and XPath constraints ([Attr = …]) — - // because a bare name there is only resolvable with the type of what - // precedes it. mxbuild does report them (CE0117 / CE0161), so say so - // rather than let a half-done rename look finished. + reportXPathRename(ctx, xres) + // Microflow expressions ($obj/Attr) are the one place left. A bare name + // there is only resolvable from the type of what precedes it, which needs + // the resolver in PROPOSAL_expression_type_checking; mxbuild reports the + // leftovers as CE0117, so say so rather than let a half-done rename look + // finished. fmt.Fprintf(ctx.Output, - "Note: uses in expressions ($obj/%s) and XPath constraints are stored as text "+ - "and were not rewritten — run 'mxcli docker check' to find them.\n", + "Note: uses in microflow expressions ($obj/%s) are stored as text and were "+ + "not rewritten — run 'mxcli docker check' to find them.\n", s.AttributeName) case ast.AlterEntityModifyAttribute: diff --git a/mdl/executor/rename_attribute_entity_rules.go b/mdl/executor/rename_attribute_entity_rules.go new file mode 100644 index 000000000..6cecde3ac --- /dev/null +++ b/mdl/executor/rename_attribute_entity_rules.go @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// renameAttributeInEntityRules moves an entity's own references to one of its +// attributes onto the attribute's new name. +// +// Access rules and validation rules both point at an attribute by qualified name +// string rather than by element ID, so renaming the attribute in the model +// leaves them behind. That matters more than it looks: UpdateEntity re-derives a +// rule's members from the attributes it can match, so a stale member is not +// merely stale — a `READ *` rule comes back with a member for the new name *and* +// the orphaned old one, which mxbuild reports as CE0066 "Entity access is out of +// date". Fixing them here, before the write, is what keeps the rule intact. +// +// The project-wide reference scan would rewrite the same strings afterwards, but +// by then the damage is done, and a scan cannot tell a duplicate from a +// legitimate second member. +func renameAttributeInEntityRules(entity *domainmodel.Entity, entityQN, oldAttr, newAttr string) { + oldQN := entityQN + "." + oldAttr + newQN := entityQN + "." + newAttr + + for _, rule := range entity.AccessRules { + if rule == nil { + continue + } + for _, ma := range rule.MemberAccesses { + if ma != nil && ma.AttributeName == oldQN { + ma.AttributeName = newQN + } + } + } + + for _, vr := range entity.ValidationRules { + if vr == nil { + continue + } + // AttributeID holds either a UUID (an entity built in this run, where the + // name is looked up at serialization time and needs no help) or the + // qualified name read from disk. + if string(vr.AttributeID) == oldQN { + vr.AttributeID = model.ID(newQN) + } + // A range rule can be bounded by another attribute of the same entity, + // including the one being renamed. MDL cannot author that form, but a + // stored rule survives a round trip and must survive a rename too. + if rng, ok := vr.Rule.(*domainmodel.RangeValidationRuleInfo); ok { + if rng.MinAttributeQualifiedName == oldQN { + rng.MinAttributeQualifiedName = newQN + } + if rng.MaxAttributeQualifiedName == oldQN { + rng.MaxAttributeQualifiedName = newQN + } + } + } +} diff --git a/mdl/executor/rename_attribute_entity_rules_test.go b/mdl/executor/rename_attribute_entity_rules_test.go new file mode 100644 index 000000000..a3eb3492e --- /dev/null +++ b/mdl/executor/rename_attribute_entity_rules_test.go @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// TestRenameAttributeInEntityRules pins that an entity's own by-name references +// to one of its attributes move with it. +// +// This is not tidiness. UpdateEntity re-derives an access rule's members from the +// attributes it can match, so a member left pointing at the old name is not +// merely stale — a READ * rule comes back carrying a member for the new name AND +// the orphan, and mxbuild reports CE0066 "Entity access is out of date". Measured +// on Mendix 11.13.0 with mdl-examples/bug-tests/910-rename-attribute-xpath-constraints.mdl: +// without this pass the rename leaves 1 error and three members for a two-attribute +// entity; with it, 0 errors and two members. +func TestRenameAttributeInEntityRules(t *testing.T) { + minAttr := "Sales.Person.FirstName" + entity := &domainmodel.Entity{ + Name: "Person", + AccessRules: []*domainmodel.AccessRule{{ + MemberAccesses: []*domainmodel.MemberAccess{ + {AttributeName: "Sales.Person.FirstName"}, + {AttributeName: "Sales.Person.LastName"}, + {AssociationName: "Sales.Order_Person"}, + nil, + }, + }, nil}, + ValidationRules: []*domainmodel.ValidationRule{ + {AttributeID: model.ID("Sales.Person.FirstName"), Type: "Required"}, + {AttributeID: model.ID("Sales.Person.LastName"), Type: "Required"}, + { + AttributeID: model.ID("Sales.Person.LastName"), + Type: "Range", + Rule: &domainmodel.RangeValidationRuleInfo{MinAttributeQualifiedName: minAttr}, + }, + nil, + }, + } + + renameAttributeInEntityRules(entity, "Sales.Person", "FirstName", "GivenName") + + ma := entity.AccessRules[0].MemberAccesses + if ma[0].AttributeName != "Sales.Person.GivenName" { + t.Errorf("the access rule member still points at %q", ma[0].AttributeName) + } + if ma[1].AttributeName != "Sales.Person.LastName" { + t.Errorf("another attribute's member moved: %q", ma[1].AttributeName) + } + if ma[2].AssociationName != "Sales.Order_Person" { + t.Errorf("an association member was touched: %q", ma[2].AssociationName) + } + + vr := entity.ValidationRules + if string(vr[0].AttributeID) != "Sales.Person.GivenName" { + t.Errorf("the validation rule still points at %q", vr[0].AttributeID) + } + if string(vr[1].AttributeID) != "Sales.Person.LastName" { + t.Errorf("another attribute's validation rule moved: %q", vr[1].AttributeID) + } + rng, ok := vr[2].Rule.(*domainmodel.RangeValidationRuleInfo) + if !ok { + t.Fatalf("the range rule lost its payload: %T", vr[2].Rule) + } + if rng.MinAttributeQualifiedName != "Sales.Person.GivenName" { + t.Errorf("an attribute-bounded range still points at %q", rng.MinAttributeQualifiedName) + } +} + +// TestRenameAttributeInEntityRulesLeavesUUIDsAlone pins that a rule whose +// AttributeID is a UUID — an entity built in this run, where the serializer looks +// the name up — is not mangled into a qualified name. +func TestRenameAttributeInEntityRulesLeavesUUIDsAlone(t *testing.T) { + const id = "0f6a2c14-8f6f-4a1e-8a1b-2f4d9b0c1e33" + entity := &domainmodel.Entity{ + Name: "Person", + ValidationRules: []*domainmodel.ValidationRule{{AttributeID: model.ID(id)}}, + } + + renameAttributeInEntityRules(entity, "Sales.Person", "FirstName", "GivenName") + + if string(entity.ValidationRules[0].AttributeID) != id { + t.Errorf("a UUID attribute reference was rewritten to %q", entity.ValidationRules[0].AttributeID) + } +} diff --git a/mdl/executor/xpath_rename.go b/mdl/executor/xpath_rename.go new file mode 100644 index 000000000..ee76ef38e --- /dev/null +++ b/mdl/executor/xpath_rename.go @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + "sort" + "strings" + + "github.com/mendixlabs/mxcli/mdl/xpathrefs" +) + +// renameAttributeInXPath rewrites the renamed attribute inside every stored XPath +// constraint in the project. +// +// dmUnitID and entityName identify the renamed entity's own domain model unit, +// which is how an access rule's constraint is attributed: the rule carries no +// entity reference, it simply lives inside one. +func renameAttributeInXPath(ctx *ExecContext, entityQN, dmUnitID, entityName, oldAttr, newAttr string) (xpathrefs.Result, error) { + model, err := buildXPathModel(ctx) + if err != nil { + return xpathrefs.Result{}, err + } + return xpathrefs.RenameAttribute(ctx.Backend, model, entityQN, dmUnitID, entityName, oldAttr, newAttr) +} + +// reportXPathRename prints what the XPath pass did and what it refused to do. +// +// The refusals are the half that matters. A constraint mxcli could not resolve is +// left exactly as it was, so the project is no worse than before — but the user +// has to be told which one, or the rename reads as complete and the breakage +// surfaces later as a CE0161 with no obvious cause. +func reportXPathRename(ctx *ExecContext, res xpathrefs.Result) { + if n := res.Total(); n > 0 { + fmt.Fprintf(ctx.Output, "Updated %d XPath constraint(s) in %d document(s)\n", n, res.Units) + } + if len(res.Skipped) == 0 { + return + } + fmt.Fprintf(ctx.Output, + "Warning: %d XPath constraint(s) name the attribute but could not be resolved, "+ + "and were left unchanged — check them by hand:\n", len(res.Skipped)) + for _, occ := range dedupeOccurrences(res.Skipped) { + fmt.Fprintf(ctx.Output, " %s: %s\n", describeOccurrence(occ), occ.Constraint) + } +} + +// describeOccurrence labels a constraint by the document it sits in, falling back +// to the unit ID when the document has no name of its own. +func describeOccurrence(occ xpathrefs.Occurrence) string { + label := occ.Document + if label == "" { + label = occ.UnitID + } + typeName := occ.UnitType + if i := strings.Index(typeName, "$"); i >= 0 { + typeName = typeName[i+1:] + } + if typeName == "" { + return label + } + return fmt.Sprintf("%s (%s)", label, typeName) +} + +// dedupeOccurrences collapses identical (document, constraint) pairs and sorts +// them, so the warning is stable across runs and a constraint repeated in one +// document is listed once. +func dedupeOccurrences(occs []xpathrefs.Occurrence) []xpathrefs.Occurrence { + seen := map[string]bool{} + out := make([]xpathrefs.Occurrence, 0, len(occs)) + for _, o := range occs { + key := o.UnitID + "\x00" + o.Constraint + if seen[key] { + continue + } + seen[key] = true + out = append(out, o) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Document != out[j].Document { + return out[i].Document < out[j].Document + } + return out[i].Constraint < out[j].Constraint + }) + return out +} diff --git a/mdl/executor/xpath_rename_model.go b/mdl/executor/xpath_rename_model.go new file mode 100644 index 000000000..cade2ca4c --- /dev/null +++ b/mdl/executor/xpath_rename_model.go @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "github.com/mendixlabs/mxcli/mdl/xpathrefs" + "github.com/mendixlabs/mxcli/model" +) + +// xpathModel answers the two questions mdl/xpathrefs asks while following an +// XPath path: is this qualified name an entity, and where does this association +// lead from here. +// +// It is a flat index built once per rename rather than a lookup per step. A +// constraint is walked once per predicate group per document, so a per-step +// backend round trip would turn one rename into thousands of domain-model reads. +type xpathModel struct { + entities map[string]bool + // assoc maps an association's qualified name to its two ends, FROM first. + assoc map[string][2]string +} + +var _ xpathrefs.Model = (*xpathModel)(nil) + +// buildXPathModel indexes every entity and association in the project by +// qualified name. +func buildXPathModel(ctx *ExecContext) (*xpathModel, error) { + m := &xpathModel{ + entities: map[string]bool{}, + assoc: map[string][2]string{}, + } + + dms, err := ctx.Backend.ListDomainModels() + if err != nil { + return nil, err + } + h, err := getHierarchy(ctx) + if err != nil { + return nil, err + } + + // Entity element ID → qualified name, so an association's ends can be + // resolved. Associations reference their ends by element ID, and a + // cross-module association's ends live in a different domain model, so the + // index has to span the whole project before any end is resolved. + byID := map[model.ID]string{} + type pending struct { + qn string + parentID, childID model.ID + } + var assocs []pending + + for _, dm := range dms { + moduleName := h.GetModuleName(h.FindModuleID(dm.ContainerID)) + if moduleName == "" { + continue + } + for _, ent := range dm.Entities { + qn := moduleName + "." + ent.Name + m.entities[qn] = true + byID[ent.ID] = qn + } + for _, a := range dm.Associations { + assocs = append(assocs, pending{moduleName + "." + a.Name, a.ParentID, a.ChildID}) + } + } + + for _, a := range assocs { + from, okF := byID[a.parentID] + to, okT := byID[a.childID] + if !okF || !okT { + // An end that cannot be resolved (an external entity, a broken + // reference) leaves the association out of the index entirely, so a + // path through it reads as unresolved and blocks the rewrite rather + // than resolving to half an answer. + continue + } + m.assoc[a.qn] = [2]string{from, to} + } + + return m, nil +} + +func (m *xpathModel) IsEntity(qn string) bool { return m.entities[qn] } + +// AssociationTarget resolves a hop in either direction: Mendix XPath traverses an +// association from its FROM end (`[Mod.Order_Person/…]` on an Order) and from its +// TO end (the same step on a Person) alike, and the stored constraint looks the +// same both ways. +func (m *xpathModel) AssociationTarget(qn, from string) (string, bool) { + ends, ok := m.assoc[qn] + if !ok || from == "" { + return "", false + } + switch from { + case ends[0]: + return ends[1], true + case ends[1]: + return ends[0], true + } + // A self-association resolves above; anything else means the path does not + // actually start where the constraint says it does. + return "", false +} diff --git a/mdl/xpathrefs/rewrite.go b/mdl/xpathrefs/rewrite.go new file mode 100644 index 000000000..4bab57f3e --- /dev/null +++ b/mdl/xpathrefs/rewrite.go @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package xpathrefs rewrites attribute names inside stored XPath constraints. +// +// An attribute is named two ways in a Mendix project. Most places store the +// fully qualified name as a string ("Mod.Entity.Attr"), which a rename can find +// by scanning for that string. XPath constraints instead name it as a bare step +// — `[Status = 'Open']` — where the same three letters mean different attributes +// on different entities, and mean nothing at all inside a string literal. A +// rename that scans for "Status" would corrupt every one of those. +// +// This package resolves each bare step to the entity it actually belongs to +// before touching anything. It needs no type inference to do it: a constraint's +// target entity is known structurally (a retrieve names its entity, a widget +// data source names its entity, an access rule belongs to one), and every +// further hop is an association or entity named in the path itself. +// +// The rewrite is deliberately **textual, gated by the parse** rather than a +// re-render of the parsed tree. Re-rendering would rewrite whitespace and +// spelling in constraints that are none of the rename's business, and any +// disagreement between mxcli's parser and its renderer would silently corrupt a +// working constraint. Here the parse only decides *whether* and *how many*; the +// edit itself replaces one identifier token and leaves every other byte alone. +package xpathrefs + +import ( + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/visitor" +) + +// Model answers the two questions walking an XPath path asks. It is deliberately +// smaller than the ModelResolver in PROPOSAL_expression_type_checking: XPath +// needs no attribute types, no enum cases and no return types, which is why this +// half does not wait for that proposal. +type Model interface { + // IsEntity reports whether qn ("Module.Name") names an entity. + IsEntity(qn string) bool + // AssociationTarget returns the entity at the other end of association qn + // when it is traversed from entity from, and whether qn is an association at + // all. Both directions resolve: Mendix XPath traverses an association from + // either end. + AssociationTarget(qn, from string) (string, bool) +} + +// rewriteConstraint returns the constraint with every bare step that resolves to +// entityQN.oldAttr replaced by newAttr. +// +// The second result is false when some part of the constraint names oldAttr but +// could not be shown to mean this entity's attribute — an unparseable predicate +// group, a path rooted in a variable, an association mxcli cannot resolve, or +// the same bare name belonging to two different entities in one group. The +// caller reports those; it must never rewrite them. A wrong rewrite corrupts a +// constraint that was working, which is strictly worse than the honest partial +// rename this package exists to finish. +func rewriteConstraint(constraint, targetEntity, entityQN, oldAttr, newAttr string, m Model) (string, bool) { + groups := visitor.SplitXPathPredicateGroups(constraint) + if len(groups) == 0 { + // Not a bracket-group constraint at all. If it mentions the name we + // cannot say anything about it, so report rather than guess. + return constraint, !mentionsIdentifier(constraint, oldAttr) + } + + out := make([]string, 0, len(groups)) + understood := true + for _, g := range groups { + rewritten, ok := rewriteGroup(g, targetEntity, entityQN, oldAttr, newAttr, m) + if !ok { + understood = false + } + out = append(out, rewritten) + } + return strings.Join(out, ""), understood +} + +// rewriteGroup rewrites one bracket group. +func rewriteGroup(group, targetEntity, entityQN, oldAttr, newAttr string, m Model) (string, bool) { + // Cheap exit: a group that never spells the name lexically cannot contain a + // reference to it, whatever it parses to. + rewritten, lexical := replaceIdentifier(group, oldAttr, newAttr) + if lexical == 0 { + return group, true + } + + expr, ok := visitor.ParseXPathConstraint(group) + if !ok || expr == nil { + // mxcli could not read this group. It mentions the name, so say so — + // passing it through silently is how a half-rename looks finished. + return group, false + } + + w := &walker{model: m, entityQN: entityQN, attr: oldAttr} + w.walk(expr, targetEntity) + + // The count invariant, and the load-bearing safety check. + // + // visitor.ParseXPathConstraint runs with ANTLR's error listeners removed, so + // it recovers from almost anything and can hand back a tree that quietly + // omits part of its input — `[LastName = 'L' FirstName]` parses, and the + // trailing step is simply not in the tree. Resolution then never sees that + // occurrence, while a lexical edit would rewrite it regardless. Requiring the + // parser and the editor to agree on how many times the name occurs is what + // makes a lenient parse safe to lean on: any disagreement means the two are + // looking at different text, so neither is trusted. + if lexical != w.hits+w.others+w.unresolved { + return group, false + } + + switch { + case w.unresolved > 0: + // Somewhere the name is spelled and the walk could not say whose + // attribute it is. Never rewrite on a guess — and never stay quiet + // either, because silence is how a half-rename looks finished. + return group, false + case w.hits == 0: + // The name occurs, but always as something else: a string literal, or + // another entity's attribute of the same name. That is a definite answer, + // so it is left alone and not reported. + return group, true + case w.others > 0: + // One bare name means this entity's attribute in one place and a + // different entity's in another. A token-level edit cannot tell them + // apart, and re-rendering to fix that is the thing this package refuses + // to do. + return group, false + } + return rewritten, true +} + +// walker counts occurrences of one attribute name, split three ways by what the +// walk could establish about each. +type walker struct { + model Model + entityQN string + attr string + // hits are occurrences shown to be the renamed entity's attribute. + hits int + // others are occurrences shown to be a *different* entity's attribute — a + // definite answer, and a reason to leave them alone rather than to complain. + others int + // unresolved are occurrences whose owning entity could not be established. + unresolved int +} + +// note records one bare identifier seen in the context of entity cur. An empty +// cur means the walk lost track of the entity, which is unresolved rather than +// "someone else's": "we do not know" must block the rewrite, not permit it. +func (w *walker) note(name, cur string) { + if name != w.attr { + return + } + switch { + case cur == "": + w.unresolved++ + case cur == w.entityQN: + w.hits++ + default: + w.others++ + } +} + +// walk visits expr with cur as the entity its bare steps are evaluated against. +func (w *walker) walk(expr ast.Expression, cur string) { + switch e := expr.(type) { + case nil: + return + case *ast.IdentifierExpr: + w.note(e.Name, cur) + case *ast.XPathPathExpr: + w.walkPath(e.Steps, cur) + case *ast.BinaryExpr: + w.walk(e.Left, cur) + w.walk(e.Right, cur) + case *ast.UnaryExpr: + w.walk(e.Operand, cur) + case *ast.ParenExpr: + w.walk(e.Inner, cur) + case *ast.FunctionCallExpr: + for _, a := range e.Arguments { + w.walk(a, cur) + } + case *ast.IfThenElseExpr: + w.walk(e.Condition, cur) + w.walk(e.ThenExpr, cur) + w.walk(e.ElseExpr, cur) + case *ast.SourceExpr: + w.walk(e.Expression, cur) + } + // Literals, variables, qualified names, tokens and constant refs name no + // bare attribute, so they need no visit. +} + +// walkPath follows a path step by step, carrying the entity each step lands on. +// +// The entity is never inferred: an association hop resolves through the model, +// an explicit entity step names itself, and anything else (a variable root, an +// unknown qualified name) sets the entity to unknown so every later step counts +// as a conflict. +func (w *walker) walkPath(steps []ast.XPathStep, cur string) { + for i, st := range steps { + next := "" + switch e := st.Expr.(type) { + case *ast.IdentifierExpr: + if i == len(steps)-1 { + // The terminal bare step is an attribute of the current entity. + w.note(e.Name, cur) + } else if e.Name == w.attr { + // A bare non-terminal step is not something this package models + // (Mendix spells the intermediate entity out). Only complain when + // it is the name we are renaming. + w.unresolved++ + } + case *ast.QualifiedNameExpr: + qn := e.QualifiedName.String() + if t, ok := w.model.AssociationTarget(qn, cur); ok { + next = t + } else if w.model.IsEntity(qn) { + next = qn + } + } + if st.Predicate != nil { + // A predicate constrains what the step reached, so it is evaluated + // against that entity — not the one the step started from. + w.walk(st.Predicate, next) + } + cur = next + } +} + +// replaceIdentifier replaces whole-token occurrences of old with new, returning +// the result and how many it replaced. +// +// Single-quoted string literals are skipped: `[Name = 'Name']` has exactly one +// identifier in it. A run that is part of a qualified name (touching a dot on +// either side) is not a bare step and is skipped too. +func replaceIdentifier(s, old, new string) (string, int) { + var b strings.Builder + b.Grow(len(s)) + count := 0 + + inString := false + for i := 0; i < len(s); { + c := s[i] + if c == '\'' { + inString = !inString + b.WriteByte(c) + i++ + continue + } + if inString || !isIdentStart(c) { + b.WriteByte(c) + i++ + continue + } + j := i + for j < len(s) && isIdentChar(s[j]) { + j++ + } + run := s[i:j] + if run == old && !touchesDot(s, i, j) { + b.WriteString(new) + count++ + } else { + b.WriteString(run) + } + i = j + } + return b.String(), count +} + +// mentionsIdentifier reports whether s contains name as a bare token outside a +// string literal. +func mentionsIdentifier(s, name string) bool { + _, n := replaceIdentifier(s, name, name) + return n > 0 +} + +// touchesDot reports whether the run s[i:j] is glued to a dot on either side, +// which makes it part of a qualified name rather than a bare step. +func touchesDot(s string, i, j int) bool { + if i > 0 && s[i-1] == '.' { + return true + } + if j < len(s) && s[j] == '.' { + return true + } + return false +} + +func isIdentStart(c byte) bool { + return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') +} + +func isIdentChar(c byte) bool { + return isIdentStart(c) || (c >= '0' && c <= '9') +} diff --git a/mdl/xpathrefs/rewrite_test.go b/mdl/xpathrefs/rewrite_test.go new file mode 100644 index 000000000..d7bd2b17b --- /dev/null +++ b/mdl/xpathrefs/rewrite_test.go @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: Apache-2.0 + +package xpathrefs + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/visitor" +) + +// testModel is a small hand-built domain: Person and Order, joined by +// Sales.Order_Person (FROM Order, TO Person). +type testModel struct{} + +func (testModel) IsEntity(qn string) bool { + switch qn { + case "Sales.Person", "Sales.Order", "Sales.Contact": + return true + } + return false +} + +func (testModel) AssociationTarget(qn, from string) (string, bool) { + // Order_Person: FROM Sales.Order, TO Sales.Person. Traversable either way. + if qn != "Sales.Order_Person" { + return "", false + } + switch from { + case "Sales.Order": + return "Sales.Person", true + case "Sales.Person": + return "Sales.Order", true + } + return "", false +} + +func rw(t *testing.T, constraint, target string) (string, bool) { + t.Helper() + return rewriteConstraint(constraint, target, "Sales.Person", "FirstName", "GivenName", testModel{}) +} + +func TestRewriteTopLevelAttribute(t *testing.T) { + got, ok := rw(t, "[FirstName = 'Ada']", "Sales.Person") + if !ok { + t.Fatal("the constraint was reported as not understood") + } + if want := "[GivenName = 'Ada']"; got != want { + t.Errorf("got %q, want %q", got, want) + } +} + +// TestRewritePreservesEverythingElse pins that the edit is a token replacement, +// not a re-render. Re-rendering would normalise the spacing and operator casing +// of a constraint the rename has no business touching, and any parser/renderer +// disagreement would corrupt it. +func TestRewritePreservesEverythingElse(t *testing.T) { + in := "[ FirstName='Ada' and LastName = 'Lovelace' ]" + got, ok := rw(t, in, "Sales.Person") + if !ok { + t.Fatal("the constraint was reported as not understood") + } + want := "[ GivenName='Ada' and LastName = 'Lovelace' ]" + if got != want { + t.Errorf("got %q, want %q", got, want) + } +} + +// TestRewriteSkipsStringLiterals pins that the name inside quotes is data, not a +// reference — the single most likely way a naive scan corrupts a project. +func TestRewriteSkipsStringLiterals(t *testing.T) { + got, ok := rw(t, "[FirstName = 'FirstName']", "Sales.Person") + if !ok { + t.Fatal("the constraint was reported as not understood") + } + if want := "[GivenName = 'FirstName']"; got != want { + t.Errorf("got %q, want %q", got, want) + } +} + +// TestRewriteSiblingGroups pins that Mendix's concatenated groups are each +// handled — the shape that silently dropped every group but the first in #772. +func TestRewriteSiblingGroups(t *testing.T) { + got, ok := rw(t, "[FirstName = 'Ada'][LastName = 'L'][FirstName != 'Bob']", "Sales.Person") + if !ok { + t.Fatal("the constraint was reported as not understood") + } + want := "[GivenName = 'Ada'][LastName = 'L'][GivenName != 'Bob']" + if got != want { + t.Errorf("got %q, want %q", got, want) + } +} + +// TestRewriteAcrossAssociationHop pins the case the qualified-name scanner can +// never reach: the constraint belongs to Order, and only the hop to Person makes +// the bare step ours. +func TestRewriteAcrossAssociationHop(t *testing.T) { + got, ok := rw(t, "[Sales.Order_Person/Sales.Person/FirstName = 'Ada']", "Sales.Order") + if !ok { + t.Fatal("the constraint was reported as not understood") + } + want := "[Sales.Order_Person/Sales.Person/GivenName = 'Ada']" + if got != want { + t.Errorf("got %q, want %q", got, want) + } +} + +// TestRewriteLeavesOtherEntitysAttributeAlone is the correctness case that makes +// this package worth having: Order has a FirstName too, and it must not move. +func TestRewriteLeavesOtherEntitysAttributeAlone(t *testing.T) { + got, ok := rw(t, "[FirstName = 'Ada']", "Sales.Order") + if !ok { + t.Error("an unambiguous constraint on another entity was reported as not understood") + } + if want := "[FirstName = 'Ada']"; got != want { + t.Errorf("another entity's attribute was renamed: got %q, want %q", got, want) + } +} + +// TestRewriteRefusesMixedMeanings pins the refusal. In one group the bare name +// means Order's attribute and Person's attribute; a token-level edit cannot tell +// them apart, so nothing is written and the caller is told. +func TestRewriteRefusesMixedMeanings(t *testing.T) { + in := "[FirstName = 'Ada' and Sales.Order_Person/Sales.Person/FirstName = 'Ada']" + got, ok := rw(t, in, "Sales.Order") + if ok { + t.Error("a group where the bare name means two different attributes was accepted") + } + if got != in { + t.Errorf("the group was rewritten anyway: got %q", got) + } +} + +// TestRewriteRefusesUnknownAssociation pins that an unresolvable hop blocks the +// rewrite rather than being assumed to land on the renamed entity. +func TestRewriteRefusesUnknownAssociation(t *testing.T) { + in := "[Sales.Mystery/FirstName = 'Ada']" + got, ok := rw(t, in, "Sales.Person") + if ok { + t.Error("a path through an unresolvable association was accepted") + } + if got != in { + t.Errorf("the group was rewritten anyway: got %q", got) + } +} + +// TestRewriteRefusesUnparseableGroup pins that a group mxcli cannot read is +// reported, not passed over in silence — silence is what makes a half-rename +// look complete. +func TestRewriteRefusesUnparseableGroup(t *testing.T) { + in := "[[FirstName]]" + got, ok := rw(t, in, "Sales.Person") + if ok { + t.Error("an unparseable group that names the attribute was accepted") + } + if got != in { + t.Errorf("the group was rewritten anyway: got %q", got) + } +} + +// TestRewriteIgnoresUnrelatedUnparseableGroup pins the other side of that: a +// group mxcli cannot read but which never mentions the attribute is not the +// rename's problem and must not be reported. +func TestRewriteIgnoresUnrelatedUnparseableGroup(t *testing.T) { + in := "[[Nonsense]]" + got, ok := rw(t, in, "Sales.Person") + if !ok { + t.Error("an unparseable group unrelated to the attribute was reported") + } + if got != in { + t.Errorf("got %q, want it untouched", got) + } +} + +// TestRewriteCountInvariantCatchesADroppedRegion is the test for the safety +// check that makes a lenient parser usable as a gate. +// +// visitor.ParseXPathConstraint runs with ANTLR's error listeners removed, so it +// recovers and can return a tree that omits part of its input — here everything +// after the first comparison. The dropped text holds a second occurrence of the +// name that the resolution therefore never saw, and a lexical edit would rewrite +// it anyway. The counts disagree, so nothing is written. +func TestRewriteCountInvariantCatchesADroppedRegion(t *testing.T) { + in := "[LastName = 'L' FirstName]" + if _, ok := visitor.ParseXPathConstraint(in); !ok { + t.Skip("the parser now rejects this shape; the invariant is exercised elsewhere") + } + got, ok := rw(t, in, "Sales.Person") + if ok { + t.Error("a group whose parse dropped an occurrence of the name was accepted") + } + if got != in { + t.Errorf("the group was rewritten anyway: got %q", got) + } +} + +// TestRewriteQualifiedNameIsNotABareStep pins that the three-part qualified form +// is the string scanner's job, not this one's — and that this package does not +// double-rewrite what the scanner already handled. +func TestRewriteQualifiedNameIsNotABareStep(t *testing.T) { + in := "[Sales.Person.FirstName = 'Ada']" + got, _ := rw(t, in, "Sales.Person") + if got != in { + t.Errorf("a qualified name was treated as a bare step: got %q", got) + } +} + +func TestRewriteNoMentionIsUntouched(t *testing.T) { + in := "[LastName = 'Lovelace']" + got, ok := rw(t, in, "Sales.Person") + if !ok { + t.Error("a constraint that never names the attribute was reported") + } + if got != in { + t.Errorf("got %q, want it untouched", got) + } +} + +// TestRewriteNonBracketConstraintIsReported pins that a stored value this +// package cannot even split into groups is reported when it names the attribute. +func TestRewriteNonBracketConstraintIsReported(t *testing.T) { + if _, ok := rw(t, "FirstName = 'Ada'", "Sales.Person"); ok { + t.Error("a constraint with no bracket group was silently accepted") + } + if _, ok := rw(t, "LastName = 'L'", "Sales.Person"); !ok { + t.Error("a constraint with no bracket group and no mention was reported") + } +} + +func TestReplaceIdentifier(t *testing.T) { + tests := []struct { + in string + want string + n int + }{ + {"[A = 'x']", "[B = 'x']", 1}, + {"[A = 'A']", "[B = 'A']", 1}, + {"[Mod.A = 'x']", "[Mod.A = 'x']", 0}, + {"[A.Sub = 'x']", "[A.Sub = 'x']", 0}, + {"[AA = 'x']", "[AA = 'x']", 0}, + {"[A_1 = 'x']", "[A_1 = 'x']", 0}, + {"[A = A]", "[B = B]", 2}, + {"[contains(A, 'x')]", "[contains(B, 'x')]", 1}, + } + for _, tc := range tests { + got, n := replaceIdentifier(tc.in, "A", "B") + if got != tc.want || n != tc.n { + t.Errorf("replaceIdentifier(%q) = (%q, %d), want (%q, %d)", tc.in, got, n, tc.want, tc.n) + } + } +} diff --git a/mdl/xpathrefs/scan.go b/mdl/xpathrefs/scan.go new file mode 100644 index 000000000..f2e3a48bb --- /dev/null +++ b/mdl/xpathrefs/scan.go @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: Apache-2.0 + +package xpathrefs + +import ( + "fmt" + + "go.mongodb.org/mongo-driver/bson" + + "github.com/mendixlabs/mxcli/mdl/types" +) + +// Store is the raw-unit surface the rewrite needs. Both engines' backends +// already provide both methods (backend.RawUnitBackend), so this needs no new +// backend interface method — and writes still go through the writer's single +// choke point, so ADR-0008 elision and identity preservation apply unchanged. +type Store interface { + ListRawUnitsByType(typePrefix string) ([]*types.RawUnit, error) + UpdateRawUnit(unitID string, contents []byte) error +} + +// Occurrence is one stored XPath constraint that named the attribute. +type Occurrence struct { + UnitID string // the document's unit ID + UnitType string // e.g. "Microflows$Microflow" + Document string // the document's Name, when it has one + TargetEntity string // the entity the constraint is evaluated against ("" if unknown) + Constraint string // the constraint as stored, before any rewrite +} + +// Result reports what the rewrite did and, just as importantly, what it refused +// to do. +type Result struct { + // Rewritten lists the constraints that changed. + Rewritten []Occurrence + // Skipped lists constraints that mention the attribute but could not be + // shown to mean it — an unreadable predicate group, a path rooted in a + // variable, an unresolvable association, or one bare name meaning two + // different entities' attributes. These are reported, never guessed at. + Skipped []Occurrence + // Units is how many documents were written. + Units int +} + +// Total returns how many constraints were rewritten. +func (r Result) Total() int { return len(r.Rewritten) } + +// RenameAttribute rewrites every stored XPath constraint that names +// entityQN.oldAttr so it names newAttr instead. +// +// domainModelUnitID and entityName identify the renamed entity's own domain +// model unit, which is how an access rule's constraint is attributed to its +// entity: an access rule holds no entity reference of its own, it simply lives +// inside one. Passing the unit ID keeps this package independent of the +// container hierarchy. +func RenameAttribute(s Store, m Model, entityQN, domainModelUnitID, entityName, oldAttr, newAttr string) (Result, error) { + var res Result + if oldAttr == "" || oldAttr == newAttr { + return res, nil + } + + units, err := s.ListRawUnitsByType("") + if err != nil { + return res, fmt.Errorf("listing units: %w", err) + } + + for _, u := range units { + if len(u.Contents) == 0 { + continue + } + var doc bson.D + if err := bson.Unmarshal(u.Contents, &doc); err != nil { + continue + } + + sc := &scanner{ + model: m, entityQN: entityQN, oldAttr: oldAttr, newAttr: newAttr, + unitID: string(u.ID), unitType: u.Type, document: docName(doc), + ownEntity: entityName, + } + sc.inOwnDomainModel = string(u.ID) == domainModelUnitID + sc.walkDoc(doc, "") + + res.Rewritten = append(res.Rewritten, sc.rewritten...) + res.Skipped = append(res.Skipped, sc.skipped...) + + if !sc.changed { + continue + } + contents, err := bson.Marshal(doc) + if err != nil { + return res, fmt.Errorf("re-encoding %s: %w", u.ID, err) + } + if err := s.UpdateRawUnit(string(u.ID), contents); err != nil { + return res, fmt.Errorf("writing %s: %w", u.ID, err) + } + res.Units++ + } + + return res, nil +} + +// scanner walks one unit's BSON, rewriting constraint values in place. +type scanner struct { + model Model + entityQN string + oldAttr string + newAttr string + + unitID string + unitType string + document string + + // ownEntity is the renamed entity's simple name, and inOwnDomainModel says + // whether this unit is the domain model holding it. Together they attribute + // an access rule's constraint to the entity that contains it. + ownEntity string + inOwnDomainModel bool + + changed bool + rewritten []Occurrence + skipped []Occurrence +} + +// Mendix spells the key two ways: the microflow retrieve source stores +// "XpathConstraint", everything else "XPathConstraint". +var constraintKeys = []string{"XPathConstraint", "XpathConstraint"} + +// walkDoc visits one BSON document. enclosing is the qualified name of the +// entity this node sits inside, or "" — it is what gives an access rule its +// target, since the rule itself names no entity. +func (s *scanner) walkDoc(doc bson.D, enclosing string) { + if s.inOwnDomainModel && isEntityNode(doc, s.ownEntity) { + enclosing = s.entityQN + } + + for i := range doc { + for _, key := range constraintKeys { + if doc[i].Key != key { + continue + } + raw, ok := doc[i].Value.(string) + if !ok || raw == "" { + continue + } + if v, changed := s.rewriteAt(raw, targetEntityOf(doc, enclosing)); changed { + doc[i].Value = v + s.changed = true + } + } + s.walkValue(doc[i].Value, enclosing) + } +} + +// rewriteAt applies the rewrite to one constraint and records the outcome. +func (s *scanner) rewriteAt(raw, target string) (string, bool) { + occ := Occurrence{ + UnitID: s.unitID, UnitType: s.unitType, Document: s.document, + TargetEntity: target, Constraint: raw, + } + if target == "" { + // Nothing anchors this constraint, so no bare step in it can be + // resolved. Report it only if it spells the name at all. + if mentionsIdentifier(raw, s.oldAttr) { + s.skipped = append(s.skipped, occ) + } + return raw, false + } + + out, understood := rewriteConstraint(raw, target, s.entityQN, s.oldAttr, s.newAttr, s.model) + if !understood { + s.skipped = append(s.skipped, occ) + } + if out == raw { + return raw, false + } + s.rewritten = append(s.rewritten, occ) + return out, true +} + +func (s *scanner) walkValue(v any, enclosing string) { + switch val := v.(type) { + case bson.D: + s.walkDoc(val, enclosing) + case bson.A: + for _, item := range val { + s.walkValue(item, enclosing) + } + case []any: + for _, item := range val { + s.walkValue(item, enclosing) + } + } +} + +// targetEntityOf reads the entity a constraint is evaluated against from the +// node that carries it, falling back to the entity the node sits inside. +// +// - "Entity" (a qualified name string) — Microflows$DatabaseRetrieveSource +// - "EntityRef".QualifiedName — the page/widget XPath data sources +// - enclosing — DomainModels$AccessRule, which carries no entity of its own +func targetEntityOf(doc bson.D, enclosing string) string { + for _, e := range doc { + switch e.Key { + case "Entity": + if s, ok := e.Value.(string); ok && s != "" { + return s + } + case "EntityRef": + if ref, ok := e.Value.(bson.D); ok { + for _, r := range ref { + if r.Key == "QualifiedName" { + if s, ok := r.Value.(string); ok && s != "" { + return s + } + } + } + } + } + } + return enclosing +} + +// entityNodeTypes are the $Type values a domain model entity node is stored +// under. +// +// "DomainModels$EntityImpl" is what is actually on disk — measured on a Mendix +// 11.13.0 project, both engines write it. "DomainModels$Entity" is the name the +// metamodel uses for an entity *reference target* (see the ref registries in +// modelsdk/gen/domainmodels), and checking for it alone silently matched nothing: +// every access rule was then reported as unanchored rather than rewritten, which +// looks like a cautious refusal instead of a bug. +var entityNodeTypes = map[string]bool{ + "DomainModels$EntityImpl": true, + "DomainModels$Entity": true, +} + +// isEntityNode reports whether doc is the domain model entity called name. +func isEntityNode(doc bson.D, name string) bool { + if name == "" { + return false + } + typ, _ := lookup(doc, "$Type").(string) + if !entityNodeTypes[typ] { + return false + } + n, _ := lookup(doc, "Name").(string) + return n == name +} + +func docName(doc bson.D) string { + n, _ := lookup(doc, "Name").(string) + return n +} + +func lookup(doc bson.D, key string) any { + for _, e := range doc { + if e.Key == key { + return e.Value + } + } + return nil +} diff --git a/mdl/xpathrefs/scan_test.go b/mdl/xpathrefs/scan_test.go new file mode 100644 index 000000000..e612c95c4 --- /dev/null +++ b/mdl/xpathrefs/scan_test.go @@ -0,0 +1,318 @@ +// SPDX-License-Identifier: Apache-2.0 + +package xpathrefs + +import ( + "testing" + + "go.mongodb.org/mongo-driver/bson" + + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/model" +) + +// memStore is an in-memory Store holding whole BSON documents. +type memStore struct { + units []*types.RawUnit + written map[string][]byte +} + +func (s *memStore) ListRawUnitsByType(string) ([]*types.RawUnit, error) { return s.units, nil } + +func (s *memStore) UpdateRawUnit(unitID string, contents []byte) error { + if s.written == nil { + s.written = map[string][]byte{} + } + s.written[unitID] = contents + return nil +} + +// constraintsIn returns every XPath constraint stored anywhere in the document. +func constraintsIn(t *testing.T, raw []byte) []string { + t.Helper() + var doc bson.D + if err := bson.Unmarshal(raw, &doc); err != nil { + t.Fatalf("unmarshalling: %v", err) + } + var out []string + var walk func(v any) + walk = func(v any) { + switch val := v.(type) { + case bson.D: + for _, e := range val { + if e.Key == "XPathConstraint" || e.Key == "XpathConstraint" { + if s, ok := e.Value.(string); ok { + out = append(out, s) + } + } + walk(e.Value) + } + case bson.A: + for _, item := range val { + walk(item) + } + } + } + walk(doc) + return out +} + +func unit(id, typ string, doc bson.D) *types.RawUnit { + b, _ := bson.Marshal(doc) + return &types.RawUnit{ID: model.ID(id), Type: typ, Contents: b} +} + +// retrieveUnit builds a microflow holding one database retrieve. The shape +// matches what mxcli writes: the constraint's target entity is the sibling +// "Entity" field, and the key is spelled "XpathConstraint" here and +// "XPathConstraint" everywhere else. +func retrieveUnit(id, name, entity, constraint string) *types.RawUnit { + return unit(id, "Microflows$Microflow", bson.D{ + {Key: "Name", Value: name}, + {Key: "ObjectCollection", Value: bson.D{ + {Key: "Objects", Value: bson.A{ + bson.D{ + {Key: "$Type", Value: "Microflows$ActionActivity"}, + {Key: "Action", Value: bson.D{ + {Key: "$Type", Value: "Microflows$RetrieveAction"}, + {Key: "RetrieveSource", Value: bson.D{ + {Key: "$Type", Value: "Microflows$DatabaseRetrieveSource"}, + {Key: "Entity", Value: entity}, + {Key: "XpathConstraint", Value: constraint}, + }}, + }}, + }, + }}, + }}, + }) +} + +// widgetUnit builds a page holding one XPath data source, whose target entity is +// a nested EntityRef rather than a plain string. +func widgetUnit(id, name, entity, constraint string) *types.RawUnit { + return unit(id, "Forms$Page", bson.D{ + {Key: "Name", Value: name}, + {Key: "Widgets", Value: bson.A{ + bson.D{ + {Key: "$Type", Value: "CustomWidgets$CustomWidgetXPathSource"}, + {Key: "EntityRef", Value: bson.D{{Key: "QualifiedName", Value: entity}}}, + {Key: "XPathConstraint", Value: constraint}, + }, + }}, + }) +} + +// domainModelUnit builds a domain model whose entities carry access rules. An +// access rule names no entity of its own — it is attributed to the entity it +// sits inside, which is the case a sibling-field lookup cannot handle. +func domainModelUnit(id string, entities ...bson.D) *types.RawUnit { + return unit(id, "DomainModels$DomainModel", bson.D{ + {Key: "Entities", Value: bson.A(func() []any { + out := make([]any, 0, len(entities)) + for _, e := range entities { + out = append(out, e) + } + return out + }())}, + }) +} + +// entityWithRule builds a domain model entity node under the $Type actually +// found on disk (measured on Mendix 11.13.0), not the metamodel's name for an +// entity reference target. +func entityWithRule(name, constraint string) bson.D { + return bson.D{ + {Key: "$Type", Value: "DomainModels$EntityImpl"}, + {Key: "Name", Value: name}, + {Key: "AccessRules", Value: bson.A{ + bson.D{ + {Key: "$Type", Value: "DomainModels$AccessRule"}, + {Key: "XPathConstraint", Value: constraint}, + }, + }}, + } +} + +func renameIn(s *memStore) (Result, error) { + return RenameAttribute(s, testModel{}, "Sales.Person", "dm-1", "Person", "FirstName", "GivenName") +} + +func TestScanRewritesRetrieveConstraint(t *testing.T) { + s := &memStore{units: []*types.RawUnit{ + retrieveUnit("mf-1", "ACT_Find", "Sales.Person", "[FirstName = 'Ada']"), + }} + + res, err := renameIn(s) + if err != nil { + t.Fatal(err) + } + if res.Total() != 1 || res.Units != 1 { + t.Fatalf("got %d rewrites in %d units, want 1 in 1 (skipped: %+v)", res.Total(), res.Units, res.Skipped) + } + if got := constraintsIn(t, s.written["mf-1"]); len(got) != 1 || got[0] != "[GivenName = 'Ada']" { + t.Errorf("stored constraint is %q", got) + } +} + +func TestScanRewritesWidgetDatasourceConstraint(t *testing.T) { + s := &memStore{units: []*types.RawUnit{ + widgetUnit("pg-1", "Overview", "Sales.Person", "[FirstName != '']"), + }} + + res, err := renameIn(s) + if err != nil { + t.Fatal(err) + } + if res.Total() != 1 { + t.Fatalf("got %d rewrites, want 1 (skipped: %+v)", res.Total(), res.Skipped) + } + if got := constraintsIn(t, s.written["pg-1"]); len(got) != 1 || got[0] != "[GivenName != '']" { + t.Errorf("stored constraint is %q", got) + } +} + +// TestScanRewritesAccessRuleInOwnDomainModel pins the case with no entity +// reference to read: the rule is attributed to the entity that contains it. +func TestScanRewritesAccessRuleInOwnDomainModel(t *testing.T) { + s := &memStore{units: []*types.RawUnit{ + domainModelUnit("dm-1", + entityWithRule("Person", "[FirstName != '']"), + entityWithRule("Order", "[FirstName != '']"), + ), + }} + + res, err := renameIn(s) + if err != nil { + t.Fatal(err) + } + if res.Total() != 1 { + t.Fatalf("got %d rewrites, want exactly the Person rule (skipped: %+v)", res.Total(), res.Skipped) + } + got := constraintsIn(t, s.written["dm-1"]) + want := []string{"[GivenName != '']", "[FirstName != '']"} + if len(got) != 2 || got[0] != want[0] || got[1] != want[1] { + t.Errorf("got %q, want %q — Order's rule must not move", got, want) + } +} + +// TestScanIgnoresAnotherModulesDomainModel pins that the entity name alone is not +// enough: a Person in another module has its own FirstName. +func TestScanIgnoresAnotherModulesDomainModel(t *testing.T) { + s := &memStore{units: []*types.RawUnit{ + domainModelUnit("dm-2", entityWithRule("Person", "[FirstName != '']")), + }} + + res, err := renameIn(s) + if err != nil { + t.Fatal(err) + } + if res.Total() != 0 { + t.Errorf("a same-named entity in another module was rewritten: %+v", res.Rewritten) + } + if len(s.written) != 0 { + t.Errorf("a unit was written with nothing to change: %v", s.written) + } +} + +// TestScanDoesNotWriteUnchangedUnits pins that a document with constraints that +// do not concern this rename is not rewritten at all. Writing it would churn the +// project's version control for no reason. +func TestScanDoesNotWriteUnchangedUnits(t *testing.T) { + s := &memStore{units: []*types.RawUnit{ + retrieveUnit("mf-1", "ACT_Find", "Sales.Person", "[LastName = 'L']"), + retrieveUnit("mf-2", "ACT_Other", "Sales.Order", "[FirstName = 'Ada']"), + }} + + res, err := renameIn(s) + if err != nil { + t.Fatal(err) + } + if res.Total() != 0 || res.Units != 0 { + t.Errorf("got %d rewrites in %d units, want none", res.Total(), res.Units) + } + if len(s.written) != 0 { + t.Errorf("units were written with nothing to change: %v", s.written) + } +} + +// TestScanReportsUnresolvableConstraint pins that a constraint naming the +// attribute which the walk cannot resolve is reported and left alone — the +// project is no worse than before, and the user is told where to look. +func TestScanReportsUnresolvableConstraint(t *testing.T) { + const in = "[Sales.Mystery/FirstName = 'Ada']" + s := &memStore{units: []*types.RawUnit{ + retrieveUnit("mf-1", "ACT_Find", "Sales.Person", in), + }} + + res, err := renameIn(s) + if err != nil { + t.Fatal(err) + } + if res.Total() != 0 { + t.Errorf("an unresolvable constraint was rewritten: %+v", res.Rewritten) + } + if len(res.Skipped) != 1 { + t.Fatalf("got %d skipped, want 1: %+v", len(res.Skipped), res.Skipped) + } + occ := res.Skipped[0] + if occ.Document != "ACT_Find" || occ.Constraint != in || occ.TargetEntity != "Sales.Person" { + t.Errorf("the report does not locate the constraint: %+v", occ) + } + if len(s.written) != 0 { + t.Errorf("the unit was written anyway: %v", s.written) + } +} + +// TestScanReportsConstraintWithNoAnchor pins that a constraint whose target +// entity cannot be determined at all is reported rather than assumed to be ours. +func TestScanReportsConstraintWithNoAnchor(t *testing.T) { + s := &memStore{units: []*types.RawUnit{ + unit("w-1", "Forms$Page", bson.D{ + {Key: "Name", Value: "Orphan"}, + {Key: "XPathConstraint", Value: "[FirstName = 'Ada']"}, + }), + }} + + res, err := renameIn(s) + if err != nil { + t.Fatal(err) + } + if res.Total() != 0 { + t.Errorf("an unanchored constraint was rewritten: %+v", res.Rewritten) + } + if len(res.Skipped) != 1 { + t.Errorf("got %d skipped, want 1: %+v", len(res.Skipped), res.Skipped) + } +} + +// TestScanPreservesTheRestOfTheDocument pins that only the constraint changes: +// the round trip through bson.D must not reorder or drop anything else. +func TestScanPreservesTheRestOfTheDocument(t *testing.T) { + s := &memStore{units: []*types.RawUnit{ + retrieveUnit("mf-1", "ACT_Find", "Sales.Person", "[FirstName = 'Ada']"), + }} + before := s.units[0].Contents + + if _, err := renameIn(s); err != nil { + t.Fatal(err) + } + + var got, want bson.D + if err := bson.Unmarshal(s.written["mf-1"], &got); err != nil { + t.Fatal(err) + } + if err := bson.Unmarshal(before, &want); err != nil { + t.Fatal(err) + } + if len(got) != len(want) { + t.Fatalf("the document gained or lost top-level fields: %d vs %d", len(got), len(want)) + } + for i := range want { + if got[i].Key != want[i].Key { + t.Errorf("field %d is %q, want %q — the field order changed", i, got[i].Key, want[i].Key) + } + } + if n, _ := got[0].Value.(string); n != "ACT_Find" { + t.Errorf("the document name changed to %q", n) + } +} From 71d3f958967479bb9006d028e23481153f4225a1 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 14:23:21 +0000 Subject: [PATCH 08/35] feat(settings): expose the remaining model settings, and stop leaking them across versions Adds FirstDayOfWeek, DecimalScale, DefaultTimeZoneCode, UseDatabaseForeignKeyConstraints, UseOQLVersion2 and SslCertificateAlgorithm to ALTER SETTINGS MODEL. The two enum-typed ones are canonicalised against generated/metamodel and a non-member is refused (MDL-SET03) rather than written through: an enum value the metamodel cannot resolve is what makes Studio Pro throw 'Sequence contains no matching element'. Fixes a defect found while checking whether that was safe. A blank Mendix 9.24 project stores 12 model settings; a blank 11.13 stores 17. Both engines' overlays wrote every known property unconditionally, so on a 9.24 project 'alter settings model BcryptCost = 11' also introduced DecimalScale = 0 and UseDatabaseForeignKeyConstraints = false -- properties that version does not store, at values that are wrong anyway (11.13 defaults them to 8 and true). One unrelated statement silently changed two other settings, and a property the type may not define is what Studio Pro refuses to open while mxbuild loads it happily. The overlay is now presence-gated and shared by both engines rather than duplicated (as Configurations already was). The executor refuses an ALTER naming a property the stored document does not carry, so the gate never becomes a silent no-op, and DESCRIBE emits only stored properties so its output replays on both versions. UseSystemContextForBackgroundTasks is read and preserved but deliberately not writable: mx check on 11.13 rejects a project holding true with CE9436 'not supported anymore', so its only legal value is its default. Verified on real 9.24 and 11.13 projects: the 9.24 key set is unchanged through a write, 11.13 takes all six with mx check at 0 errors, both describe outputs replay, re-running is byte-identical with MXCLI_ALWAYS_WRITE=1 as the control, and the regression test reproduces the leak when the guard is stubbed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01KTdvGwVewkdgNQGDCxpQPZ --- .claude/skills/mendix/project-settings.md | 29 +- cmd/mxcli/syntax/features_misc.go | 15 +- .../2026-08-17-banking-app-findings-triage.md | 52 ++- .../14-project-settings-examples.mdl | 25 ++ mdl/backend/modelsdk/settings_read.go | 4 + mdl/backend/modelsdk/settings_write.go | 15 +- mdl/executor/cmd_settings.go | 191 +++++++++- mdl/executor/cmd_settings_model_test.go | 353 ++++++++++++++++++ mdl/executor/cmd_settings_optimistic_test.go | 139 ------- mdl/executor/cmd_settings_validation_test.go | 8 + mdl/executor/validate_settings.go | 26 +- mdl/settingsoverlay/model_settings_test.go | 161 ++++++++ mdl/settingsoverlay/settingsoverlay.go | 80 ++++ model/types.go | 4 + sdk/mpr/parser_settings.go | 11 +- sdk/mpr/writer_settings.go | 20 +- 16 files changed, 935 insertions(+), 198 deletions(-) create mode 100644 mdl/executor/cmd_settings_model_test.go delete mode 100644 mdl/executor/cmd_settings_optimistic_test.go create mode 100644 mdl/settingsoverlay/model_settings_test.go diff --git a/.claude/skills/mendix/project-settings.md b/.claude/skills/mendix/project-settings.md index e5b2c14ca..765836316 100644 --- a/.claude/skills/mendix/project-settings.md +++ b/.claude/skills/mendix/project-settings.md @@ -34,9 +34,27 @@ alter settings model JavaVersion = 'Java21'; -- or '21'; see note below alter settings model RoundingMode = 'HalfUp'; alter settings model AllowUserMultipleSessions = true; alter settings model ScheduledEventTimeZoneCode = 'Etc/UTC'; +alter settings model DefaultTimeZoneCode = 'Europe/Amsterdam'; +alter settings model FirstDayOfWeek = 'Monday'; -- Default, Monday..Sunday +alter settings model DecimalScale = 8; alter settings model EnableDataStorageOptimisticLocking = true; +alter settings model UseDatabaseForeignKeyConstraints = true; +alter settings model UseOQLVersion2 = true; +alter settings model SslCertificateAlgorithm = 'PKIX'; -- PKIX or SunX509 ``` +**Not every project stores every setting.** Mendix adds model settings over time — +a blank 9.24 project stores 12 of them, a blank 11.13 project stores 17. mxcli +refuses an `alter` naming one the project does not store rather than introducing +it, because Studio Pro refuses to open a model carrying a property its type does +not define (and `mx check` does *not* catch that). `describe settings` emits only +what the project actually stores, so its output always replays. + +**`UseSystemContextForBackgroundTasks` is read but not writable.** Mendix withdrew +it: `mx check` on 11.13 rejects a project holding `true` with +**CE9436** *"The project setting 'System context tasks' is not supported anymore."* +mxcli preserves whatever the project stores and offers no way to change it. + **Optimistic locking** is App Settings → Runtime → *Optimistic locking* in Studio Pro. With it on, the runtime tracks an `MxObjectVersion` on every persistable entity and a commit whose version no longer matches the database throws @@ -83,9 +101,14 @@ alter settings configuration 'Default' ``` `HttpPortNumber`, `ServerPortNumber`, `BcryptCost`, `DefaultTaskParallelism` and -`WorkflowEngineParallelism` are Integer-typed, and `AllowUserMultipleSessions` and -`EnableDataStorageOptimisticLocking` are -Boolean. An unparseable value is rejected by `mxcli check` (MDL-SET01 / MDL-SET02) +`WorkflowEngineParallelism` and `DecimalScale` are Integer-typed; +`AllowUserMultipleSessions`, `EnableDataStorageOptimisticLocking`, +`UseDatabaseForeignKeyConstraints` and `UseOQLVersion2` are Boolean; `FirstDayOfWeek` +and `SslCertificateAlgorithm` are enumerations, matched case-insensitively and +stored in Mendix's own spelling (MDL-SET03 rejects a non-member rather than writing +it through — an unresolvable enum value is what makes Studio Pro throw +"Sequence contains no matching element"). +An unparseable value is rejected by `mxcli check` (MDL-SET01 / MDL-SET02) and by the write itself — it is no longer silently ignored. Quoted numbers are fine: `HttpPortNumber = '8080'` and `HttpPortNumber = 8080` are equivalent. diff --git a/cmd/mxcli/syntax/features_misc.go b/cmd/mxcli/syntax/features_misc.go index d558e74a3..5ae377706 100644 --- a/cmd/mxcli/syntax/features_misc.go +++ b/cmd/mxcli/syntax/features_misc.go @@ -210,11 +210,20 @@ CREATE CONFIGURATION 'Production' -- MODEL accepts these keys (an unknown one is refused and lists them): -- AfterStartupMicroflow, BeforeShutdownMicroflow, HealthCheckMicroflow, -- HashAlgorithm, BcryptCost, JavaVersion, RoundingMode, --- AllowUserMultipleSessions, ScheduledEventTimeZoneCode, --- EnableDataStorageOptimisticLocking +-- AllowUserMultipleSessions, ScheduledEventTimeZoneCode, DefaultTimeZoneCode, +-- FirstDayOfWeek, DecimalScale, EnableDataStorageOptimisticLocking, +-- UseDatabaseForeignKeyConstraints, UseOQLVersion2, SslCertificateAlgorithm +-- -- EnableDataStorageOptimisticLocking is Studio Pro's App Settings → Runtime → -- "Optimistic locking": it makes a stale commit fail instead of silently --- overwriting, which is the fix for a read-then-write race in a microflow.`, +-- overwriting, which is the fix for a read-then-write race in a microflow. +-- FirstDayOfWeek is Default or Monday..Sunday; SslCertificateAlgorithm is +-- PKIX or SunX509. Both are matched case-insensitively. +-- +-- Which of these a project stores depends on its Mendix version (a blank 9.24 +-- project has 12, a blank 11.13 has 17). An ALTER naming one the project does +-- not store is refused rather than introducing it, and DESCRIBE SETTINGS emits +-- only the stored ones so its output always replays.`, SeeAlso: []string{"settings.show"}, }) diff --git a/docs/12-bug-reports/2026-08-17-banking-app-findings-triage.md b/docs/12-bug-reports/2026-08-17-banking-app-findings-triage.md index 46b562960..159657dd7 100644 --- a/docs/12-bug-reports/2026-08-17-banking-app-findings-triage.md +++ b/docs/12-bug-reports/2026-08-17-banking-app-findings-triage.md @@ -370,12 +370,56 @@ The unknown-key error now lists the accepted keys. The report tried and got a bare `unknown model setting` each time; the first of those now answers with the real name. +**The sibling settings are now exposed too**, in the same shape: `FirstDayOfWeek`, +`DecimalScale`, `DefaultTimeZoneCode`, `UseDatabaseForeignKeyConstraints`, +`UseOQLVersion2` and `SslCertificateAlgorithm`. The two enumeration-typed ones are +canonicalised against `generated/metamodel` and a non-member is refused +(MDL-SET03) rather than written through — an enum value the metamodel cannot +resolve is what makes Studio Pro throw "Sequence contains no matching element". + +`UseSystemContextForBackgroundTasks` is deliberately **read-only**. `mx check` on +11.13 rejects a project holding `true` with **CE9436** *"The project setting +'System context tasks' is not supported anymore"* — its only legal value on a +current version is its default, so an ALTER could only ever produce a project that +does not build. mxcli preserves whatever is stored and offers no way to change it. +Found by running `mx check` after the change; nothing else would have caught it. + Still on the "needs Studio Pro" list and **not** addressed here: strict mode (SEC005), which the report also flags and which weakens XPath constraint -enforcement (CVE-2023-23835). Worth checking whether it is equally reachable — -`Settings$ModelSettings` on 11.13 also carries `UseSystemContextForBackgroundTasks`, -`UseOQLVersion2`, `UseDatabaseForeignKeyConstraints`, `DecimalScale`, -`FirstDayOfWeek` and `SslCertificateAlgorithm`, none of which MDL exposes either. +enforcement (CVE-2023-23835). + +### 10b. NEW, found while doing the above: writes leaked properties across versions + +Not in the report. Exposing the sibling settings meant checking whether the +overlay was safe, and it was not. + +**A blank Mendix 9.24 project stores 12 model settings; a blank 11.13 stores 17.** +Both engines' overlays wrote all of their known properties unconditionally, so on +a 9.24 project: + +``` +alter settings model BcryptCost = 11 +``` + +…also introduced `DecimalScale = 0` and `UseDatabaseForeignKeyConstraints = false`, +neither of which that version stores. Both were wrong as values too (11.13 defaults +them to 8 and `true`), so one unrelated one-line statement silently changed two +other settings. And a property the type may not define is the #759 shape: Studio Pro +resolves every stored property against the type's property list and throws +"Sequence contains no matching element", while mxbuild loads it happily — so the +build is not a safety net. + +Fixed by making the overlay **presence-gated** (`settingsoverlay.SetModelSettings`, +now shared by both engines rather than duplicated, as `Configurations` already was), +with the executor refusing an ALTER naming a property the stored document does not +carry so the gate never becomes a silent no-op, and `DESCRIBE SETTINGS` emitting +only stored properties so its output still replays on both versions. + +Measured: a 9.24 project keeps exactly its 12 keys through a write; an 11.13 +project takes all six new settings with `mx check` at 0 errors; both projects' +describe output replays; re-running is byte-identical with `MXCLI_ALWAYS_WRITE=1` +as the control. The regression test reproduces the exact symptom when the guard is +stubbed out. Same list, not investigated here: strict mode (SEC005), which the report also flags as Studio-Pro-only and which weakens XPath constraint enforcement diff --git a/mdl-examples/doctype-tests/14-project-settings-examples.mdl b/mdl-examples/doctype-tests/14-project-settings-examples.mdl index a72f18b10..cea4e711d 100644 --- a/mdl-examples/doctype-tests/14-project-settings-examples.mdl +++ b/mdl-examples/doctype-tests/14-project-settings-examples.mdl @@ -70,6 +70,31 @@ alter settings model */ alter settings model EnableDataStorageOptimisticLocking = true; +/** + * Example 1.5: The remaining runtime model settings + * + * FirstDayOfWeek (Default, Monday..Sunday) and SslCertificateAlgorithm (PKIX, + * SunX509) are enumerations -- matched case-insensitively and stored in Mendix's + * own spelling. A non-member is refused rather than written through, because an + * enum value the metamodel cannot resolve is what makes Studio Pro throw + * "Sequence contains no matching element". + * + * Which of these a project stores depends on its Mendix version: a blank 9.24 + * project stores 12 model settings, a blank 11.13 stores 17. An alter naming one + * this project does not store is refused rather than introducing it. + */ +alter settings model + FirstDayOfWeek = 'Monday', + DecimalScale = 8, + UseDatabaseForeignKeyConstraints = true, + UseOQLVersion2 = true, + SslCertificateAlgorithm = 'PKIX', + DefaultTimeZoneCode = 'Europe/Amsterdam'; + +-- Note: UseSystemContextForBackgroundTasks is read and preserved but NOT +-- writable -- Mendix withdrew it, and mx check on 11.13 rejects a project +-- holding true with CE9436 "not supported anymore". + -- MARK: Configuration Settings -- ============================================================================ diff --git a/mdl/backend/modelsdk/settings_read.go b/mdl/backend/modelsdk/settings_read.go index 47942bd69..3a8f18efa 100644 --- a/mdl/backend/modelsdk/settings_read.go +++ b/mdl/backend/modelsdk/settings_read.go @@ -165,10 +165,14 @@ func modelSettingsFromGen(p *genSet.RuntimeSettings) *model.ModelSettings { JavaVersion: javaVersionOf(p), RoundingMode: p.RoundingMode(), ScheduledEventTimeZoneCode: p.ScheduledEventTimeZoneCode(), + DefaultTimeZoneCode: p.DefaultTimeZoneCode(), FirstDayOfWeek: p.FirstDayOfWeek(), DecimalScale: int(p.DecimalScale()), EnableDataStorageOptimisticLocking: p.EnableDataStorageOptimisticLocking(), UseDatabaseForeignKeyConstraints: p.UseDatabaseForeignKeyConstraints(), + UseOQLVersion2: p.UseOQLVersion2(), + UseSystemContextForBackgroundTasks: p.UseSystemContextForBackgroundTasks(), + SslCertificateAlgorithm: p.SslCertificateAlgorithm(), } setBase(&ms.BaseElement, p, "Settings$ModelSettings") return ms diff --git a/mdl/backend/modelsdk/settings_write.go b/mdl/backend/modelsdk/settings_write.go index f5a0a1b8d..5ca535ab8 100644 --- a/mdl/backend/modelsdk/settings_write.go +++ b/mdl/backend/modelsdk/settings_write.go @@ -86,18 +86,5 @@ func (b *Backend) UpdateProjectSettings(ps *model.ProjectSettings) error { } func overlayModelSettings(ms *model.ModelSettings, raw map[string]any) map[string]any { - raw["AfterStartupMicroflow"] = ms.AfterStartupMicroflow - raw["BeforeShutdownMicroflow"] = ms.BeforeShutdownMicroflow - raw["HealthCheckMicroflow"] = ms.HealthCheckMicroflow - raw["AllowUserMultipleSessions"] = ms.AllowUserMultipleSessions - raw["HashAlgorithm"] = ms.HashAlgorithm - raw["BcryptCost"] = settingsoverlay.SafeInt64(ms.BcryptCost) - settingsoverlay.SetJavaVersion(raw, ms.JavaVersion) - raw["RoundingMode"] = ms.RoundingMode - raw["ScheduledEventTimeZoneCode"] = ms.ScheduledEventTimeZoneCode - raw["FirstDayOfWeek"] = ms.FirstDayOfWeek - raw["DecimalScale"] = settingsoverlay.SafeInt64(ms.DecimalScale) - raw["EnableDataStorageOptimisticLocking"] = ms.EnableDataStorageOptimisticLocking - raw["UseDatabaseForeignKeyConstraints"] = ms.UseDatabaseForeignKeyConstraints - return raw + return settingsoverlay.SetModelSettings(ms, raw) } diff --git a/mdl/executor/cmd_settings.go b/mdl/executor/cmd_settings.go index 1a5afb9ae..42fffe640 100644 --- a/mdl/executor/cmd_settings.go +++ b/mdl/executor/cmd_settings.go @@ -11,6 +11,7 @@ import ( "github.com/mendixlabs/mxcli/generated/metamodel" "github.com/mendixlabs/mxcli/mdl/ast" mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" + "github.com/mendixlabs/mxcli/mdl/settingsoverlay" "github.com/mendixlabs/mxcli/model" ) @@ -116,25 +117,50 @@ func describeSettings(ctx *ExecContext, configName string) error { // Model settings if ps.Model != nil { ms := ps.Model + // Emit only the properties this project's Mendix version actually stores, + // so the output replays: the executor refuses an ALTER naming one the + // document does not carry (see refuseIfNotStored), and the versions differ + // by five properties between 9.24 and 11.13. + stored := storedModelSettings(ps) var parts []string - if ms.AfterStartupMicroflow != "" { - parts = append(parts, fmt.Sprintf(" AfterStartupMicroflow = '%s'", ms.AfterStartupMicroflow)) - } - if ms.BeforeShutdownMicroflow != "" { - parts = append(parts, fmt.Sprintf(" BeforeShutdownMicroflow = '%s'", ms.BeforeShutdownMicroflow)) - } - if ms.HealthCheckMicroflow != "" { - parts = append(parts, fmt.Sprintf(" HealthCheckMicroflow = '%s'", ms.HealthCheckMicroflow)) + // add emits a property only if this project's Mendix version stores it, so + // the output replays: the executor refuses an ALTER naming a property the + // document does not carry (refuseIfNotStored), and the set differs by five + // properties between a blank 9.24 and a blank 11.13 project. + add := func(key, format string, args ...any) { + if stored != nil && !settingsoverlay.Has(stored, key) { + return + } + parts = append(parts, " "+fmt.Sprintf(format, args...)) } - parts = append(parts, fmt.Sprintf(" HashAlgorithm = '%s'", ms.HashAlgorithm)) - parts = append(parts, fmt.Sprintf(" BcryptCost = %d", ms.BcryptCost)) - parts = append(parts, fmt.Sprintf(" JavaVersion = '%s'", ms.JavaVersion)) - parts = append(parts, fmt.Sprintf(" RoundingMode = '%s'", ms.RoundingMode)) - parts = append(parts, fmt.Sprintf(" AllowUserMultipleSessions = %t", ms.AllowUserMultipleSessions)) - parts = append(parts, fmt.Sprintf(" EnableDataStorageOptimisticLocking = %t", ms.EnableDataStorageOptimisticLocking)) - if ms.ScheduledEventTimeZoneCode != "" { - parts = append(parts, fmt.Sprintf(" ScheduledEventTimeZoneCode = '%s'", ms.ScheduledEventTimeZoneCode)) + // addIfSet additionally skips a property holding no value, so a blank + // project's describe output is not a wall of empty strings. + addIfSet := func(key, format string, v string) { + if v != "" { + add(key, format, v) + } } + + addIfSet("AfterStartupMicroflow", "AfterStartupMicroflow = '%s'", ms.AfterStartupMicroflow) + addIfSet("BeforeShutdownMicroflow", "BeforeShutdownMicroflow = '%s'", ms.BeforeShutdownMicroflow) + addIfSet("HealthCheckMicroflow", "HealthCheckMicroflow = '%s'", ms.HealthCheckMicroflow) + add("HashAlgorithm", "HashAlgorithm = '%s'", ms.HashAlgorithm) + add("BcryptCost", "BcryptCost = %d", ms.BcryptCost) + // JavaVersion is stored under either JavaVersion or JavaMajorVersion; emit it + // when the document carries whichever spelling, in mxcli's single input name. + if settingsoverlay.JavaVersionKey(stored) != "" || stored == nil { + parts = append(parts, fmt.Sprintf(" JavaVersion = '%s'", ms.JavaVersion)) + } + add("RoundingMode", "RoundingMode = '%s'", ms.RoundingMode) + add("AllowUserMultipleSessions", "AllowUserMultipleSessions = %t", ms.AllowUserMultipleSessions) + add("EnableDataStorageOptimisticLocking", "EnableDataStorageOptimisticLocking = %t", ms.EnableDataStorageOptimisticLocking) + add("UseDatabaseForeignKeyConstraints", "UseDatabaseForeignKeyConstraints = %t", ms.UseDatabaseForeignKeyConstraints) + add("UseOQLVersion2", "UseOQLVersion2 = %t", ms.UseOQLVersion2) + add("DecimalScale", "DecimalScale = %d", ms.DecimalScale) + addIfSet("FirstDayOfWeek", "FirstDayOfWeek = '%s'", ms.FirstDayOfWeek) + addIfSet("SslCertificateAlgorithm", "SslCertificateAlgorithm = '%s'", ms.SslCertificateAlgorithm) + addIfSet("ScheduledEventTimeZoneCode", "ScheduledEventTimeZoneCode = '%s'", ms.ScheduledEventTimeZoneCode) + addIfSet("DefaultTimeZoneCode", "DefaultTimeZoneCode = '%s'", ms.DefaultTimeZoneCode) fmt.Fprintf(ctx.Output, "alter settings model\n%s;\n\n", strings.Join(parts, ",\n")) } @@ -179,6 +205,13 @@ func describeSettings(ctx *ExecContext, configName string) error { // // Hand-maintained alongside the switch: add a case, add it here. // TestModelSettingKeys_AllAccepted fails when a listed key is not accepted. +// +// Deliberately absent: UseSystemContextForBackgroundTasks. The property is still +// stored, and mxcli reads and round-trips it, but Mendix has withdrawn it — +// `mx check` on 11.13 rejects a project holding true with +// CE9436 "The project setting 'System context tasks' is not supported anymore." +// Its only legal value on a current version is its default, so an ALTER for it +// could only ever produce a project that does not build. var modelSettingKeys = []string{ "AfterStartupMicroflow", "BeforeShutdownMicroflow", @@ -189,7 +222,94 @@ var modelSettingKeys = []string{ "RoundingMode", "AllowUserMultipleSessions", "ScheduledEventTimeZoneCode", + "DefaultTimeZoneCode", + "FirstDayOfWeek", + "DecimalScale", "EnableDataStorageOptimisticLocking", + "UseDatabaseForeignKeyConstraints", + "UseOQLVersion2", + "SslCertificateAlgorithm", +} + +// firstDayOfWeekValues and sslCertificateAlgorithmValues are the members of the +// corresponding Mendix enumerations as Studio Pro spells them in BSON. Passing an +// unrecognised string through is the mendixlabs/mxcli#759 shape: the metamodel +// cannot resolve it and Studio Pro throws "Sequence contains no matching element", +// while mxbuild loads the project fine. +// +// generated/metamodel is a snapshot of Mendix 11.6, so a member added in a later +// version would be rejected here. That is the safer direction — the error names +// what is accepted, so an over-strict refusal is obvious rather than silent. +var firstDayOfWeekValues = []string{ + string(metamodel.SettingsFirstDayOfWeekDefault), + string(metamodel.SettingsFirstDayOfWeekMonday), + string(metamodel.SettingsFirstDayOfWeekTuesday), + string(metamodel.SettingsFirstDayOfWeekWednesday), + string(metamodel.SettingsFirstDayOfWeekThursday), + string(metamodel.SettingsFirstDayOfWeekFriday), + string(metamodel.SettingsFirstDayOfWeekSaturday), + string(metamodel.SettingsFirstDayOfWeekSunday), +} + +var sslCertificateAlgorithmValues = []string{ + string(metamodel.SettingsSslCertificateAlgorithmPKIX), + string(metamodel.SettingsSslCertificateAlgorithmSunX509), +} + +// settingsEnumValues maps "
/" to the enumeration members accepted for +// it, for the properties validated as settingsKindEnum. +var settingsEnumValues = map[string][]string{ + "model/FirstDayOfWeek": firstDayOfWeekValues, + "model/SslCertificateAlgorithm": sslCertificateAlgorithmValues, +} + +// settingsEnum canonicalises an enumeration-typed settings value to the member +// Mendix stores, matching case-insensitively. Anything unrecognised is rejected +// rather than written through — same contract as settingsDatabaseType. +func settingsEnum(key, valStr string, members []string) (string, error) { + want := strings.TrimSpace(valStr) + for _, m := range members { + if strings.EqualFold(m, want) { + return m, nil + } + } + return "", mdlerrors.NewValidationf("%s must be one of %s, got %q", + key, strings.Join(members, ", "), valStr) +} + +// storedModelSettings returns the raw Settings$ModelSettings part, so the handler +// can tell "this project does not store that property" from "that property is +// false". Returns nil when the part is not in RawParts. +func storedModelSettings(ps *model.ProjectSettings) map[string]any { + for _, part := range ps.RawParts { + if t, _ := part["$Type"].(string); t == "Settings$ModelSettings" { + return part + } + } + return nil +} + +// refuseIfNotStored rejects an ALTER naming a property this project's Mendix +// version does not store. The overlay is presence-gated (it will not introduce a +// property the type may not define), so without this check the statement would +// report success and change nothing — the silent no-op shape of #805. +// +// JavaVersion is exempt: it is stored under either JavaVersion or +// JavaMajorVersion, and settingsoverlay.JavaVersionKey resolves which. +func refuseIfNotStored(raw map[string]any, key string) error { + if raw == nil || key == "JavaVersion" { + return nil + } + if settingsoverlay.Has(raw, key) { + return nil + } + return mdlerrors.NewUnsupported(fmt.Sprintf( + "this project does not store the model setting %s\n"+ + " Mendix adds model settings over time — a blank 9.24 project stores 12 of them, "+ + "a blank 11.13 project stores 17.\n"+ + " mxcli will not introduce a property the project's Mendix version may not define: "+ + "Studio Pro refuses to open a model carrying one, and mxbuild does not catch it.\n"+ + " hint: set it in Studio Pro once, or upgrade the project", key)) } // alterSettings modifies project settings based on ALTER SETTINGS statement. @@ -209,8 +329,15 @@ func alterSettings(ctx *ExecContext, stmt *ast.AlterSettingsStmt) error { if ps.Model == nil { return mdlerrors.NewNotFound("settings section", "model") } + storedModel := storedModelSettings(ps) for key, val := range stmt.Properties { valStr := settingsValueToString(val) + // The overlay will not introduce a property the stored document does + // not carry, so refuse here rather than reporting a success that + // changes nothing. + if err := refuseIfNotStored(storedModel, key); err != nil { + return err + } switch key { case "AfterStartupMicroflow": ps.Model.AfterStartupMicroflow = valStr @@ -252,6 +379,38 @@ func alterSettings(ctx *ExecContext, stmt *ast.AlterSettingsStmt) error { return err } ps.Model.EnableDataStorageOptimisticLocking = v + case "DefaultTimeZoneCode": + ps.Model.DefaultTimeZoneCode = valStr + case "FirstDayOfWeek": + v, err := settingsEnum(key, valStr, firstDayOfWeekValues) + if err != nil { + return err + } + ps.Model.FirstDayOfWeek = v + case "SslCertificateAlgorithm": + v, err := settingsEnum(key, valStr, sslCertificateAlgorithmValues) + if err != nil { + return err + } + ps.Model.SslCertificateAlgorithm = v + case "DecimalScale": + v, err := settingsInt(key, valStr) + if err != nil { + return err + } + ps.Model.DecimalScale = v + case "UseDatabaseForeignKeyConstraints": + v, err := settingsBool(key, valStr) + if err != nil { + return err + } + ps.Model.UseDatabaseForeignKeyConstraints = v + case "UseOQLVersion2": + v, err := settingsBool(key, valStr) + if err != nil { + return err + } + ps.Model.UseOQLVersion2 = v default: return mdlerrors.NewUnsupported(fmt.Sprintf( "unknown model setting: %s\n valid keys: %s", diff --git a/mdl/executor/cmd_settings_model_test.go b/mdl/executor/cmd_settings_model_test.go new file mode 100644 index 000000000..e11829206 --- /dev/null +++ b/mdl/executor/cmd_settings_model_test.go @@ -0,0 +1,353 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/model" +) + +// TestAlterSettingsModel_OptimisticLocking covers the gap reported by the +// mxcli-banking app: Mendix ships optimistic locking as an app setting (App +// Settings → Runtime), which is the mitigation for a read-then-write balance +// race in a transfer microflow, and ALTER SETTINGS MODEL refused every spelling +// of it with "unknown model setting". The property was already read and written +// by both engines — only the executor's assignment switch was missing. +// +// The stored key is EnableDataStorageOptimisticLocking, verified against a +// Studio-Pro-created project on both Mendix 9.24 and 11.13. Unlike JavaVersion +// (renamed to JavaMajorVersion between 11.6 and 11.12) this one did not move, so +// a single spelling is correct for every version mxcli supports. +func TestAlterSettingsModel_OptimisticLocking(t *testing.T) { + tests := []struct { + name string + given string + want bool + }{ + {name: "enable", given: "true", want: true}, + {name: "disable", given: "false", want: false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var written *model.ProjectSettings + ctx, _ := newMockCtx(t, withBackend(captureSettingsBackend(&written))) + + err := alterSettings(ctx, &ast.AlterSettingsStmt{ + Section: "model", + Properties: map[string]any{"EnableDataStorageOptimisticLocking": tc.given}, + }) + if err != nil { + t.Fatalf("alterSettings: %v", err) + } + if written == nil { + t.Fatal("no settings written") + } + if got := written.Model.EnableDataStorageOptimisticLocking; got != tc.want { + t.Errorf("EnableDataStorageOptimisticLocking = %v, want %v", got, tc.want) + } + }) + } +} + +// A non-boolean value must be refused rather than silently skipped — the same +// silent-no-op shape as mendixlabs/mxcli#805, where a bad Integer value skipped +// the assignment while the handler reported success. +func TestAlterSettingsModel_OptimisticLockingRejectsNonBool(t *testing.T) { + wrote := false + ctx, _ := newMockCtx(t, withBackend(settingsBackend(&wrote))) + + err := alterSettings(ctx, &ast.AlterSettingsStmt{ + Section: "model", + Properties: map[string]any{"EnableDataStorageOptimisticLocking": "yes-please"}, + }) + if err == nil { + t.Fatal("expected an error for a non-boolean value, got nil") + } + if !strings.Contains(err.Error(), "EnableDataStorageOptimisticLocking") { + t.Errorf("error should name the offending key, got: %v", err) + } + if wrote { + t.Error("a rejected statement must not write") + } +} + +// DESCRIBE must emit the property so a describe → edit → exec round trip can +// carry it. Without this the setting is writable but invisible, which is how a +// user discovers it does not exist. +func TestDescribeSettings_EmitsOptimisticLocking(t *testing.T) { + var written *model.ProjectSettings + ctx, buf := newMockCtx(t, withBackend(captureSettingsBackend(&written))) + + if err := describeSettings(ctx, ""); err != nil { + t.Fatalf("describeSettings: %v", err) + } + assertContainsStr(t, buf.String(), "EnableDataStorageOptimisticLocking") +} + +// TestModelSettingKeys_AllAccepted is the drift guard for the hand-maintained +// modelSettingKeys list: a name that appears in the error message's "valid keys" +// but is not actually assigned by the switch would send the next reader down the +// same dead end the bare message did. +func TestModelSettingKeys_AllAccepted(t *testing.T) { + for _, key := range modelSettingKeys { + t.Run(key, func(t *testing.T) { + wrote := false + ctx, _ := newMockCtx(t, withBackend(settingsBackend(&wrote))) + + // A value valid for every kind in the list: parses as a string, and + // the typed keys are covered by TestTypedSettingsKeys_MatchExecutor. + val := "1" + if kind, ok := typedSettingsKeys["model"][key]; ok && kind == settingsKindBool { + val = "true" + } + err := alterSettings(ctx, &ast.AlterSettingsStmt{ + Section: "model", + Properties: map[string]any{key: val}, + }) + if err != nil && strings.Contains(err.Error(), "unknown model setting") { + t.Errorf("%s is listed as valid but the executor rejects it: %v", key, err) + } + }) + } +} + +// The unknown-setting error must list what IS accepted. The banking app tried +// OptimisticLocking, UseOptimisticLocking and EnableOptimisticLocking in turn +// and got a bare "unknown model setting" each time, with nothing pointing at the +// real name. +func TestAlterSettingsModel_UnknownKeyListsValidKeys(t *testing.T) { + wrote := false + ctx, _ := newMockCtx(t, withBackend(settingsBackend(&wrote))) + + err := alterSettings(ctx, &ast.AlterSettingsStmt{ + Section: "model", + Properties: map[string]any{"OptimisticLocking": "true"}, + }) + if err == nil { + t.Fatal("expected an error for an unknown key, got nil") + } + if !strings.Contains(err.Error(), "EnableDataStorageOptimisticLocking") { + t.Errorf("error should point at the real key name, got: %v", err) + } + if wrote { + t.Error("a rejected statement must not write") + } +} + +// modelSettingsBackend serves a project whose Settings$ModelSettings part carries +// exactly the given keys, so the presence gate can be exercised. The real backends +// populate RawParts from the stored document; settingsBackend does not carry a +// model part at all, which is the "cannot tell" case the gate treats permissively. +func modelSettingsBackend(out **model.ProjectSettings, keys ...string) *mock.MockBackend { + part := map[string]any{"$Type": "Settings$ModelSettings"} + for _, k := range keys { + part[k] = "" + } + return &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + GetProjectSettingsFunc: func() (*model.ProjectSettings, error) { + return &model.ProjectSettings{ + Model: &model.ModelSettings{HashAlgorithm: "BCrypt", BcryptCost: 10}, + RawParts: []map[string]any{part}, + }, nil + }, + UpdateProjectSettingsFunc: func(ps *model.ProjectSettings) error { + *out = ps + return nil + }, + } +} + +// TestAlterSettingsModel_NewSettings covers the six properties added alongside +// optimistic locking. Each was already parsed and written by both engines and +// simply had no executor case, so MDL could not reach it. +func TestAlterSettingsModel_NewSettings(t *testing.T) { + tests := []struct { + key string + given string + check func(*model.ModelSettings) any + want any + }{ + {"DecimalScale", "4", func(m *model.ModelSettings) any { return m.DecimalScale }, 4}, + {"UseDatabaseForeignKeyConstraints", "false", func(m *model.ModelSettings) any { return m.UseDatabaseForeignKeyConstraints }, false}, + {"UseOQLVersion2", "false", func(m *model.ModelSettings) any { return m.UseOQLVersion2 }, false}, + {"DefaultTimeZoneCode", "Etc/UTC", func(m *model.ModelSettings) any { return m.DefaultTimeZoneCode }, "Etc/UTC"}, + {"FirstDayOfWeek", "Monday", func(m *model.ModelSettings) any { return m.FirstDayOfWeek }, "Monday"}, + {"SslCertificateAlgorithm", "SunX509", func(m *model.ModelSettings) any { return m.SslCertificateAlgorithm }, "SunX509"}, + } + + for _, tc := range tests { + t.Run(tc.key, func(t *testing.T) { + var written *model.ProjectSettings + ctx, _ := newMockCtx(t, withBackend(modelSettingsBackend(&written, tc.key))) + + if err := alterSettings(ctx, &ast.AlterSettingsStmt{ + Section: "model", + Properties: map[string]any{tc.key: tc.given}, + }); err != nil { + t.Fatalf("alterSettings: %v", err) + } + if written == nil { + t.Fatal("no settings written") + } + if got := tc.check(written.Model); got != tc.want { + t.Errorf("%s = %#v, want %#v", tc.key, got, tc.want) + } + }) + } +} + +// The two enumeration-typed settings must be canonicalised to the member Mendix +// stores and refuse anything else, rather than writing a string through that the +// metamodel cannot resolve (the mendixlabs/mxcli#759 shape). +func TestAlterSettingsModel_EnumsCanonicaliseAndRefuse(t *testing.T) { + tests := []struct { + key string + given string + want string // "" = must be refused + }{ + {"FirstDayOfWeek", "monday", "Monday"}, + {"FirstDayOfWeek", "Monday", "Monday"}, + {"FirstDayOfWeek", "Caturday", ""}, + {"SslCertificateAlgorithm", "pkix", "PKIX"}, + {"SslCertificateAlgorithm", "SunX509", "SunX509"}, + {"SslCertificateAlgorithm", "MD5", ""}, + } + + for _, tc := range tests { + t.Run(tc.key+"/"+tc.given, func(t *testing.T) { + var written *model.ProjectSettings + ctx, _ := newMockCtx(t, withBackend(modelSettingsBackend(&written, tc.key))) + + err := alterSettings(ctx, &ast.AlterSettingsStmt{ + Section: "model", + Properties: map[string]any{tc.key: tc.given}, + }) + + if tc.want == "" { + if err == nil { + t.Fatalf("accepted %q, which is not a member of the enumeration", tc.given) + } + if written != nil { + t.Error("a rejected statement must not write") + } + return + } + if err != nil { + t.Fatalf("alterSettings: %v", err) + } + got := written.Model.FirstDayOfWeek + if tc.key == "SslCertificateAlgorithm" { + got = written.Model.SslCertificateAlgorithm + } + if got != tc.want { + t.Errorf("%s = %q, want %q (the enum member Mendix stores)", tc.key, got, tc.want) + } + }) + } +} + +// TestAlterSettingsModel_RefusesPropertyThisVersionDoesNotStore is the executor +// half of the presence gate. The overlay will not introduce a property the stored +// document does not carry, so without this refusal the statement would report +// success and change nothing. +func TestAlterSettingsModel_RefusesPropertyThisVersionDoesNotStore(t *testing.T) { + var written *model.ProjectSettings + // A Mendix 9.24-shaped document: no UseOQLVersion2. + ctx, _ := newMockCtx(t, withBackend(modelSettingsBackend(&written, "HashAlgorithm", "BcryptCost"))) + + err := alterSettings(ctx, &ast.AlterSettingsStmt{ + Section: "model", + Properties: map[string]any{"UseOQLVersion2": "true"}, + }) + if err == nil { + t.Fatal("expected a refusal for a property this project does not store, got nil") + } + if !strings.Contains(err.Error(), "UseOQLVersion2") { + t.Errorf("error should name the property, got: %v", err) + } + if written != nil { + t.Error("a refused statement must not write") + } +} + +// The gate must not block a property the document does carry. +func TestAlterSettingsModel_AllowsStoredProperty(t *testing.T) { + var written *model.ProjectSettings + ctx, _ := newMockCtx(t, withBackend(modelSettingsBackend(&written, "UseOQLVersion2"))) + + if err := alterSettings(ctx, &ast.AlterSettingsStmt{ + Section: "model", + Properties: map[string]any{"UseOQLVersion2": "false"}, + }); err != nil { + t.Fatalf("alterSettings: %v", err) + } + if written == nil { + t.Fatal("no settings written") + } +} + +// DESCRIBE must emit only what the project stores, or replaying its output hits +// the refusal above. +func TestDescribeSettings_OmitsPropertiesThisVersionDoesNotStore(t *testing.T) { + var written *model.ProjectSettings + ctx, buf := newMockCtx(t, withBackend(modelSettingsBackend(&written, + "HashAlgorithm", "BcryptCost", "EnableDataStorageOptimisticLocking"))) + + if err := describeSettings(ctx, ""); err != nil { + t.Fatalf("describeSettings: %v", err) + } + out := buf.String() + assertContainsStr(t, out, "EnableDataStorageOptimisticLocking") + for _, absent := range []string{"UseOQLVersion2", "DecimalScale", "SslCertificateAlgorithm"} { + if strings.Contains(out, absent) { + t.Errorf("described %s, which this project does not store — the output will not replay", absent) + } + } +} + +// TestAlterSettingsModel_WithdrawnSettingNotWritable: Mendix withdrew +// UseSystemContextForBackgroundTasks — `mx check` on 11.13 rejects a project +// holding true with CE9436 "not supported anymore" — so mxcli must not offer it +// for writing even though it still parses and round-trips the stored value. +func TestAlterSettingsModel_WithdrawnSettingNotWritable(t *testing.T) { + var written *model.ProjectSettings + ctx, _ := newMockCtx(t, withBackend(modelSettingsBackend(&written, "UseSystemContextForBackgroundTasks"))) + + err := alterSettings(ctx, &ast.AlterSettingsStmt{ + Section: "model", + Properties: map[string]any{"UseSystemContextForBackgroundTasks": "true"}, + }) + if err == nil { + t.Fatal("expected a refusal: setting this to true fails mx check with CE9436") + } + if written != nil { + t.Error("a refused statement must not write") + } +} + +// ...but it must still survive a write that touches other settings, rather than +// being dropped or reset. +func TestAlterSettingsModel_WithdrawnSettingSurvivesOtherWrites(t *testing.T) { + var written *model.ProjectSettings + ctx, _ := newMockCtx(t, withBackend(modelSettingsBackend(&written, + "BcryptCost", "UseSystemContextForBackgroundTasks"))) + + if err := alterSettings(ctx, &ast.AlterSettingsStmt{ + Section: "model", + Properties: map[string]any{"BcryptCost": "13"}, + }); err != nil { + t.Fatalf("alterSettings: %v", err) + } + if written == nil { + t.Fatal("no settings written") + } + if written.Model.BcryptCost != 13 { + t.Errorf("BcryptCost = %d, want 13", written.Model.BcryptCost) + } +} diff --git a/mdl/executor/cmd_settings_optimistic_test.go b/mdl/executor/cmd_settings_optimistic_test.go deleted file mode 100644 index 3869e4019..000000000 --- a/mdl/executor/cmd_settings_optimistic_test.go +++ /dev/null @@ -1,139 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package executor - -import ( - "strings" - "testing" - - "github.com/mendixlabs/mxcli/mdl/ast" - "github.com/mendixlabs/mxcli/model" -) - -// TestAlterSettingsModel_OptimisticLocking covers the gap reported by the -// mxcli-banking app: Mendix ships optimistic locking as an app setting (App -// Settings → Runtime), which is the mitigation for a read-then-write balance -// race in a transfer microflow, and ALTER SETTINGS MODEL refused every spelling -// of it with "unknown model setting". The property was already read and written -// by both engines — only the executor's assignment switch was missing. -// -// The stored key is EnableDataStorageOptimisticLocking, verified against a -// Studio-Pro-created project on both Mendix 9.24 and 11.13. Unlike JavaVersion -// (renamed to JavaMajorVersion between 11.6 and 11.12) this one did not move, so -// a single spelling is correct for every version mxcli supports. -func TestAlterSettingsModel_OptimisticLocking(t *testing.T) { - tests := []struct { - name string - given string - want bool - }{ - {name: "enable", given: "true", want: true}, - {name: "disable", given: "false", want: false}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - var written *model.ProjectSettings - ctx, _ := newMockCtx(t, withBackend(captureSettingsBackend(&written))) - - err := alterSettings(ctx, &ast.AlterSettingsStmt{ - Section: "model", - Properties: map[string]any{"EnableDataStorageOptimisticLocking": tc.given}, - }) - if err != nil { - t.Fatalf("alterSettings: %v", err) - } - if written == nil { - t.Fatal("no settings written") - } - if got := written.Model.EnableDataStorageOptimisticLocking; got != tc.want { - t.Errorf("EnableDataStorageOptimisticLocking = %v, want %v", got, tc.want) - } - }) - } -} - -// A non-boolean value must be refused rather than silently skipped — the same -// silent-no-op shape as mendixlabs/mxcli#805, where a bad Integer value skipped -// the assignment while the handler reported success. -func TestAlterSettingsModel_OptimisticLockingRejectsNonBool(t *testing.T) { - wrote := false - ctx, _ := newMockCtx(t, withBackend(settingsBackend(&wrote))) - - err := alterSettings(ctx, &ast.AlterSettingsStmt{ - Section: "model", - Properties: map[string]any{"EnableDataStorageOptimisticLocking": "yes-please"}, - }) - if err == nil { - t.Fatal("expected an error for a non-boolean value, got nil") - } - if !strings.Contains(err.Error(), "EnableDataStorageOptimisticLocking") { - t.Errorf("error should name the offending key, got: %v", err) - } - if wrote { - t.Error("a rejected statement must not write") - } -} - -// DESCRIBE must emit the property so a describe → edit → exec round trip can -// carry it. Without this the setting is writable but invisible, which is how a -// user discovers it does not exist. -func TestDescribeSettings_EmitsOptimisticLocking(t *testing.T) { - var written *model.ProjectSettings - ctx, buf := newMockCtx(t, withBackend(captureSettingsBackend(&written))) - - if err := describeSettings(ctx, ""); err != nil { - t.Fatalf("describeSettings: %v", err) - } - assertContainsStr(t, buf.String(), "EnableDataStorageOptimisticLocking") -} - -// TestModelSettingKeys_AllAccepted is the drift guard for the hand-maintained -// modelSettingKeys list: a name that appears in the error message's "valid keys" -// but is not actually assigned by the switch would send the next reader down the -// same dead end the bare message did. -func TestModelSettingKeys_AllAccepted(t *testing.T) { - for _, key := range modelSettingKeys { - t.Run(key, func(t *testing.T) { - wrote := false - ctx, _ := newMockCtx(t, withBackend(settingsBackend(&wrote))) - - // A value valid for every kind in the list: parses as a string, and - // the typed keys are covered by TestTypedSettingsKeys_MatchExecutor. - val := "1" - if kind, ok := typedSettingsKeys["model"][key]; ok && kind == settingsKindBool { - val = "true" - } - err := alterSettings(ctx, &ast.AlterSettingsStmt{ - Section: "model", - Properties: map[string]any{key: val}, - }) - if err != nil && strings.Contains(err.Error(), "unknown model setting") { - t.Errorf("%s is listed as valid but the executor rejects it: %v", key, err) - } - }) - } -} - -// The unknown-setting error must list what IS accepted. The banking app tried -// OptimisticLocking, UseOptimisticLocking and EnableOptimisticLocking in turn -// and got a bare "unknown model setting" each time, with nothing pointing at the -// real name. -func TestAlterSettingsModel_UnknownKeyListsValidKeys(t *testing.T) { - wrote := false - ctx, _ := newMockCtx(t, withBackend(settingsBackend(&wrote))) - - err := alterSettings(ctx, &ast.AlterSettingsStmt{ - Section: "model", - Properties: map[string]any{"OptimisticLocking": "true"}, - }) - if err == nil { - t.Fatal("expected an error for an unknown key, got nil") - } - if !strings.Contains(err.Error(), "EnableDataStorageOptimisticLocking") { - t.Errorf("error should point at the real key name, got: %v", err) - } - if wrote { - t.Error("a rejected statement must not write") - } -} diff --git a/mdl/executor/cmd_settings_validation_test.go b/mdl/executor/cmd_settings_validation_test.go index f65c61113..9ae473623 100644 --- a/mdl/executor/cmd_settings_validation_test.go +++ b/mdl/executor/cmd_settings_validation_test.go @@ -390,6 +390,14 @@ func TestTypedSettingsKeys_MatchExecutor(t *testing.T) { good = "true" case settingsKindDatabaseType: good = "PostgreSql" + case settingsKindEnum: + // The first member of the property's own enumeration; the + // table has no single valid value across enum-typed keys. + members := settingsEnumValues[section+"/"+key] + if len(members) == 0 { + t.Fatalf("%s/%s is settingsKindEnum but has no members in settingsEnumValues", section, key) + } + good = members[0] } if got := ValidateSettings(&ast.AlterSettingsStmt{ Section: section, diff --git a/mdl/executor/validate_settings.go b/mdl/executor/validate_settings.go index c5838ae8e..560f8b6b1 100644 --- a/mdl/executor/validate_settings.go +++ b/mdl/executor/validate_settings.go @@ -19,6 +19,9 @@ const ( settingsKindInt settingsValueKind = iota settingsKindBool settingsKindDatabaseType + // settingsKindEnum covers any other enumeration-typed property; its accepted + // members are looked up per section/key in settingsEnumValues. + settingsKindEnum ) // typedSettingsKeys maps a lower-cased ALTER SETTINGS section to the properties @@ -28,8 +31,13 @@ const ( var typedSettingsKeys = map[string]map[string]settingsValueKind{ "model": { "BcryptCost": settingsKindInt, + "DecimalScale": settingsKindInt, "AllowUserMultipleSessions": settingsKindBool, "EnableDataStorageOptimisticLocking": settingsKindBool, + "UseDatabaseForeignKeyConstraints": settingsKindBool, + "UseOQLVersion2": settingsKindBool, + "FirstDayOfWeek": settingsKindEnum, + "SslCertificateAlgorithm": settingsKindEnum, }, "workflows": { "DefaultTaskParallelism": settingsKindInt, @@ -50,18 +58,18 @@ func ValidateSettings(stmt *ast.AlterSettingsStmt) []linter.Violation { if !ok { return nil } - return validateTypedSettings(keys, stmt.Properties, + return validateTypedSettings(strings.ToLower(stmt.Section), keys, stmt.Properties, "alter settings "+strings.ToLower(stmt.Section)) } // ValidateCreateConfiguration reports the same for CREATE CONFIGURATION, which // accepts the configuration properties directly. func ValidateCreateConfiguration(stmt *ast.CreateConfigurationStmt) []linter.Violation { - return validateTypedSettings(typedSettingsKeys["configuration"], stmt.Properties, + return validateTypedSettings("configuration", typedSettingsKeys["configuration"], stmt.Properties, fmt.Sprintf("create configuration '%s'", stmt.Name)) } -func validateTypedSettings(keys map[string]settingsValueKind, props map[string]any, what string) []linter.Violation { +func validateTypedSettings(section string, keys map[string]settingsValueKind, props map[string]any, what string) []linter.Violation { if len(keys) == 0 || len(props) == 0 { return nil } @@ -113,6 +121,18 @@ func validateTypedSettings(keys map[string]settingsValueKind, props map[string]a Suggestion: fmt.Sprintf("Use one of: %s.", strings.Join(databaseTypes, ", ")), }) } + case settingsKindEnum: + members := settingsEnumValues[section+"/"+key] + if _, err := settingsEnum(key, valStr, members); err != nil { + out = append(out, linter.Violation{ + RuleID: "MDL-SET03", + Severity: linter.SeverityError, + Location: loc, + Message: fmt.Sprintf("%s: %s must be one of %s, got %q", + what, key, strings.Join(members, ", "), valStr), + Suggestion: fmt.Sprintf("Use one of: %s.", strings.Join(members, ", ")), + }) + } } } return out diff --git a/mdl/settingsoverlay/model_settings_test.go b/mdl/settingsoverlay/model_settings_test.go new file mode 100644 index 000000000..37b9da509 --- /dev/null +++ b/mdl/settingsoverlay/model_settings_test.go @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settingsoverlay + +import ( + "testing" + + "github.com/mendixlabs/mxcli/model" +) + +// mendix924Part is the Settings$ModelSettings a blank Mendix 9.24 project stores, +// key-for-key. It carries 12 properties; a blank 11.13 project carries 17. The +// five it lacks — DecimalScale, JavaMajorVersion, SslCertificateAlgorithm, +// UseDatabaseForeignKeyConstraints, UseOQLVersion2 — are the ones this test is +// about. +func mendix924Part() map[string]any { + return map[string]any{ + "$Type": "Settings$ModelSettings", + "AfterStartupMicroflow": "", + "AllowUserMultipleSessions": true, + "BcryptCost": int64(10), + "BeforeShutdownMicroflow": "", + "DefaultTimeZoneCode": "", + "EnableDataStorageOptimisticLocking": false, + "FirstDayOfWeek": "Default", + "HashAlgorithm": "BCrypt", + "HealthCheckMicroflow": "", + "RoundingMode": "HalfUp", + "ScheduledEventTimeZoneCode": "Etc/UTC", + "UseSystemContextForBackgroundTasks": false, + } +} + +// TestSetModelSettings_DoesNotIntroduceAbsentProperties is the regression test for +// the leak found while triaging the mxcli-banking findings: writing any model +// setting to a Mendix 9.24 project also wrote DecimalScale = 0 and +// UseDatabaseForeignKeyConstraints = false, neither of which that version stores. +// +// Both were wrong as values too — 11.13 defaults them to 8 and true — so one +// unrelated statement silently changed two other settings. And a property the +// type may not define is the mendixlabs/mxcli#759 shape: Studio Pro throws +// "Sequence contains no matching element" while mxbuild loads the model happily, +// so the build is not a safety net. +func TestSetModelSettings_DoesNotIntroduceAbsentProperties(t *testing.T) { + raw := mendix924Part() + before := len(raw) + + // A model carrying values for every property mxcli knows, as the reader would + // produce it: the absent ones hold their parse defaults. + ms := &model.ModelSettings{ + HashAlgorithm: "BCrypt", + BcryptCost: 11, + DecimalScale: 0, + UseDatabaseForeignKeyConstraints: true, + UseOQLVersion2: true, + SslCertificateAlgorithm: "", + JavaVersion: "Java21", + } + SetModelSettings(ms, raw) + + for _, key := range []string{ + "DecimalScale", + "UseDatabaseForeignKeyConstraints", + "UseOQLVersion2", + "SslCertificateAlgorithm", + "JavaVersion", + "JavaMajorVersion", + } { + if v, has := raw[key]; has { + t.Errorf("introduced %s = %#v into a document that did not carry it", key, v) + } + } + if len(raw) != before { + t.Errorf("property count %d -> %d; the overlay must not add or remove keys", before, len(raw)) + } +} + +// The gate must not become a blanket refusal to write: a property the document +// does carry is still updated. +func TestSetModelSettings_UpdatesPresentProperties(t *testing.T) { + raw := mendix924Part() + ms := &model.ModelSettings{ + HashAlgorithm: "SHA256", + BcryptCost: 13, + AllowUserMultipleSessions: false, + EnableDataStorageOptimisticLocking: true, + FirstDayOfWeek: "Monday", + UseSystemContextForBackgroundTasks: true, + } + SetModelSettings(ms, raw) + + for key, want := range map[string]any{ + "HashAlgorithm": "SHA256", + "BcryptCost": int64(13), + "AllowUserMultipleSessions": false, + "EnableDataStorageOptimisticLocking": true, + "FirstDayOfWeek": "Monday", + "UseSystemContextForBackgroundTasks": true, + } { + if raw[key] != want { + t.Errorf("%s = %#v, want %#v", key, raw[key], want) + } + } +} + +// An 11.13 document carries the whole set, so every property is writable there. +func TestSetModelSettings_WritesFullPropertySetWhenStored(t *testing.T) { + raw := mendix924Part() + raw["DecimalScale"] = int64(8) + raw["JavaMajorVersion"] = "21" + raw["SslCertificateAlgorithm"] = "PKIX" + raw["UseDatabaseForeignKeyConstraints"] = true + raw["UseOQLVersion2"] = true + + ms := &model.ModelSettings{ + DecimalScale: 4, + JavaVersion: "Java21", + SslCertificateAlgorithm: "SunX509", + UseDatabaseForeignKeyConstraints: false, + UseOQLVersion2: false, + } + SetModelSettings(ms, raw) + + for key, want := range map[string]any{ + "DecimalScale": int64(4), + "SslCertificateAlgorithm": "SunX509", + "UseDatabaseForeignKeyConstraints": false, + "UseOQLVersion2": false, + // Written back under the key the document uses, in that key's value format. + "JavaMajorVersion": "21", + } { + if raw[key] != want { + t.Errorf("%s = %#v, want %#v", key, raw[key], want) + } + } + if _, has := raw["JavaVersion"]; has { + t.Error("wrote the 11.6 JavaVersion spelling into a document using JavaMajorVersion") + } +} + +// ModelSettingsKeys is hand-maintained beside the overlay; a name in it that the +// overlay does not actually write would misdescribe what mxcli round-trips. +func TestModelSettingsKeys_AllWritten(t *testing.T) { + for _, key := range ModelSettingsKeys { + t.Run(key, func(t *testing.T) { + // Seed the document with the key so the presence gate lets it through. + raw := map[string]any{"$Type": "Settings$ModelSettings"} + stored := key + if key == "JavaVersion" { + stored = JavaMajorVersionKey + } + raw[stored] = "sentinel" + + SetModelSettings(&model.ModelSettings{JavaVersion: "Java21"}, raw) + + if raw[stored] == "sentinel" { + t.Errorf("%s is listed in ModelSettingsKeys but the overlay never writes it", key) + } + }) + } +} diff --git a/mdl/settingsoverlay/settingsoverlay.go b/mdl/settingsoverlay/settingsoverlay.go index 615eafe5f..cf79272d9 100644 --- a/mdl/settingsoverlay/settingsoverlay.go +++ b/mdl/settingsoverlay/settingsoverlay.go @@ -210,6 +210,86 @@ func JavaVersionValue(key, v string) string { return "Java" + major } +// ModelSettingsKeys names every Settings$ModelSettings property mxcli parses and +// writes back, in the order DESCRIBE emits them. Which of these a document +// actually carries depends on the Mendix version — a blank 9.24 project stores 12 +// of them, a blank 11.13 project stores 17 — so the overlay is presence-gated (see +// SetModelSettings) and callers must not assume any particular key exists. +var ModelSettingsKeys = []string{ + "AfterStartupMicroflow", + "BeforeShutdownMicroflow", + "HealthCheckMicroflow", + "AllowUserMultipleSessions", + "HashAlgorithm", + "BcryptCost", + "JavaVersion", // stored as JavaVersion or JavaMajorVersion; see JavaVersionKey + "RoundingMode", + "ScheduledEventTimeZoneCode", + "DefaultTimeZoneCode", + "FirstDayOfWeek", + "DecimalScale", + "EnableDataStorageOptimisticLocking", + "UseDatabaseForeignKeyConstraints", + "UseOQLVersion2", + "UseSystemContextForBackgroundTasks", + "SslCertificateAlgorithm", +} + +// Has reports whether a raw BSON part carries a property at all. An absent +// optional property is not the same as one holding a zero value: Mendix fills it +// in from the type's default on load, so "absent" means "this version's default", +// not "false" or "0". +func Has(raw map[string]any, key string) bool { + _, ok := raw[key] + return ok +} + +// setIfPresent writes a property back only if the stored document already carries +// it. Introducing one it does not carry is the mendixlabs/mxcli#759 failure shape: +// Studio Pro resolves every stored property against the type's property list and +// throws "Sequence contains no matching element" on one the type does not define, +// while mxbuild's deserializer tolerates it — so the build is not a safety net. +// +// Measured: before this gate, `alter settings model BcryptCost = 11` against a +// Mendix 9.24 project introduced DecimalScale = 0 and +// UseDatabaseForeignKeyConstraints = false, neither of which that version stores. +// Both were also wrong as values — 11.13 defaults them to 8 and true — so an +// unrelated one-line statement silently changed two other settings. +func setIfPresent(raw map[string]any, key string, v any) { + if Has(raw, key) { + raw[key] = v + } +} + +// SetModelSettings overlays parsed model settings onto the Settings$ModelSettings +// part they were read from. Shared by both write engines so the two cannot drift +// (the codec engine in mdl/backend/modelsdk and the legacy engine in sdk/mpr). +// +// Every property is presence-gated. A stored part always carries the core ones, so +// the gate is invisible there; it is load-bearing for the version-variable tail. +// The executor refuses an ALTER naming a property the document does not carry, so +// the gate never turns a user's request into a silent no-op. +func SetModelSettings(ms *model.ModelSettings, raw map[string]any) map[string]any { + setIfPresent(raw, "AfterStartupMicroflow", ms.AfterStartupMicroflow) + setIfPresent(raw, "BeforeShutdownMicroflow", ms.BeforeShutdownMicroflow) + setIfPresent(raw, "HealthCheckMicroflow", ms.HealthCheckMicroflow) + setIfPresent(raw, "AllowUserMultipleSessions", ms.AllowUserMultipleSessions) + setIfPresent(raw, "HashAlgorithm", ms.HashAlgorithm) + setIfPresent(raw, "BcryptCost", SafeInt64(ms.BcryptCost)) + SetJavaVersion(raw, ms.JavaVersion) // already presence-gated, and key-aware + setIfPresent(raw, "RoundingMode", ms.RoundingMode) + setIfPresent(raw, "ScheduledEventTimeZoneCode", ms.ScheduledEventTimeZoneCode) + setIfPresent(raw, "DefaultTimeZoneCode", ms.DefaultTimeZoneCode) + setIfPresent(raw, "FirstDayOfWeek", ms.FirstDayOfWeek) + setIfPresent(raw, "DecimalScale", SafeInt64(ms.DecimalScale)) + setIfPresent(raw, "EnableDataStorageOptimisticLocking", ms.EnableDataStorageOptimisticLocking) + setIfPresent(raw, "UseDatabaseForeignKeyConstraints", ms.UseDatabaseForeignKeyConstraints) + setIfPresent(raw, "UseOQLVersion2", ms.UseOQLVersion2) + setIfPresent(raw, "UseSystemContextForBackgroundTasks", ms.UseSystemContextForBackgroundTasks) + setIfPresent(raw, "SslCertificateAlgorithm", ms.SslCertificateAlgorithm) + return raw +} + // ConstantValues rebuilds a configuration's ConstantValues list, updating each // override in the slot it is already stored in so its value shape survives. // Studio Pro and mxbuild only read the nested SharedOrPrivateValue; a flat "Value" diff --git a/model/types.go b/model/types.go index f58e5a770..3ed465321 100644 --- a/model/types.go +++ b/model/types.go @@ -1001,10 +1001,14 @@ type ModelSettings struct { JavaVersion string `json:"javaVersion,omitempty"` RoundingMode string `json:"roundingMode,omitempty"` ScheduledEventTimeZoneCode string `json:"scheduledEventTimeZoneCode,omitempty"` + DefaultTimeZoneCode string `json:"defaultTimeZoneCode,omitempty"` FirstDayOfWeek string `json:"firstDayOfWeek,omitempty"` DecimalScale int `json:"decimalScale,omitempty"` EnableDataStorageOptimisticLocking bool `json:"enableDataStorageOptimisticLocking"` UseDatabaseForeignKeyConstraints bool `json:"useDatabaseForeignKeyConstraints"` + UseOQLVersion2 bool `json:"useOQLVersion2"` + UseSystemContextForBackgroundTasks bool `json:"useSystemContextForBackgroundTasks"` + SslCertificateAlgorithm string `json:"sslCertificateAlgorithm,omitempty"` } // ConventionSettings represents Settings$ConventionSettings. diff --git a/sdk/mpr/parser_settings.go b/sdk/mpr/parser_settings.go index 1af1f7240..3b02512f4 100644 --- a/sdk/mpr/parser_settings.go +++ b/sdk/mpr/parser_settings.go @@ -159,10 +159,19 @@ func parseModelSettings(raw map[string]any) *model.ModelSettings { ms.JavaVersion = settingsoverlay.JavaVersion(raw) ms.RoundingMode = extractString(raw["RoundingMode"]) ms.ScheduledEventTimeZoneCode = extractString(raw["ScheduledEventTimeZoneCode"]) + ms.DefaultTimeZoneCode = extractString(raw["DefaultTimeZoneCode"]) ms.FirstDayOfWeek = extractString(raw["FirstDayOfWeek"]) ms.DecimalScale = extractInt(raw["DecimalScale"]) ms.EnableDataStorageOptimisticLocking = extractBool(raw["EnableDataStorageOptimisticLocking"], false) - ms.UseDatabaseForeignKeyConstraints = extractBool(raw["UseDatabaseForeignKeyConstraints"], false) + // The defaults below are only reached when the key is absent, which happens on + // older Mendix versions that do not store the property (a blank 9.24 project + // has none of UseOQLVersion2 / UseDatabaseForeignKeyConstraints / DecimalScale / + // SslCertificateAlgorithm). The overlay is presence-gated, so a value read from + // a default here is never written back — see settingsoverlay.SetModelSettings. + ms.UseDatabaseForeignKeyConstraints = extractBool(raw["UseDatabaseForeignKeyConstraints"], true) + ms.UseOQLVersion2 = extractBool(raw["UseOQLVersion2"], true) + ms.UseSystemContextForBackgroundTasks = extractBool(raw["UseSystemContextForBackgroundTasks"], false) + ms.SslCertificateAlgorithm = extractString(raw["SslCertificateAlgorithm"]) return ms } diff --git a/sdk/mpr/writer_settings.go b/sdk/mpr/writer_settings.go index ad1529244..03e3c1b12 100644 --- a/sdk/mpr/writer_settings.go +++ b/sdk/mpr/writer_settings.go @@ -86,22 +86,12 @@ func (w *Writer) serializeProjectSettings(ps *model.ProjectSettings) ([]byte, er return marshalUnitIDFirst(doc) } -// serializeModelSettings updates the raw BSON map with modified model settings fields. +// serializeModelSettings overlays the modified model settings onto the raw BSON +// part. The overlay is shared with the codec engine so the two write paths cannot +// drift (see mdl/settingsoverlay), and is presence-gated so a write never +// introduces a property this Mendix version does not store. func serializeModelSettings(ms *model.ModelSettings, raw map[string]any) map[string]any { - raw["AfterStartupMicroflow"] = ms.AfterStartupMicroflow - raw["BeforeShutdownMicroflow"] = ms.BeforeShutdownMicroflow - raw["HealthCheckMicroflow"] = ms.HealthCheckMicroflow - raw["AllowUserMultipleSessions"] = ms.AllowUserMultipleSessions - raw["HashAlgorithm"] = ms.HashAlgorithm - raw["BcryptCost"] = safeInt64(ms.BcryptCost) - settingsoverlay.SetJavaVersion(raw, ms.JavaVersion) - raw["RoundingMode"] = ms.RoundingMode - raw["ScheduledEventTimeZoneCode"] = ms.ScheduledEventTimeZoneCode - raw["FirstDayOfWeek"] = ms.FirstDayOfWeek - raw["DecimalScale"] = safeInt64(ms.DecimalScale) - raw["EnableDataStorageOptimisticLocking"] = ms.EnableDataStorageOptimisticLocking - raw["UseDatabaseForeignKeyConstraints"] = ms.UseDatabaseForeignKeyConstraints - return raw + return settingsoverlay.SetModelSettings(ms, raw) } // serializeConfigurationSettings overlays the modified configuration settings onto From c1fbbe3ea40b669fc0232127c9a0a4aaf9534323 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 14:53:28 +0000 Subject: [PATCH 09/35] docs: refute the view-entity finding by measuring it on Mendix 11.13 mxbuild 11.13.0 is cached now, so finding #14 was tested rather than reasoned about. It does not reproduce: SHOW ENTITIES lists the view as Type: View, DESCRIBE round-trips it and its OQL, GRANT succeeds, a microflow RETURNS LIST OF it, '--' comments in the OQL body parse, and CREATE OR MODIFY prunes members the OQL no longer produces. The whole chain passes mx check with 0 errors. No fix landed in the window that would explain it. The reporter's own notes describe a view that had been dropped and never recreated, which would make 'entity not found' from GRANT and RETURNS, plus the absence from SHOW ENTITIES, correct answers rather than three bugs. This matters because the finding is what killed their dashboard design. Two things in the area are real, and recorded: a view entity needs UseOQLVersion2 = true or the build fails CE6779 (a setting that was not reachable from MDL until the previous commit), and cast() parses without a length but not with one, so a pass-through String(unlimited) column still cannot be narrowed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01KTdvGwVewkdgNQGDCxpQPZ --- .../2026-08-17-banking-app-findings-triage.md | 74 ++++++++++++++++++- 1 file changed, 70 insertions(+), 4 deletions(-) diff --git a/docs/12-bug-reports/2026-08-17-banking-app-findings-triage.md b/docs/12-bug-reports/2026-08-17-banking-app-findings-triage.md index 159657dd7..79cdfda20 100644 --- a/docs/12-bug-reports/2026-08-17-banking-app-findings-triage.md +++ b/docs/12-bug-reports/2026-08-17-banking-app-findings-triage.md @@ -33,7 +33,7 @@ project. Claims that could not be measured here say so, and say why. | 11 | `DROP USER ROLE` leaves demo users dangling | **Confirmed defect** | yes | | 12 | `SHOW MESSAGE` clause-order error is misleading | **Confirmed UX defect** | yes | | 13 | `ALTER MODULE/PAGE SET DOCUMENTATION` is a parse error | **Confirmed gap** | yes | -| 14 | OQL view entities are write-only in MDL | **Not reproduced here** — needs 11.x | no | +| 14 | OQL view entities are write-only in MDL | **Refuted** — measured on 11.13, every operation works | n/a | | — | Everything else (Mendix platform behaviour) | **Correct, not our bug** | n/a | --- @@ -477,7 +477,66 @@ complaint: mxcli's own linter raises a finding mxcli gives you no way to fix. --- -## 14. OQL view entities — reported, not reproduced here +## 14. OQL view entities — REPRODUCED AND REFUTED + +**Update, same day.** mxbuild 11.13.0 is now cached, so this was tested properly on +a real Mendix 11.13 project rather than reasoned about. **The finding does not +reproduce. Every operation it reports as impossible works on current main**, and +the resulting project builds clean. + +Test: a module with two persistent entities and an association, a view entity over +a join with `sum()`/`GROUP BY`, a module role granted read on it, and a microflow +returning a list of it. + +| Reported as | Measured on 11.13 / main | +|---|---| +| `SHOW ENTITIES IN ` does not list it | **Lists it**, `Type: View` | +| `GRANT … ON ` → `entity not found` | **Granted**, `read *` | +| `RETURNS LIST OF ` → `entity not found for return type` | **Microflow created** | +| `--` comments in the OQL body fail the parse | **Parses**, and `exec` creates the view | +| `CREATE OR MODIFY` does not prune dropped members (CE6770) | **Prunes**; add-then-remove cycle → 0 errors | +| — | `DESCRIBE ENTITY` round-trips the view and its OQL | +| — | **`mx check`: 0 errors** with the whole chain wired up | + +No fix landed in the window that would explain it: the only comment-handling fix +(`cd2ac98`, comments inside expressions) is from 2026-08-09, six days *before* the +reporter's nightly, and the two view-entity commits dated 2026-08-17 are OData +pushdown work that does not touch these paths. + +The likeliest explanation is in the reporter's own notes. They record that a +"0 errors" build was reported for **a view that had been dropped and never +recreated** — and if the view entity is not in the model, then `entity not found` +from `GRANT` and from `RETURNS LIST OF`, plus its absence from `SHOW ENTITIES`, are +all *correct* answers rather than three separate bugs. One missing document +explains the whole finding. That cannot be proven from here, but it fits every +symptom, and nothing else does. + +**This matters beyond the triage:** the conclusion "a view entity cannot feed a +page written with mxcli" is what killed the reporter's dashboard design and pushed +it onto N+1 non-persistent `DashboardCard` objects. It is worth revisiting — as is +`DS_AdminSummary`'s six-queries-in-six-counts, which the report itself calls the +strongest remaining case for a view. + +### What is real in this area + +- **A view entity requires `UseOQLVersion2 = true`.** Otherwise the build fails + **CE6779** *"View Entity 'X' is only allowed in the domain model when 'OQL + version 2' is set to 'Yes' in the runtime settings."* Found by accident: the + earlier settings work had flipped it off on the test project. Until finding #10's + change, **that setting was not reachable from MDL at all** — so a project with it + off could not be made to accept a view entity without Studio Pro. Worth stating + in the skill, because the error names the runtime setting rather than anything + the author wrote. +- **`cast()` cannot narrow a length.** The report says `cast()` "is not in the MDL + grammar"; that is not quite right. `cast(c.Name as String)` parses; + `cast(c.Name as String(200))` does not. So the underlying limitation is real — a + pass-through `String(unlimited)` column still cannot be narrowed — but the gap is + a missing length argument on an existing function, not a missing function. +- **Associations to a view entity** are still unauthorable from MDL (CE6771 for an + explicit one; Mendix's own route is to select the associated object's `.ID`). + Untested here, and independent of the above. + +## 14b. Original triage note (superseded by the measurement above) **Reported:** slice 6, as the headline finding that killed the dashboard design. mxcli can CREATE a view entity and Mendix accepts it (0 errors), after which MDL @@ -570,5 +629,12 @@ four stale tests during a regression run. harmful: CONV010's 11 false positives buried 2 real findings. 5. **#6/#6b** — three documented spellings of `NON-PERSISTENT`, two of which do not parse, one of which ships to every user project. -6. **#14** — reproduce on 11.13 before designing anything. -7. **#2, #9, #11, #12, #13** — individually small. (**#10 is done.**) +6. **#2, #9, #11, #12, #13** — individually small. + +**Done since this document was written:** + +- **#10** — optimistic locking and its five sibling settings, plus the + cross-version property leak (10b) found while checking that the additions were + safe. +- **#14** — reproduced on a real Mendix 11.13 project and **refuted**; the + dashboard design it blocked is worth revisiting. From fe87c1a2e35e1a4e4014b729f2360da53f3d0a02 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 13:57:45 +0000 Subject: [PATCH 10/35] docs: settle CASE/enum-split support, fix all seven contradicting surfaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit write-microflows.md documented `case … when … end case;` as supported (with CE0079/MDL056/MDL008 detail), then declared "CASE/SWITCH not supported — use nested IF" 1000 lines later. Five other surfaces sided with the wrong half, and the generated project CLAUDE.md gave a third shape that parses in neither. Settled against the binary rather than by argument, one variant per claim: case $Status when Open, Pending then … when (empty) then … end case; -> passes case $Order/Status when Open then … else … -> MDL008 case $Order/Status as s … -> mismatched input 'as' when Module.Enum.Value … -> mismatched input '.' when 'Active' … -> extraneous input ''Active'' So CASE is supported, in microflows and nanoflows alike, for enum splits only: bare enum values, a branch per value including `(empty)`, no `else`, no alias. Notably the old "WRONG" example failed for an unrelated reason (string-literal values), which is what made the false claim look confirmed. Corrected: write-microflows.md (the unsupported section now pins the three spellings that really fail), check-syntax.md, MDL_QUICK_REFERENCE (which also contradicted itself), docs-site control-flow (given a proper CASE section — it had no CASE docs at all, only two denials), appendixes/{quick-reference,common-mistakes}, and init_claudemd.go. Pinned both directions in mdl-examples/bug-tests/ so `make check-mdl` catches drift back: a positive fixture for the documented forms and a .fail.mdl for the quoted/qualified/alias spellings. Symptom row appended to fix-issue.md, including why the existing skill-MDL gate could not have caught this (it skips microflow bodies). --- .claude/skills/fix-issue.md | 1 + .claude/skills/mendix/check-syntax.md | 2 +- .claude/skills/mendix/write-microflows.md | 46 +++++++++---- cmd/mxcli/init_claudemd.go | 2 +- docs-site/src/appendixes/common-mistakes.md | 2 +- docs-site/src/appendixes/quick-reference.md | 2 +- docs-site/src/language/control-flow.md | 35 +++++++++- docs/01-project/MDL_QUICK_REFERENCE.md | 4 +- ...907-case-enum-split-invalid-forms.fail.mdl | 60 +++++++++++++++++ .../907-case-enum-split-is-supported.mdl | 65 +++++++++++++++++++ 10 files changed, 197 insertions(+), 22 deletions(-) create mode 100644 mdl-examples/bug-tests/907-case-enum-split-invalid-forms.fail.mdl create mode 100644 mdl-examples/bug-tests/907-case-enum-split-is-supported.mdl diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index 6ebd0d603..f989d102d 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -525,3 +525,4 @@ extracting `OffsetExpression`/`LimitExpression`. | CE7375 "must be published and be the key when associations are exposed as an associated object id" on a service publishing no associations | `PublishAssociations` is the representation, not a yes/no — `No` selects "as an associated object id", which needs the system ID as key | `mdl/executor/validate_odata_service_shape.go` | Set `PublishAssociations: Yes` ("as a link", and the default when omitted). MDL-ODATA06 warns at check time | | `mx check` exits with `System.ArgumentOutOfRangeException: length ('-1')` and no error code | an OData service `Path` with no slash in it; mxbuild throws out of its own validator | `mdl/executor/validate_odata_service_shape.go` | Path must not start with `/` and must end with one (CE6550/CE6552). MDL-ODATA05 catches all three | | `ALTER ENTITY … MODIFY ATTRIBUTE X SET DEFAULT ` (or any unrecognised word in the type position, including a typo like `Integr`) silently retypes the attribute to `Enumeration()`, and the project then **cannot be loaded at all** — `mx check` dies before validating with `System.ArgumentNullException: Value cannot be null. (Parameter 'value') at EnumerationAttributeType.set_EnumerationId` | `MODIFY ATTRIBUTE` requires a `dataType`, whose last grammar alternative is a bare `qualifiedName` (entity reference), so any word matches and the visitor maps it to `TypeEnumeration`. The executor wrote it with no enumeration behind it. CREATE ENTITY had guarded this since #552; the guard was **inline in the create handler** and never applied to MODIFY | `mdl/executor/attribute_type_ref.go` (new), `mdl/executor/cmd_entities.go` (`AlterEntityModifyAttribute` case) | Extract the create-path guard to `validateAttributeTypeRef` and call it from the modify path **before** mutating the attribute. Add `DROP DEFAULT ON ATTRIBUTE ` so clearing a default has a spelling that does not go through the type slot, and point the refusal at it. Check `mxcli syntax` too: the topic documented `MODIFY ATTRIBUTE AttrName SET DEFAULT val` — the corrupting form — which is how the reporter got there. When a guard is added inline to one handler, grep for sibling handlers that take the same input. mendixlabs/mxcli#910 | +| A skill claims a construct is unsupported that the grammar accepts (and documents the working form elsewhere in the same file) — an agent writes valid MDL, hits the contradicting claim, and undoes it. Seen for `case … end case` in `write-microflows.md` (supported at §CASE Statements, "not supported" under UNSUPPORTED Syntax), with `check-syntax.md`, `MDL_QUICK_REFERENCE.md`, three `docs-site` pages and the generated project CLAUDE.md (a third shape, `CASE $Var/Attr AS x WHEN Enum.Value`) all disagreeing | Nothing validates prose claims against the grammar. `scripts/check-skill-mdl.sh` extracts fenced blocks but deliberately **skips microflow bodies** (fragment-heavy), so every syntax claim about microflow statements is unguarded. The wrong "WRONG:" example also used string-literal case values, which fail for their own reason — making the claim look confirmed | Docs only: `.claude/skills/mendix/write-microflows.md` + `check-syntax.md`, `docs/01-project/MDL_QUICK_REFERENCE.md`, `docs-site/src/language/control-flow.md` + `appendixes/{quick-reference,common-mistakes}.md`, `cmd/mxcli/init_claudemd.go` (the generated table) | Settle it against the binary before editing, one variant per claim (`bin/mxcli check` on the documented form, the quoted form, the qualified form, the `AS` form, the `else` form) — a "WRONG" example that fails for an unrelated reason is not evidence. Then fix **every** surface: grep `end case\|END CASE` across `.claude/`, `docs/`, `docs-site/` and `cmd/mxcli/*.go`, not just the file in the report (7 surfaces, the issue named 3). Pin both directions in `mdl-examples/bug-tests/` — `907-case-enum-split-is-supported.mdl` (must pass) and `907-case-enum-split-invalid-forms.fail.mdl` (must fail) — so `make check-mdl` catches drift back. Issue #907 | diff --git a/.claude/skills/mendix/check-syntax.md b/.claude/skills/mendix/check-syntax.md index e8586afa4..1a983454a 100644 --- a/.claude/skills/mendix/check-syntax.md +++ b/.claude/skills/mendix/check-syntax.md @@ -46,7 +46,7 @@ Before writing any MDL, verify these requirements: **NOT Supported (will cause errors):** - `set $var = call microflow ...` - Use `$var = call microflow ...` (no SET) - `while ... end while` - Use `loop` with lists -- `case ... when ... end case` - Use nested `if` +- `case ... when 'String' ...` - Case values are bare enum identifiers, never quoted or qualified; `case ... when Value then ... end case;` itself IS supported (enum splits only), and takes no `else` (MDL008) and no `AS` alias - `TRY ... CATCH` - Use `on error` blocks - `break` / `continue` - Not implemented - `commit message 'text'` - Not in current grammar (session command only) diff --git a/.claude/skills/mendix/write-microflows.md b/.claude/skills/mendix/write-microflows.md index 8005d9a47..1266ff841 100644 --- a/.claude/skills/mendix/write-microflows.md +++ b/.claude/skills/mendix/write-microflows.md @@ -1444,28 +1444,46 @@ begin end loop; ``` -### CASE/SWITCH Statement +### CASE with string values, `else`, or an alias + +`case … end case` **is supported** — see [CASE Statements (Enum Split)](#case-statements-enum-split) +above for the correct form. What is not supported is the SQL-flavoured spelling of +it: quoted values, an `else` fallback, and an `AS` alias all fail. ```mdl --- WRONG: CASE/SWITCH not supported -case $status +-- WRONG: case values are not string literals (parse error) +case $Order/Status when 'Active' then set $Result = 1; - when 'Inactive' then set $Result = 2; +end case; + +-- WRONG: case values are not qualified (parse error) +case $Order/Status + when MyModule.Status.Active then set $Result = 1; +end case; + +-- WRONG: no AS alias (parse error: mismatched input 'as' expecting WHEN) +case $Order/Status as s + when Active then set $Result = 1; +end case; + +-- WRONG: no else branch (MDL008 → mxbuild CE0079 + CE0773) +case $Order/Status + when Active then set $Result = 1; else set $Result = 0; end case; --- CORRECT: Use nested IF statements -if $status = 'Active' then - set $Result = 1; -else - if $status = 'Inactive' then - set $Result = 2; - else - set $Result = 0; - end if; -end if; +-- CORRECT: bare enum values, one branch per value, including (empty) +case $Order/Status + when Active then set $Result = 1; + when Inactive then set $Result = 2; + when (empty) then set $Result = 0; +end case; ``` +An enum split is the *only* thing `case` does — it branches on an enumeration, not +on arbitrary expressions. For anything else (a string comparison, a numeric range), +use nested `if … else … end if`. + ### TRY/CATCH Block ```mdl diff --git a/cmd/mxcli/init_claudemd.go b/cmd/mxcli/init_claudemd.go index 7e486dfff..6d486010b 100644 --- a/cmd/mxcli/init_claudemd.go +++ b/cmd/mxcli/init_claudemd.go @@ -520,7 +520,7 @@ func generateClaudeMD(projectName, mprFile string) string { w("|-----------|--------|\n") w("| Variable declaration (primitive only) | " + bt + "DECLARE $Var Type = value;" + bt + " |\n") w("| Assignment | " + bt + "SET $Var = expression;" + bt + " |\n") - w("| Enum split (CASE) | " + bt + "CASE $Var/Attr AS x WHEN Enum.Value THEN ... END CASE" + bt + " |\n") + w("| Enum split (CASE) | " + bt + "CASE $Var/Attr WHEN Value[, Value] THEN ... WHEN (empty) THEN ... END CASE;" + bt + " (bare enum values, no `ELSE`, no alias) |\n") w("| Create object | " + bt + "$Var = CREATE Module.Entity (Attr = value);" + bt + " |\n") w("| Change object | " + bt + "CHANGE $Entity (Attr = value);" + bt + " |\n") w("| Commit | " + bt + "COMMIT $Entity [WITH EVENTS] [REFRESH];" + bt + " |\n") diff --git a/docs-site/src/appendixes/common-mistakes.md b/docs-site/src/appendixes/common-mistakes.md index f90688acc..5a46299e9 100644 --- a/docs-site/src/appendixes/common-mistakes.md +++ b/docs-site/src/appendixes/common-mistakes.md @@ -125,7 +125,7 @@ These constructs will cause parse errors: | Unsupported | Use Instead | |-------------|-------------| -| `CASE ... WHEN ... END CASE` | Nested `IF ... ELSE ... END IF` | +| `CASE ... WHEN 'String' ... ELSE ...` | Bare enum values, one branch per value including `(empty)` — `CASE` itself IS supported for enum splits | | `TRY ... CATCH ... END TRY` | `ON ERROR { ... }` blocks on specific activities | ## Boolean Attributes Must Have Defaults diff --git a/docs-site/src/appendixes/quick-reference.md b/docs-site/src/appendixes/quick-reference.md index f2699ee64..bc73a9736 100644 --- a/docs-site/src/appendixes/quick-reference.md +++ b/docs-site/src/appendixes/quick-reference.md @@ -175,7 +175,7 @@ AUTHENTICATION Basic, Session | Unsupported | Use Instead | Notes | |-------------|-------------|-------| -| `CASE ... WHEN ... END CASE` | Nested `IF ... ELSE ... END IF` | Switch not implemented | +| `CASE ... WHEN 'String' ... ELSE ...` | Bare enum values, one branch per value | `CASE` itself IS supported for **enum splits**; what fails is quoted/qualified values, an `ELSE` branch (MDL008), and an `AS` alias | | `TRY ... CATCH ... END TRY` | `ON ERROR { ... }` blocks | Use error handlers on specific activities | **Notes:** diff --git a/docs-site/src/language/control-flow.md b/docs-site/src/language/control-flow.md index 8b42be9ff..fb0595430 100644 --- a/docs-site/src/language/control-flow.md +++ b/docs-site/src/language/control-flow.md @@ -26,7 +26,9 @@ END IF; ### Nested IF -Since MDL does not support `CASE`/`WHEN` (switch statements), use nested `IF...ELSE` blocks: +For multi-way branching on anything other than an enumeration, use nested `IF...ELSE` +blocks. (To branch on an **enumeration**, use [CASE (Enum Split)](#case-enum-split) +instead — it maps to a Mendix enum split rather than a chain of decisions.) ```sql IF $Order/Status = 'Draft' THEN @@ -178,13 +180,42 @@ COMMIT $Order WITH EVENTS ON ERROR ROLLBACK; > **Note:** `ON ERROR` is not supported on `EXECUTE DATABASE QUERY` activities. +## CASE (Enum Split) + +`CASE` branches on an **enumeration** and compiles to a Mendix enum split. It is not a +general-purpose switch: the source is an enum attribute or variable, and the values are +bare enum member names. + +```sql +CASE $Order/Status + WHEN Draft, Submitted THEN + LOG INFO 'Not shipped yet'; + WHEN Approved THEN + LOG INFO 'Ready to ship'; + WHEN Shipped, (empty) THEN + LOG INFO 'Nothing to do'; +END CASE; +``` + +Rules, each of which `mxcli check` enforces: + +| Rule | Why | +|------|-----| +| Values are **bare identifiers** — not `'Quoted'`, not `Module.Enum.Value` | Parse error otherwise | +| One branch per enum value, **including `(empty)`** | A missing `(empty)` is **MDL056**; mxbuild reports **CE0079** for any uncovered value. Required even when the attribute is `not null` | +| **No `ELSE`** | **MDL008**. An enum split is exclusive with one outgoing flow per value; mxbuild reports CE0079 per uncovered value *and* CE0773 on the else flow | +| **No `AS` alias** | Parse error: `mismatched input 'as' expecting WHEN` | + +Several values may share a branch (`WHEN Draft, Submitted THEN …`). `CASE` works in +nanoflows on the same terms. + ## Unsupported Control Flow The following constructs are **not** supported in MDL and will cause parse errors: | Unsupported | Use Instead | |-------------|-------------| -| `CASE ... WHEN ... END CASE` | Nested `IF ... ELSE ... END IF` | +| `CASE ... WHEN 'String' ... ELSE ...` | Bare enum values and a branch per value — see [CASE (Enum Split)](#case-enum-split); `CASE` itself is supported | | `TRY ... CATCH ... END TRY` | `ON ERROR { ... }` blocks on individual activities | ## Complete Example diff --git a/docs/01-project/MDL_QUICK_REFERENCE.md b/docs/01-project/MDL_QUICK_REFERENCE.md index fc8ffd027..372031a45 100644 --- a/docs/01-project/MDL_QUICK_REFERENCE.md +++ b/docs/01-project/MDL_QUICK_REFERENCE.md @@ -477,7 +477,7 @@ it is for pages. | Annotation | `@annotation 'text'` | Visual note attached to next activity | | Free annotation | `@annotation 'text'` before `@position(...)` | Free-floating visual note preserved by order | | IF | `if condition then ... [else ...] end if;` | | -| Enum split | `case $Var when Value then ... end case;` | Enumeration decision branches | +| Enum split | `case $Var when Value then ... end case;` | Enumeration decision branches. Bare enum values (never quoted or qualified), one branch per value **including `(empty)`** (MDL056), no `else` (MDL008), no `AS` alias | | Type split | `split type $Var case Module.Entity ... end split;` | Runtime specialization branches | | Cast | `cast $SpecificVar;` | Downcast inside a type split branch | | LOOP | `loop $item in $list begin ... end loop;` | FOR EACH over list | @@ -492,7 +492,7 @@ it is for pages. | Unsupported | Use Instead | Notes | |-------------|-------------|-------| -| `case ... when ... end case` | Nested `if ... else ... end if` | Switch not implemented | +| `case ... when 'String' ... else ...` | Bare enum values, one branch per value | `case` itself IS supported for **enum splits** (see above); what fails is quoted/qualified values, an `else` branch, and an `AS` alias | | `TRY ... CATCH ... end TRY` | `on error { ... }` blocks | Use error handlers on specific activities | **Notes:** diff --git a/mdl-examples/bug-tests/907-case-enum-split-invalid-forms.fail.mdl b/mdl-examples/bug-tests/907-case-enum-split-invalid-forms.fail.mdl new file mode 100644 index 000000000..154204b91 --- /dev/null +++ b/mdl-examples/bug-tests/907-case-enum-split-invalid-forms.fail.mdl @@ -0,0 +1,60 @@ +-- ============================================================================ +-- #907 — the CASE spellings that genuinely do NOT work +-- ============================================================================ +-- +-- NEGATIVE TEST (.fail.mdl): `mxcli check` MUST reject this file. +-- +-- The companion 907-case-enum-split-is-supported.mdl pins the working form. +-- This file pins the three spellings the docs used to imply were the whole +-- story ("CASE not supported"), so the correction cannot drift back: +-- +-- 1. quoted string values — `when 'Active' then` +-- parse error: extraneous input ''Active'' expecting … +-- 2. qualified values — `when Bug907.Status.Open then` +-- parse error: mismatched input '.' expecting {THEN, ','} +-- 3. an `AS` alias — `case $Order/Status as s` +-- parse error: mismatched input 'as' expecting WHEN +-- +-- (`else` is the fourth invalid form — it parses but is rejected as MDL008. +-- It is already covered by mdl009-enum-split-empty-branch.fail.mdl.) +-- +-- The first statement alone is enough to fail the file; the rest document the +-- other two spellings in the same place. + +CREATE OR MODIFY ENUMERATION Bug907.Status ( + Open 'Open', + Closed 'Closed' +); + +CREATE OR MODIFY MICROFLOW Bug907.QuotedValues ($Order: Bug907.Order) +RETURNS Boolean +BEGIN + case $Order/Status + when 'Open' then + return true; + when (empty) then + return false; + end case; +END; + +CREATE OR MODIFY MICROFLOW Bug907.QualifiedValues ($Order: Bug907.Order) +RETURNS Boolean +BEGIN + case $Order/Status + when Bug907.Status.Open then + return true; + when (empty) then + return false; + end case; +END; + +CREATE OR MODIFY MICROFLOW Bug907.AliasClause ($Order: Bug907.Order) +RETURNS Boolean +BEGIN + case $Order/Status as s + when Open then + return true; + when (empty) then + return false; + end case; +END; diff --git a/mdl-examples/bug-tests/907-case-enum-split-is-supported.mdl b/mdl-examples/bug-tests/907-case-enum-split-is-supported.mdl new file mode 100644 index 000000000..d3390b970 --- /dev/null +++ b/mdl-examples/bug-tests/907-case-enum-split-is-supported.mdl @@ -0,0 +1,65 @@ +-- ============================================================================ +-- #907 — CASE / enum split IS supported (documentation contradicted itself) +-- ============================================================================ +-- +-- POSITIVE TEST: `mxcli check` MUST accept this file. +-- +-- Six doc surfaces claimed "CASE/SWITCH not supported — use nested IF", while +-- write-microflows.md documented the working form two sections earlier and +-- the generated project CLAUDE.md gave a third shape (`AS x` + `Enum.Value`) +-- that parses in neither. An agent reading top-to-bottom wrote a CASE split, +-- then hit "not supported" later and burned a validation cycle undoing it. +-- +-- This fixture pins the forms the docs now promise. The invalid spellings are +-- pinned separately in 907-case-enum-split-invalid-forms.fail.mdl. + +CREATE OR MODIFY ENUMERATION Bug907.Status ( + Open 'Open', + Pending 'Pending', + Closed 'Closed' +); + +CREATE OR MODIFY PERSISTENT ENTITY Bug907.Order ( + "Reference": String(50), + "Status": Enumeration(Bug907.Status) +); + +-- Source is a variable; one branch per value; several values share a branch. +CREATE OR MODIFY MICROFLOW Bug907.SplitOnVariable ($Status: Enumeration(Bug907.Status)) +RETURNS Boolean +BEGIN + case $Status + when Open, Pending then + return true; + when Closed then + return false; + when (empty) then + return false; + end case; +END; + +-- Source is an attribute path; `(empty)` shares a branch with a real value. +CREATE OR MODIFY MICROFLOW Bug907.SplitOnAttributePath ($Order: Bug907.Order) +RETURNS String +BEGIN + case $Order/Status + when Open then + return 'open'; + when Pending, Closed then + return 'other'; + when (empty) then + return 'unset'; + end case; +END; + +-- Nanoflows take CASE on the same terms. +CREATE OR MODIFY NANOFLOW Bug907.SplitInNanoflow ($Status: Enumeration(Bug907.Status)) +RETURNS Boolean +BEGIN + case $Status + when Open, Pending then + return true; + when Closed, (empty) then + return false; + end case; +END; From 85b3c330933ee8ae72080c0f683de280f27810a1 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 14:13:24 +0000 Subject: [PATCH 11/35] docs: pin where an enum may be a string literal, measured per context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The skills said "NEVER use a string literal for an enumeration"; docs-site used `Status = 'Draft'` in a dozen CREATE/CHANGE examples and called it "plain strings when the context is unambiguous". Both are half right, and `mxcli check` passes the good and the bad form alike, so the build is where a reader finds out which half applied to them. Measured on a real project rather than argued — mxcli new 11.13.0, one microflow per construct, `mxcli docker check` read per construct: if $O/Status = 'Draft' then … CE0117 "Error(s) in expression." log warning 'x: ' + $O/Status; CE0117 "Error(s) in expression." change $O (Status = 'Submitted'); clean create M.E (Status = 'Draft'); clean DEFAULT 'Draft' on the attribute clean retrieve … where [Status='Draft'] clean getCaption(…) / toString(…) concat clean The string form is accepted wherever the slot is already enum-typed, and rejected in an expression, where it is a String compared to an Enumeration. So only comparisons and concatenations were fixed: control-flow.md (its nested-IF example had both defects — three string comparisons and an enum concatenated into a LOG message), create-microflow.md, and expressions.md, whose "unambiguous context" sentence now names the four accepted slots and the two that are CE0117. A sweep-and-replace over every `Status = '…'` would have "corrected" ten sites that build fine, including the XPath one where the string is the only correct form. The skill gains the same table, plus getCaption()/toString() for putting an enum into a string, and a note that check cannot catch either failing form (the type checker is still a proposal). No .fail.mdl is possible here, so the verdicts live in mdl-examples/bug-tests/enum-string-literal-contexts.mdl, pinning the accepted forms so a later fix does not widen into them. Control: adding it to the probe project left the error count unchanged at the two deliberate failures. --- .claude/skills/fix-issue.md | 1 + .claude/skills/mendix/write-microflows.md | 29 ++++++++ docs-site/src/language/control-flow.md | 45 ++++++++++-- docs-site/src/language/expressions.md | 21 +++++- .../reference/microflow/create-microflow.md | 4 +- .../enum-string-literal-contexts.mdl | 73 +++++++++++++++++++ 6 files changed, 161 insertions(+), 12 deletions(-) create mode 100644 mdl-examples/bug-tests/enum-string-literal-contexts.mdl diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index f989d102d..e6dfcb6bc 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -526,3 +526,4 @@ extracting `OffsetExpression`/`LimitExpression`. | `mx check` exits with `System.ArgumentOutOfRangeException: length ('-1')` and no error code | an OData service `Path` with no slash in it; mxbuild throws out of its own validator | `mdl/executor/validate_odata_service_shape.go` | Path must not start with `/` and must end with one (CE6550/CE6552). MDL-ODATA05 catches all three | | `ALTER ENTITY … MODIFY ATTRIBUTE X SET DEFAULT ` (or any unrecognised word in the type position, including a typo like `Integr`) silently retypes the attribute to `Enumeration()`, and the project then **cannot be loaded at all** — `mx check` dies before validating with `System.ArgumentNullException: Value cannot be null. (Parameter 'value') at EnumerationAttributeType.set_EnumerationId` | `MODIFY ATTRIBUTE` requires a `dataType`, whose last grammar alternative is a bare `qualifiedName` (entity reference), so any word matches and the visitor maps it to `TypeEnumeration`. The executor wrote it with no enumeration behind it. CREATE ENTITY had guarded this since #552; the guard was **inline in the create handler** and never applied to MODIFY | `mdl/executor/attribute_type_ref.go` (new), `mdl/executor/cmd_entities.go` (`AlterEntityModifyAttribute` case) | Extract the create-path guard to `validateAttributeTypeRef` and call it from the modify path **before** mutating the attribute. Add `DROP DEFAULT ON ATTRIBUTE ` so clearing a default has a spelling that does not go through the type slot, and point the refusal at it. Check `mxcli syntax` too: the topic documented `MODIFY ATTRIBUTE AttrName SET DEFAULT val` — the corrupting form — which is how the reporter got there. When a guard is added inline to one handler, grep for sibling handlers that take the same input. mendixlabs/mxcli#910 | | A skill claims a construct is unsupported that the grammar accepts (and documents the working form elsewhere in the same file) — an agent writes valid MDL, hits the contradicting claim, and undoes it. Seen for `case … end case` in `write-microflows.md` (supported at §CASE Statements, "not supported" under UNSUPPORTED Syntax), with `check-syntax.md`, `MDL_QUICK_REFERENCE.md`, three `docs-site` pages and the generated project CLAUDE.md (a third shape, `CASE $Var/Attr AS x WHEN Enum.Value`) all disagreeing | Nothing validates prose claims against the grammar. `scripts/check-skill-mdl.sh` extracts fenced blocks but deliberately **skips microflow bodies** (fragment-heavy), so every syntax claim about microflow statements is unguarded. The wrong "WRONG:" example also used string-literal case values, which fail for their own reason — making the claim look confirmed | Docs only: `.claude/skills/mendix/write-microflows.md` + `check-syntax.md`, `docs/01-project/MDL_QUICK_REFERENCE.md`, `docs-site/src/language/control-flow.md` + `appendixes/{quick-reference,common-mistakes}.md`, `cmd/mxcli/init_claudemd.go` (the generated table) | Settle it against the binary before editing, one variant per claim (`bin/mxcli check` on the documented form, the quoted form, the qualified form, the `AS` form, the `else` form) — a "WRONG" example that fails for an unrelated reason is not evidence. Then fix **every** surface: grep `end case\|END CASE` across `.claude/`, `docs/`, `docs-site/` and `cmd/mxcli/*.go`, not just the file in the report (7 surfaces, the issue named 3). Pin both directions in `mdl-examples/bug-tests/` — `907-case-enum-split-is-supported.mdl` (must pass) and `907-case-enum-split-invalid-forms.fail.mdl` (must fail) — so `make check-mdl` catches drift back. Issue #907 | +| Docs disagree on whether an enumeration may be written as a string literal — the skills say "NEVER use a string literal", docs-site uses `Status = 'Draft'` in a dozen CREATE/CHANGE examples and calls it "plain strings when the context is unambiguous". Following either one wholesale is wrong, and `mxcli check` passes **both** the good and the bad form, so the build is where you find out | It is context-dependent and nobody had measured it. A string is accepted wherever the slot is **already enum-typed**; in an **expression** it is a String compared to an Enumeration | Docs only: `.claude/skills/mendix/write-microflows.md` (Enumeration Comparisons), `docs-site/src/language/{control-flow,expressions}.md`, `docs-site/src/reference/microflow/create-microflow.md` | Measure per context on a real project before touching a single example — one microflow per construct, `mxcli docker check` read per construct (`mxcli new EP --version 11.13.0 --skip-build`, then `exec` + `docker check`). Verified on 11.13.0: **CE0117** for `if $O/Status = 'Draft'` and for `'x' + $O/Status`; **clean** for `change`/`create` member values, attribute `DEFAULT 'Draft'`, XPath `[Status = 'Draft']`, and `getCaption()`/`toString()` in a concat. So fix **only** comparisons and concatenations — a sweep-and-replace over every `Status = '…'` would have "corrected" ~10 of 12 sites that are fine. No `.fail.mdl` is possible (check passes the failing form; the type checker is still a proposal — `docs/11-proposals/PROPOSAL_expression_type_checking.md`), so the verdicts live in `mdl-examples/bug-tests/enum-string-literal-contexts.mdl`, which pins the ACCEPTED forms and builds clean (control: adding it to a probe project left the error count unchanged at the 2 deliberate failures) | diff --git a/.claude/skills/mendix/write-microflows.md b/.claude/skills/mendix/write-microflows.md index 1266ff841..18383790a 100644 --- a/.claude/skills/mendix/write-microflows.md +++ b/.claude/skills/mendix/write-microflows.md @@ -406,6 +406,35 @@ end if; -- IF $Task/Status = 'Completed' THEN -- INCORRECT! ``` +Putting an enumeration **into** a string is the same mistake — concatenating it +directly is rejected, so render it first with `getCaption()` (the caption) or +`toString()` (the value name): + +```mdl +-- CORRECT +log warning 'Unexpected status: ' + getCaption($Order/Status); + +-- WRONG +log warning 'Unexpected status: ' + $Order/Status; +``` + +**Where the string form is and is not accepted** (verified against mxbuild 11.13.0 — +one microflow per row, `mx check` read per construct): + +| Context | `'Draft'` | Note | +|---------|-----------|------| +| Comparison in a decision — `if $O/Status = 'Draft'` | ❌ **CE0117** | The one that bites | +| Concatenation — `'x' + $O/Status` | ❌ **CE0117** | Use `getCaption()` / `toString()` | +| `change $O (Status = 'Draft')` | ✅ accepted | Slot is already enum-typed | +| `create M.E (Status = 'Draft')` | ✅ accepted | Same | +| Attribute `DEFAULT 'Draft'` | ✅ accepted | Documented as the legacy form | +| XPath constraint `[Status = 'Draft']` | ✅ accepted | Enums are strings at DB level | + +`mxcli check` does **not** flag the two failing rows (it does not type expressions — +see `docs/11-proposals/PROPOSAL_expression_type_checking.md`), so a script can pass +`check` and fail the build. The qualified form is valid in every row above: use it +everywhere and the distinction never has to be remembered. + **Checking for empty enumeration:** ```mdl if $entity/status = empty then diff --git a/docs-site/src/language/control-flow.md b/docs-site/src/language/control-flow.md index fb0595430..d4dd881d4 100644 --- a/docs-site/src/language/control-flow.md +++ b/docs-site/src/language/control-flow.md @@ -31,21 +31,52 @@ blocks. (To branch on an **enumeration**, use [CASE (Enum Split)](#case-enum-spl instead — it maps to a Mendix enum split rather than a chain of decisions.) ```sql -IF $Order/Status = 'Draft' THEN - CHANGE $Order (Status = 'Submitted'); +IF $Order/TotalAmount > 10000 THEN + CHANGE $Order (DiscountPercentage = 15); ELSE - IF $Order/Status = 'Submitted' THEN - CHANGE $Order (Status = 'Approved'); + IF $Order/TotalAmount > 5000 THEN + CHANGE $Order (DiscountPercentage = 10); ELSE - IF $Order/Status = 'Approved' THEN - CHANGE $Order (Status = 'Shipped'); + IF $Order/TotalAmount > 1000 THEN + CHANGE $Order (DiscountPercentage = 5); ELSE - LOG WARNING 'Unexpected order status: ' + $Order/Status; + CHANGE $Order (DiscountPercentage = 0); END IF; END IF; END IF; ``` +### Comparing Enumerations + +An enumeration is compared against its **qualified value**, never a string literal. +A string literal here fails the build with **CE0117** *"Error(s) in expression"* — +`mxcli check` does not catch it (see +[expression type checking](https://github.com/mendixlabs/mxcli/blob/main/docs/11-proposals/PROPOSAL_expression_type_checking.md)), +so the first sign is a failed build. + +```sql +-- CORRECT +IF $Order/Status = Sales.OrderStatus.Draft THEN ... END IF; + +-- WRONG: CE0117 "Error(s) in expression." +IF $Order/Status = 'Draft' THEN ... END IF; +``` + +The same applies to putting an enumeration into a string — concatenating it directly +is CE0117, so render it first: + +```sql +-- CORRECT +LOG WARNING 'Unexpected status: ' + getCaption($Order/Status); + +-- WRONG: CE0117 +LOG WARNING 'Unexpected status: ' + $Order/Status; +``` + +The string form **is** accepted outside comparisons — in a `CREATE`/`CHANGE` member +value, an attribute `DEFAULT`, and an XPath constraint (where enums live as strings at +the database level). The qualified form works everywhere, so prefer it. + ### Complex Conditions Conditions support `AND`, `OR`, and parentheses: diff --git a/docs-site/src/language/expressions.md b/docs-site/src/language/expressions.md index 7e52f13fb..06a3afa93 100644 --- a/docs-site/src/language/expressions.md +++ b/docs-site/src/language/expressions.md @@ -170,7 +170,9 @@ IF $Order/Status = Sales.OrderStatus.Confirmed THEN END IF; ``` -Alternatively, enumeration values can be referenced as plain strings when the context is unambiguous: +A plain string is accepted where the slot is already typed as the enumeration — a +`CREATE`/`CHANGE` member value, an attribute `DEFAULT`, and an XPath constraint (enums +are stored as strings at the database level): ```sql $Order = CREATE Sales.Order ( @@ -178,6 +180,19 @@ $Order = CREATE Sales.Order ( ); ``` +**It is not accepted in an expression.** Comparing an enumeration to a string, or +concatenating one into a string, fails the build with **CE0117** *"Error(s) in +expression"* — and `mxcli check` does not catch it, so the build is where you find out: + +```sql +IF $Order/Status = 'Draft' THEN ... -- CE0117 +LOG WARNING 'Status: ' + $Order/Status; -- CE0117 +IF $Order/Status = Sales.OrderStatus.Draft THEN ... -- correct +LOG WARNING 'Status: ' + getCaption($Order/Status); -- correct +``` + +The qualified form works in every context, so prefer it throughout. + ## Expression Contexts Expressions appear in several places within microflow activities: @@ -185,9 +200,9 @@ Expressions appear in several places within microflow activities: ### In Conditions (IF, WHILE, WHERE) ```sql -IF $Order/TotalAmount > 0 AND $Order/Status != 'Cancelled' THEN ... +IF $Order/TotalAmount > 0 AND $Order/Status != Sales.OrderStatus.Cancelled THEN ... WHILE $Counter < $MaxRetries BEGIN ... END WHILE; -RETRIEVE $Active FROM Sales.Order WHERE Status = 'Active'; +RETRIEVE $Active FROM Sales.Order WHERE Status = 'Active'; -- XPath: string is correct here ``` ### In Attribute Assignments (CREATE, CHANGE) diff --git a/docs-site/src/reference/microflow/create-microflow.md b/docs-site/src/reference/microflow/create-microflow.md index b9ffb7b5c..a7b64d3d8 100644 --- a/docs-site/src/reference/microflow/create-microflow.md +++ b/docs-site/src/reference/microflow/create-microflow.md @@ -197,8 +197,8 @@ CREATE MICROFLOW Sales.ACT_ApproveOrder (DECLARE $Order: Sales.Order) RETURN Boolean BEGIN - IF $Order/Status = 'Pending' THEN - CHANGE $Order (Status = 'Approved'); + IF $Order/Status = Sales.OrderStatus.Pending THEN + CHANGE $Order (Status = Sales.OrderStatus.Approved); COMMIT $Order WITH EVENTS; LOG INFO NODE 'OrderProcessing' 'Order approved'; RETURN true; diff --git a/mdl-examples/bug-tests/enum-string-literal-contexts.mdl b/mdl-examples/bug-tests/enum-string-literal-contexts.mdl new file mode 100644 index 000000000..1d0ccf953 --- /dev/null +++ b/mdl-examples/bug-tests/enum-string-literal-contexts.mdl @@ -0,0 +1,73 @@ +-- ============================================================================ +-- Enum vs. string literal — which contexts mxbuild accepts +-- ============================================================================ +-- +-- POSITIVE TEST: `mxcli check` MUST accept this file. Every statement below is +-- also verified to build clean. +-- +-- The docs disagreed about this: the skills said "NEVER use a string literal +-- for an enumeration", while docs-site used `Status = 'Draft'` in a dozen +-- CREATE/CHANGE examples and called it "referencing plain strings when the +-- context is unambiguous". Both were half right, so a reader could not tell +-- which of their own statements were wrong. +-- +-- Settled on a real project (mxcli new EP --version 11.13.0, one microflow per +-- construct, `mxcli docker check` read per construct): +-- +-- if $O/Status = 'Draft' then … -> [error] CE0117 "Error(s) in expression." +-- log warning 'x: ' + $O/Status; -> [error] CE0117 "Error(s) in expression." +-- change $O (Status = 'Submitted'); -> clean +-- create M.E (Status = 'Draft'); -> clean +-- "Status": Enumeration(…) DEFAULT 'Draft' -> clean +-- retrieve … where [Status = 'Draft'] -> clean +-- getCaption($O/Status) / toString($O/Status) in a concat -> clean +-- +-- So the string form is accepted wherever the slot is ALREADY enum-typed, and +-- rejected in an expression, where it is a String compared to an Enumeration. +-- `mxcli check` catches neither failing form — it does not type expressions +-- (docs/11-proposals/PROPOSAL_expression_type_checking.md) — which is why this +-- fixture carries the verdicts in comments rather than as a .fail.mdl. +-- +-- This file pins the ACCEPTED forms, so the failing-form fix is never "widened" +-- into rewriting the clean ones. + +CREATE OR MODIFY ENUMERATION EnumCtx.OrderStatus ( + Draft 'Draft', + Submitted 'Submitted', + Approved 'Approved' +); + +-- DEFAULT with a string literal: accepted (the legacy form). +CREATE OR MODIFY PERSISTENT ENTITY EnumCtx.Order ( + "Reference": String(50), + "Status": Enumeration(EnumCtx.OrderStatus) DEFAULT 'Draft' +); + +-- CREATE and CHANGE member values: the slot is enum-typed, string accepted. +CREATE OR MODIFY MICROFLOW EnumCtx.AssignWithString () +RETURNS Boolean +BEGIN + $New = create EnumCtx.Order (Reference = 'R1', Status = 'Draft'); + change $New (Status = 'Submitted'); + return true; +END; + +-- XPath constraint: enums are strings at the database level. +CREATE OR MODIFY MICROFLOW EnumCtx.RetrieveWithString () +RETURNS Boolean +BEGIN + retrieve $Orders from EnumCtx.Order where [Status = 'Draft']; + return true; +END; + +-- Comparison and concatenation: qualified value, and getCaption() for the string. +CREATE OR MODIFY MICROFLOW EnumCtx.CompareQualified ($Order: EnumCtx.Order) +RETURNS Boolean +BEGIN + if $Order/Status = EnumCtx.OrderStatus.Draft then + change $Order (Status = EnumCtx.OrderStatus.Submitted); + return true; + end if; + log warning 'Unexpected status: ' + getCaption($Order/Status); + return false; +END; From 218d4fe74155f5848a36d7d2a324dc4d168b9015 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 15:51:59 +0000 Subject: [PATCH 12/35] feat(exec): refuse a script whose checks report an error mxcli exec ran no semantic checks. Parse errors stopped it, but the ~24 Validate* checks lived inline in cmd_check.go with exactly one caller, so a script mxcli check rejected was applied by exec anyway. The mxcli-banking report hit this with a page whose invalid widget property was silently dropped -- a picker that renders and does nothing. 'Run check before exec' was a convention nothing enforced, on a tool built for unattended agent use. exec now runs the same checks first and refuses when any reports an error: nothing is written, because exec applies statements one at a time and cannot roll back, so a known-bad script would leave the model partly updated. Warnings print and do not block. --no-check opts out. The checks moved to executor.ValidateProgram rather than being copied, because a second copy would drift -- the same failure that produced the CONV010 and QUAL004 false positives and the duplicated settings overlays. mxcli check now calls it too, and its output is byte-identical across all 20 doctype-test scripts. A check wired into nothing is invisible: it reports no violations and reads as a clean project. TestValidateProgram_WiresEveryWholeProgramValidator therefore asserts structurally that every exported Validate*(prog *ast.Program, ...) is called from ValidateProgram, and fails when one is dropped. Verified on a real 11.13 project: a combobox with an unknown property is refused with nothing written, the same script under --no-check writes the page with the property silently dropped, and a warning-only script still applies. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01KTdvGwVewkdgNQGDCxpQPZ --- .claude/skills/fix-issue.md | 1 + .claude/skills/mendix/check-syntax.md | 22 ++++ cmd/mxcli/cmd_check.go | 128 +--------------------- cmd/mxcli/cmd_exec.go | 32 ++++++ mdl/executor/validate_program.go | 151 ++++++++++++++++++++++++++ mdl/executor/validate_program_test.go | 142 ++++++++++++++++++++++++ 6 files changed, 352 insertions(+), 124 deletions(-) create mode 100644 mdl/executor/validate_program.go create mode 100644 mdl/executor/validate_program_test.go diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index b9670ad25..91d2af4a2 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -528,3 +528,4 @@ extracting `OffsetExpression`/`LimitExpression`. | `ALTER ENTITY … MODIFY ATTRIBUTE X SET DEFAULT ` (or any unrecognised word in the type position, including a typo like `Integr`) silently retypes the attribute to `Enumeration()`, and the project then **cannot be loaded at all** — `mx check` dies before validating with `System.ArgumentNullException: Value cannot be null. (Parameter 'value') at EnumerationAttributeType.set_EnumerationId` | `MODIFY ATTRIBUTE` requires a `dataType`, whose last grammar alternative is a bare `qualifiedName` (entity reference), so any word matches and the visitor maps it to `TypeEnumeration`. The executor wrote it with no enumeration behind it. CREATE ENTITY had guarded this since #552; the guard was **inline in the create handler** and never applied to MODIFY | `mdl/executor/attribute_type_ref.go` (new), `mdl/executor/cmd_entities.go` (`AlterEntityModifyAttribute` case) | Extract the create-path guard to `validateAttributeTypeRef` and call it from the modify path **before** mutating the attribute. Add `DROP DEFAULT ON ATTRIBUTE ` so clearing a default has a spelling that does not go through the type slot, and point the refusal at it. Check `mxcli syntax` too: the topic documented `MODIFY ATTRIBUTE AttrName SET DEFAULT val` — the corrupting form — which is how the reporter got there. When a guard is added inline to one handler, grep for sibling handlers that take the same input. mendixlabs/mxcli#910 | | A skill claims a construct is unsupported that the grammar accepts (and documents the working form elsewhere in the same file) — an agent writes valid MDL, hits the contradicting claim, and undoes it. Seen for `case … end case` in `write-microflows.md` (supported at §CASE Statements, "not supported" under UNSUPPORTED Syntax), with `check-syntax.md`, `MDL_QUICK_REFERENCE.md`, three `docs-site` pages and the generated project CLAUDE.md (a third shape, `CASE $Var/Attr AS x WHEN Enum.Value`) all disagreeing | Nothing validates prose claims against the grammar. `scripts/check-skill-mdl.sh` extracts fenced blocks but deliberately **skips microflow bodies** (fragment-heavy), so every syntax claim about microflow statements is unguarded. The wrong "WRONG:" example also used string-literal case values, which fail for their own reason — making the claim look confirmed | Docs only: `.claude/skills/mendix/write-microflows.md` + `check-syntax.md`, `docs/01-project/MDL_QUICK_REFERENCE.md`, `docs-site/src/language/control-flow.md` + `appendixes/{quick-reference,common-mistakes}.md`, `cmd/mxcli/init_claudemd.go` (the generated table) | Settle it against the binary before editing, one variant per claim (`bin/mxcli check` on the documented form, the quoted form, the qualified form, the `AS` form, the `else` form) — a "WRONG" example that fails for an unrelated reason is not evidence. Then fix **every** surface: grep `end case\|END CASE` across `.claude/`, `docs/`, `docs-site/` and `cmd/mxcli/*.go`, not just the file in the report (7 surfaces, the issue named 3). Pin both directions in `mdl-examples/bug-tests/` — `907-case-enum-split-is-supported.mdl` (must pass) and `907-case-enum-split-invalid-forms.fail.mdl` (must fail) — so `make check-mdl` catches drift back. Issue #907 | | Docs disagree on whether an enumeration may be written as a string literal — the skills say "NEVER use a string literal", docs-site uses `Status = 'Draft'` in a dozen CREATE/CHANGE examples and calls it "plain strings when the context is unambiguous". Following either one wholesale is wrong, and `mxcli check` passes **both** the good and the bad form, so the build is where you find out | It is context-dependent and nobody had measured it. A string is accepted wherever the slot is **already enum-typed**; in an **expression** it is a String compared to an Enumeration | Docs only: `.claude/skills/mendix/write-microflows.md` (Enumeration Comparisons), `docs-site/src/language/{control-flow,expressions}.md`, `docs-site/src/reference/microflow/create-microflow.md` | Measure per context on a real project before touching a single example — one microflow per construct, `mxcli docker check` read per construct (`mxcli new EP --version 11.13.0 --skip-build`, then `exec` + `docker check`). Verified on 11.13.0: **CE0117** for `if $O/Status = 'Draft'` and for `'x' + $O/Status`; **clean** for `change`/`create` member values, attribute `DEFAULT 'Draft'`, XPath `[Status = 'Draft']`, and `getCaption()`/`toString()` in a concat. So fix **only** comparisons and concatenations — a sweep-and-replace over every `Status = '…'` would have "corrected" ~10 of 12 sites that are fine. No `.fail.mdl` is possible (check passes the failing form; the type checker is still a proposal — `docs/11-proposals/PROPOSAL_expression_type_checking.md`), so the verdicts live in `mdl-examples/bug-tests/enum-string-literal-contexts.mdl`, which pins the ACCEPTED forms and builds clean (control: adding it to a probe project left the error count unchanged at the 2 deliberate failures) | +| `mxcli exec` applies a script that `mxcli check` rejects with an error — e.g. a page written with an invalid widget property, which is then silently dropped, so the widget renders but does nothing | `exec` ran no semantic checks at all. Parse errors stopped it; the ~24 `Validate*` checks lived inline in `cmd/mxcli/cmd_check.go` and had exactly one caller. "Run check before exec" was a convention nothing enforced, on a tool built for unattended agent use | `mdl/executor/validate_program.go` (the shared list), `cmd/mxcli/cmd_exec.go` (the gate), `cmd/mxcli/cmd_check.go` | `executor.ValidateProgram` is now the single definition of what `check` checks, and `exec` refuses to run when it reports an **error** (warnings print and do not block). `--no-check` opts out. **A check wired into nothing is invisible** — it reports no violations and reads as a clean project — so `TestValidateProgram_WiresEveryWholeProgramValidator` asserts structurally that every exported `Validate*(prog *ast.Program, …)` is called from `ValidateProgram`; per-statement helpers take a statement and are excluded. When adding a check, add it to `ValidateProgram` and both commands get it | diff --git a/.claude/skills/mendix/check-syntax.md b/.claude/skills/mendix/check-syntax.md index 1a983454a..01e05a7b4 100644 --- a/.claude/skills/mendix/check-syntax.md +++ b/.claude/skills/mendix/check-syntax.md @@ -9,6 +9,28 @@ This skill ensures MDL scripts are validated before presenting them to users or - Executing MDL scripts via `mxcli exec` - Committing MDL files to version control +## `exec` refuses what `check` rejects + +`mxcli exec` runs the same semantic checks before writing anything. A script whose +checks report an **error** is not executed at all — nothing is written — because +`exec` applies statements one at a time and cannot roll back, so a known-bad +script would leave the model partly updated. Warnings are printed and do not stop +the run. + +```bash +mxcli exec script.mdl -p app.mpr # checked, then applied +mxcli exec script.mdl -p app.mpr --no-check # applied regardless +``` + +This does **not** replace running `check` yourself. `check` is faster, needs no +write connection, and reports the warnings worth reading before you commit to a +run. What the gate guarantees is narrower and still valuable: a script that slips +past you cannot half-apply. + +It also does not mean the script is *correct*. `mxcli check` validates MDL syntax +and mxcli's own rules; it does not validate the Mendix model. Run +`mx check` (or `mxcli docker check -p app.mpr`) after applying a slice. + ## Pre-Flight Validation Checklist Before writing any MDL, verify these requirements: diff --git a/cmd/mxcli/cmd_check.go b/cmd/mxcli/cmd_check.go index d311d9444..2d6e506f4 100644 --- a/cmd/mxcli/cmd_check.go +++ b/cmd/mxcli/cmd_check.go @@ -7,7 +7,6 @@ import ( "os" "strings" - "github.com/mendixlabs/mxcli/mdl/ast" "github.com/mendixlabs/mxcli/mdl/executor" "github.com/mendixlabs/mxcli/mdl/linter" "github.com/mendixlabs/mxcli/mdl/visitor" @@ -105,129 +104,10 @@ Examples: fmt.Printf("✓ Syntax OK (%d statements)\n", len(prog.Statements)) } - // Validate statements (doesn't require project connection) - var violations []linter.Violation - for _, stmt := range prog.Statements { - // Check enumeration values for reserved words - if enumStmt, ok := stmt.(*ast.CreateEnumerationStmt); ok { - violations = append(violations, executor.ValidateEnumeration(enumStmt)...) - } - // Check entity attributes for reserved system names - if entityStmt, ok := stmt.(*ast.CreateEntityStmt); ok { - violations = append(violations, executor.ValidateEntity(entityStmt)...) - } - // Apply the same per-attribute checks to ALTER ENTITY ADD ATTRIBUTE - if alterStmt, ok := stmt.(*ast.AlterEntityStmt); ok { - violations = append(violations, executor.ValidateAlterEntity(alterStmt)...) - } - // Check microflow body for common issues - if mfStmt, ok := stmt.(*ast.CreateMicroflowStmt); ok { - violations = append(violations, executor.ValidateMicroflow(mfStmt)...) - } - // Check workflow for constructs MxBuild rejects (missing page, - // single-outcome-with-activities, invalid decision outcome names) - if wfStmt, ok := stmt.(*ast.CreateWorkflowStmt); ok { - violations = append(violations, executor.ValidateWorkflow(wfStmt)...) - } - // Check GRANT for member rights Mendix cannot store - if grantStmt, ok := stmt.(*ast.GrantEntityAccessStmt); ok { - violations = append(violations, executor.ValidateGrantEntityAccess(grantStmt)...) - } - // Check typed ALTER SETTINGS / CREATE CONFIGURATION property values - if setStmt, ok := stmt.(*ast.AlterSettingsStmt); ok { - violations = append(violations, executor.ValidateSettings(setStmt)...) - } - if cfgStmt, ok := stmt.(*ast.CreateConfigurationStmt); ok { - violations = append(violations, executor.ValidateCreateConfiguration(cfgStmt)...) - } - // Check database connection credentials: a literal where Mendix - // stores a constant reference writes an UNOPENABLE project. - if dbStmt, ok := stmt.(*ast.CreateDatabaseConnectionStmt); ok { - violations = append(violations, executor.ValidateDatabaseConnection(dbStmt)...) - } - // Check view entity OQL - if viewStmt, ok := stmt.(*ast.CreateViewEntityStmt); ok { - if viewStmt.Query.RawQuery != "" { - violations = append(violations, executor.ValidateOQLSyntax(viewStmt.Query.RawQuery)...) - violations = append(violations, executor.ValidateOQLTypes(viewStmt.Query.RawQuery, viewStmt.Attributes)...) - } - } - } - - // Check for intra-script duplicate definitions (CREATE X … CREATE X without DROP) - violations = append(violations, executor.CheckScriptDuplicates(prog)...) - - // Validate design properties against the project's theme registry - // (themesource design-properties.json) — flags unknown keys and invalid - // option values, listing the allowed values. Only runs with --project. - violations = append(violations, executor.ValidateDesignProperties(prog, projectPath)...) - - // Validate pluggable widget properties against widget definitions — - // catches typos in property keys before MxBuild does. Uses built-in - // definitions alone when no project is given; with --project, also - // loads project-installed .def.json files for full coverage. - violations = append(violations, executor.ValidateWidgetProperties(prog, projectPath)...) - - // Warn (MPR010) when an edit/new form (a parameter-bound DataView) is not - // wrapped in a layout grid — its label/input widths only render correctly - // inside a layoutgrid. Same rule as the MPR010 lint rule, surfaced at - // authoring time on the AST. - violations = append(violations, executor.ValidatePageLayoutGrid(prog)...) - - // Flag control-bar buttons that pass $currentObject — a control bar is - // not row-scoped, so the argument is unbound (CE1571) at build time. - violations = append(violations, executor.ValidatePageButtonContext(prog)...) - - // Flag a database-connection TYPE Studio Pro does not offer. mxcli writes - // the string through and mxbuild does not check it, so a wrong value - // builds green and simply does not connect. - violations = append(violations, executor.ValidateDatabaseConnectionType(prog)...) - - // Flag OData property names nothing below will act on. The grammar takes - // any `name: value` pair, so a typo used to be discarded in silence and - // the model quietly lacked what the author asked for. - violations = append(violations, executor.ValidateODataProperties(prog)...) - - // Flag a microflow-backed OData resource whose read microflow cannot keep - // the promises the service makes for it. A read microflow has no - // System.HttpResponse parameter, so it cannot answer 400 — its contract - // has to be declared correctly up front, and nothing else checks that. - violations = append(violations, executor.ValidateODataReadContract(prog)...) - - // Flag `authentication microflow` with no microflow named. The grammar - // makes the name optional, so this parses and executes into a service - // Mendix refuses to build (CE0333). - violations = append(violations, executor.ValidateODataAuth(prog)...) - - // Flag two service shapes mxbuild rejects — a Path that breaks its - // slash rules, and the PublishAssociations mode whose name invites - // exactly the wrong value. A Path with no slash at all is the reason - // this is worth a check: mxbuild throws out of its own validator with - // no error code, so there is nothing to look up. - violations = append(violations, executor.ValidateODataServiceShape(prog)...) - - // Flag a page whose widgets point at a page created further down the same - // script. `exec` resolves page references in statement order and is not - // transactional, so this fails after earlier statements are already - // written. --references catches it too, but the ordering needs no project - // when the target is created by a plain CREATE (#9). - violations = append(violations, executor.ValidateScriptPageOrder(prog)...) - - // Flag a document-access GRANT naming a role from another module — Mendix - // rejects it with CE0148. Needs no project, so it runs here rather than - // under --references, where it would only fire with -p (#836). - violations = append(violations, executor.ValidateGrantRoles(prog)...) - - // Flag a REST client operation whose Body/Response mapping clause has no - // `{ ... }` body — Mendix cannot reference a mapping document from an - // operation, so the mapping would be dropped in silence (#843). - violations = append(violations, executor.ValidateRestClientMappings(prog)...) - - // Flag a scheduled event whose Repeat and fields disagree (a Multiplier on - // a Daily repeat, an HourOfDay of 99). Decidable from the statement, so it - // runs here rather than at exec, where the script would already have - // passed check. - violations = append(violations, executor.ValidateScheduledEvents(prog)...) + // Every semantic check lives in executor.ValidateProgram, so `mxcli exec` + // refuses exactly what `mxcli check` reports. Adding a check there gives + // both commands it at once. + violations := executor.ValidateProgram(prog, projectPath) if isStructured { // Always emit structured output (even when clean) diff --git a/cmd/mxcli/cmd_exec.go b/cmd/mxcli/cmd_exec.go index b40c15d22..03464a95d 100644 --- a/cmd/mxcli/cmd_exec.go +++ b/cmd/mxcli/cmd_exec.go @@ -8,6 +8,7 @@ import ( "os" "github.com/mendixlabs/mxcli/mdl/executor" + "github.com/mendixlabs/mxcli/mdl/linter" "github.com/mendixlabs/mxcli/mdl/visitor" "github.com/spf13/cobra" ) @@ -17,6 +18,12 @@ var execCmd = &cobra.Command{ Short: "Execute an MDL script file", Long: `Execute an MDL script file containing MDL commands. +Before anything is written, the script is put through the same semantic checks +as "mxcli check". If any of them reports an error, nothing is executed: exec +applies statements one at a time and cannot roll back, so running a script with +a known error leaves the model partly updated. Warnings are printed and do not +stop the run. Use --no-check to apply a script anyway. + By default execution stops at the first error. With --continue-on-error, every statement is attempted; each failure is reported (prefixed with its statement number) and execution continues, exiting non-zero if any statement failed. This @@ -31,6 +38,7 @@ Example: mxcli exec setup.mdl mxcli exec -p app.mpr script.mdl mxcli exec -p app.mpr script.mdl --continue-on-error + mxcli exec -p app.mpr script.mdl --no-check mxcli exec -p app.mpr - <<'EOF' SHOW STRUCTURE DEPTH 1; EOF @@ -40,6 +48,7 @@ Example: filePath := args[0] projectPath, _ := cmd.Flags().GetString("project") continueOnError, _ := cmd.Flags().GetBool("continue-on-error") + skipCheck, _ := cmd.Flags().GetBool("no-check") // Read the script (a path, or "-" for stdin) content, err := readMDLSource(filePath) @@ -73,6 +82,27 @@ Example: os.Exit(1) } + // Pre-flight: refuse a script whose semantic checks report an error, + // rather than writing part of it and leaving the model to mxbuild. + // exec is not transactional, so "run it and see" means a half-applied + // model. Warnings are printed and do not stop the run. + if !skipCheck { + violations := executor.ValidateProgram(prog, projectPath) + if len(violations) > 0 { + formatter := linter.GetFormatter(linter.OutputFormatText, true) + formatter.Format(violations, os.Stderr) + } + if summary := linter.Summarize(violations); summary.Errors > 0 { + fmt.Fprintf(os.Stderr, + "\nRefusing to execute: %d error(s) above. Nothing was written.\n"+ + " exec applies statements one at a time and cannot roll back, so a script\n"+ + " with a known error would leave the model partly updated.\n"+ + " Fix them, or re-run with --no-check to apply the script anyway.\n", + summary.Errors) + os.Exit(1) + } + } + if continueOnError { res, err := exec.ExecuteProgramContinueOnError(prog, os.Stderr) if err != nil && !errors.Is(err, executor.ErrExit) { @@ -97,6 +127,8 @@ Example: } func init() { + execCmd.Flags().Bool("no-check", false, + "Skip the pre-flight semantic checks and apply the script even if mxcli check would report errors") execCmd.Flags().Bool("continue-on-error", false, "Run every statement, reporting each failure instead of halting at the first (exits non-zero if any failed) — makes a partially-applied script re-runnable") } diff --git a/mdl/executor/validate_program.go b/mdl/executor/validate_program.go new file mode 100644 index 000000000..d0f74a52e --- /dev/null +++ b/mdl/executor/validate_program.go @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/linter" +) + +// ValidateProgram runs every semantic check mxcli can make from the parsed +// script alone, plus the few that additionally consult a project when one is +// given. It is the single definition of "what `mxcli check` checks". +// +// It exists as one function because it has two callers that must not diverge: +// `mxcli check`, which reports the violations, and `mxcli exec`, which refuses +// to apply a script whose violations include an error. Those were previously +// unconnected — `check` held this list inline and `exec` ran none of it — so a +// script `check` rejected was applied by `exec` anyway (mxcli-banking findings, +// slice 2: "a page with an invalid widget property was written to the model"). +// Keeping the list in one place is what makes "check before exec" enforceable +// rather than a convention. +// +// projectPath may be empty; the checks that need a project skip themselves. +// Parse errors are the caller's business — this operates on a built program. +func ValidateProgram(prog *ast.Program, projectPath string) []linter.Violation { + // Statement-level checks that need no project connection. + var violations []linter.Violation + for _, stmt := range prog.Statements { + // Check enumeration values for reserved words + if enumStmt, ok := stmt.(*ast.CreateEnumerationStmt); ok { + violations = append(violations, ValidateEnumeration(enumStmt)...) + } + // Check entity attributes for reserved system names + if entityStmt, ok := stmt.(*ast.CreateEntityStmt); ok { + violations = append(violations, ValidateEntity(entityStmt)...) + } + // Apply the same per-attribute checks to ALTER ENTITY ADD ATTRIBUTE + if alterStmt, ok := stmt.(*ast.AlterEntityStmt); ok { + violations = append(violations, ValidateAlterEntity(alterStmt)...) + } + // Check microflow body for common issues + if mfStmt, ok := stmt.(*ast.CreateMicroflowStmt); ok { + violations = append(violations, ValidateMicroflow(mfStmt)...) + } + // Check workflow for constructs MxBuild rejects (missing page, + // single-outcome-with-activities, invalid decision outcome names) + if wfStmt, ok := stmt.(*ast.CreateWorkflowStmt); ok { + violations = append(violations, ValidateWorkflow(wfStmt)...) + } + // Check GRANT for member rights Mendix cannot store + if grantStmt, ok := stmt.(*ast.GrantEntityAccessStmt); ok { + violations = append(violations, ValidateGrantEntityAccess(grantStmt)...) + } + // Check typed ALTER SETTINGS / CREATE CONFIGURATION property values + if setStmt, ok := stmt.(*ast.AlterSettingsStmt); ok { + violations = append(violations, ValidateSettings(setStmt)...) + } + if cfgStmt, ok := stmt.(*ast.CreateConfigurationStmt); ok { + violations = append(violations, ValidateCreateConfiguration(cfgStmt)...) + } + // Check database connection credentials: a literal where Mendix + // stores a constant reference writes an UNOPENABLE project. + if dbStmt, ok := stmt.(*ast.CreateDatabaseConnectionStmt); ok { + violations = append(violations, ValidateDatabaseConnection(dbStmt)...) + } + // Check view entity OQL + if viewStmt, ok := stmt.(*ast.CreateViewEntityStmt); ok { + if viewStmt.Query.RawQuery != "" { + violations = append(violations, ValidateOQLSyntax(viewStmt.Query.RawQuery)...) + violations = append(violations, ValidateOQLTypes(viewStmt.Query.RawQuery, viewStmt.Attributes)...) + } + } + } + + // Check for intra-script duplicate definitions (CREATE X … CREATE X without DROP) + violations = append(violations, CheckScriptDuplicates(prog)...) + + // Validate design properties against the project's theme registry + // (themesource design-properties.json) — flags unknown keys and invalid + // option values, listing the allowed values. Only runs with --project. + violations = append(violations, ValidateDesignProperties(prog, projectPath)...) + + // Validate pluggable widget properties against widget definitions — + // catches typos in property keys before MxBuild does. Uses built-in + // definitions alone when no project is given; with --project, also + // loads project-installed .def.json files for full coverage. + violations = append(violations, ValidateWidgetProperties(prog, projectPath)...) + + // Warn (MPR010) when an edit/new form (a parameter-bound DataView) is not + // wrapped in a layout grid — its label/input widths only render correctly + // inside a layoutgrid. Same rule as the MPR010 lint rule, surfaced at + // authoring time on the AST. + violations = append(violations, ValidatePageLayoutGrid(prog)...) + + // Flag control-bar buttons that pass $currentObject — a control bar is + // not row-scoped, so the argument is unbound (CE1571) at build time. + violations = append(violations, ValidatePageButtonContext(prog)...) + + // Flag a database-connection TYPE Studio Pro does not offer. mxcli writes + // the string through and mxbuild does not check it, so a wrong value + // builds green and simply does not connect. + violations = append(violations, ValidateDatabaseConnectionType(prog)...) + + // Flag OData property names nothing below will act on. The grammar takes + // any `name: value` pair, so a typo used to be discarded in silence and + // the model quietly lacked what the author asked for. + violations = append(violations, ValidateODataProperties(prog)...) + + // Flag a microflow-backed OData resource whose read microflow cannot keep + // the promises the service makes for it. A read microflow has no + // System.HttpResponse parameter, so it cannot answer 400 — its contract + // has to be declared correctly up front, and nothing else checks that. + violations = append(violations, ValidateODataReadContract(prog)...) + + // Flag `authentication microflow` with no microflow named. The grammar + // makes the name optional, so this parses and executes into a service + // Mendix refuses to build (CE0333). + violations = append(violations, ValidateODataAuth(prog)...) + + // Flag two service shapes mxbuild rejects — a Path that breaks its + // slash rules, and the PublishAssociations mode whose name invites + // exactly the wrong value. A Path with no slash at all is the reason + // this is worth a check: mxbuild throws out of its own validator with + // no error code, so there is nothing to look up. + violations = append(violations, ValidateODataServiceShape(prog)...) + + // Flag a page whose widgets point at a page created further down the same + // script. `exec` resolves page references in statement order and is not + // transactional, so this fails after earlier statements are already + // written. --references catches it too, but the ordering needs no project + // when the target is created by a plain CREATE (#9). + violations = append(violations, ValidateScriptPageOrder(prog)...) + + // Flag a document-access GRANT naming a role from another module — Mendix + // rejects it with CE0148. Needs no project, so it runs here rather than + // under --references, where it would only fire with -p (#836). + violations = append(violations, ValidateGrantRoles(prog)...) + + // Flag a REST client operation whose Body/Response mapping clause has no + // `{ ... }` body — Mendix cannot reference a mapping document from an + // operation, so the mapping would be dropped in silence (#843). + violations = append(violations, ValidateRestClientMappings(prog)...) + + // Flag a scheduled event whose Repeat and fields disagree (a Multiplier on + // a Daily repeat, an HourOfDay of 99). Decidable from the statement, so it + // runs here rather than at exec, where the script would already have + // passed check. + violations = append(violations, ValidateScheduledEvents(prog)...) + + return violations +} diff --git a/mdl/executor/validate_program_test.go b/mdl/executor/validate_program_test.go new file mode 100644 index 000000000..dd253c313 --- /dev/null +++ b/mdl/executor/validate_program_test.go @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "strings" + "testing" + + mdlast "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/linter" + "github.com/mendixlabs/mxcli/mdl/visitor" +) + +func build(t *testing.T, src string) *mdlast.Program { + t.Helper() + prog, errs := visitor.Build(src) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + return prog +} + +func ruleSeverities(vs []linter.Violation) map[string]linter.Severity { + out := map[string]linter.Severity{} + for _, v := range vs { + out[v.RuleID] = v.Severity + } + return out +} + +// TestValidateProgram_ReportsWidgetPropertyError is the case from the +// mxcli-banking report: a widget property the widget does not have. `mxcli check` +// reported it as an error and `mxcli exec` applied the script anyway, writing a +// page whose property was silently dropped — a picker that does nothing. +// +// Both commands now run this one function, so the error `check` prints is exactly +// the error `exec` refuses on. +func TestValidateProgram_ReportsWidgetPropertyError(t *testing.T) { + prog := build(t, ` +create page M.P (title: 'P', layout: Atlas_Core.Atlas_Default) +{ + listview lv (datasource: database M.E) { + combobox cmbX (label: 'X', attribute: Name, onChangeEvent: 'nope') + } +}`) + + got := ruleSeverities(ValidateProgram(prog, "")) + sev, found := got["MDL-WIDGET01"] + if !found { + t.Fatalf("no MDL-WIDGET01 for an unknown widget property; got %v", got) + } + if sev != linter.SeverityError { + t.Errorf("MDL-WIDGET01 severity = %v, want error — exec only refuses on errors", sev) + } +} + +// A clean script must produce no errors, or the gate would block correct work. +func TestValidateProgram_CleanScriptHasNoErrors(t *testing.T) { + prog := build(t, ` +create module M; +create persistent entity M.Thing ( Name: String(100) ); +`) + for _, v := range ValidateProgram(prog, "") { + if v.Severity == linter.SeverityError { + t.Errorf("clean script produced an error: %s %s", v.RuleID, v.Message) + } + } +} + +// TestValidateProgram_WiresEveryWholeProgramValidator is the drift guard. +// +// ValidateProgram was extracted from an inline block in cmd/mxcli/cmd_check.go so +// that `exec` could run the same checks. The failure mode it replaces is a check +// that exists but is wired into nothing — which is invisible, because a check that +// never runs reports no violations and looks like a clean project. +// +// The rule is structural rather than a hand-kept list: any exported +// Validate(prog *ast.Program, …) in this package is a whole-program +// check and must be called from ValidateProgram. Per-statement helpers +// (ValidateMicroflowBody, ValidateWidgetPropertiesForStatement, …) take a +// statement instead and are deliberately excluded — they are called from inside +// the whole-program validators and from the LSP. +func TestValidateProgram_WiresEveryWholeProgramValidator(t *testing.T) { + body, err := os.ReadFile("validate_program.go") + if err != nil { + t.Fatalf("read validate_program.go: %v", err) + } + wired := string(body) + + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + checked := 0 + + for _, path := range files { + if strings.HasSuffix(path, "_test.go") || path == "validate_program.go" { + continue + } + f, err := parser.ParseFile(fset, path, nil, 0) + if err != nil { + t.Fatalf("parse %s: %v", path, err) + } + for _, decl := range f.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Recv != nil || !strings.HasPrefix(fn.Name.Name, "Validate") || !fn.Name.IsExported() { + continue + } + if len(fn.Type.Params.List) == 0 || !isProgramParam(fn.Type.Params.List[0].Type) { + continue // per-statement helper, not a whole-program check + } + checked++ + if !strings.Contains(wired, fn.Name.Name+"(") { + t.Errorf("%s (%s) takes a *ast.Program but ValidateProgram never calls it — "+ + "the check would never run for `mxcli check` or `mxcli exec`", fn.Name.Name, path) + } + } + } + + // Guard the guard: if the signature heuristic ever stops matching anything, + // this test would pass while checking nothing. + if checked < 10 { + t.Fatalf("only matched %d whole-program validators; the detection is broken", checked) + } +} + +// isProgramParam reports whether a parameter type is *ast.Program (as written in +// the executor sources, i.e. a starred selector ending in ".Program"). +func isProgramParam(expr ast.Expr) bool { + star, ok := expr.(*ast.StarExpr) + if !ok { + return false + } + sel, ok := star.X.(*ast.SelectorExpr) + return ok && sel.Sel.Name == "Program" +} From c464f18d34974dad8b77b3a58979df3ccc6094d1 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 19:12:44 +0000 Subject: [PATCH 13/35] fix(pages): write OnChange on every input widget, not just textbox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An `OnChange:` authored on a checkbox, radiobuttons, combobox, dropdown, textarea or datepicker parsed cleanly, passed `mxcli check`, and executed with a success report — while the property was never written. In the browser the control changed state and issued zero /xas/ requests. Three independent drops, any one of which was sufficient: 1. The page builder read `w.GetOnChange()` in `buildTextBoxV3` only, so the property died between the AST and the model for the other five. All six now route through one `applyOnChangeV3` helper, so a widget added later cannot forget it. 2. The legacy writer hardcoded `serializeClientAction(nil)` for TextArea, DatePicker, CheckBox, RadioButtons and DropDown — a populated action would not have reached BSON under `--engine legacy` either. 3. For pluggable widgets `actionSourceForKey` matched only the bare keys `onClick`/`onChange`, but Mendix's Combobox names its slot `onChangeEvent`, so no action mapping was emitted at all. One Event/Action suffix is now stripped before matching — narrowly, so the Combobox's `onChangeFilterInputEvent` and `onChangeDatabaseEvent` (distinct properties with no MDL surface) stay unmapped rather than having one `OnChange:` write three actions. The shipped combobox.def.json overrides any .mpk-derived def, so the generator fix alone would never have reached a real combobox; the mapping is added there too, in both modes (modes are exclusive). Generator version bumped 14 → 15 so existing projects regenerate. DESCRIBE is wired at the same time for textarea / datepicker / checkbox / radiobuttons: DESCRIBE is how this class of drop gets found, and it could only see OnChange on textbox. (`dropdown` has no DESCRIBE case at all — a separate gap, left alone.) Control: reverting the combobox def alone fails TestCombobox_OnChangeMappedInBothModes with the reported symptom. Reported as FINDINGS #14 in ako/mxcli-owid. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017s476QkXr9CFMvKspVzcvu --- .claude/skills/fix-issue.md | 1 + .../bug-tests/widget-onchange-dropped.mdl | 95 +++++++++++++ .../cmd_pages_builder_onchange_test.go | 75 ++++++++++ mdl/executor/cmd_pages_builder_v3_widgets.go | 53 ++++++- .../cmd_pages_describe_onchange_test.go | 55 ++++++++ mdl/executor/cmd_pages_describe_output.go | 12 ++ mdl/executor/cmd_pages_describe_parse.go | 4 + mdl/executor/widget_defs.go | 12 +- mdl/executor/widget_engine.go | 7 +- mdl/executor/widget_onchange_event_test.go | 131 ++++++++++++++++++ mdl/executor/widget_registry_test.go | 10 +- .../widgets/definitions/combobox.def.json | 6 +- sdk/mpr/writer_widgets_input.go | 10 +- sdk/widgets/definitions/combobox.def.json | 6 +- 14 files changed, 456 insertions(+), 21 deletions(-) create mode 100644 mdl-examples/bug-tests/widget-onchange-dropped.mdl create mode 100644 mdl/executor/cmd_pages_builder_onchange_test.go create mode 100644 mdl/executor/cmd_pages_describe_onchange_test.go create mode 100644 mdl/executor/widget_onchange_event_test.go diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index 14828122a..a532f6304 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -526,3 +526,4 @@ extracting `OffsetExpression`/`LimitExpression`. | `mx check` exits with `System.ArgumentOutOfRangeException: length ('-1')` and no error code | an OData service `Path` with no slash in it; mxbuild throws out of its own validator | `mdl/executor/validate_odata_service_shape.go` | Path must not start with `/` and must end with one (CE6550/CE6552). MDL-ODATA05 catches all three | | Authoring a pluggable widget with an **object list** (Accordion group, Pop-up menu item, chart series) raises **CE0463** "the definition of this widget has changed" on the widget itself, on a widget created seconds ago against the current package | An object-list item's **required TextTemplate** that the author left unset was serialized as `null`. `emptyClientTemplateRules` in `widgetobj/builder.go` is a hardcoded per-widget table covering only DataGrid columns, so every other object-list widget fell through it. Required-ness is genuinely absent from the widget XML for these properties — and the widget schema **defaults `required` to true** — so the Accordion's `headerText` is mandatory even though nothing says so | `mdl/backend/widgetobj/builder.go` (`isUnsetRequiredTextTemplate`, `buildDefaultTextClientTemplateProperty`), plus the entry plumbing in `mdl/types/widget_property_type.go`, `modelsdk/widgets/loader.go`, `sdk/pages/pages_widgets_advanced.go`, `mdl/backend/modelsdk/widget_pluggable_write.go` (`convertPropTypeIDs`) | Serialize a required unset TextTemplate with the widget's **shipped ``**, read off the template ValueType's `Translations` beside the `Required` flag. **Both weaker forms were measured and fail**: `null` is CE0463, and an *empty* `Forms$ClientTemplate` is **CE4899** "Property 'Groups/1/Text' is required" — so the intuitive "emit an empty template" fix only moves the error. Populating is what `mx update-widgets` itself writes (verified: mxcli and the reference now emit the same `'Header'`/`'Koptekst'`). Scope it to **required** properties: filling every TextTemplate is the documented way to take CE0463 from 33 to 127. Note `PropertyTypeIDEntry` exists in THREE places (`mdl/types` — canonical and aliased by `modelsdk/widgets`; `sdk/pages` — what the builder consumes; `sdk/widgets` — the legacy engine's own), so a field added for this must be carried through `convertPropTypeIDs` or it silently never arrives. Repro `mdl-examples/bug-tests/891-accordion-group-nested-widgets.mdl`. Issue #891 | | `ALTER ENTITY … MODIFY ATTRIBUTE X SET DEFAULT ` (or any unrecognised word in the type position, including a typo like `Integr`) silently retypes the attribute to `Enumeration()`, and the project then **cannot be loaded at all** — `mx check` dies before validating with `System.ArgumentNullException: Value cannot be null. (Parameter 'value') at EnumerationAttributeType.set_EnumerationId` | `MODIFY ATTRIBUTE` requires a `dataType`, whose last grammar alternative is a bare `qualifiedName` (entity reference), so any word matches and the visitor maps it to `TypeEnumeration`. The executor wrote it with no enumeration behind it. CREATE ENTITY had guarded this since #552; the guard was **inline in the create handler** and never applied to MODIFY | `mdl/executor/attribute_type_ref.go` (new), `mdl/executor/cmd_entities.go` (`AlterEntityModifyAttribute` case) | Extract the create-path guard to `validateAttributeTypeRef` and call it from the modify path **before** mutating the attribute. Add `DROP DEFAULT ON ATTRIBUTE ` so clearing a default has a spelling that does not go through the type slot, and point the refusal at it. Check `mxcli syntax` too: the topic documented `MODIFY ATTRIBUTE AttrName SET DEFAULT val` — the corrupting form — which is how the reporter got there. When a guard is added inline to one handler, grep for sibling handlers that take the same input. mendixlabs/mxcli#910 | +| An `OnChange:` on a `checkbox`, `radiobuttons`, `combobox`, `dropdown`, `textarea` or `datepicker` does nothing. The MDL parses, `mxcli check` passes, `exec` reports success, and `describe page` shows the widget with its Label + Attribute and **no OnChange**. In the browser the control changes state and issues **zero `/xas/` requests** — no server round-trip at all. `textbox` and `actionbutton` keep their actions, so it reads as a page problem rather than a per-widget one | Three independent drops, any one of which is sufficient: (1) the page builder read `w.GetOnChange()` in `buildTextBoxV3` **only**, so the property died between AST and model for the other five; (2) the legacy writer hardcoded `serializeClientAction(nil)` for TextArea/DatePicker/CheckBox/RadioButtons/DropDown; (3) for pluggable widgets `actionSourceForKey` matched only the bare keys `onClick`/`onChange`, but Mendix's Combobox names its slot **`onChangeEvent`**, so no action mapping was emitted — and the shipped `combobox.def.json`, which **overrides any .mpk-derived def**, had none either | `mdl/executor/cmd_pages_builder_v3_widgets.go` (`applyOnChangeV3`), `sdk/mpr/writer_widgets_input.go`, `mdl/executor/widget_defs.go` (`actionSourceForKey`), `sdk/widgets/definitions/combobox.def.json` + `modelsdk/widgets/definitions/combobox.def.json`, `mdl/executor/cmd_pages_describe_parse.go` + `_output.go` | Route every input widget's OnChange through one helper rather than repeating the block, so the next widget added cannot forget it. Strip **one** `Event`/`Action` suffix before matching an action slot — narrowly: a Combobox also carries `onChangeFilterInputEvent` and `onChangeDatabaseEvent`, which have no MDL surface and must stay unmapped or one `OnChange:` writes three actions. A hand-written built-in `.def.json` beats the generator, so fixing the generator alone changes nothing for COMBOBOX/GALLERY/DATAGRID/filters — patch both, and put the mapping in **every** mode (modes are exclusive). Wire DESCRIBE at the same time: DESCRIBE is how a dropped property gets *found*, and it could only see this one on `textbox`. `dropdown` has no DESCRIBE case at all (separate gap). Repro `mdl-examples/bug-tests/widget-onchange-dropped.mdl`. FINDINGS #14 | diff --git a/mdl-examples/bug-tests/widget-onchange-dropped.mdl b/mdl-examples/bug-tests/widget-onchange-dropped.mdl new file mode 100644 index 000000000..45a05d050 --- /dev/null +++ b/mdl-examples/bug-tests/widget-onchange-dropped.mdl @@ -0,0 +1,95 @@ +-- ============================================================================ +-- OnChange was silently dropped on every input widget except textbox +-- ============================================================================ +-- +-- Symptom (before fix): +-- `OnChange: MICROFLOW Mod.ACT_Apply(...)` on a checkbox, radiobuttons, +-- combobox, dropdown, textarea or datepicker parsed cleanly, `mxcli check` +-- passed, and `exec` reported success — but the property was never written. +-- `describe page` showed the widget with its Label and Attribute and no +-- OnChange, and in the browser clicking the control produced zero /xas/ +-- requests: no server round-trip at all. `textbox` and `actionbutton` kept +-- their actions, which is what made the drop look like a page-level problem +-- rather than a per-widget one. +-- +-- Root cause (three layers, each independently sufficient): +-- 1. The page builder read `w.GetOnChange()` in `buildTextBoxV3` only. The +-- other five builders never looked at the property, so it died between the +-- AST and the model. +-- 2. The legacy writer hardcoded `serializeClientAction(nil)` for TextArea, +-- DatePicker, CheckBox, RadioButtons and DropDown, so even a populated +-- OnChangeAction would not have reached BSON under `--engine legacy`. +-- 3. For pluggable widgets, `actionSourceForKey` matched only the bare keys +-- `onClick`/`onChange`. Mendix's Combobox names its slot `onChangeEvent`, +-- so no action mapping was emitted for it at all — and the shipped +-- combobox.def.json (which overrides any .mpk-derived one) had none either. +-- +-- After fix: +-- All six input widgets route OnChange through `applyOnChangeV3`; the legacy +-- writer serializes the real action; `actionSourceForKey` strips one +-- Event/Action suffix (narrowly — Combobox's `onChangeFilterInputEvent` and +-- `onChangeDatabaseEvent` stay unmapped); and combobox.def.json maps +-- `onChangeEvent` in BOTH modes, since modes are exclusive. +-- DESCRIBE also reads and renders OnChange for textarea / datepicker / +-- checkbox / radiobuttons, so the round trip can confirm itself. +-- +-- Verify: +-- mxcli exec mdl-examples/bug-tests/widget-onchange-dropped.mdl -p app.mpr +-- mxcli -p app.mpr -c "describe page Ctrl.ControlPanel" +-- -> every control below must show its `OnChange:`. +-- ============================================================================ + +create module Ctrl; +/ +create non-persistent entity Ctrl.Filter ( + Enabled: Boolean, + Topic: String, + Notes: String, + AsOf: DateTime +); +/ +create microflow Ctrl.ACT_Apply ($Filter: Ctrl.Filter) +begin + change $Filter (Notes = 'applied') refresh; +end; +/ +-- Each control below carries an OnChange. Before the fix only a `textbox` kept +-- it; the other five wrote nothing and the page still built green. +create or replace page Ctrl.ControlPanel ( + layout: Atlas_Core.Atlas_Default +) { + layoutgrid lgMain { + row rowMain { + column colMain (desktopwidth: autofill) { + dataview dvFilter (datasource: microflow Ctrl.ACT_Apply) { + textbox txtTopic ( + Label: 'Topic', Attribute: Topic, + OnChange: MICROFLOW Ctrl.ACT_Apply(Filter: $currentObject) + ) + textarea taNotes ( + Label: 'Notes', Attribute: Notes, + OnChange: MICROFLOW Ctrl.ACT_Apply(Filter: $currentObject) + ) + datepicker dpAsOf ( + Label: 'As of', Attribute: AsOf, + OnChange: MICROFLOW Ctrl.ACT_Apply(Filter: $currentObject) + ) + checkbox cbEnabled ( + Label: 'Enabled', Attribute: Enabled, + OnChange: MICROFLOW Ctrl.ACT_Apply(Filter: $currentObject) + ) + radiobuttons rbTopic ( + Label: 'Topic', Attribute: Topic, + OnChange: MICROFLOW Ctrl.ACT_Apply(Filter: $currentObject) + ) + -- The pluggable case, and the one originally reported. Its slot is named + -- `onChangeEvent`, not `onChange`, which is why it needed a fix of its own. + combobox cboTopic ( + Label: 'Topic', Attribute: Topic, + OnChange: MICROFLOW Ctrl.ACT_Apply(Filter: $currentObject) + ) + } + } + } + } +} diff --git a/mdl/executor/cmd_pages_builder_onchange_test.go b/mdl/executor/cmd_pages_builder_onchange_test.go new file mode 100644 index 000000000..a1c154607 --- /dev/null +++ b/mdl/executor/cmd_pages_builder_onchange_test.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/sdk/pages" +) + +// mkOnChangeWidget builds a V3 widget of the given MDL type carrying an +// `OnChange:` action. `close` is used because it resolves without touching the +// backend — the point under test is whether the property survives the builder, +// not which action it is. +func mkOnChangeWidget(mdlType, name string) *ast.WidgetV3 { + return &ast.WidgetV3{ + Type: mdlType, + Name: name, + Properties: map[string]any{ + "Attribute": "Value", + "Label": "L", + "OnChange": &ast.ActionV3{Type: "close"}, + }, + } +} + +// onChangeOf returns the OnChangeAction of any input widget that has one. +func onChangeOf(t *testing.T, w pages.Widget) pages.ClientAction { + t.Helper() + switch x := w.(type) { + case *pages.TextBox: + return x.OnChangeAction + case *pages.TextArea: + return x.OnChangeAction + case *pages.DatePicker: + return x.OnChangeAction + case *pages.DropDown: + return x.OnChangeAction + case *pages.CheckBox: + return x.OnChangeAction + case *pages.RadioButtons: + return x.OnChangeAction + default: + t.Fatalf("widget type %T has no OnChangeAction field", w) + return nil + } +} + +// TestBuildWidgetV3_OnChangeSurvivesBuilder covers ledger #14: an `OnChange:` +// authored on checkbox / radiobuttons / dropdown / textarea / datepicker parsed +// and executed without error, but the builder never read it — the property was +// dropped between the AST and the model, so the rendered control produced no +// server round-trip at all. Only `textbox` read it. +func TestBuildWidgetV3_OnChangeSurvivesBuilder(t *testing.T) { + for _, mdlType := range []string{ + "textbox", "textarea", "datepicker", "dropdown", "checkbox", "radiobuttons", + } { + t.Run(mdlType, func(t *testing.T) { + mod := mkModule("Mod") + h := mkHierarchy(mod) + withContainer(h, mod.ID, mod.ID) + pb := newPageBuilder(&mock.MockBackend{}, h, "Mod") + + w, err := pb.buildWidgetV3(mkOnChangeWidget(mdlType, "w1")) + if err != nil { + t.Fatalf("buildWidgetV3(%s): %v", mdlType, err) + } + if act := onChangeOf(t, w); act == nil { + t.Fatalf("%s: OnChange was dropped by the builder (OnChangeAction is nil)", mdlType) + } + }) + } +} diff --git a/mdl/executor/cmd_pages_builder_v3_widgets.go b/mdl/executor/cmd_pages_builder_v3_widgets.go index 920a1f623..9a3d98e62 100644 --- a/mdl/executor/cmd_pages_builder_v3_widgets.go +++ b/mdl/executor/cmd_pages_builder_v3_widgets.go @@ -309,6 +309,26 @@ func (pb *pageBuilder) buildListViewV3(w *ast.WidgetV3) (*pages.ListView, error) return lv, nil } +// applyOnChangeV3 resolves a widget's `OnChange:` client action into dst. +// +// Every input widget carrying an OnChangeAction must call this. Before ledger +// #14 only `buildTextBoxV3` read GetOnChange(), so an OnChange authored on a +// checkbox / radiobuttons / dropdown / textarea / datepicker parsed, checked and +// executed clean while the property was dropped between the AST and the model — +// the control rendered correctly and produced no server round-trip at all. +func (pb *pageBuilder) applyOnChangeV3(w *ast.WidgetV3, dst *pages.ClientAction) error { + action := w.GetOnChange() + if action == nil { + return nil + } + act, err := pb.buildClientActionV3(action) + if err != nil { + return err + } + *dst = act + return nil +} + func (pb *pageBuilder) buildTextBoxV3(w *ast.WidgetV3) (*pages.TextBox, error) { tb := &pages.TextBox{ BaseWidget: pages.BaseWidget{ @@ -342,12 +362,8 @@ func (pb *pageBuilder) buildTextBoxV3(w *ast.WidgetV3) (*pages.TextBox, error) { } // Handle OnChange (the "On change" client action) - if action := w.GetOnChange(); action != nil { - act, err := pb.buildClientActionV3(action) - if err != nil { - return nil, err - } - tb.OnChangeAction = act + if err := pb.applyOnChangeV3(w, &tb.OnChangeAction); err != nil { + return nil, err } if err := pb.registerWidgetName(w.Name, tb.ID); err != nil { @@ -378,6 +394,11 @@ func (pb *pageBuilder) buildTextAreaV3(w *ast.WidgetV3) (*pages.TextArea, error) ta.Label = label } + // Handle OnChange (the "On change" client action) + if err := pb.applyOnChangeV3(w, &ta.OnChangeAction); err != nil { + return nil, err + } + if err := pb.registerWidgetName(w.Name, ta.ID); err != nil { return nil, err } @@ -406,6 +427,11 @@ func (pb *pageBuilder) buildDatePickerV3(w *ast.WidgetV3) (*pages.DatePicker, er dp.Label = label } + // Handle OnChange (the "On change" client action) + if err := pb.applyOnChangeV3(w, &dp.OnChangeAction); err != nil { + return nil, err + } + if err := pb.registerWidgetName(w.Name, dp.ID); err != nil { return nil, err } @@ -434,6 +460,11 @@ func (pb *pageBuilder) buildDropdownV3(w *ast.WidgetV3) (*pages.DropDown, error) dd.Label = label } + // Handle OnChange (the "On change" client action) + if err := pb.applyOnChangeV3(w, &dd.OnChangeAction); err != nil { + return nil, err + } + if err := pb.registerWidgetName(w.Name, dd.ID); err != nil { return nil, err } @@ -462,6 +493,11 @@ func (pb *pageBuilder) buildCheckBoxV3(w *ast.WidgetV3) (*pages.CheckBox, error) cb.Label = label } + // Handle OnChange (the "On change" client action) + if err := pb.applyOnChangeV3(w, &cb.OnChangeAction); err != nil { + return nil, err + } + if err := pb.registerWidgetName(w.Name, cb.ID); err != nil { return nil, err } @@ -487,6 +523,11 @@ func (pb *pageBuilder) buildRadioButtonsV3(w *ast.WidgetV3) (*pages.RadioButtons rb.AttributePath = pb.resolveAttributePath(attr) } + // Handle OnChange (the "On change" client action) + if err := pb.applyOnChangeV3(w, &rb.OnChangeAction); err != nil { + return nil, err + } + if err := pb.registerWidgetName(w.Name, rb.ID); err != nil { return nil, err } diff --git a/mdl/executor/cmd_pages_describe_onchange_test.go b/mdl/executor/cmd_pages_describe_onchange_test.go new file mode 100644 index 000000000..86cbb608e --- /dev/null +++ b/mdl/executor/cmd_pages_describe_onchange_test.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" +) + +// TestParseRawWidget_OnChangeOnEveryInputWidget covers the read half of ledger +// #14. DESCRIBE PAGE is how the dropped OnChange was found, and it could only +// find it on `textbox` — every other input widget parsed its Label and Attribute +// and ignored OnChangeAction, so a page that *did* carry the action still +// described as if it did not. That makes DESCRIBE unable to confirm its own +// round trip for these five widgets. +func TestParseRawWidget_OnChangeOnEveryInputWidget(t *testing.T) { +// `dropdown` is absent: DESCRIBE has no Forms$DropDown case at all, which is a +// separate gap from this one and is left alone here. + types := map[string]string{ + "textbox": "Forms$TextBox", + "textarea": "Forms$TextArea", + "datepicker": "Forms$DatePicker", + "checkbox": "Forms$CheckBox", + "radiobuttons": "Forms$RadioButtonGroup", + } + + for mdlName, bsonType := range types { + t.Run(mdlName, func(t *testing.T) { + ctx, _ := newMockCtx(t) + + raw := map[string]any{ + "$Type": bsonType, + "Name": "w1", + "OnChangeAction": map[string]any{ + "$Type": "Forms$MicroflowAction", + "MicroflowSettings": map[string]any{ + "$Type": "Forms$MicroflowSettings", + "Microflow": "MyFirstModule.ACT_Apply", + }, + }, + } + + got := parseRawWidget(ctx, raw) + if len(got) != 1 { + t.Fatalf("expected 1 widget, got %d", len(got)) + } + if got[0].OnChange == "" { + t.Fatalf("%s: OnChangeAction was not read back — DESCRIBE cannot see it", bsonType) + } + if !strings.Contains(got[0].OnChange, "ACT_Apply") { + t.Errorf("%s: OnChange = %q, want it to name the microflow", bsonType, got[0].OnChange) + } + }) + } +} diff --git a/mdl/executor/cmd_pages_describe_output.go b/mdl/executor/cmd_pages_describe_output.go index 8d783d966..fb84a79d0 100644 --- a/mdl/executor/cmd_pages_describe_output.go +++ b/mdl/executor/cmd_pages_describe_output.go @@ -461,6 +461,9 @@ func outputWidgetMDLV3(ctx *ExecContext, w rawWidget, indent int) { if w.Content != "" { props = append(props, fmt.Sprintf("Attribute: %s", w.Content)) } + if w.OnChange != "" { + props = append(props, fmt.Sprintf("OnChange: %s", w.OnChange)) + } props = appendAppearanceProps(props, w) formatWidgetProps(ctx.Output, prefix, header, props, "\n") @@ -473,6 +476,9 @@ func outputWidgetMDLV3(ctx *ExecContext, w rawWidget, indent int) { if w.Content != "" { props = append(props, fmt.Sprintf("Attribute: %s", w.Content)) } + if w.OnChange != "" { + props = append(props, fmt.Sprintf("OnChange: %s", w.OnChange)) + } props = appendAppearanceProps(props, w) formatWidgetProps(ctx.Output, prefix, header, props, "\n") @@ -485,6 +491,9 @@ func outputWidgetMDLV3(ctx *ExecContext, w rawWidget, indent int) { if w.Content != "" { props = append(props, fmt.Sprintf("Attribute: %s", w.Content)) } + if w.OnChange != "" { + props = append(props, fmt.Sprintf("OnChange: %s", w.OnChange)) + } props = appendAppearanceProps(props, w) formatWidgetProps(ctx.Output, prefix, header, props, "\n") @@ -509,6 +518,9 @@ func outputWidgetMDLV3(ctx *ExecContext, w rawWidget, indent int) { if !w.ShowLabel { props = append(props, "ShowLabel: No") } + if w.OnChange != "" { + props = append(props, fmt.Sprintf("OnChange: %s", w.OnChange)) + } props = appendAppearanceProps(props, w) formatWidgetProps(ctx.Output, prefix, header, props, "\n") diff --git a/mdl/executor/cmd_pages_describe_parse.go b/mdl/executor/cmd_pages_describe_parse.go index ca42ddb25..0722287d0 100644 --- a/mdl/executor/cmd_pages_describe_parse.go +++ b/mdl/executor/cmd_pages_describe_parse.go @@ -268,17 +268,20 @@ func parseRawWidget(ctx *ExecContext, w map[string]any, parentEntityContext ...s case "Forms$TextArea", "Pages$TextArea": widget.Caption = extractLabelText(ctx, w) widget.Content = extractAttributeRef(ctx, w) + widget.OnChange = extractOnChangeAction(ctx, w) return []rawWidget{widget} case "Forms$DatePicker", "Pages$DatePicker": widget.Caption = extractLabelText(ctx, w) widget.Content = extractAttributeRef(ctx, w) + widget.OnChange = extractOnChangeAction(ctx, w) return []rawWidget{widget} case "Forms$RadioButtons", "Pages$RadioButtons", "Forms$RadioButtonGroup", "Pages$RadioButtonGroup": widget.Type = "Forms$RadioButtons" // Normalize type widget.Caption = extractLabelText(ctx, w) widget.Content = extractAttributeRef(ctx, w) + widget.OnChange = extractOnChangeAction(ctx, w) return []rawWidget{widget} case "Forms$CheckBox", "Pages$CheckBox": @@ -287,6 +290,7 @@ func parseRawWidget(ctx *ExecContext, w map[string]any, parentEntityContext ...s widget.Editable = extractEditable(ctx, w) widget.ReadOnlyStyle = extractReadOnlyStyle(ctx, w) widget.ShowLabel = extractShowLabel(ctx, w) + widget.OnChange = extractOnChangeAction(ctx, w) return []rawWidget{widget} case "CustomWidgets$CustomWidget": diff --git a/mdl/executor/widget_defs.go b/mdl/executor/widget_defs.go index 97749b18b..68711f21c 100644 --- a/mdl/executor/widget_defs.go +++ b/mdl/executor/widget_defs.go @@ -564,8 +564,18 @@ var propertyAliases = map[string]map[string][]string{ // today are wired: `onClick` → the widget's Action property, `onChange` → // OnChange. Other action slots (e.g. DataGrid2 `onSelectionChange`) have no MDL // surface yet, so they return "" and no mapping is emitted. +// Mendix's own pluggable widgets suffix their action slots — the Combobox names +// its on-change slot `onChangeEvent`, not `onChange` — so the bare key is not +// enough. One `Event`/`Action` suffix is stripped before matching (ledger #14). +// The stripping is deliberately narrow: a Combobox also carries +// `onChangeFilterInputEvent` and `onChangeDatabaseEvent`, which are separate +// properties with no MDL surface and must stay unmapped, or one `OnChange:` +// would write three different actions. func actionSourceForKey(key string) string { - switch strings.ToLower(key) { + k := strings.ToLower(key) + k = strings.TrimSuffix(k, "event") + k = strings.TrimSuffix(k, "action") + switch k { case "onclick": return "OnClick" case "onchange": diff --git a/mdl/executor/widget_engine.go b/mdl/executor/widget_engine.go index 27e3da195..6eb1fa64e 100644 --- a/mdl/executor/widget_engine.go +++ b/mdl/executor/widget_engine.go @@ -79,7 +79,12 @@ const defaultSlotContainer = "template" // in the generated def, so the checker warns "recognized but not persisted" // instead of silently dropping or falsely rejecting it (general guard for the // ledger #67 class). Bump forces regeneration to carry the new field. -const WidgetDefGeneratorVersion = 14 +// 15 — match an action slot after stripping one `Event`/`Action` suffix, so +// Mendix's own suffixed slots map (Combobox `onChangeEvent`). Version 13 +// matched the bare keys only, so `OnChange:` on a combobox emitted no +// mapping and was dropped with no error (FINDINGS #14). Bump forces +// regeneration so existing projects pick up the suffixed slots. +const WidgetDefGeneratorVersion = 15 // WidgetDefinition describes how to construct a pluggable widget from MDL syntax. // Loaded from embedded JSON definition files (*.def.json). diff --git a/mdl/executor/widget_onchange_event_test.go b/mdl/executor/widget_onchange_event_test.go new file mode 100644 index 000000000..64146857e --- /dev/null +++ b/mdl/executor/widget_onchange_event_test.go @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/sdk/widgets/mpk" +) + +// TestActionSourceForKey_EventSuffixedSlots covers the pluggable half of ledger +// #14. `actionSourceForKey` matched only the bare keys `onClick` / `onChange`, +// but Mendix's own pluggable widgets suffix their action slots: the Combobox +// names its on-change slot `onChangeEvent`. No mapping was emitted for it, so an +// `OnChange:` authored on a `combobox` was dropped with no error — the same +// silent drop as the built-in input widgets, one layer down. +// +// The Combobox's other change-shaped slots (`onChangeFilterInputEvent`, +// `onChangeDatabaseEvent`) are distinct properties with no MDL surface and must +// stay unmapped, or `OnChange:` would write three different actions. +func TestActionSourceForKey_EventSuffixedSlots(t *testing.T) { + cases := map[string]string{ + "onChange": "OnChange", + "onChangeEvent": "OnChange", + "onChangeAction": "OnChange", + "onClick": "OnClick", + "onClickEvent": "OnClick", + "onChangeFilterInputEvent": "", + "onChangeDatabaseEvent": "", + "onEnterEvent": "", + "onSelectionChange": "", + } + for key, want := range cases { + if got := actionSourceForKey(key); got != want { + t.Errorf("actionSourceForKey(%q) = %q, want %q", key, got, want) + } + } +} + +// TestGenerateDefJSON_ComboboxOnChangeEvent asserts the mapping actually reaches +// a generated definition for a Combobox-shaped MPK, not just the key matcher. +func TestGenerateDefJSON_ComboboxOnChangeEvent(t *testing.T) { + def := GenerateDefJSON(&mpk.WidgetDefinition{ + ID: "com.mendix.widget.web.combobox.Combobox", + Name: "Combo box", + Properties: []mpk.PropertyDef{ + {Key: "attributeEnumeration", Type: "attribute"}, + {Key: "onChangeEvent", Type: "action"}, + {Key: "onChangeFilterInputEvent", Type: "action"}, + {Key: "onEnterEvent", Type: "action"}, + }, + }, "COMBOBOX") + + var mapped []string + for _, m := range def.PropertyMappings { + if m.Operation == "action" { + mapped = append(mapped, m.PropertyKey+"→"+m.Source) + } + } + if len(mapped) != 1 || mapped[0] != "onChangeEvent→OnChange" { + t.Fatalf("action mappings = %v, want exactly [onChangeEvent→OnChange]", mapped) + } +} + +// TestCombobox_OnChangeMappedInBothModes pins the shipped built-in definition, +// which is hand-written and therefore not covered by the generator fix above: +// the built-in def overrides any .mpk-derived one, so without a mapping here the +// generator change would never reach a real combobox. Modes are exclusive, so +// the mapping has to be present in each — the reported drop (ledger #14) was on +// an enumeration-bound combobox, i.e. the default mode. +func TestCombobox_OnChangeMappedInBothModes(t *testing.T) { + reg := LoadWidgetRegistry("") + if reg == nil { + t.Fatal("built-in widget registry not available") + } + def, ok := reg.Get("COMBOBOX") + if !ok { + t.Fatal("no COMBOBOX definition in the built-in registry") + } + engine := &PluggableWidgetEngine{} + + widgets := map[string]*ast.WidgetV3{ + "enumeration": {Name: "cb", Type: "combobox", Properties: map[string]any{ + "Attribute": "TopicSel", + "OnChange": &ast.ActionV3{Type: "close"}, + }}, + "association": {Name: "cb", Type: "combobox", Properties: map[string]any{ + "Association": "Mod.A_B", + "CaptionAttribute": "Name", + "DataSource": &ast.DataSourceV3{Type: "database", Reference: "Mod.B"}, + "OnChange": &ast.ActionV3{Type: "close"}, + }}, + } + + for mode, w := range widgets { + t.Run(mode, func(t *testing.T) { + mappings, _, err := engine.selectMappings(def, w) + if err != nil { + t.Fatalf("selectMappings: %v", err) + } + var onChange *PropertyMapping + for i := range mappings { + if mappings[i].Operation == "action" && mappings[i].Source == "OnChange" { + onChange = &mappings[i] + } + } + if onChange == nil { + t.Fatal("no OnChange action mapping — an OnChange: on a combobox is dropped silently") + } + if onChange.PropertyKey != "onChangeEvent" { + t.Fatalf("OnChange maps to %q, want the Combobox's own slot \"onChangeEvent\"", onChange.PropertyKey) + } + + // The mapping is only half of it: resolveMapping must actually build the + // client action out of the AST, or the engine writes a nil action. + mod := mkModule("Mod") + h := mkHierarchy(mod) + withContainer(h, mod.ID, mod.ID) + engine.pageBuilder = newPageBuilder(&mock.MockBackend{}, h, "Mod") + ctx, err := engine.resolveMapping(*onChange, w) + if err != nil { + t.Fatalf("resolveMapping: %v", err) + } + if ctx.Action == nil { + t.Fatal("resolveMapping produced no client action for OnChange") + } + }) + } +} diff --git a/mdl/executor/widget_registry_test.go b/mdl/executor/widget_registry_test.go index e2a276aa0..cf1dbe6af 100644 --- a/mdl/executor/widget_registry_test.go +++ b/mdl/executor/widget_registry_test.go @@ -381,16 +381,18 @@ func TestRegistryComboboxModes(t *testing.T) { if def.Modes[0].Condition != "hasDataSource" { t.Errorf("association mode condition = %q, want hasDataSource", def.Modes[0].Condition) } - if len(def.Modes[0].PropertyMappings) != 4 { - t.Errorf("association mode mappings = %d, want 4", len(def.Modes[0].PropertyMappings)) + // 4 options-source mappings + the onChangeEvent action slot (ledger #14). + if len(def.Modes[0].PropertyMappings) != 5 { + t.Errorf("association mode mappings = %d, want 5", len(def.Modes[0].PropertyMappings)) } // Second mode: default (no condition) if def.Modes[1].Name != "default" { t.Errorf("second mode name = %q, want default", def.Modes[1].Name) } - if len(def.Modes[1].PropertyMappings) != 1 { - t.Errorf("default mode mappings = %d, want 1", len(def.Modes[1].PropertyMappings)) + // attributeEnumeration + the onChangeEvent action slot (ledger #14). + if len(def.Modes[1].PropertyMappings) != 2 { + t.Errorf("default mode mappings = %d, want 2", len(def.Modes[1].PropertyMappings)) } } diff --git a/modelsdk/widgets/definitions/combobox.def.json b/modelsdk/widgets/definitions/combobox.def.json index 5c77fbb4f..ba79255b8 100644 --- a/modelsdk/widgets/definitions/combobox.def.json +++ b/modelsdk/widgets/definitions/combobox.def.json @@ -18,14 +18,16 @@ {"propertyKey": "optionsSourceType", "value": "association", "operation": "primitive"}, {"propertyKey": "optionsSourceAssociationDataSource", "source": "DataSource", "operation": "datasource"}, {"propertyKey": "attributeAssociation", "source": "Association", "operation": "association"}, - {"propertyKey": "optionsSourceAssociationCaptionAttribute", "source": "CaptionAttribute", "operation": "attribute"} + {"propertyKey": "optionsSourceAssociationCaptionAttribute", "source": "CaptionAttribute", "operation": "attribute"}, + {"propertyKey": "onChangeEvent", "source": "OnChange", "operation": "action"} ] }, { "name": "default", "description": "Enumeration mode", "propertyMappings": [ - {"propertyKey": "attributeEnumeration", "source": "Attribute", "operation": "attribute"} + {"propertyKey": "attributeEnumeration", "source": "Attribute", "operation": "attribute"}, + {"propertyKey": "onChangeEvent", "source": "OnChange", "operation": "action"} ] } ] diff --git a/sdk/mpr/writer_widgets_input.go b/sdk/mpr/writer_widgets_input.go index d14ef60c5..8757bcc7e 100644 --- a/sdk/mpr/writer_widgets_input.go +++ b/sdk/mpr/writer_widgets_input.go @@ -63,7 +63,7 @@ func serializeTextArea(ta *pages.TextArea) bson.D { {Key: "Name", Value: ta.Name}, {Key: "NativeAccessibilitySettings", Value: nil}, {Key: "NumberOfLines", Value: int64(5)}, - {Key: "OnChangeAction", Value: serializeClientAction(nil)}, + {Key: "OnChangeAction", Value: serializeClientAction(ta.OnChangeAction)}, {Key: "OnEnterAction", Value: serializeClientAction(nil)}, {Key: "OnLeaveAction", Value: serializeClientAction(nil)}, {Key: "PlaceholderTemplate", Value: serializeEmptyPlaceholderTemplate()}, @@ -93,7 +93,7 @@ func serializeDatePicker(dp *pages.DatePicker) bson.D { {Key: "LabelTemplate", Value: serializeLabelTemplate(dp.Label)}, {Key: "Name", Value: dp.Name}, {Key: "NativeAccessibilitySettings", Value: nil}, - {Key: "OnChangeAction", Value: serializeClientAction(nil)}, + {Key: "OnChangeAction", Value: serializeClientAction(dp.OnChangeAction)}, {Key: "OnEnterAction", Value: serializeClientAction(nil)}, {Key: "PlaceholderTemplate", Value: serializeEmptyPlaceholderTemplate()}, {Key: "ReadOnlyStyle", Value: "Inherit"}, @@ -117,7 +117,7 @@ func serializeCheckBox(cb *pages.CheckBox) bson.D { {Key: "LabelTemplate", Value: serializeLabelTemplate(cb.Label)}, {Key: "Name", Value: cb.Name}, {Key: "NativeAccessibilitySettings", Value: nil}, - {Key: "OnChangeAction", Value: serializeClientAction(nil)}, + {Key: "OnChangeAction", Value: serializeClientAction(cb.OnChangeAction)}, {Key: "OnEnterAction", Value: serializeClientAction(nil)}, {Key: "ReadOnlyStyle", Value: "Inherit"}, {Key: "ScreenReaderLabel", Value: nil}, @@ -141,7 +141,7 @@ func serializeRadioButtons(rb *pages.RadioButtons) bson.D { {Key: "LabelTemplate", Value: serializeLabelTemplate(rb.Label)}, {Key: "Name", Value: rb.Name}, {Key: "NativeAccessibilitySettings", Value: nil}, - {Key: "OnChangeAction", Value: serializeClientAction(nil)}, + {Key: "OnChangeAction", Value: serializeClientAction(rb.OnChangeAction)}, {Key: "OnEnterAction", Value: serializeClientAction(nil)}, {Key: "Orientation", Value: "Horizontal"}, {Key: "ReadOnlyStyle", Value: "Inherit"}, @@ -171,7 +171,7 @@ func serializeDropDown(dd *pages.DropDown) bson.D { {Key: "LabelTemplate", Value: serializeLabelTemplate(dd.Label)}, {Key: "Name", Value: dd.Name}, {Key: "NativeAccessibilitySettings", Value: nil}, - {Key: "OnChangeAction", Value: serializeClientAction(nil)}, + {Key: "OnChangeAction", Value: serializeClientAction(dd.OnChangeAction)}, {Key: "OnEnterAction", Value: serializeClientAction(nil)}, {Key: "OnLeaveAction", Value: serializeClientAction(nil)}, {Key: "ReadOnlyStyle", Value: "Inherit"}, diff --git a/sdk/widgets/definitions/combobox.def.json b/sdk/widgets/definitions/combobox.def.json index 5c77fbb4f..ba79255b8 100644 --- a/sdk/widgets/definitions/combobox.def.json +++ b/sdk/widgets/definitions/combobox.def.json @@ -18,14 +18,16 @@ {"propertyKey": "optionsSourceType", "value": "association", "operation": "primitive"}, {"propertyKey": "optionsSourceAssociationDataSource", "source": "DataSource", "operation": "datasource"}, {"propertyKey": "attributeAssociation", "source": "Association", "operation": "association"}, - {"propertyKey": "optionsSourceAssociationCaptionAttribute", "source": "CaptionAttribute", "operation": "attribute"} + {"propertyKey": "optionsSourceAssociationCaptionAttribute", "source": "CaptionAttribute", "operation": "attribute"}, + {"propertyKey": "onChangeEvent", "source": "OnChange", "operation": "action"} ] }, { "name": "default", "description": "Enumeration mode", "propertyMappings": [ - {"propertyKey": "attributeEnumeration", "source": "Attribute", "operation": "attribute"} + {"propertyKey": "attributeEnumeration", "source": "Attribute", "operation": "attribute"}, + {"propertyKey": "onChangeEvent", "source": "OnChange", "operation": "action"} ] } ] From be223c55566c2f7023774006ff803f145138876a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 19:27:18 +0000 Subject: [PATCH 14/35] fix(settings): name an unknown model setting instead of blaming the Mendix version The presence gate added in #163 runs before the switch that rejects an unrecognised key, so a typo was answered with this project does not store the model setting NotARealSetting ... hint: set it in Studio Pro once, or upgrade the project which sends the reader after a version problem that does not exist. That is a regression against the message the key list was added for: a bare 'unknown model setting' had already sent the mxcli-banking app through three wrong guesses at the optimistic-locking property. An unknown key and a real key this version does not store are different problems with different fixes, and the presence check cannot tell them apart -- so check membership first. The version-specific message stays reachable for a real key, which the second test pins. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01KTdvGwVewkdgNQGDCxpQPZ --- .../objectlist_required_texttemplate_test.go | 5 +- mdl/executor/cmd_settings.go | 23 +++++++++ mdl/executor/cmd_settings_model_test.go | 48 +++++++++++++++++++ mdl/types/widget_property_type.go | 12 ++--- 4 files changed, 80 insertions(+), 8 deletions(-) diff --git a/mdl/backend/widgetobj/objectlist_required_texttemplate_test.go b/mdl/backend/widgetobj/objectlist_required_texttemplate_test.go index 162290c44..064a0902f 100644 --- a/mdl/backend/widgetobj/objectlist_required_texttemplate_test.go +++ b/mdl/backend/widgetobj/objectlist_required_texttemplate_test.go @@ -10,8 +10,9 @@ // declare object lists with texttemplate items. // // Both weaker forms were measured and rejected: -// null -> CE0463 "the definition of this widget has changed" -// empty template -> CE4899 "Property 'Groups/1/Text' is required" +// +// null -> CE0463 "the definition of this widget has changed" +// empty template -> CE4899 "Property 'Groups/1/Text' is required" // // Only the widget's own shipped translations satisfy both, which is what // `mx update-widgets` writes. diff --git a/mdl/executor/cmd_settings.go b/mdl/executor/cmd_settings.go index 42fffe640..d4565a83c 100644 --- a/mdl/executor/cmd_settings.go +++ b/mdl/executor/cmd_settings.go @@ -231,6 +231,18 @@ var modelSettingKeys = []string{ "SslCertificateAlgorithm", } +// isKnownModelSetting reports whether ALTER SETTINGS MODEL has a case for a key. +// Kept beside modelSettingKeys so the membership test and the "valid keys" list +// in the error can never disagree. +func isKnownModelSetting(key string) bool { + for _, k := range modelSettingKeys { + if k == key { + return true + } + } + return false +} + // firstDayOfWeekValues and sslCertificateAlgorithmValues are the members of the // corresponding Mendix enumerations as Studio Pro spells them in BSON. Passing an // unrecognised string through is the mendixlabs/mxcli#759 shape: the metamodel @@ -332,6 +344,17 @@ func alterSettings(ctx *ExecContext, stmt *ast.AlterSettingsStmt) error { storedModel := storedModelSettings(ps) for key, val := range stmt.Properties { valStr := settingsValueToString(val) + // Order matters: a key mxcli does not know at all is a different + // problem from a real key this Mendix version does not store, and the + // presence check cannot tell them apart. Running it first answered a + // typo with "this project does not store NotARealSetting … upgrade the + // project", sending the reader after a version problem that does not + // exist instead of at the misspelling in front of them. + if !isKnownModelSetting(key) { + return mdlerrors.NewUnsupported(fmt.Sprintf( + "unknown model setting: %s\n valid keys: %s", + key, strings.Join(modelSettingKeys, ", "))) + } // The overlay will not introduce a property the stored document does // not carry, so refuse here rather than reporting a success that // changes nothing. diff --git a/mdl/executor/cmd_settings_model_test.go b/mdl/executor/cmd_settings_model_test.go index e11829206..1ffa36dca 100644 --- a/mdl/executor/cmd_settings_model_test.go +++ b/mdl/executor/cmd_settings_model_test.go @@ -351,3 +351,51 @@ func TestAlterSettingsModel_WithdrawnSettingSurvivesOtherWrites(t *testing.T) { t.Errorf("BcryptCost = %d, want 13", written.Model.BcryptCost) } } + +// TestAlterSettingsModel_UnknownKeyNamesItself: an unknown key and a real key +// this Mendix version does not store are different problems with different +// fixes, and the presence check cannot tell them apart. Running the presence +// check first answered a typo with "this project does not store +// NotARealSetting … upgrade the project", pointing at a version problem that +// does not exist. +func TestAlterSettingsModel_UnknownKeyNamesItself(t *testing.T) { + var written *model.ProjectSettings + ctx, _ := newMockCtx(t, withBackend(modelSettingsBackend(&written, "BcryptCost"))) + + err := alterSettings(ctx, &ast.AlterSettingsStmt{ + Section: "model", + Properties: map[string]any{"NotARealSetting": "true"}, + }) + if err == nil { + t.Fatal("expected a refusal for an unknown model setting") + } + if !strings.Contains(err.Error(), "unknown model setting") { + t.Errorf("error should say the key is unknown, got: %v", err) + } + if !strings.Contains(err.Error(), "valid keys") { + t.Errorf("error should list the valid keys, got: %v", err) + } + if strings.Contains(err.Error(), "does not store") { + t.Errorf("error blames the project's Mendix version for a key that does not exist: %v", err) + } + if written != nil { + t.Error("a refused statement must not write") + } +} + +// The version-specific message must still be reachable for a real key. +func TestAlterSettingsModel_KnownKeyStillReportsVersionGap(t *testing.T) { + var written *model.ProjectSettings + ctx, _ := newMockCtx(t, withBackend(modelSettingsBackend(&written, "BcryptCost"))) + + err := alterSettings(ctx, &ast.AlterSettingsStmt{ + Section: "model", + Properties: map[string]any{"UseOQLVersion2": "true"}, + }) + if err == nil { + t.Fatal("expected a refusal: this project does not store UseOQLVersion2") + } + if !strings.Contains(err.Error(), "does not store") { + t.Errorf("a real key absent from the document should get the version message, got: %v", err) + } +} diff --git a/mdl/types/widget_property_type.go b/mdl/types/widget_property_type.go index 0bf23969f..262296447 100644 --- a/mdl/types/widget_property_type.go +++ b/mdl/types/widget_property_type.go @@ -12,18 +12,18 @@ type PropertyTranslation struct { // PropertyTypeIDEntry holds the IDs for a property type from a cloned pluggable widget template. // This is an engine-internal struct used by WidgetObjectBuilder; it is not a BSON wire type. type PropertyTypeIDEntry struct { - PropertyTypeID string - ValueTypeID string - DefaultValue string // Default value from the template's ValueType - ValueType string // Type of value (Boolean, Integer, String, DataSource, etc.) - Required bool // Whether this property is required + PropertyTypeID string + ValueTypeID string + DefaultValue string // Default value from the template's ValueType + ValueType string // Type of value (Boolean, Integer, String, DataSource, etc.) + Required bool // Whether this property is required // DefaultTranslations are the widget-shipped for this property. // A REQUIRED TextTemplate the author leaves unset must be serialized WITH this // text: a null there is CE0463 "the definition of this widget has changed", // and an empty Forms$ClientTemplate is CE4899 "Property … is required" (#891). // This is what `mx update-widgets` itself writes. DefaultTranslations []PropertyTranslation - DataSourceProperty string // Non-empty when this attribute is linked to another DataSource property + DataSourceProperty string // Non-empty when this attribute is linked to another DataSource property // For object list properties (IsList=true with ObjectType), these hold nested IDs ObjectTypeID string // ID of the nested ObjectType (for object lists like columns) NestedPropertyIDs map[string]PropertyTypeIDEntry // Property IDs within the nested ObjectType From a40fb0f213cc251f41c57acc50114d2e33eea543 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 19:39:00 +0000 Subject: [PATCH 15/35] =?UTF-8?q?feat(microflows):=20IN=20QUEUE=20?= =?UTF-8?q?=E2=80=94=20run=20a=20call=20activity=20on=20a=20task=20queue?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mendix can run both a Call microflow and a Call Java action activity on a task queue; it is a property of the call activity, not a separate construct. MDL could express it on neither, so a queue could be created and never bound to anything. The gen setters existed and nothing called them. CALL MICROFLOW Ops.ACT_Process(Order = $Order) IN QUEUE Ops.Work; CALL JAVA ACTION Ops.RefreshData(Url = $Url) IN QUEUE Ops.Work; The clause takes the same position on both — after the argument list, before any ON ERROR — and DESCRIBE renders it back, so the binding round-trips. What is written is a Queues$QueueSettings child, NOT the call's sibling `Queue` string. gen carries a `Queue` ByNameRef on both call types; generated/metamodel (the arbiter) has neither, and a call carrying only `Queue` is inert. Writing it would have produced a call that looks queued and is not. The rewrite guard changes shape rather than going away. A CREATE OR REPLACE/MODIFY still rebuilds the microflow from the statement, so: - a script that restates every stored queue is allowed — that is now the normal way to edit a microflow with a queued call, and before this clause existed there was nothing to restate it with; - a script that drops one is refused, naming the clause that fixes it; - a stored retry policy (Queues$QueueFixedRetry / ExponentialRetry) is still refused outright — MDL has no syntax for it, so a rewrite would reset it (guard-don't-drop, ADR-0005). The restatement walk is reflective so a newly added statement nesting cannot silently stop being searched. `check --references` resolves the queue name, because the build-time failure (CE1613) is reported against the call activity rather than the script and is expensive to trace from a build log. Verified end to end on mxbuild 11.13.0, from a project created by `mxcli new`: - queued CALL MICROFLOW and CALL JAVA ACTION: 0 errors; - dropping the queue produces CE1613 naming both the queue and each activity — which is the proof mxbuild resolves what we wrote, not merely tolerates it; - a queued Java action returning anything but Nothing is CE7038. mxcli defaults CREATE JAVA ACTION to Boolean, so `returns void` is required, not optional. Documented in the skill, the syntax topic and the example. Reported as FINDINGS #24 in ako/mxcli-owid, which proposed one setter call apiece; that would have written the inert property. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017s476QkXr9CFMvKspVzcvu --- .../mendix/scheduled-events-and-queues.md | 85 ++++++++-- .claude/skills/mendix/write-microflows.md | 11 ++ cmd/mxcli/syntax/features_microflow.go | 8 +- cmd/mxcli/syntax/features_misc.go | 17 +- docs/01-project/MDL_FEATURE_MATRIX.md | 2 +- docs/01-project/MDL_QUICK_REFERENCE.md | 2 + mdl-examples/doctype-tests/queued-calls.mdl | 67 ++++++++ mdl/ast/ast_microflow.go | 2 + .../modelsdk/microflow_read_actions.go | 18 ++ mdl/backend/modelsdk/microflow_write.go | 32 ++++ mdl/executor/cmd_microflows_builder_calls.go | 50 ++++++ .../cmd_microflows_builder_queue_test.go | 126 ++++++++++++++ mdl/executor/cmd_microflows_create.go | 2 +- mdl/executor/cmd_microflows_format_action.go | 23 ++- .../cmd_pages_describe_onchange_test.go | 4 +- mdl/executor/helpers.go | 19 +++ mdl/executor/validate.go | 24 ++- mdl/executor/validate_queued_calls.go | 156 +++++++++++++++--- mdl/executor/validate_queued_calls_test.go | 110 +++++++++++- mdl/grammar/domains/MDLMicroflow.g4 | 11 +- mdl/visitor/visitor_microflow_actions.go | 25 +++ sdk/microflows/microflows_actions.go | 19 +++ sdk/mpr/writer_microflow_actions.go | 20 ++- 23 files changed, 770 insertions(+), 63 deletions(-) create mode 100644 mdl-examples/doctype-tests/queued-calls.mdl create mode 100644 mdl/executor/cmd_microflows_builder_queue_test.go diff --git a/.claude/skills/mendix/scheduled-events-and-queues.md b/.claude/skills/mendix/scheduled-events-and-queues.md index 187b06ecf..40ba31c43 100644 --- a/.claude/skills/mendix/scheduled-events-and-queues.md +++ b/.claude/skills/mendix/scheduled-events-and-queues.md @@ -6,7 +6,7 @@ Use this skill when the user wants to: - Run a microflow on a schedule ("every night at 4", "hourly", "cron", "batch job") - Inspect or change an existing scheduled event - Limit how many background tasks run at once (a task queue) -- Understand why `mxcli` refuses to rewrite a microflow that has a queued call +- Run a microflow or Java action call on a queue (`IN QUEUE`) **These two features are unrelated.** A scheduled event does **not** go through a task queue. Its own concurrency control is `OnOverlap`. @@ -116,7 +116,8 @@ create scheduled event Ops.QuarterEnd ( ## Task Queues -A task queue bounds how many queued microflow calls run at once. +A task queue bounds how many queued calls run at once. Binding a call to it +is a separate step — see [`IN QUEUE`](#binding-a-call-to-a-queue--in-queue). ```sql list queues; @@ -147,22 +148,80 @@ thing and an arbitrary expression is legal. | `HourOfDay: 24` | `it must be between 0 and 23` | Hours are 0–23; midnight is `0` | | Expecting a queue to throttle a scheduled event | Nothing changes | They are unrelated — use `OnOverlap` | -## Rewriting a Microflow with a Queued Call Is Refused +## Binding a Call to a Queue — `IN QUEUE` -MDL cannot yet author a *queued call* — the binding lives on the call activity -inside a microflow, not on the queue. So `create or replace|modify microflow` is -refused when the stored microflow has one: +The queue document only *defines* the concurrency limit. What actually runs work +in the background is the binding on the **call activity**, and Mendix allows it +on exactly two: *Call microflow* and *Call Java action*. In MDL that is a +trailing `in queue` clause, in the same position on both — after the argument +list, before any `on error`: +```sql +create or modify microflow Ops.ACT_Enqueue () +begin + call microflow Ops.ACT_Process(Order = $Order) in queue Ops.OrderProcessing; + call java action Ops.RefreshData(Url = $Url) in queue Ops.OrderProcessing; +end; +``` + +`describe microflow` renders the clause back, so the binding round-trips. + +### Two traps, both verified on mxbuild 11.13.0 + +**A queued Java action must return Nothing.** Anything else fails the build with +**CE7038** *"A Java action used for background execution must have a return type +of 'Nothing'."* mxcli's default return type for `create java action` is +**Boolean**, so `returns void` is required, not optional: + +```sql +create java action Ops.RefreshData(Url: string not null) returns void +as $$ return; $$; ``` -Error: microflow Ops.ACT_Caller has 1 call(s) bound to a task queue (Ops.MyQueue), -and rewriting it would silently drop that binding + +**The queue must exist.** A missing one is **CE1613** *"The selected task queue +… no longer exists"*, reported against the **call activity** — it names the +activity, not the script, so a typo is expensive to trace from the build log. +`mxcli check --references` resolves the name first and reports it against the +statement instead. + +That CE1613 is also the proof the binding is real: drop the queue on a project +mxcli wrote and the error appears, naming both the queue and the activity. + +### A Rewrite Must Restate the Queue + +`create or replace|modify microflow` rebuilds the microflow from the statement, +so a binding the script does not restate is gone. mxcli refuses rather than drop +it: + +``` +Error: microflow Ops.ACT_Caller has 1 call(s) bound to a task queue that this +script does not restate (Ops.MyQueue), and rewriting it would silently drop the +binding. +``` + +Add `in queue Ops.MyQueue` to the call and the rewrite goes through. Without the +refusal the binding was written back as null and the project then looked +*healthier* than before — `mx check` stopped reporting CE1613, because the +configuration the error was about had been deleted. + +One thing is still refused: a **retry policy** on a queued call +(`Queues$QueueFixedRetry` / `Queues$QueueExponentialRetry`). MDL has no syntax +for it, so a rewrite cannot preserve it — change that microflow in Studio Pro. + +### When a Java Action Is the Better Tool + +The activity property gives queueing and nothing else. The runtime API, reachable +from a Java action, gives queueing *and* retry, and can queue a Java action +directly with no wrapper microflow: + +```java +Core.userActionCall("Ops.RefreshData") + .withParams(url) + .withExponentialRetry(5, Duration.ofSeconds(2), Duration.ofMinutes(2)) + .executeInBackground(ctx, "Ops.OrderProcessing"); ``` -This is deliberate. Change that microflow in Studio Pro, or remove the task queue -from the call first. Without the refusal the binding was written back as null and -the project then looked *healthier* than before — `mx check` stopped reporting -`CE1613 "The selected task queue no longer exists"`, because the configuration -the error was about had been deleted. +Use `Core.microflowCall(...)` when the unit of work really is a microflow. ## Validation Checklist diff --git a/.claude/skills/mendix/write-microflows.md b/.claude/skills/mendix/write-microflows.md index 8005d9a47..2045f3d0a 100644 --- a/.claude/skills/mendix/write-microflows.md +++ b/.claude/skills/mendix/write-microflows.md @@ -1013,8 +1013,19 @@ call microflow Module.SendNotification(message = $message); -- Call with error handling $Result = call microflow Module.ExternalService(data = $data) on error continue; + +-- Run the call on a task queue (background execution). The clause goes after +-- the arguments and before any ON ERROR, and works on CALL JAVA ACTION too. +call microflow Module.ACT_Refresh() in queue Module.RefreshQueue; +call java action Module.RefreshData(Url = $Url) in queue Module.RefreshQueue; ``` +**Queued calls** — the queue must already exist (`create queue Module.RefreshQueue +(Parallelism: 2)`), and a queued **Java action must `returns void`** or the build +fails with CE7038. Rewriting a microflow that has a queued call must restate the +`in queue` clause; a rewrite that omits it is refused rather than silently +dropping the binding. See `.claude/skills/mendix/scheduled-events-and-queues.md`. + ### ❌ INCORRECT Syntax ```mdl diff --git a/cmd/mxcli/syntax/features_microflow.go b/cmd/mxcli/syntax/features_microflow.go index 669655540..408987557 100644 --- a/cmd/mxcli/syntax/features_microflow.go +++ b/cmd/mxcli/syntax/features_microflow.go @@ -148,12 +148,12 @@ func init() { Path: "microflow.call", Summary: "Call microflows and Java actions with parameters", Keywords: []string{ - "call microflow", "call java action", "invoke", + "call microflow", "call java action", "invoke", "in queue", "queued call", "background execution", "sub-microflow", "java action", "parameter passing", }, - Syntax: "$Result = CALL MICROFLOW Module.Name (Param = value);\n$Result = CALL JAVA ACTION Module.Name (Param = value);", - Example: "$IsValid = CALL MICROFLOW MyModule.ValidateOrder (\n Order = $NewOrder\n);\n\n$Token = CALL JAVA ACTION MyModule.GenerateToken (\n UserId = $User/Id\n);", - SeeAlso: []string{"java-action", "microflow.create"}, + Syntax: "$Result = CALL MICROFLOW Module.Name (Param = value) [IN QUEUE Module.Queue];\n$Result = CALL JAVA ACTION Module.Name (Param = value) [IN QUEUE Module.Queue];", + Example: "$IsValid = CALL MICROFLOW MyModule.ValidateOrder (\n Order = $NewOrder\n);\n\n$Token = CALL JAVA ACTION MyModule.GenerateToken (\n UserId = $User/Id\n);\n\n-- Run the call on a task queue (background execution). The queue must\n-- exist; a queued CALL JAVA ACTION must return Nothing, or the build fails\n-- with CE7038.\nCALL MICROFLOW MyModule.ACT_Refresh () IN QUEUE MyModule.RefreshQueue;\nCALL JAVA ACTION MyModule.RefreshData (Url = $Url) IN QUEUE MyModule.RefreshQueue;", + SeeAlso: []string{"java-action", "microflow.create", "queue"}, }) Register(SyntaxFeature{ diff --git a/cmd/mxcli/syntax/features_misc.go b/cmd/mxcli/syntax/features_misc.go index d6fa7d893..11aad2371 100644 --- a/cmd/mxcli/syntax/features_misc.go +++ b/cmd/mxcli/syntax/features_misc.go @@ -231,9 +231,14 @@ Properties: = per runtime instance. Documentation free text. -Binding a call to a queue is not yet expressible in MDL. Because a rebuild -would drop an existing binding, mxcli REFUSES to CREATE OR REPLACE/MODIFY a -microflow whose stored calls are queued — change those in Studio Pro.`, +Bind a call to a queue with the IN QUEUE clause on CALL MICROFLOW or +CALL JAVA ACTION (see: mxcli syntax microflow.call): + + CALL MICROFLOW Ops.ACT_Process(Order = $Order) IN QUEUE Ops.OrderProcessing; + +A rewrite that does NOT restate a stored binding is refused, because it would +drop it silently. A retry policy on a queued call has no MDL spelling and is +also refused rather than reset — change those in Studio Pro.`, Example: `CREATE QUEUE Ops.OrderProcessing ( Parallelism: 3, ClusterWide: true @@ -248,6 +253,12 @@ CREATE OR MODIFY QUEUE Ops.OrderProcessing ( ClusterWide: true ); +-- Bind a call to it. A queued Java action must return Nothing (CE7038). +CREATE OR MODIFY MICROFLOW Ops.ACT_Enqueue () +BEGIN + CALL MICROFLOW Ops.ACT_Process(Order = $Order) IN QUEUE Ops.OrderProcessing; +END; + SHOW QUEUES IN Ops; DESCRIBE QUEUE Ops.OrderProcessing; DROP QUEUE Ops.Mail;`, diff --git a/docs/01-project/MDL_FEATURE_MATRIX.md b/docs/01-project/MDL_FEATURE_MATRIX.md index 0dbd5e26e..80c631624 100644 --- a/docs/01-project/MDL_FEATURE_MATRIX.md +++ b/docs/01-project/MDL_FEATURE_MATRIX.md @@ -179,6 +179,7 @@ live distinction is **MPR vs MCP**. | **Navigation** | Y | Y | Y | - | - | Y | 11 | N | Y | Y | Y | Y | Y | N | N | Y | N | | **Business Events** | Y | Y | Y | N | Y | N | 13 | N | Y | N | Y | N | Y | N | Y | Y | N | | **Project Settings** | Y | Y | - | - | - | Y | N | N | Y | Y | Y | N | Y | N | N | Y | P | +| **Task Queues** | Y | Y | Y | Y | Y | N | 21 | Y | Y | N | N | Y | Y | N | Y | Y | N | ## Security Features @@ -358,7 +359,6 @@ Document types that exist in Mendix but have no MDL support: | **Module settings** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | Module-level configuration | | **Image collection** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | Image document collections | | **Icon collection** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | Icon/glyph collections | -| **Task queue** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | Background task queue config | | **Rules** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | Microflow rules (decision logic) | | **Regular expressions** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | Reusable regex definitions | | **Scheduled events** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | Timer-triggered microflows | diff --git a/docs/01-project/MDL_QUICK_REFERENCE.md b/docs/01-project/MDL_QUICK_REFERENCE.md index fc8ffd027..50f9f1c0a 100644 --- a/docs/01-project/MDL_QUICK_REFERENCE.md +++ b/docs/01-project/MDL_QUICK_REFERENCE.md @@ -454,6 +454,8 @@ it is for pages. | Retrieve (Assoc) | `retrieve $list from $Parent/Module.AssocName;` | Retrieve by association | | Add to list | `add expression to $list;` | Also accepts existing `add $item to $list;` form | | Call microflow | `$Result = call microflow Module.Name (Param = $value);` | | +| Call microflow on a queue | `call microflow Module.Name (Param = $value) in queue Module.Queue;` | Background execution; the queue must exist (CE1613) | +| Call Java action on a queue | `call java action Module.Name (Param = $value) in queue Module.Queue;` | The Java action must `returns void`, else CE7038 | | Call nanoflow | `$Result = call nanoflow Module.Name (Param = $value);` | | | Call JS action | `$Result = call javascript action Module.Name (Param = $value);` | JavaScript action (nanoflow/microflow) | | Call Java action | `$Result = call java action Module.Name (Param = $value);` | Java action (microflow only) | diff --git a/mdl-examples/doctype-tests/queued-calls.mdl b/mdl-examples/doctype-tests/queued-calls.mdl new file mode 100644 index 000000000..8a9ddf8bd --- /dev/null +++ b/mdl-examples/doctype-tests/queued-calls.mdl @@ -0,0 +1,67 @@ +-- ============================================================================ +-- Queued calls — IN QUEUE on CALL MICROFLOW and CALL JAVA ACTION +-- ============================================================================ +-- +-- A task queue runs a call in the background with bounded concurrency. In +-- Mendix it is a property of the CALL ACTIVITY, not a separate construct, and +-- it applies to exactly two activities: Call microflow and Call Java action. +-- `IN QUEUE Module.Queue` is that property. +-- +-- What is actually written is a `Queues$QueueSettings` child on the call. The +-- call's sibling `Queue` string (which modelsdk/gen exposes and +-- generated/metamodel does not) is NOT written: measured on 11.13, a call +-- carrying only `Queue` is inert — mx check does not read it. +-- +-- Two traps, both verified on mxbuild 11.13.0: +-- +-- 1. A queued CALL JAVA ACTION must return Nothing. Anything else is +-- CE7038 "A Java action used for background execution must have a return +-- type of 'Nothing'." mxcli's default return type for CREATE JAVA ACTION +-- is Boolean, so `returns void` is required here, not optional. +-- 2. The queue must exist. A missing one is CE1613 on the CALL ACTIVITY — +-- it names the activity, not the script — so `mxcli check --references` +-- resolves the name first and reports it against the statement. +-- +-- Rewriting a microflow that already has a queued call must RESTATE the queue. +-- A CREATE OR MODIFY that omits it is refused rather than silently dropping the +-- binding (guard-don't-drop, ADR-0005); before `IN QUEUE` existed there was no +-- way to restate it, so every such rewrite was refused. A stored retry policy +-- has no MDL spelling and is still refused. +-- +-- Verify: +-- mxcli exec mdl-examples/doctype-tests/queued-calls.mdl -p app.mpr +-- mx check app.mpr # 0 errors +-- mxcli -p app.mpr -c "describe microflow Ops.ACT_Enqueue" # shows IN QUEUE +-- mxcli -p app.mpr -c "drop queue Ops.RefreshQueue" +-- mx check app.mpr # CE1613 on the call activity — proves it is read +-- ============================================================================ + +create module Ops; +/ +create queue Ops.RefreshQueue (Parallelism: 2, ClusterWide: true); +/ +create microflow Ops.ACT_Work ($Note: String) +begin + log info node 'Ops' 'working'; +end; +/ +-- A queued Java action must return Nothing (CE7038 otherwise). +create java action Ops.RefreshData(Url: string not null) returns void +as $$ +return; +$$; +/ +create or modify microflow Ops.ACT_Enqueue () +begin + -- Both call activities take the same clause, in the same position: + -- after the argument list, before any ON ERROR. + call microflow Ops.ACT_Work(Note = 'queued') in queue Ops.RefreshQueue; + call java action Ops.RefreshData(Url = 'https://example.org') in queue Ops.RefreshQueue; +end; +/ +-- An unqueued call is unchanged: no QueueSettings is written at all, which is +-- what Studio Pro stores and what every existing microflow round-trips as. +create or modify microflow Ops.ACT_Direct () +begin + call microflow Ops.ACT_Work(Note = 'inline'); +end; diff --git a/mdl/ast/ast_microflow.go b/mdl/ast/ast_microflow.go index 150610d7e..a6aa61924 100644 --- a/mdl/ast/ast_microflow.go +++ b/mdl/ast/ast_microflow.go @@ -438,6 +438,7 @@ type CallMicroflowStmt struct { OutputVariable string // Optional output variable MicroflowName QualifiedName // Microflow to call Arguments []CallArgument // Arguments + Queue *QualifiedName // Optional IN QUEUE clause (task queue to run the call on) ErrorHandling *ErrorHandlingClause // Optional ON ERROR clause Annotations *ActivityAnnotations // Optional @position, @caption, @color, @annotation } @@ -460,6 +461,7 @@ type CallJavaActionStmt struct { OutputVariable string // Optional output variable ActionName QualifiedName // Java action name Arguments []CallArgument // Arguments + Queue *QualifiedName // Optional IN QUEUE clause (task queue to run the call on) ErrorHandling *ErrorHandlingClause // Optional ON ERROR clause Annotations *ActivityAnnotations // Optional @position, @caption, @color, @annotation } diff --git a/mdl/backend/modelsdk/microflow_read_actions.go b/mdl/backend/modelsdk/microflow_read_actions.go index 019bc1f8a..aaac40c73 100644 --- a/mdl/backend/modelsdk/microflow_read_actions.go +++ b/mdl/backend/modelsdk/microflow_read_actions.go @@ -135,6 +135,7 @@ func actionFromGen(el element.Element) microflows.MicroflowAction { call.ParameterMappings = append(call.ParameterMappings, m) } } + call.QueueSettings = queueSettingsFromRaw(mc.Raw()) out.MicroflowCall = call } return out @@ -291,6 +292,7 @@ func actionFromGen(el element.Element) microflows.MicroflowAction { if b, ok := raw.Lookup("UseReturnVariable").BooleanOK(); ok { out.UseReturnVariable = b } + out.QueueSettings = queueSettingsFromRaw(raw) if arr, ok := raw.Lookup("ParameterMappings").ArrayOK(); ok { vals, _ := arr.Values() for _, v := range vals { @@ -1122,3 +1124,19 @@ func rawDocElements(raw bson.Raw, key string) []bson.Raw { } return out } + +// queueSettingsFromRaw reads a call's Queues$QueueSettings child back into the +// semantic model. Retry is carried as raw storage rather than decoded: MDL +// cannot author one, and the rewrite guard needs to know it is there so it can +// refuse rather than drop it (guard-don't-drop, ADR-0005). +func queueSettingsFromRaw(raw bson.Raw) *microflows.QueueSettings { + doc, ok := raw.Lookup("QueueSettings").DocumentOK() + if !ok { + return nil + } + qs := µflows.QueueSettings{Queue: rawStr(doc, "Queue")} + if v, err := doc.LookupErr("Retry"); err == nil && v.Type != bson.TypeNull { + qs.Retry = v + } + return qs +} diff --git a/mdl/backend/modelsdk/microflow_write.go b/mdl/backend/modelsdk/microflow_write.go index f981248ae..611fcc443 100644 --- a/mdl/backend/modelsdk/microflow_write.go +++ b/mdl/backend/modelsdk/microflow_write.go @@ -57,6 +57,14 @@ func init() { codec.RegisterTypeDefaults("Microflows$JavaActionCallAction", codec.TypeDefaults{ NullFields: []string{"QueueSettings"}, }) + // A QueueSettings written by `IN QUEUE` names the queue and configures no + // retry. Retry (Queues$QueueFixedRetry / Queues$QueueExponentialRetry) has no + // MDL surface; serializing it as null mirrors how the call itself serializes + // an absent QueueSettings. A stored retry is never overwritten by this — + // checkNoQueuedCalls refuses the rewrite instead (guard-don't-drop). + codec.RegisterTypeDefaults("Queues$QueueSettings", codec.TypeDefaults{ + NullFields: []string{"Retry"}, + }) codec.RegisterListMarker("Microflows$JavaActionParameterMapping", 2) codec.RegisterTypeDefaults("Microflows$TypedTemplate", codec.TypeDefaults{ MandatoryListMarkers: map[string]int32{"Arguments": 2}, @@ -480,6 +488,9 @@ func microflowActionToGen(action microflows.MicroflowAction) element.Element { m.SetArgument(pm.Argument) mc.AddParameterMappings(m) } + if qs := queueSettingsToGen(a.MicroflowCall.QueueSettings); qs != nil { + mc.SetQueueSettings(qs) + } g.SetMicroflowCall(mc) } g.SetOutputVariableName(a.ResultVariableName) // BSON key "ResultVariableName" @@ -544,6 +555,9 @@ func microflowActionToGen(action microflows.MicroflowAction) element.Element { mappings = append(mappings, m) } addPartList(g, "ParameterMappings", mappings) + if qs := queueSettingsToGen(a.QueueSettings); qs != nil { + addPart(g, "QueueSettings", qs) + } return g case *microflows.JavaScriptActionCallAction: // Built directly like JavaActionCallAction: the JS action call binds @@ -950,6 +964,24 @@ func codeActionParameterValueToGen(v microflows.CodeActionParameterValue) elemen return nil } +// queueSettingsToGen builds the Queues$QueueSettings child that binds a call +// activity to a task queue. Returns nil for an unqueued call, so the call's +// registered NullFields default writes QueueSettings: null as before. +// +// Only the QueueSettings element is written — NOT the call's sibling `Queue` +// string. gen carries a `Queue` ByNameRef on both call types, but +// generated/metamodel (the arbiter) has neither, and measured on 11.13 a call +// carrying only `Queue` is inert: mx check does not read it. Writing both would +// mean two places to keep in sync, one of which nothing consumes. +func queueSettingsToGen(qs *microflows.QueueSettings) element.Element { + if qs == nil || qs.Queue == "" { + return nil + } + g := newElem("Queues$QueueSettings", string(qs.ID)) + addStr(g, "Queue", qs.Queue) + return g +} + func newElem(typeName, id string) *element.Base { b := &element.Base{} b.SetTypeName(typeName) diff --git a/mdl/executor/cmd_microflows_builder_calls.go b/mdl/executor/cmd_microflows_builder_calls.go index 35d35d40c..8bbd5b29e 100644 --- a/mdl/executor/cmd_microflows_builder_calls.go +++ b/mdl/executor/cmd_microflows_builder_calls.go @@ -109,6 +109,54 @@ func (fb *flowBuilder) addLogMessageAction(s *ast.LogStmt) model.ID { } // addCallMicroflowAction creates a CALL MICROFLOW statement. +// buildQueueSettings resolves an `IN QUEUE Module.Name` clause into the +// Queues$QueueSettings element that binds a call activity to a task queue. +// +// The queue must exist: mxbuild reports CE1613 ("The selected task queue … no +// longer exists") on the call activity otherwise, and that error names the +// activity rather than the script, which makes a typo here expensive to trace. +// +// `what` names the statement in the error (e.g. "CALL MICROFLOW"). +func (fb *flowBuilder) buildQueueSettings(q *ast.QualifiedName, what string) *microflows.QueueSettings { + if q == nil { + return nil + } + qn := q.Module + "." + q.Name + if !fb.queueExists(q.Module, q.Name) { + fb.addError("%s ... IN QUEUE '%s': task queue not found in the project (create it with `CREATE QUEUE %s (Parallelism: 1)`)", what, qn, qn) + } + return µflows.QueueSettings{ + BaseElement: model.BaseElement{ID: model.ID(types.GenerateID())}, + Queue: qn, + } +} + +// queueExists reports whether a Queues$Queue with the given module-qualified +// name is in the project. A backend that cannot answer is treated as "exists", +// matching microflowExists — this check exists to catch typos, not to be the +// last line of defence. +func (fb *flowBuilder) queueExists(moduleName, name string) bool { + if fb.backend == nil { + return true + } + queues, err := fb.backend.ListQueues() + if err != nil { + return true + } + for _, q := range queues { + if !strings.EqualFold(q.Name, name) { + continue + } + if fb.hierarchy == nil { + return true + } + if strings.EqualFold(fb.hierarchy.GetModuleName(fb.hierarchy.FindModuleID(q.ContainerID)), moduleName) { + return true + } + } + return false +} + func (fb *flowBuilder) addCallMicroflowAction(s *ast.CallMicroflowStmt) model.ID { mfQN := s.MicroflowName.Module + "." + s.MicroflowName.Name @@ -134,6 +182,7 @@ func (fb *flowBuilder) addCallMicroflowAction(s *ast.CallMicroflowStmt) model.ID BaseElement: model.BaseElement{ID: model.ID(types.GenerateID())}, Microflow: mfQN, ParameterMappings: mappings, + QueueSettings: fb.buildQueueSettings(s.Queue, "CALL MICROFLOW"), } action := µflows.MicroflowCallAction{ BaseElement: model.BaseElement{ID: model.ID(types.GenerateID())}, @@ -350,6 +399,7 @@ func (fb *flowBuilder) addCallJavaActionAction(s *ast.CallJavaActionStmt) model. ErrorHandlingType: fb.ehType(s.ErrorHandling), JavaAction: actionQN, ParameterMappings: mappings, + QueueSettings: fb.buildQueueSettings(s.Queue, "CALL JAVA ACTION"), ResultVariableName: s.OutputVariable, UseReturnVariable: s.OutputVariable != "", } diff --git a/mdl/executor/cmd_microflows_builder_queue_test.go b/mdl/executor/cmd_microflows_builder_queue_test.go new file mode 100644 index 000000000..1140d2a17 --- /dev/null +++ b/mdl/executor/cmd_microflows_builder_queue_test.go @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// FINDINGS #24: Mendix can run a Call microflow and a Call Java action activity +// on a task queue, and MDL could express it on neither — the gen setters existed +// and nothing called them. These pin the authoring path end to end within the +// executor: grammar → AST → builder → semantic model. +// +// The finding proposed writing the call's `Queue` string. That property is +// inert (see queueSettingsToGen); the binding mxbuild reads is the +// Queues$QueueSettings child, which is what these assert. + +func TestCallMicroflow_InQueue_BuildsQueueSettings(t *testing.T) { + fb := &flowBuilder{} + fb.addCallMicroflowAction(&ast.CallMicroflowStmt{ + MicroflowName: ast.QualifiedName{Module: "Q", Name: "ACT_Work"}, + Queue: &ast.QualifiedName{Module: "Q", Name: "RefreshQueue"}, + }) + + act := lastCallAction[*microflows.MicroflowCallAction](t, fb) + if act.MicroflowCall == nil { + t.Fatal("no MicroflowCall built") + } + qs := act.MicroflowCall.QueueSettings + if qs == nil { + t.Fatal("IN QUEUE produced no QueueSettings — the binding is dropped") + } + if qs.Queue != "Q.RefreshQueue" { + t.Errorf("QueueSettings.Queue = %q, want %q", qs.Queue, "Q.RefreshQueue") + } + if qs.ID == "" { + t.Error("QueueSettings has no $ID") + } +} + +func TestCallJavaAction_InQueue_BuildsQueueSettings(t *testing.T) { + fb := &flowBuilder{} + fb.addCallJavaActionAction(&ast.CallJavaActionStmt{ + ActionName: ast.QualifiedName{Module: "Q", Name: "RefreshData"}, + Queue: &ast.QualifiedName{Module: "Q", Name: "RefreshQueue"}, + }) + + act := lastCallAction[*microflows.JavaActionCallAction](t, fb) + if act.QueueSettings == nil { + t.Fatal("IN QUEUE produced no QueueSettings — the binding is dropped") + } + if act.QueueSettings.Queue != "Q.RefreshQueue" { + t.Errorf("QueueSettings.Queue = %q, want %q", act.QueueSettings.Queue, "Q.RefreshQueue") + } +} + +// An unqueued call must keep writing no QueueSettings at all: the call type's +// registered NullFields default serializes it as null, which is what Studio Pro +// stores and what every existing microflow round-trips as. +func TestCallMicroflow_WithoutQueue_HasNoQueueSettings(t *testing.T) { + fb := &flowBuilder{} + fb.addCallMicroflowAction(&ast.CallMicroflowStmt{ + MicroflowName: ast.QualifiedName{Module: "Q", Name: "ACT_Work"}, + }) + act := lastCallAction[*microflows.MicroflowCallAction](t, fb) + if act.MicroflowCall.QueueSettings != nil { + t.Fatalf("unqueued call gained a QueueSettings: %+v", act.MicroflowCall.QueueSettings) + } +} + +// lastCallAction returns the Action of the last activity the builder appended. +func lastCallAction[T microflows.MicroflowAction](t *testing.T, fb *flowBuilder) T { + t.Helper() + var zero T + if len(fb.objects) == 0 { + t.Fatal("builder appended no objects") + return zero + } + activity, ok := fb.objects[len(fb.objects)-1].(*microflows.ActionActivity) + if !ok { + t.Fatalf("last object is %T, want *microflows.ActionActivity", fb.objects[len(fb.objects)-1]) + return zero + } + act, ok := activity.Action.(T) + if !ok { + t.Fatalf("action is %T, want %T", activity.Action, zero) + return zero + } + return act +} + +// TestFormatAction_RendersInQueue covers the DESCRIBE half. Without it a +// describe of a queued microflow emits a script whose re-execution silently +// unqueues the call — and, because the rewrite guard now allows a script that +// restates every stored queue, that script would be accepted. +func TestFormatAction_RendersInQueue(t *testing.T) { + mfAct := µflows.MicroflowCallAction{ + MicroflowCall: µflows.MicroflowCall{ + Microflow: "Q.ACT_Work", + QueueSettings: µflows.QueueSettings{Queue: "Q.RefreshQueue"}, + }, + } + jaAct := µflows.JavaActionCallAction{ + JavaAction: "Q.RefreshData", + QueueSettings: µflows.QueueSettings{Queue: "Q.RefreshQueue"}, + } + + for name, act := range map[string]microflows.MicroflowAction{"microflow": mfAct, "javaaction": jaAct} { + t.Run(name, func(t *testing.T) { + got := formatAction(nil, act, nil, nil) + if !strings.Contains(got, "in queue Q.RefreshQueue") { + t.Fatalf("formatted as %q, want it to carry `in queue Q.RefreshQueue`", got) + } + }) + } + + // An unqueued call must not grow the clause. + plain := µflows.MicroflowCallAction{MicroflowCall: µflows.MicroflowCall{Microflow: "Q.ACT_Work"}} + if got := formatAction(nil, plain, nil, nil); strings.Contains(got, "in queue") { + t.Fatalf("unqueued call formatted as %q", got) + } +} diff --git a/mdl/executor/cmd_microflows_create.go b/mdl/executor/cmd_microflows_create.go index 81aa0238d..18783e561 100644 --- a/mdl/executor/cmd_microflows_create.go +++ b/mdl/executor/cmd_microflows_create.go @@ -95,7 +95,7 @@ func execCreateMicroflow(ctx *ExecContext, s *ast.CreateMicroflowStmt) error { // Refuse before writing if the stored microflow has a call bound to a task // queue: the rebuild would null it out and nothing downstream would notice. if existingID != "" { - if err := checkNoQueuedCalls(ctx, existingID, qualifiedName); err != nil { + if err := checkNoQueuedCalls(ctx, existingID, qualifiedName, s); err != nil { return err } } diff --git a/mdl/executor/cmd_microflows_format_action.go b/mdl/executor/cmd_microflows_format_action.go index 168a812a0..4ed5f68d0 100644 --- a/mdl/executor/cmd_microflows_format_action.go +++ b/mdl/executor/cmd_microflows_format_action.go @@ -580,10 +580,14 @@ func formatAction( paramStr = strings.Join(params, ", ") } + queue := "" + if a.MicroflowCall != nil { + queue = queueClauseMDL(a.MicroflowCall.QueueSettings) + } if a.UseReturnVariable && a.ResultVariableName != "" { - return fmt.Sprintf("$%s = call microflow %s(%s);", a.ResultVariableName, mfName, paramStr) + return fmt.Sprintf("$%s = call microflow %s(%s)%s;", a.ResultVariableName, mfName, paramStr, queue) } - return fmt.Sprintf("call microflow %s(%s);", mfName, paramStr) + return fmt.Sprintf("call microflow %s(%s)%s;", mfName, paramStr, queue) case *microflows.NanoflowCallAction: nfName := "" @@ -663,10 +667,11 @@ func formatAction( paramStr = strings.Join(params, ", ") } + queue := queueClauseMDL(a.QueueSettings) if a.UseReturnVariable && a.ResultVariableName != "" { - return fmt.Sprintf("$%s = call java action %s(%s);", a.ResultVariableName, javaActionName, paramStr) + return fmt.Sprintf("$%s = call java action %s(%s)%s;", a.ResultVariableName, javaActionName, paramStr, queue) } - return fmt.Sprintf("call java action %s(%s);", javaActionName, paramStr) + return fmt.Sprintf("call java action %s(%s)%s;", javaActionName, paramStr, queue) case *microflows.CallExternalAction: serviceName := a.ConsumedODataService @@ -1973,3 +1978,13 @@ func enrichXPathGroup(group string, enumAttrs map[string]string) string { } return "[" + xpathExprToMDLString(enrichXPathExprWithEnums(expr, enumAttrs)) + "]" } + +// queueClauseMDL renders the `IN QUEUE Module.Name` clause of a queued call, or +// "" for an unqueued one. A DESCRIBE that omitted it would emit a script whose +// re-execution silently unqueues the call. +func queueClauseMDL(qs *microflows.QueueSettings) string { + if qs == nil || qs.Queue == "" { + return "" + } + return " in queue " + qs.Queue +} diff --git a/mdl/executor/cmd_pages_describe_onchange_test.go b/mdl/executor/cmd_pages_describe_onchange_test.go index 86cbb608e..e9f10a530 100644 --- a/mdl/executor/cmd_pages_describe_onchange_test.go +++ b/mdl/executor/cmd_pages_describe_onchange_test.go @@ -14,8 +14,8 @@ import ( // described as if it did not. That makes DESCRIBE unable to confirm its own // round trip for these five widgets. func TestParseRawWidget_OnChangeOnEveryInputWidget(t *testing.T) { -// `dropdown` is absent: DESCRIBE has no Forms$DropDown case at all, which is a -// separate gap from this one and is left alone here. + // `dropdown` is absent: DESCRIBE has no Forms$DropDown case at all, which is a + // separate gap from this one and is left alone here. types := map[string]string{ "textbox": "Forms$TextBox", "textarea": "Forms$TextArea", diff --git a/mdl/executor/helpers.go b/mdl/executor/helpers.go index 1c898371e..7c6a0f52c 100644 --- a/mdl/executor/helpers.go +++ b/mdl/executor/helpers.go @@ -406,6 +406,25 @@ func buildMicroflowQualifiedNames(ctx *ExecContext) map[string]bool { return result } +// buildQueueQualifiedNames returns the set of task queue qualified names in the +// project, lower-cased — Mendix name resolution is case-insensitive and the +// caller compares an author-written name against it. +func buildQueueQualifiedNames(ctx *ExecContext) map[string]bool { + result := make(map[string]bool) + h, err := getHierarchy(ctx) + if err != nil { + return result + } + queues, err := ctx.Backend.ListQueues() + if err != nil { + return result + } + for _, q := range queues { + result[strings.ToLower(h.GetQualifiedName(q.ContainerID, q.Name))] = true + } + return result +} + // buildNanoflowQualifiedNames returns a set of all nanoflow qualified names in the project. func buildNanoflowQualifiedNames(ctx *ExecContext) map[string]bool { result := make(map[string]bool) diff --git a/mdl/executor/validate.go b/mdl/executor/validate.go index 10caf855a..8292b972c 100644 --- a/mdl/executor/validate.go +++ b/mdl/executor/validate.go @@ -626,6 +626,15 @@ func validateFlowBodyReferences(ctx *ExecContext, body []ast.MicroflowStatement, } } + if len(refs.queues) > 0 { + known := buildQueueQualifiedNames(ctx) + for _, ref := range refs.queues { + if !known[strings.ToLower(ref)] { + errors = append(errors, fmt.Sprintf("task queue not found: %s (referenced by in queue)", ref)) + } + } + } + if len(refs.microflows) > 0 { known := buildMicroflowQualifiedNames(ctx) for _, ref := range refs.microflows { @@ -825,6 +834,7 @@ type flowRefCollector struct { javaScriptActions []codeActionCallRef entities []entityRef retrieves []retrieveConstraintRef + queues []string } // codeActionCallRef is a Java / JavaScript action call: the action's qualified @@ -900,10 +910,20 @@ type retrieveConstraintRef struct { constraint string // bracketed XPath constraint, e.g. "[System.owner = '[%CurrentUser%]']" } +// addQueue records an `IN QUEUE Module.Name` target. A queue that does not exist +// builds a dangling reference that only fails at build time, as CE1613 on the +// call activity rather than on the script — so it is worth catching in +// `check --references`. +func (c *flowRefCollector) addQueue(q *ast.QualifiedName) { + if q != nil && q.Module != "" { + c.queues = append(c.queues, q.Module+"."+q.Name) + } +} + func (c *flowRefCollector) empty() bool { return len(c.pages) == 0 && len(c.microflows) == 0 && len(c.nanoflows) == 0 && len(c.javaActions) == 0 && len(c.javaScriptActions) == 0 && len(c.entities) == 0 && - len(c.retrieves) == 0 + len(c.retrieves) == 0 && len(c.queues) == 0 } func (c *flowRefCollector) collectFromStatements(stmts []ast.MicroflowStatement) { @@ -917,6 +937,7 @@ func (c *flowRefCollector) collectFromStatements(stmts []ast.MicroflowStatement) if s.MicroflowName.Module != "" { c.microflows = append(c.microflows, s.MicroflowName.String()) } + c.addQueue(s.Queue) case *ast.CallNanoflowStmt: if s.NanoflowName.Module != "" { c.nanoflows = append(c.nanoflows, s.NanoflowName.String()) @@ -927,6 +948,7 @@ func (c *flowRefCollector) collectFromStatements(stmts []ast.MicroflowStatement) name: s.ActionName.String(), argNames: callArgNames(s.Arguments), }) } + c.addQueue(s.Queue) case *ast.CallJavaScriptActionStmt: if s.ActionName.Module != "" { c.javaScriptActions = append(c.javaScriptActions, codeActionCallRef{ diff --git a/mdl/executor/validate_queued_calls.go b/mdl/executor/validate_queued_calls.go index fbbf2ea3f..09f16785e 100644 --- a/mdl/executor/validate_queued_calls.go +++ b/mdl/executor/validate_queued_calls.go @@ -4,52 +4,127 @@ package executor import ( "fmt" + "reflect" "sort" "strings" + "github.com/mendixlabs/mxcli/mdl/ast" mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" "github.com/mendixlabs/mxcli/model" ) -// checkNoQueuedCalls refuses to rewrite a microflow that has a call bound to a -// task queue, because the rewrite would silently drop that binding. +// checkNoQueuedCalls refuses a microflow rewrite that would silently drop a +// call's task-queue binding. // -// A CREATE OR REPLACE/MODIFY rebuilds the microflow from the statement, and both -// engines hardcode QueueSettings to null on a call — correct for a newly -// authored call, wrong for one that was already queued: +// A CREATE OR REPLACE/MODIFY rebuilds the microflow from the statement, so any +// binding the script does not restate is gone. Nothing signals the loss +// afterwards: measured on Mendix 11.13, with the binding present `mx check` +// reports CE1613 ("The selected task queue … no longer exists") on the call +// activity; after a dropping rewrite it reports 0 errors. So mxcli "fixes" the +// build by deleting the user's configuration, and the project then looks +// healthy (guard-don't-drop, ADR-0005). // -// codec.RegisterTypeDefaults("Microflows$MicroflowCall", codec.TypeDefaults{ -// NullFields: []string{"QueueSettings"}, ... +// `IN QUEUE` makes the binding authorable, so a script that restates every +// stored queue is allowed through — that is the normal way to edit a microflow +// with a queued call. Two things still refuse: // -// Nothing signals the loss afterwards. Measured on Mendix 11.13: with the -// binding present `mx check` reports CE1613 ("The selected task queue … no -// longer exists") on the call activity; after the rewrite it reports 0 errors. -// So mxcli "fixes" the build by deleting the user's configuration, and the -// project then looks healthy. -// -// MDL cannot yet author a queued call, so the binding cannot be restated in the -// script either — refusing is the only option that does not lose data -// (guard-don't-drop, ADR-0005). Remove this guard when `in queue` exists and the -// rebuild carries the binding through. -func checkNoQueuedCalls(ctx *ExecContext, microflowID model.ID, qualifiedName string) error { +// - A stored queue the new script does not name. Restating some and dropping +// others is far more likely a mistake than an intent. +// - A stored QueueSettings carrying a Retry (Queues$QueueFixedRetry / +// Queues$QueueExponentialRetry). MDL has no syntax for a retry policy, so +// the rewrite cannot preserve one, and re-running an "unchanged" script +// would quietly reset it. +func checkNoQueuedCalls(ctx *ExecContext, microflowID model.ID, qualifiedName string, stmt *ast.CreateMicroflowStmt) error { raw, err := ctx.Backend.GetRawUnit(microflowID) if err != nil { // Unreadable stored unit is not this guard's business; the rewrite path // reports its own errors. return nil } - queues := queuedCallTargets(raw) - if len(queues) == 0 { + stored := queuedCallTargets(raw) + if len(stored) == 0 { + return nil + } + + if retries := storedQueueRetries(raw); len(retries) > 0 { + sort.Strings(retries) + return mdlerrors.NewUnsupported(fmt.Sprintf( + "microflow %s has %d queued call(s) with a retry policy (%s), and MDL cannot express one — "+ + "rewriting the microflow would reset it.\n"+ + " Change the microflow in Studio Pro, or remove the retry from the call first.", + qualifiedName, len(retries), strings.Join(retries, ", "))) + } + + restated := authoredQueueTargets(stmt) + var lost []string + for _, q := range stored { + if !restated[strings.ToLower(q)] { + lost = append(lost, q) + } + } + if len(lost) == 0 { return nil } - sort.Strings(queues) + sort.Strings(lost) return mdlerrors.NewUnsupported(fmt.Sprintf( - "microflow %s has %d call(s) bound to a task queue (%s), and rewriting it would silently "+ - "drop that binding — MDL cannot express a queued call yet, so the queue cannot be restated "+ - "in this script.\n"+ - " Change the microflow in Studio Pro, or remove the task queue from the call first "+ - "(the binding lives on the call activity, not the microflow).", - qualifiedName, len(queues), strings.Join(queues, ", "))) + "microflow %s has %d call(s) bound to a task queue that this script does not restate (%s), "+ + "and rewriting it would silently drop the binding.\n"+ + " Add the queue to the call — `CALL MICROFLOW Mod.Target(...) IN QUEUE %s` (same clause on "+ + "CALL JAVA ACTION) — or change the microflow in Studio Pro.", + qualifiedName, len(lost), strings.Join(lost, ", "), lost[0])) +} + +// authoredQueueTargets returns the lower-cased queue names the incoming +// statement binds calls to, found by walking the whole statement tree — call +// statements nest inside IF / LOOP / error handlers, and a hand-written switch +// over statement types silently misses whichever nesting was added last. +func authoredQueueTargets(stmt *ast.CreateMicroflowStmt) map[string]bool { + out := map[string]bool{} + if stmt == nil { + return out + } + collectAuthoredQueues(reflect.ValueOf(stmt), out, map[uintptr]bool{}) + return out +} + +func collectAuthoredQueues(v reflect.Value, out map[string]bool, seen map[uintptr]bool) { + switch v.Kind() { + case reflect.Ptr, reflect.Interface: + if v.IsNil() { + return + } + if v.Kind() == reflect.Ptr { + if seen[v.Pointer()] { + return + } + seen[v.Pointer()] = true + switch s := v.Interface().(type) { + case *ast.CallMicroflowStmt: + addQueueName(s.Queue, out) + case *ast.CallJavaActionStmt: + addQueueName(s.Queue, out) + } + } + collectAuthoredQueues(v.Elem(), out, seen) + case reflect.Slice, reflect.Array: + for i := 0; i < v.Len(); i++ { + collectAuthoredQueues(v.Index(i), out, seen) + } + case reflect.Struct: + for i := 0; i < v.NumField(); i++ { + if v.Type().Field(i).PkgPath != "" { + continue // unexported + } + collectAuthoredQueues(v.Field(i), out, seen) + } + } +} + +func addQueueName(q *ast.QualifiedName, out map[string]bool) { + if q == nil { + return + } + out[strings.ToLower(q.Module+"."+q.Name)] = true } // queuedCallTargets walks a stored microflow document and returns the queue @@ -86,6 +161,33 @@ func queuedCallTargets(v any) []string { return dedupeStrings(out) } +// storedQueueRetries returns the queue names whose QueueSettings carries a retry +// policy. `IN QUEUE` writes Retry as null, so a non-null one can only have come +// from Studio Pro and has no MDL spelling. +func storedQueueRetries(v any) []string { + var out []string + switch t := v.(type) { + case map[string]any: + if qs, ok := t["QueueSettings"].(map[string]any); ok && qs != nil { + if retry, ok := qs["Retry"]; ok && retry != nil { + name, _ := qs["Queue"].(string) + if name == "" { + name = "(unnamed queue)" + } + out = append(out, name) + } + } + for _, val := range t { + out = append(out, storedQueueRetries(val)...) + } + case []any: + for _, el := range t { + out = append(out, storedQueueRetries(el)...) + } + } + return dedupeStrings(out) +} + func dedupeStrings(in []string) []string { if len(in) < 2 { return in diff --git a/mdl/executor/validate_queued_calls_test.go b/mdl/executor/validate_queued_calls_test.go index b44d4066c..dc9203292 100644 --- a/mdl/executor/validate_queued_calls_test.go +++ b/mdl/executor/validate_queued_calls_test.go @@ -85,7 +85,7 @@ func TestCheckNoQueuedCalls_Refuses(t *testing.T) { } ctx, _ := newMockCtx(t, withBackend(mb)) - err := checkNoQueuedCalls(ctx, "mf-1", "Q.ACT_Caller") + err := checkNoQueuedCalls(ctx, "mf-1", "Q.ACT_Caller", nil) if err == nil { t.Fatal("expected a refusal for a microflow with a queued call") } @@ -105,7 +105,7 @@ func TestCheckNoQueuedCalls_AllowsUnqueued(t *testing.T) { }, } ctx, _ := newMockCtx(t, withBackend(mb)) - if err := checkNoQueuedCalls(ctx, "mf-1", "Q.ACT_Caller"); err != nil { + if err := checkNoQueuedCalls(ctx, "mf-1", "Q.ACT_Caller", nil); err != nil { t.Fatalf("unqueued microflow must still be rewritable: %v", err) } } @@ -114,7 +114,7 @@ func TestCheckNoQueuedCalls_AllowsUnqueued(t *testing.T) { // own errors, and failing here would block writes for an unrelated reason. func TestCheckNoQueuedCalls_UnreadableUnitDoesNotBlock(t *testing.T) { ctx, _ := newMockCtx(t) // default mock: GetRawUnit is not configured, so it errors - if err := checkNoQueuedCalls(ctx, "mf-1", "Q.ACT_Caller"); err != nil { + if err := checkNoQueuedCalls(ctx, "mf-1", "Q.ACT_Caller", nil); err != nil { t.Fatalf("unreadable unit must not block the write: %v", err) } } @@ -158,3 +158,107 @@ func TestCreateOrModifyMicroflow_RefusesQueuedCall(t *testing.T) { t.Errorf("error should name the queue that would be lost:\n%s", err) } } + +// `IN QUEUE` makes the binding authorable, so the guard changes shape: a script +// that restates every stored queue is the normal way to edit a microflow with a +// queued call and must go through. Before the clause existed, every such rewrite +// was refused because there was nothing to restate it with. +func TestCheckNoQueuedCalls_AllowsWhenScriptRestatesQueue(t *testing.T) { + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + GetRawUnitFunc: func(id model.ID) (map[string]any, error) { + return storedCall(map[string]any{"Queue": "Q.MyQueue"}, nil), nil + }, + } + ctx, _ := newMockCtx(t, withBackend(mb)) + + restating := &ast.CreateMicroflowStmt{ + Name: ast.QualifiedName{Module: "Q", Name: "ACT_Caller"}, + Body: []ast.MicroflowStatement{ + &ast.CallMicroflowStmt{ + MicroflowName: ast.QualifiedName{Module: "Q", Name: "Target"}, + Queue: &ast.QualifiedName{Module: "Q", Name: "MyQueue"}, + }, + }, + } + if err := checkNoQueuedCalls(ctx, "mf-1", "Q.ACT_Caller", restating); err != nil { + t.Fatalf("a script that restates the queue must be allowed: %v", err) + } + + // Dropping it is still refused, and the message must name the clause that + // fixes it — the whole point of the guard is that the loss is invisible. + dropping := &ast.CreateMicroflowStmt{ + Name: ast.QualifiedName{Module: "Q", Name: "ACT_Caller"}, + Body: []ast.MicroflowStatement{ + &ast.CallMicroflowStmt{MicroflowName: ast.QualifiedName{Module: "Q", Name: "Target"}}, + }, + } + err := checkNoQueuedCalls(ctx, "mf-1", "Q.ACT_Caller", dropping) + if err == nil { + t.Fatal("a rewrite that drops the binding must still be refused") + } + for _, want := range []string{"Q.MyQueue", "IN QUEUE"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error message missing %q:\n%s", want, err.Error()) + } + } +} + +// A call nested inside IF/LOOP/error-handler bodies still counts as restating. +// The walk is reflective precisely so a newly added nesting cannot silently stop +// being searched — a hand-written switch would. +func TestAuthoredQueueTargets_FindsNestedCalls(t *testing.T) { + stmt := &ast.CreateMicroflowStmt{ + Body: []ast.MicroflowStatement{ + &ast.IfStmt{ + ThenBody: []ast.MicroflowStatement{ + &ast.LoopStmt{ + Body: []ast.MicroflowStatement{ + &ast.CallJavaActionStmt{ + ActionName: ast.QualifiedName{Module: "Q", Name: "Work"}, + Queue: &ast.QualifiedName{Module: "Q", Name: "Deep"}, + }, + }, + }, + }, + }, + }, + } + got := authoredQueueTargets(stmt) + if !got["q.deep"] { + t.Fatalf("nested IN QUEUE not found: %v", got) + } +} + +// A stored retry policy has no MDL spelling, so restating the queue is not +// enough — the rewrite would reset the retry. That must still refuse. +func TestCheckNoQueuedCalls_RefusesStoredRetry(t *testing.T) { + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + GetRawUnitFunc: func(id model.ID) (map[string]any, error) { + return storedCall(map[string]any{ + "$Type": "Queues$QueueSettings", + "Queue": "Q.MyQueue", + "Retry": map[string]any{"$Type": "Queues$QueueFixedRetry", "Retries": 3}, + }, nil), nil + }, + } + ctx, _ := newMockCtx(t, withBackend(mb)) + + restating := &ast.CreateMicroflowStmt{ + Name: ast.QualifiedName{Module: "Q", Name: "ACT_Caller"}, + Body: []ast.MicroflowStatement{ + &ast.CallMicroflowStmt{ + MicroflowName: ast.QualifiedName{Module: "Q", Name: "Target"}, + Queue: &ast.QualifiedName{Module: "Q", Name: "MyQueue"}, + }, + }, + } + err := checkNoQueuedCalls(ctx, "mf-1", "Q.ACT_Caller", restating) + if err == nil { + t.Fatal("a stored retry policy must refuse the rewrite even when the queue is restated") + } + if !strings.Contains(err.Error(), "retry") { + t.Errorf("error should name the retry:\n%s", err.Error()) + } +} diff --git a/mdl/grammar/domains/MDLMicroflow.g4 b/mdl/grammar/domains/MDLMicroflow.g4 index e204bf9b3..1495e7c57 100644 --- a/mdl/grammar/domains/MDLMicroflow.g4 +++ b/mdl/grammar/domains/MDLMicroflow.g4 @@ -356,7 +356,14 @@ logTemplateParam: templateParam; // $Result = CALL MICROFLOW MfTest.M001_HelloWorld(); or CALL MICROFLOW MfTest.M001_HelloWorld(); callMicroflowStatement - : (VARIABLE EQUALS)? CALL MICROFLOW qualifiedName LPAREN callArgumentList? RPAREN onErrorClause? + : (VARIABLE EQUALS)? CALL MICROFLOW qualifiedName LPAREN callArgumentList? RPAREN queueClause? onErrorClause? + ; + +// IN QUEUE Module.QueueName — runs the call on a task queue (Queues$QueueSettings +// on the call activity). Valid on CALL MICROFLOW and CALL JAVA ACTION only; +// Mendix has no queued nanoflow or JavaScript action. +queueClause + : IN QUEUE qualifiedName ; callNanoflowStatement @@ -365,7 +372,7 @@ callNanoflowStatement // $Result = CALL JAVA ACTION CustomActivities.ExecuteOQL(OqlStatement = '...'); callJavaActionStatement - : (VARIABLE EQUALS)? CALL JAVA ACTION qualifiedName LPAREN callArgumentList? RPAREN onErrorClause? + : (VARIABLE EQUALS)? CALL JAVA ACTION qualifiedName LPAREN callArgumentList? RPAREN queueClause? onErrorClause? ; // $Result = CALL JAVASCRIPT ACTION Module.JSAction(Param = 'value'); diff --git a/mdl/visitor/visitor_microflow_actions.go b/mdl/visitor/visitor_microflow_actions.go index a9c487e77..ffd9d6bd6 100644 --- a/mdl/visitor/visitor_microflow_actions.go +++ b/mdl/visitor/visitor_microflow_actions.go @@ -360,6 +360,25 @@ func appendSourceExpressionSuffix( return &ast.SourceExpr{Expression: innerExpr, Source: source + suffix} } +// buildQueueClause converts an `IN QUEUE Module.Name` clause into a qualified +// name, or returns nil when the call is not queued. Shared by CALL MICROFLOW and +// CALL JAVA ACTION — the only two activities Mendix can run on a task queue. +func buildQueueClause(ctx parser.IQueueClauseContext) *ast.QualifiedName { + if ctx == nil { + return nil + } + qc, ok := ctx.(*parser.QueueClauseContext) + if !ok { + return nil + } + qn := qc.QualifiedName() + if qn == nil { + return nil + } + name := buildQualifiedName(qn) + return &name +} + // buildCallMicroflowStatement converts CALL MICROFLOW statement context to CallMicroflowStmt. // Grammar: (VARIABLE EQUALS)? CALL MICROFLOW qualifiedName LPAREN callArgumentList? RPAREN func buildCallMicroflowStatement(ctx parser.ICallMicroflowStatementContext) *ast.CallMicroflowStmt { @@ -385,6 +404,9 @@ func buildCallMicroflowStatement(ctx parser.ICallMicroflowStatementContext) *ast stmt.Arguments = buildCallArgumentList(argList) } + // IN QUEUE Module.Name — runs the call on a task queue. + stmt.Queue = buildQueueClause(callCtx.QueueClause()) + // Check for ON ERROR clause if errClause := callCtx.OnErrorClause(); errClause != nil { stmt.ErrorHandling = buildOnErrorClause(errClause) @@ -451,6 +473,9 @@ func buildCallJavaActionStatement(ctx parser.ICallJavaActionStatementContext) *a stmt.Arguments = buildCallArgumentList(argList) } + // IN QUEUE Module.Name — runs the call on a task queue. + stmt.Queue = buildQueueClause(callCtx.QueueClause()) + // Check for ON ERROR clause if errClause := callCtx.OnErrorClause(); errClause != nil { stmt.ErrorHandling = buildOnErrorClause(errClause) diff --git a/sdk/microflows/microflows_actions.go b/sdk/microflows/microflows_actions.go index b51799ec5..cbaec08f7 100644 --- a/sdk/microflows/microflows_actions.go +++ b/sdk/microflows/microflows_actions.go @@ -552,6 +552,24 @@ type MicroflowCall struct { model.BaseElement Microflow string `json:"microflow,omitempty"` // Qualified name string ParameterMappings []*MicroflowCallParameterMapping `json:"parameterMappings,omitempty"` + QueueSettings *QueueSettings `json:"queueSettings,omitempty"` +} + +// QueueSettings binds a call activity to a task queue (Queues$QueueSettings). +// +// The load-bearing property is this element, NOT the call's sibling `Queue` +// string: measured on Mendix 11.13, a call carrying only `Queue` with +// QueueSettings null draws no complaint from mx check at all, while one carrying +// QueueSettings does (CE1613 when the named queue is missing). `generated/metamodel` +// — the arbiter — has no top-level `Queue` on either call type, only this. +// +// Retry is a Queues$QueueRetry (fixed or exponential) that MDL cannot author. +// It is carried as raw storage so a rewrite preserves whatever Studio Pro wrote +// rather than silently dropping it (guard-don't-drop, ADR-0005). +type QueueSettings struct { + model.BaseElement + Queue string `json:"queue,omitempty"` // Qualified name of the Queues$Queue + Retry any `json:"retry,omitempty"` // Opaque Queues$QueueRetry, preserved verbatim } // MicroflowCallParameterMapping maps a parameter to an argument. @@ -567,6 +585,7 @@ type JavaActionCallAction struct { ErrorHandlingType ErrorHandlingType `json:"errorHandlingType,omitempty"` JavaAction string `json:"javaAction,omitempty"` // Qualified name string ParameterMappings []*JavaActionParameterMapping `json:"parameterMappings,omitempty"` + QueueSettings *QueueSettings `json:"queueSettings,omitempty"` ResultVariableName string `json:"resultVariableName,omitempty"` UseReturnVariable bool `json:"useReturnVariable"` } diff --git a/sdk/mpr/writer_microflow_actions.go b/sdk/mpr/writer_microflow_actions.go index 4ad8c2767..3af311a9e 100644 --- a/sdk/mpr/writer_microflow_actions.go +++ b/sdk/mpr/writer_microflow_actions.go @@ -242,7 +242,7 @@ func serializeMicroflowAction(action microflows.MicroflowAction) bson.D { } else { mfCall = append(mfCall, bson.E{Key: "ParameterMappings", Value: bson.A{int32(2)}}) // Empty array with marker } - mfCall = append(mfCall, bson.E{Key: "QueueSettings", Value: nil}) + mfCall = append(mfCall, bson.E{Key: "QueueSettings", Value: serializeQueueSettings(a.MicroflowCall.QueueSettings)}) doc = append(doc, bson.E{Key: "MicroflowCall", Value: mfCall}) } doc = append(doc, @@ -292,7 +292,7 @@ func serializeMicroflowAction(action microflows.MicroflowAction) bson.D { {Key: "$Type", Value: "Microflows$JavaActionCallAction"}, {Key: "ErrorHandlingType", Value: stringOrDefault(string(a.ErrorHandlingType), "Rollback")}, {Key: "JavaAction", Value: a.JavaAction}, - {Key: "QueueSettings", Value: nil}, + {Key: "QueueSettings", Value: serializeQueueSettings(a.QueueSettings)}, {Key: "ResultVariableName", Value: a.ResultVariableName}, {Key: "UseReturnVariable", Value: a.UseReturnVariable}, } @@ -1606,3 +1606,19 @@ func importMappingRange(h *microflows.ResultHandlingMapping) bson.D { {Key: "SingleObject", Value: microflows.RangeSingleObjectOf(h)}, } } + +// serializeQueueSettings renders the Queues$QueueSettings child that binds a call +// activity to a task queue, or nil for an unqueued call (which is what Studio Pro +// stores). Retry has no MDL surface and is always null here; a stored retry is +// never overwritten, because checkNoQueuedCalls refuses the rewrite instead. +func serializeQueueSettings(qs *microflows.QueueSettings) any { + if qs == nil || qs.Queue == "" { + return nil + } + return bson.D{ + {Key: "$ID", Value: idToBsonBinary(string(qs.ID))}, + {Key: "$Type", Value: "Queues$QueueSettings"}, + {Key: "Queue", Value: qs.Queue}, + {Key: "Retry", Value: nil}, + } +} From f288106f2f413b9ba420ad28591f35dbd7e6d5aa Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 19:51:59 +0000 Subject: [PATCH 16/35] mxcli test: implement @verify, and fix the OQL path it runs on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FINDINGS #48: no OQL — well-formed or not, true or not, against a real entity or an invented one — could make a @verify fail. It was parsed into the TestCase and read by nothing but --list. Same silent-absence defect as the dropped @expect, in the annotation covering the harder half of testing a Mendix app: most microflows are side effects, so asserting on the rows one wrote is the only way to test it. @verify now runs its OQL against the app after the microflow returns and keeps three outcomes apart — holds, does not hold (FAIL, with the value that came back), cannot be evaluated (ERROR, counted with the failures). Three things are enforced rather than left to trip you up, each learned by running it against a real app: * @cleanup rollback is refused. It is the default, and it undoes the test's writes before the query could see them, so the assertion would run against the pre-test state and report a confident wrong answer. * The result must be one row and one column. Picking a cell out of a table would be a guess, and a guess here is the same silent wrong answer being fixed. * The legacy after-startup runner refuses a suite that uses @verify: its tests run during boot, so there is no point at which to query the app. The expected value is split off at the last comparison operator outside quotes and parentheses, then required to be a literal — without that check the split silently eats a `where x = 1` and sends a truncated query. **The OQL path itself was broken, and the first real run is what found it.** Unit tests against a stubbed endpoint proved the logic and would have shipped it unusable. Two bugs, both in cmd/mxcli/docker/oql.go: * `mxcli oql` failed on every Mendix older than 11.11. The 11.11+ REST route /dev/preview_execute_oql does not exist there, and the admin API does not 404 it — it dispatches the POST as an ordinary admin request, finds no "action" field, and answers HTTP 200 with {"result":1, "message":"Action not found"}. The fallback to the legacy M2EE action keyed on the 404 alone, so the legacy action — which works fine on those runtimes — was never tried, and the user was told to upgrade mxcli. Measured on 11.6.6: before, no query ran at all; after, `select count(*) as n from Mod.E` returns a row. * A rejected query was reported as 0 rows. The legacy action puts the error inside the feedback with a **success** result code, so M2EEError() says nothing and the error body parsed as an empty result. When the dev route is absent the legacy action is the real attempt, so its error is the one reported — otherwise an unknown entity came back as "upgrade mxcli", which is neither true nor actionable. Verified end to end against a booted Mendix 11.6.6 app, seven canaries from the finding: a true @verify PASSes, two false ones FAIL with the observed count, and four unevaluatable ones ERROR — including the unknown entity now carrying the runtime's own message. Stubbing runVerifies to return early puts every canary back to a wrongly green PASS. Note for suite authors: Mendix OQL requires every selected column to have a name, so write `select count(*) as n`, not `select count(*)`. A bare one is an ERROR, not a pass. Repro: mdl-examples/bug-tests/expect-vacuous-assertions.test.mdl Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CqQLyxnppZGfqPYbUHvSWG --- .claude/skills/fix-issue.md | 2 + .claude/skills/mendix/test-microflows.md | 66 ++- cmd/mxcli/docker/oql.go | 93 ++++- cmd/mxcli/docker/oql_test.go | 75 ++++ cmd/mxcli/syntax/features_misc.go | 11 +- cmd/mxcli/testrunner/assertions_test.go | 24 +- cmd/mxcli/testrunner/parser.go | 50 ++- cmd/mxcli/testrunner/runner.go | 24 ++ cmd/mxcli/testrunner/runner_attach.go | 13 +- cmd/mxcli/testrunner/runner_endpoint.go | 19 +- cmd/mxcli/testrunner/verify.go | 385 ++++++++++++++++++ cmd/mxcli/testrunner/verify_test.go | 293 +++++++++++++ cmd/mxcli/testrunner/watch.go | 2 +- docs-site/src/tools/running-tests.md | 58 ++- .../expect-vacuous-assertions.test.mdl | 68 +++- 15 files changed, 1094 insertions(+), 89 deletions(-) create mode 100644 cmd/mxcli/testrunner/verify.go create mode 100644 cmd/mxcli/testrunner/verify_test.go diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index b9670ad25..78d8addef 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -521,6 +521,8 @@ extracting `OffsetExpression`/`LimitExpression`. | A microflow computes a **different number than the expression says**, while `mxcli check`, `mx check` and the build are all green. An additive chain comes back from `DESCRIBE MICROFLOW` with its `+` and `-` exchanged — `$A - $B + 1` stored as `$A + $B - 1`. All-plus and all-minus chains are fine, as is `-` against `*` | `buildAdditiveExpression` read `AllPLUS()` and `AllMINUS()` as two separate token lists and emitted **every plus before every minus**, discarding source order. The precise rule is "the chain is re-sorted, all `+` ahead of all `-`" — sharper than "a `-` followed by a `+` swaps", and it predicts which cases survive | `mdl/visitor/visitor_microflow_expression.go` (`buildAdditiveExpression`) | **The fix already existed 20 lines below**: `buildMultiplicativeExpression` walks `GetChildren()` in order and builds its operator list correctly, so the additive case is that pattern copied — no new mechanism. The corruption is in the **stored model**, not in DESCRIBE: `strings` on the `.mxunit` shows the swapped text, which is why the running app computes it. A rewritten expression is perfectly valid, so no validator can catch this class — the only test that works is round-trip equality, not "does it apply cleanly". **The control cases carry the weight**: `$A - $B - 1` and `$A + $B - 1` pass both before and after, so a test built only from failing cases would have passed against code that sorted all minuses first instead. Verified by reverting the fix and confirming exactly the four swapped cases fail. Test `mdl/visitor/visitor_additive_order_test.go`; example `mdl-examples/bug-tests/additive-operator-order.mdl`. Reported in mxcli-ledger FINDINGS #105 | | `mxcli test` reports **PASS for an assertion that must fail** — `@expect 1 = 2`, `@expect length($result) = 999`, `@expect find($result, 'Z') >= 0` with the needle absent. Nothing in the output distinguishes a real assertion from a vacuous one, so a suite certifies work as verified while asserting only that the microflow did not throw. Mutation testing is what exposes it: mutants that return an obviously wrong value survive the suite | The `@expect` annotation was matched with a regex for one shape — `@expect $var (=|<>) ` — and `FindStringSubmatch` returning nil produced **no assertion at all** rather than an error. A test with zero assertions passes if its body completes. So the narrow support was not the defect; the silence was | `cmd/mxcli/testrunner/parser.go` (`expectPattern`, `parseAnnotations`), `cmd/mxcli/testrunner/expect.go` (new — `ParseExpect`, the validating parser), `cmd/mxcli/testrunner/generator_endpoint.go` + `generator.go` (emit the condition, not a rebuilt equality), `cmd/mxcli/testrunner/results.go` (`expectErrorResult`) | Capture the **whole** annotation body and hand it to a validating parser; anything it cannot compile becomes an `ExpectErrors` entry, the test is not generated at all, and the runner reports `StatusError` (which `FailCount` counts, so the exit code is non-zero). The parser is a strict recursive-descent pass over `exprcheck.Lex` — **not** `mdl/exprcheck`'s own parser, which recovers and emits hints, exactly the wrong behaviour here. Two measurements pinned the emitted expression against mxbuild 11.6.6: `<>` really is CE0117 (so the rewrite to `!=` is load-bearing, not cosmetic) and a wrong-typed comparison really is caught (`$result = 3` → CE0117), which is what makes the 0-error run on the 11 generated shapes mean something. **Generalisable**: when a pattern-matching parser can match *less* than its input, the non-match branch is a silent-failure path — audit every `if m := re.FindStringSubmatch(...); m != nil` whose else-branch does nothing. Repro `mdl-examples/bug-tests/expect-vacuous-assertions.mdl`. mxcli-sudoku FINDINGS #46 | | A test suite's green is unreadable: a test that asserts **nothing** prints the same `PASS` as one with six assertions, and `@verify` — documented as an OQL post-condition — is parsed and evaluated by nothing at all. After @expect started failing closed, the cheapest way back to green is to delete the assertion, and the output cannot tell that apart from a repair | Two silent-absence paths rather than the silent-drop path fixed in the row above. `TestResult` carried no assertion count, so nothing downstream could report one; and `TestCase.Verify` was populated by `parseAnnotations` and read by nothing but `--list` — `grep -n '\.Verify' cmd/mxcli/testrunner/*.go` returns the parser and the lister, no runner | `cmd/mxcli/testrunner/results.go` (`TestResult.Assertions`/`SourceFile`, `newResult`, `vacuousResult`, `resultNote`, `VacuousCount`), `cmd/mxcli/testrunner/parser.go` (`AssertionCount`, `AssertionErrors`, the @verify rejection), `cmd/mxcli/testrunner/junit.go` (`junitClassName`, assertions property), `cmd/mxcli/main.go` + `cmd_test_run.go` (`--require-assertions`) | Count assertions on the test case and carry them onto every result through **one** constructor (`newResult`) — the previous code built `TestResult` literals at five sites, which is exactly how a new field gets populated in one path and silently missing in another. Report the count on the ordinary result line, not behind `--verbose`: the whole lesson of #46 is that the *default* output must distinguish a test that asserted from one that did not. Vacuous tests warn by default and error under `--require-assertions`, because a smoke test is legitimate but an indistinguishable one is not. **Generalisable**: when auditing an annotation/config field for dead ends, grep for *readers*, not writers — a field with a parser and no consumer is a feature the docs promise and the code does not deliver, and it fails silently by construction. mxcli-sudoku FINDINGS #46 (follow-up) | +| `@verify` cannot fail. No OQL — well-formed or not, true or not, against a real entity or an invented one — makes a test fail: `@verify select count(*) as n from Mod.Game = 999999` reports PASS on a table with one row, and so does `this is not a query` | The annotation was parsed into `TestCase.Verify` and read by nothing but `--list`. Same silent-absence class as the dropped `@expect`, in the annotation covering the *harder* half of Mendix testing: most microflows are side effects, so asserting on rows written is the only way to test them | `cmd/mxcli/testrunner/verify.go` (new — `ParseVerify`, `scalarOf`, `compare`, `runVerifies`), `cmd/mxcli/testrunner/parser.go` (`checkVerifyCleanup`), `runner_endpoint.go` + `runner_attach.go` + `watch.go` (`adminOptions` on `testTarget`), `runner.go` (`rejectVerifyOnLegacyRunner`) | Run the OQL over the admin API after the microflow returns, compare against a literal, and keep three outcomes apart: holds → unchanged, does not hold → FAIL with the observed value, cannot be evaluated → ERROR. Three things are enforced rather than left to trip you up, each learned by running it: **`@cleanup rollback` is refused** (the default undoes the writes before the query could see them, so it would assert against the pre-test state); **the result must be one row × one column** (picking a cell out of a table is a guess); and the **legacy after-startup runner refuses the whole suite**, since its tests run during boot with no seam to query at. Split the expected value off at the **last** comparison operator outside quotes and parens, then require the RHS to be a literal — without that check the split silently eats a `where x = 1` and sends a truncated query. **Generalisable**: unit tests with a stubbed endpoint proved the logic and would have shipped it broken — the first real run against a booted app found the OQL endpoint unreachable on that Mendix and the alias requirement, neither of which any stub could show. mxcli-sudoku FINDINGS #48 | +| `mxcli oql` fails on every Mendix older than 11.11 with *"Action not found ... upgrade mxcli"*, and separately reports a **rejected query as `0 rows`** rather than as an error | Two independent misreadings of the runtime's replies. (1) The 11.11+ REST route `/dev/preview_execute_oql` does not exist on older runtimes, and the admin API does not 404 it — it dispatches the POST as an ordinary admin request, finds no `action` field, and answers **HTTP 200** with `{"result":1,"message":"Action not found"}`. The fallback to the legacy M2EE action keyed on the 404 alone, so the legacy action — which works fine there — was never tried. (2) That legacy action reports a bad query as `{"feedback":{"error":"..."},"result":0}` — inside the feedback, with a **success** result code — so `M2EEError()` (which keys off the result) says nothing and the error body parses as an empty result | `cmd/mxcli/docker/oql.go` (`legacyOQL`, `oqlDevErrorKind`, the `error` field in `parseOQLFeedback`'s envelope) | Fall back to the legacy action on *either* absence signal, and surface `feedback.error` regardless of the result code. Measured on 11.6.6: before, `mxcli oql` could not run any query; after, `select count(*) as n from Mod.E` returns a row, and a bad query is an error instead of `0 rows`. **Generalisable**: when a fallback is keyed on one specific failure signal, check what the *other* end actually sends — an HTTP-level 404 and an application-level "not found" are different wires, and a success code next to an error message is common enough to assume it happens. Found while wiring @verify (FINDINGS #48); related to #39 | | Windows Defender flags the mxcli **Windows** release binary as `Trojan:Script/Sabsik.EN.A!ml`; enterprise EDR (Defender for Endpoint, CrowdStrike, SentinelOne) blocks it harder. Not the generic unsigned-Go-binary false positive of #185 | The binary genuinely embedded **chisel**, a dual-use tunnelling/pivoting tool (SSH over WebSocket), on every platform — although the tunnel only ever runs inside a Linux container. `run --hub` linked `chisel/client`, `tunnel-hub` linked `chisel/server`, so windows/darwin carried 32 packages incl. the whole `x/crypto/ssh` stack for a feature they cannot use | `cmd/mxcli/docker/tunnel_linux.go` + `tunnel_other.go` (client seam), `cmd/mxcli/tunnelhub/control_linux.go` + `control_other.go` (server seam), `scripts/check-tunnel-deps.sh` (guard) | **Never obfuscate, pack or rename to dodge the scanner** — attacker tradecraft, and it makes the binary less trustworthy, not more. **Code signing does not fix this class**: a signed binary containing chisel is still flagged behaviourally; signing only addresses #185's generic false positive. The fix is to stop shipping the capability where it is unused: one interface per seam, `_linux.go` impl + `!linux` stub, commands still registered everywhere but failing with an actionable message. **Prove absence three ways, and know that `go tool nm` is not one of them** — release ldflags `-s -w` strip the symbol table, so nm reports "no symbols" whether or not the code is linked and would give a false pass; use `go list -deps`, `go version -m`, and `strings` (nm only on a deliberately unstripped build). **Guard against the transitive path, not the name**: match the module list (`x/crypto/ssh`, `gorilla/websocket`, `armon/go-socks5`, `jpillora/*`) so re-entry without the word "chisel" still trips it, and assert a **positive control** (chisel IS in the linux graph) so the check cannot pass vacuously. Verified by re-adding the import and watching the guard fail on all four windows/darwin targets. Result: -13.5 MB (-14.7%) on windows+darwin, linux unchanged. See ADR-0009 | | CE7375 "must be published and be the key when associations are exposed as an associated object id" on a service publishing no associations | `PublishAssociations` is the representation, not a yes/no — `No` selects "as an associated object id", which needs the system ID as key | `mdl/executor/validate_odata_service_shape.go` | Set `PublishAssociations: Yes` ("as a link", and the default when omitted). MDL-ODATA06 warns at check time | | `mx check` exits with `System.ArgumentOutOfRangeException: length ('-1')` and no error code | an OData service `Path` with no slash in it; mxbuild throws out of its own validator | `mdl/executor/validate_odata_service_shape.go` | Path must not start with `/` and must end with one (CE6550/CE6552). MDL-ODATA05 catches all three | diff --git a/.claude/skills/mendix/test-microflows.md b/.claude/skills/mendix/test-microflows.md index 7711d104e..0e4b7d28a 100644 --- a/.claude/skills/mendix/test-microflows.md +++ b/.claude/skills/mendix/test-microflows.md @@ -155,7 +155,7 @@ The markdown format turns your tests into living documentation. | `@expect` | Assert a Mendix condition | `@expect $result = 'John Doe'` | | `@expect` | Assert an entity attribute | `@expect $product/Name = 'TestProduct'` | | `@expect` | Assert with a built-in | `@expect length($result) = 81` | -| `@verify` | **Not implemented** — rejected as an error | see below | +| `@verify` | OQL post-condition on the database | `@verify select count(*) as n from Mod.E = 1` | | `@throws` | Expect error | `@throws 'validation failed'` | | `@cleanup` | Rollback strategy | `@cleanup rollback` (default) or `@cleanup none` | @@ -204,24 +204,62 @@ The value is omitted rather than guessed when neither side of the comparison establishes a type (`@expect $a = $b`), because Mendix's expression engine is typed and a wrong guess would break the build instead of the test. -### `@verify` is not implemented, and says so +### `@verify` — asserting on what the microflow wrote -`@verify` was documented here as an OQL post-condition. It is parsed and **no -runner has ever evaluated one**, so a test whose only assertion was a `@verify` -asserted nothing. It is now rejected: +`@expect` can only see what a microflow **returned**. Most Mendix microflows are +side effects, so `@verify` is how you assert on the rows one left behind: an OQL +query, a comparison, and the value it must satisfy. + +```mdl +/** + * @test dealing a board writes 81 cells + * @cleanup none + * @expect $result = 'ok' + * @verify select count(*) as n from Sudoku.Cell = 81 + * @verify select count(*) as n from Sudoku.Cell where Value = 0 > 0 + */ +$result = CALL MICROFLOW Sudoku.ACT_DealGame(); +/ +``` + +The query runs against the app **after** the microflow returns, over the same +admin API `mxcli oql` uses. Three rules follow from that, and each is enforced +rather than left to trip you up: + +- **`@cleanup none` is required.** `rollback` is the default, and it undoes the + test's writes before the query could see them — so a `@verify` on a rollback + test is **refused**, not run against the pre-test state. +- **The query must return exactly one row and one column.** Comparing a table to + a literal would mean guessing which cell was meant. Aggregate it + (`select count(*)`), or select one attribute of one row. +- **The expected value is a literal** — a number, a quoted string, `true`/`false` + or `empty`. It is split off at the **last** comparison operator outside quotes + and parentheses, so a `where Value = 5` in the query is left alone. +- **Every selected column needs a name.** Mendix's OQL rejects a bare + `select count(*)` with *"All OQL select columns must have a name"*, so write + `select count(*) as n`. That comes back as an ERROR, not a pass. + +Operators: `=`, `!=` (`<>` accepted), `<`, `<=`, `>`, `>=`. Numbers compare +numerically even though the runtime returns them as strings. + +**A `@verify` that cannot be evaluated is an ERROR, never a pass** — an unknown +entity, malformed OQL, a non-scalar result, or something that was never a query: + +``` +ERROR dealing a board writes 81 cells + @verify select count(*) as n from Sudoku.NoSuch = 1: OQL error: Unknown entity +``` + +and a false one fails with the value that came back: ``` -ERROR writes a row - @verify select count(*) …: @verify is not implemented — no runner - evaluates it, so it would assert nothing. Assert on the microflow's own - result with @expect instead +FAIL dealing a board writes 81 cells + expected select count(*) as n from Sudoku.Cell = 81, actual: 27 ``` -That is the same rule as for an uncompilable `@expect`, applied to the same -class of problem: an annotation that looks like an assertion and is silently -ignored is worse than one that is missing. To check a database post-condition -today, have the microflow under test return the value and assert on it with -`@expect`, or query the app separately with `mxcli oql`. +`@verify` needs the test endpoint, so it runs under `--local` (the default) and +`--attach`. The Docker / `--legacy-runner` path **refuses** a suite using it: +its tests execute during boot, so there is no point at which to query the app. ### A test that asserts nothing says so diff --git a/cmd/mxcli/docker/oql.go b/cmd/mxcli/docker/oql.go index b41b10395..f0d955cd0 100644 --- a/cmd/mxcli/docker/oql.go +++ b/cmd/mxcli/docker/oql.go @@ -67,18 +67,21 @@ func ExecuteOQL(opts OQLOptions, query string) (*OQLResult, error) { // Mendix 11.11+ serves OQL preview as a REST endpoint // (POST /dev/preview_execute_oql with the params as the body, returning - // {"data":[...]} directly). Try it first; on older runtimes it 404s and we - // fall back to the legacy M2EE action (POST / with {"action","params"}). + // {"data":[...]} directly). Try it first and fall back to the legacy M2EE + // action (POST / with {"action","params"}) when it is not there. + // + // "Not there" has two shapes, and only one of them is an HTTP 404. A runtime + // older than 11.11 has no /dev/ route at all, so the admin API dispatches the + // POST as an ordinary admin request, finds no "action" field in the body, and + // answers **200** with {"result":,"message":"Action not found"}. + // Treating only the 404 as absence meant the legacy action — which works + // perfectly on those runtimes — was never tried, so `mxcli oql` failed on + // every Mendix before 11.11 with a message telling the user to upgrade mxcli. + // Measured against 11.6.6: the dev path returns that 200, and the legacy + // action answers the same query. raw, err := previewOQLDev(m2eeOpts, params) if errors.Is(err, errDevEndpointNotFound) { - resp, lerr := CallM2EE(m2eeOpts, "preview_execute_oql", params) - if lerr != nil { - return nil, lerr - } - if errMsg := resp.M2EEError(); errMsg != "" { - return nil, fmt.Errorf("OQL error: %s", errMsg) - } - return parseOQLFeedback(resp.RawFeedback) + return legacyOQL(m2eeOpts, params) } if err != nil { return nil, err @@ -87,13 +90,45 @@ func ExecuteOQL(opts OQLOptions, query string) (*OQLResult, error) { // The dev endpoint reports query failures as HTTP 200 with an {"error":"..."} // body (no "data"), so a bad query must be surfaced here rather than parsed // as an empty result. - if errMsg := oqlDevError(raw); errMsg != "" { + errMsg, absent := oqlDevErrorKind(raw) + if absent { + // The dev route is not mounted, so the legacy action is the real attempt + // and its answer is the one that matters — including when it is an error. + // Reporting the dev route's "Action not found" instead would blame the + // transport for a query the runtime rejected on its merits: an unknown + // entity would come back as "upgrade mxcli", which is neither true nor + // actionable. + res, lerr := legacyOQL(m2eeOpts, params) + if lerr == nil { + return res, nil + } + // Unless the legacy action is missing too — then the live-preview + // servlets really are absent, and the dev message carries the hint that + // says so. + if !strings.Contains(strings.ToLower(lerr.Error()), "action not found") { + return nil, lerr + } + } + if errMsg != "" { return nil, fmt.Errorf("OQL error: %s", errMsg) } return parseOQLFeedback(raw) } +// legacyOQL runs the query through the M2EE admin action, which is how every +// runtime before 11.11 serves OQL preview. +func legacyOQL(opts M2EEOptions, params map[string]any) (*OQLResult, error) { + resp, err := CallM2EE(opts, "preview_execute_oql", params) + if err != nil { + return nil, err + } + if errMsg := resp.M2EEError(); errMsg != "" { + return nil, fmt.Errorf("OQL error: %s", errMsg) + } + return parseOQLFeedback(resp.RawFeedback) +} + // oqlDevError returns the message from a dev-endpoint error response, or "" when // the body is a valid result. Two error shapes are handled: // - {"error":"..."} — a query error (bad OQL) reported by the preview servlet. @@ -104,8 +139,16 @@ func ExecuteOQL(opts OQLOptions, query string) (*OQLResult, error) { // "data" and no "result"; without this check these would be silently parsed // as 0 rows. func oqlDevError(raw json.RawMessage) string { + msg, _ := oqlDevErrorKind(raw) + return msg +} + +// oqlDevErrorKind is oqlDevError plus whether the response means the /dev/ route +// is not mounted at all — the admin dispatcher's "Action not found", which is +// the signal to try the legacy action rather than to give up. +func oqlDevErrorKind(raw json.RawMessage) (string, bool) { if len(raw) == 0 { - return "" + return "", false } var env struct { Error string `json:"error"` @@ -114,16 +157,17 @@ func oqlDevError(raw json.RawMessage) string { Data json.RawMessage `json:"data"` } if err := json.Unmarshal(raw, &env); err != nil { - return "" + return "", false } if env.Error != "" { - return env.Error + return env.Error, false } if len(env.Data) == 0 && env.Result != nil && *env.Result != 0 { msg := env.Message if msg == "" { msg = fmt.Sprintf("runtime returned result %d", *env.Result) } + absent := strings.Contains(strings.ToLower(msg), "action not found") // "Action not found" means the OQL preview servlet isn't mounted — the app // must be started with the live-preview dev flags (mxcli docker does this). // A common cause is a stale docker-compose.yml: `mxcli docker init` skips an @@ -134,9 +178,9 @@ func oqlDevError(raw json.RawMessage) string { " If it was started with `mxcli run --local`, upgrade mxcli to a build that boots the local runtime with live preview (nightly-93 and earlier do not)." + " If it runs under docker and your .docker/ predates this fix, regenerate it with `mxcli docker init --force`, then `mxcli docker build && mxcli docker up`." } - return msg + return msg, absent } - return "" + return "", false } // parseOQLFeedback extracts OQL results from the raw M2EE feedback JSON, @@ -146,14 +190,27 @@ func parseOQLFeedback(rawFeedback json.RawMessage) (*OQLResult, error) { return &OQLResult{}, nil } - // Parse the feedback to extract the data field as raw JSON + // Parse the feedback to extract the data field as raw JSON. + // + // The error field matters as much as the data one: the legacy admin action + // reports a bad query as {"feedback":{"error":"..."},"result":0} — inside the + // feedback, with a **successful** result code. M2EEError() keys off the + // result, so it says nothing, and without this check the error body parses as + // an empty result and a rejected query is reported as "0 rows". Measured + // against 11.6.6: `select count(*)` without an alias comes back exactly that + // way ("All OQL select columns must have a name"). var envelope struct { - Data json.RawMessage `json:"data"` + Data json.RawMessage `json:"data"` + Error string `json:"error"` } if err := json.Unmarshal(rawFeedback, &envelope); err != nil { return nil, fmt.Errorf("parsing feedback: %w", err) } + if envelope.Error != "" { + return nil, fmt.Errorf("OQL error: %s", envelope.Error) + } + if len(envelope.Data) == 0 { return &OQLResult{}, nil } diff --git a/cmd/mxcli/docker/oql_test.go b/cmd/mxcli/docker/oql_test.go index 7d12bb90e..85ac33def 100644 --- a/cmd/mxcli/docker/oql_test.go +++ b/cmd/mxcli/docker/oql_test.go @@ -6,8 +6,11 @@ import ( "bytes" "encoding/json" "fmt" + "io" "net/http" "net/http/httptest" + "net/url" + "strconv" "strings" "testing" ) @@ -605,3 +608,75 @@ func indexOf(s, sub string) int { } return -1 } + +// TestParseOQLFeedbackSurfacesAQueryError pins that a rejected query is an error +// rather than an empty result. +// +// The legacy admin action reports a bad query inside the feedback with a +// successful result code, so M2EEError() says nothing about it. Parsing that +// body as data reported "0 rows" for a query the runtime refused — a wrong +// answer delivered as a benign one. +func TestParseOQLFeedbackSurfacesAQueryError(t *testing.T) { + raw := []byte(`{"error":"All OQL select columns must have a name"}`) + res, err := parseOQLFeedback(raw) + if err == nil { + t.Fatalf("a rejected query parsed as a result: %+v", res) + } + if !strings.Contains(err.Error(), "must have a name") { + t.Errorf("error = %v, want the runtime's message", err) + } +} + +// TestOQLDevErrorKindDetectsAnAbsentRoute pins the fallback signal. A runtime +// older than 11.11 has no /dev/ route, so the admin API dispatches the POST as +// an ordinary request and answers 200 with "Action not found" — not a 404. Only +// treating the 404 as absence meant the legacy action, which works on those +// runtimes, was never tried. +func TestOQLDevErrorKindDetectsAnAbsentRoute(t *testing.T) { + msg, absent := oqlDevErrorKind([]byte(`{"result":1,"message":"Action not found."}`)) + if !absent { + t.Errorf("Action not found was not recognised as an absent route (msg=%q)", msg) + } + if _, absent := oqlDevErrorKind([]byte(`{"error":"syntax error near FROM"}`)); absent { + t.Error("a query error was mistaken for an absent route") + } + if _, absent := oqlDevErrorKind([]byte(`{"data":[{"c":"1"}]}`)); absent { + t.Error("a successful result was mistaken for an absent route") + } +} + +// TestExecuteOQLPrefersTheLegacyError pins which of two failures is reported. +// +// When the /dev/ route is absent the legacy action is the real attempt, so its +// error is the informative one. Reporting the dev route's "Action not found" +// instead blames the transport for a query the runtime rejected on its merits — +// an unknown entity would come back as "upgrade mxcli". +func TestExecuteOQLPrefersTheLegacyError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if strings.HasPrefix(r.URL.Path, "/dev/") { + // Pre-11.11: no /dev/ route, dispatched as an ordinary admin request. + fmt.Fprint(w, `{"result":1,"message":"Action not found."}`) + return + } + fmt.Fprint(w, `{"feedback":{"error":"Unknown entity Mod.NoSuch"},"result":0}`) + })) + defer srv.Close() + + u, _ := url.Parse(srv.URL) + port, _ := strconv.Atoi(u.Port()) + _, err := ExecuteOQL(OQLOptions{ + Host: u.Hostname(), Port: port, Direct: true, + Stdout: io.Discard, Stderr: io.Discard, + }, "select count(*) as n from Mod.NoSuch") + + if err == nil { + t.Fatal("a rejected query returned a result") + } + if !strings.Contains(err.Error(), "Unknown entity") { + t.Errorf("error = %v, want the runtime's own message", err) + } + if strings.Contains(err.Error(), "upgrade mxcli") { + t.Errorf("the transport hint masked the query error: %v", err) + } +} diff --git a/cmd/mxcli/syntax/features_misc.go b/cmd/mxcli/syntax/features_misc.go index 5ae377706..290e696c9 100644 --- a/cmd/mxcli/syntax/features_misc.go +++ b/cmd/mxcli/syntax/features_misc.go @@ -643,9 +643,14 @@ Annotations: reports the observed value alongside the expectation whenever the assertion pins its type. @throws 'message' Expect error - @verify NOT IMPLEMENTED — rejected as an error. Nothing - evaluates it, so it would assert nothing. Return - the value from the microflow and use @expect. + @verify Assert on the DATABASE after the microflow ran: + @verify select count(*) as n from Mod.Cell = 81 + @verify select count(*) as n from Mod.Cell > 0 + The query must return one row and one column, and + the test needs @cleanup none — rollback would undo + the writes before the query could see them. An + unevaluatable @verify is an ERROR, never a pass. + --local / --attach only. @cleanup rollback|none What happens to the test's database writes. rollback (the default) wraps the test in a transaction and rolls it back, so nothing it diff --git a/cmd/mxcli/testrunner/assertions_test.go b/cmd/mxcli/testrunner/assertions_test.go index 70176d838..4a4ad6e0b 100644 --- a/cmd/mxcli/testrunner/assertions_test.go +++ b/cmd/mxcli/testrunner/assertions_test.go @@ -22,9 +22,8 @@ func TestAssertionCountIsReported(t *testing.T) { {"two expects", TestCase{Expects: []Expect{expectOf("$a = 1"), expectOf("$b = 2")}}, 2}, {"throws is an assertion", TestCase{Throws: "boom"}, 1}, {"nothing at all", TestCase{}, 0}, - // @verify is parsed and never executed, so it asserts nothing and must - // not be counted as if it did. - {"verify does not count", TestCase{Verify: []string{"select 1 = 1"}}, 0}, + // @verify is evaluated now, so it counts. + {"verify counts", TestCase{Verify: []Verify{{Raw: "select count(*) from Mod.E = 1"}}}, 1}, } for _, c := range cases { if got := c.tc.AssertionCount(); got != c.want { @@ -75,25 +74,6 @@ func TestRequireAssertionsMakesVacuousTestsErrors(t *testing.T) { } } -// TestVerifyIsRejectedRatherThanIgnored. @verify is documented in the skill's -// annotation table as an OQL post-condition, is parsed into the TestCase, and is -// then read by nothing but `--list`. That is exactly the shape of the defect -// this whole change exists to remove: an annotation that looks like an assertion -// and asserts nothing. Until it is implemented it must be an error. -func TestVerifyIsRejectedRatherThanIgnored(t *testing.T) { - doc := `/** - * @test writes a row - * @verify select count(*) from Mod.E where Code = 'X' = 1 - */` - a := parseAnnotations(doc) - if len(a.AssertionErrors) != 1 { - t.Fatalf("AssertionErrors: got %d, want 1 — @verify was silently ignored", len(a.AssertionErrors)) - } - if !strings.Contains(a.AssertionErrors[0], "@verify") { - t.Errorf("AssertionErrors[0] = %q, want it to name @verify", a.AssertionErrors[0]) - } -} - // TestJUnitCarriesSourceFileAndAssertions pins the CI-side reporting: a failure // in a multi-file run has to say which file it came from, and the assertion // count has to survive into the report a CI actually renders. diff --git a/cmd/mxcli/testrunner/parser.go b/cmd/mxcli/testrunner/parser.go index 3d289f117..2895dd1b2 100644 --- a/cmd/mxcli/testrunner/parser.go +++ b/cmd/mxcli/testrunner/parser.go @@ -24,7 +24,7 @@ type TestCase struct { // and never run: an assertion that cannot be evaluated must not be able to // report a pass. AssertionErrors []string - Verify []string // @verify OQL queries + Verify []Verify // @verify OQL post-conditions Setup string // @setup block reference Cleanup string // @cleanup strategy ("rollback" or "none") Throws string // @throws expected error message @@ -34,11 +34,11 @@ type TestCase struct { // AssertionCount reports how many assertions the test actually makes. // -// @expect and @throws each assert something a runner evaluates. @verify does -// not — it is parsed and never executed, which is why it is rejected at parse -// time rather than counted here. +// @expect, @verify and @throws each assert something a runner evaluates. An +// annotation that is parsed and never executed must not be counted here — that +// is what made @verify look like an assertion while asserting nothing. func (tc TestCase) AssertionCount() int { - n := len(tc.Expects) + n := len(tc.Expects) + len(tc.Verify) if tc.Throws != "" { n++ } @@ -295,7 +295,7 @@ type annotations struct { Test string Expects []Expect AssertionErrors []string - Verify []string + Verify []Verify Setup string Cleanup string Throws string @@ -345,17 +345,12 @@ func parseAnnotations(doc string) annotations { } } if m := verifyPattern.FindStringSubmatch(line); m != nil { - // @verify is parsed here, listed by --list, and read by nothing - // else — no runner has ever executed one. A documented annotation - // that looks like an assertion and asserts nothing is the same - // defect as the @expect shapes that used to be dropped, so it gets - // the same answer: an error, not silence. When it is implemented, - // this branch becomes the OQL post-condition it claims to be. - a.Verify = append(a.Verify, strings.TrimSpace(m[1])) - a.AssertionErrors = append(a.AssertionErrors, fmt.Sprintf( - "@verify %s: @verify is not implemented — no runner evaluates it, "+ - "so it would assert nothing. Assert on the microflow's own "+ - "result with @expect instead", strings.TrimSpace(m[1]))) + v, err := ParseVerify(m[1]) + if err != nil { + a.AssertionErrors = append(a.AssertionErrors, err.Error()) + } else { + a.Verify = append(a.Verify, v) + } } if m := setupPattern.FindStringSubmatch(line); m != nil { a.Setup = strings.TrimSpace(m[1]) @@ -368,5 +363,26 @@ func parseAnnotations(doc string) annotations { } } + checkVerifyCleanup(&a) return a } + +// checkVerifyCleanup refuses a @verify on a test whose writes are rolled back. +// +// @verify asserts on rows the microflow wrote, and @cleanup rollback — the +// default — undoes them when the call returns, before anything can look. The +// query would run against the pre-test state and report a confident wrong +// answer, which is the failure mode this annotation is being fixed for. So it is +// refused, with the one-line change that makes it work. +func checkVerifyCleanup(a *annotations) { + if len(a.Verify) == 0 || a.Cleanup != CleanupRollback { + return + } + for _, v := range a.Verify { + a.AssertionErrors = append(a.AssertionErrors, fmt.Sprintf( + "@verify %s: this test uses @cleanup rollback (the default), so its writes are "+ + "undone before the query runs and it would assert against the pre-test "+ + "state. Add @cleanup none to the test", v.Raw)) + } + a.Verify = nil +} diff --git a/cmd/mxcli/testrunner/runner.go b/cmd/mxcli/testrunner/runner.go index 5c27ce536..4277c625f 100644 --- a/cmd/mxcli/testrunner/runner.go +++ b/cmd/mxcli/testrunner/runner.go @@ -165,9 +165,30 @@ func Run(opts RunOptions) (*SuiteResult, error) { if opts.Local && !opts.LegacyRunner { return runEndpoint(opts, suite, timeout, w) } + // @verify needs a seam after each test's call, and a reachable admin API to + // query — neither of which the after-startup runner has: its tests execute + // during boot and its results are recovered from the log. Refuse rather than + // run the suite with those assertions quietly skipped. + if err := rejectVerifyOnLegacyRunner(suite); err != nil { + return nil, err + } return runAfterStartup(opts, suite, timeout, w) } +// rejectVerifyOnLegacyRunner refuses a suite the after-startup runner cannot +// fully evaluate. +func rejectVerifyOnLegacyRunner(suite *TestSuite) error { + for _, tc := range suite.Tests { + if len(tc.Verify) > 0 { + return fmt.Errorf( + "test %q uses @verify, which the after-startup runner cannot evaluate: "+ + "its tests run during boot, so there is no point at which to query the app. "+ + "Run with --local (the default test endpoint) or --attach", tc.Name) + } + } + return nil +} + // validateOptions rejects combinations that cannot work, with a message that // says what to do instead. Watching depends on re-invoking tests without a // restart, which only the test endpoint can do. @@ -385,6 +406,9 @@ func ListTests(files []string, w io.Writer) error { for _, exp := range tc.Expects { fmt.Fprintf(w, " @expect %s\n", exp.Raw) } + for _, v := range tc.Verify { + fmt.Fprintf(w, " @verify %s\n", v.Raw) + } if tc.Throws != "" { fmt.Fprintf(w, " @throws '%s'\n", tc.Throws) } diff --git a/cmd/mxcli/testrunner/runner_attach.go b/cmd/mxcli/testrunner/runner_attach.go index 53565477c..9d817f391 100644 --- a/cmd/mxcli/testrunner/runner_attach.go +++ b/cmd/mxcli/testrunner/runner_attach.go @@ -64,6 +64,17 @@ func attach(opts RunOptions, w io.Writer) (*attachedApp, error) { // since both produce the same deployment from the same source. func (a *attachedApp) endpoint() *endpointClient { return a.client } +// adminOptions reaches the attached app's admin API — the same plane +// applyModelChange drives, and where @verify's OQL runs. +func (a *attachedApp) adminOptions() docker.M2EEOptions { + return docker.M2EEOptions{ + Host: "127.0.0.1", + Port: a.hs.AdminPort, + Token: a.hs.AdminPass, + Direct: true, + } +} + func (a *attachedApp) applyModelChange(projectPath string) (string, error) { build, err := a.serve.Build(docker.BuildRequest{Target: docker.TargetDeploy, ProjectFilePath: projectPath}) if err != nil { @@ -134,7 +145,7 @@ func runAttached(opts RunOptions, suite *TestSuite, timeout time.Duration, w io. return runAttachedWatch(opts, app, suite, timeout, w, finish, func(s *TestSuite) { injected = s }) } - result, err := runSuite(app.client, suite, opts, w) + result, err := runSuite(app.client, app.adminOptions(), suite, opts, w) if err != nil { return finish(nil, err) } diff --git a/cmd/mxcli/testrunner/runner_endpoint.go b/cmd/mxcli/testrunner/runner_endpoint.go index 419e4d59a..c8119ed5e 100644 --- a/cmd/mxcli/testrunner/runner_endpoint.go +++ b/cmd/mxcli/testrunner/runner_endpoint.go @@ -56,6 +56,11 @@ func (s *testAppSession) stop() { type testTarget interface { // endpoint is the client for the app's test endpoint. endpoint() *endpointClient + // adminOptions reaches the app's M2EE admin API, which is where @verify's + // OQL runs. It is a different plane and a different secret from the test + // endpoint: the endpoint token invokes microflows, the admin password + // queries the database. + adminOptions() docker.M2EEOptions // applyModelChange rebuilds the project and applies it, returning a label for // what it took ("reload"/"restart"). It returns only once the endpoint is // reachable again, so the caller can invoke a test straight after. @@ -64,6 +69,10 @@ type testTarget interface { func (s *testAppSession) endpoint() *endpointClient { return s.client } +func (s *testAppSession) adminOptions() docker.M2EEOptions { + return s.app.Runtime.AdminOptions() +} + // applyModelChange rebuilds through the serve server this session owns. // // A restart is fine here — the session owns the runtime — but it re-runs @@ -93,14 +102,14 @@ func runViaEndpoint(opts RunOptions, suite *TestSuite, token string, timeout tim return nil, err } defer sess.stop() - return runSuite(sess.client, suite, opts, w) + return runSuite(sess.client, sess.adminOptions(), suite, opts, w) } // runSuite invokes every test in the suite against a booted app and collects the // verdicts. It never returns an error for a test-level problem — a missing // microflow or a failed request is that test's result, so one bad test cannot // hide every other. -func runSuite(client *endpointClient, suite *TestSuite, opts RunOptions, w io.Writer) (*SuiteResult, error) { +func runSuite(client *endpointClient, admin docker.M2EEOptions, suite *TestSuite, opts RunOptions, w io.Writer) (*SuiteResult, error) { // Ask the app which test microflows it actually has. A test whose microflow // is missing is reported as an error against that test rather than failing // the run. @@ -159,6 +168,12 @@ func runSuite(client *endpointClient, suite *TestSuite, opts RunOptions, w io.Wr } res := toResult(tc, rr) + // @verify runs only once the microflow has been and gone: it asserts on + // what the app wrote, not on what it returned. A test that already + // failed keeps its verdict — the first failure is the informative one. + if res.Status == StatusPass { + runVerifies(&res, tc, admin, opts.ProjectPath) + } result.Tests = append(result.Tests, res) if opts.Verbose { fmt.Fprintf(w, " %s %s (%s)%s\n", res.Status, res.Name, diff --git a/cmd/mxcli/testrunner/verify.go b/cmd/mxcli/testrunner/verify.go new file mode 100644 index 000000000..5277f06d2 --- /dev/null +++ b/cmd/mxcli/testrunner/verify.go @@ -0,0 +1,385 @@ +// SPDX-License-Identifier: Apache-2.0 + +package testrunner + +import ( + "fmt" + "io" + "strconv" + "strings" + + "github.com/mendixlabs/mxcli/cmd/mxcli/docker" +) + +// Verify is one @verify assertion: an OQL query, a comparison operator, and the +// value the query's single result must satisfy. +// +// @verify select count(*) as n from Sudoku.Cell = 81 +// +// @verify exists for the half of a Mendix app @expect cannot reach. Most +// microflows are side effects — they write rows and return nothing useful — so +// asserting on what the database holds afterwards is the only way to test them. +// That is exactly why it must be able to fail: it covers the least testable +// surface in the app, and until now no query, true or false, well-formed or +// not, could make it do so. +type Verify struct { + // Raw is the annotation body as written, for messages. + Raw string + // Query is the OQL sent to the running app. + Query string + // Operator is the comparison, normalised (`<>` becomes `!=`). + Operator string + // Expected is the literal the result is compared against, as written. + Expected string +} + +// ParseVerify parses one @verify annotation body. +// +// The shape is ` `, and the split is the **last** comparison +// operator outside quotes and parentheses: the expectation is a literal at the +// very end, so anything earlier belongs to the query — a WHERE clause's own `=`, +// or one inside a subquery. +// +// Both halves are then checked, because the split alone is not enough to know it +// was right. A right-hand side that is not a literal means the operator found +// was part of the query, and continuing would send the runtime a truncated +// query — silently asserting on something other than what was written. +func ParseVerify(raw string) (Verify, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return Verify{}, fmt.Errorf("@verify needs an OQL query and an expected value") + } + + pos, op := lastTopLevelComparison(raw) + if pos < 0 { + return Verify{}, fmt.Errorf( + "@verify %s: no comparison found — a @verify is an OQL query followed by "+ + "an expected value, e.g. `@verify select count(*) as n from Mod.Entity = 1`", raw) + } + + query := strings.TrimSpace(raw[:pos]) + expected := strings.TrimSpace(raw[pos+len(op):]) + + if !isOQLLiteral(expected) { + return Verify{}, fmt.Errorf( + "@verify %s: %q is not a value to compare against — the expected value must be "+ + "a number, a quoted string, or true/false", raw, expected) + } + if err := looksLikeOQL(query); err != nil { + return Verify{}, fmt.Errorf("@verify %s: %w", raw, err) + } + + if op == "<>" { + op = "!=" + } + return Verify{Raw: raw, Query: query, Operator: op, Expected: expected}, nil +} + +// looksLikeOQL rejects an obviously non-query left-hand side before it is ever +// sent to the runtime. +// +// This is a cheap sanity check, not a parser — the runtime is the authority on +// whether OQL is valid, and an error from it is reported as an error. What this +// catches is the annotation that was never a query at all (`this is not a +// query = 1`), where the runtime's message would be less use than saying so +// here. +func looksLikeOQL(query string) error { + if query == "" { + return fmt.Errorf("the query is empty") + } + fields := strings.Fields(strings.ToLower(query)) + if len(fields) == 0 || fields[0] != "select" { + return fmt.Errorf("the query must start with SELECT") + } + for _, f := range fields { + if f == "from" || strings.HasPrefix(f, "from(") { + return nil + } + } + return fmt.Errorf("the query has no FROM clause") +} + +// comparisonTokens are matched longest-first so `<=` is not read as `<`. +var comparisonTokens = []string{"!=", "<>", "<=", ">=", "=", "<", ">"} + +// lastTopLevelComparison returns the offset and text of the last comparison +// operator that is not inside a string literal or parentheses, or (-1, ""). +func lastTopLevelComparison(s string) (int, string) { + bestPos, bestOp := -1, "" + depth := 0 + inString := false + + for i := 0; i < len(s); { + c := s[i] + if inString { + if c == '\'' { + // '' is an escaped quote inside an OQL string literal. + if i+1 < len(s) && s[i+1] == '\'' { + i += 2 + continue + } + inString = false + } + i++ + continue + } + switch { + case c == '\'': + inString = true + i++ + continue + case c == '(': + depth++ + i++ + continue + case c == ')': + if depth > 0 { + depth-- + } + i++ + continue + } + if depth == 0 { + if op := matchComparison(s[i:]); op != "" { + bestPos, bestOp = i, op + i += len(op) + continue + } + } + i++ + } + return bestPos, bestOp +} + +func matchComparison(s string) string { + for _, op := range comparisonTokens { + if strings.HasPrefix(s, op) { + return op + } + } + return "" +} + +// isOQLLiteral reports whether the expectation is something that can be compared +// against — a number, a quoted string, or a boolean. +func isOQLLiteral(s string) bool { + if s == "" { + return false + } + if len(s) >= 2 && s[0] == '\'' && s[len(s)-1] == '\'' { + return true + } + switch strings.ToLower(s) { + case "true", "false", "empty", "null": + return true + } + _, err := strconv.ParseFloat(s, 64) + return err == nil +} + +// scalarOf reduces an OQL result to the single value the assertion compares +// against, or explains why it cannot. +// +// A @verify compares one value with one literal, so anything other than exactly +// one row and one column is an error. Picking a cell out of a table would be a +// guess, and a guess here is the same silent wrong answer the annotation is +// being fixed for. +func (v Verify) scalarOf(columns []string, rows [][]any) (any, error) { + switch { + case len(rows) == 0: + return nil, fmt.Errorf("the query returned no rows; @verify needs exactly one row and one column") + case len(rows) > 1: + return nil, fmt.Errorf("the query returned %d rows; @verify needs exactly one row and one column "+ + "(aggregate it, e.g. select count(*))", len(rows)) + case len(columns) != 1: + return nil, fmt.Errorf("the query returned %d columns; @verify needs exactly one row and one column", + len(columns)) + case len(rows[0]) != 1: + return nil, fmt.Errorf("the query returned a row of %d values; @verify needs exactly one row and one column", + len(rows[0])) + } + return rows[0][0], nil +} + +// compare evaluates the assertion against the value the query returned. It also +// returns the value rendered for the failure message, because a @verify that +// says only what was expected leaves you no better off than before. +func (v Verify) compare(actual any) (bool, string, error) { + shown := renderOQLValue(actual) + + // The runtime is asked for numbers as strings (numberHandling: asString), so + // a count comes back as "81" and must still compare numerically against 81. + if want, err := strconv.ParseFloat(strings.TrimSpace(v.Expected), 64); err == nil { + got, ok := numericValue(actual) + if !ok { + return false, shown, fmt.Errorf( + "the query returned %s, which cannot be compared numerically with %s", shown, v.Expected) + } + return compareOrdered(got, want, v.Operator), shown, nil + } + + switch strings.ToLower(v.Expected) { + case "true", "false": + want := strings.EqualFold(v.Expected, "true") + got, ok := actual.(bool) + if !ok { + // A boolean may arrive as the string "true"/"false". + s := strings.ToLower(strings.TrimSpace(shown)) + if s != "true" && s != "false" { + return false, shown, fmt.Errorf( + "the query returned %s, which is not a boolean", shown) + } + got = s == "true" + } + return compareEquality(got == want, v.Operator, shown) + case "empty", "null": + return compareEquality(actual == nil, v.Operator, shown) + } + + // A quoted string literal. + want := strings.ReplaceAll(strings.Trim(v.Expected, "'"), "''", "'") + if actual == nil { + return compareEquality(false, v.Operator, shown) + } + return compareOrderedStrings(shown, want, v.Operator), shown, nil +} + +// compareEquality handles the operators that make sense for a value with no +// ordering: a boolean, or a null check. +// +// shown is threaded through rather than dropped — a FAIL that reports the +// expectation with an empty "actual" is the failure message this change exists +// to improve. +func compareEquality(equal bool, op, shown string) (bool, string, error) { + switch op { + case "=": + return equal, shown, nil + case "!=": + return !equal, shown, nil + } + return false, shown, fmt.Errorf("%s cannot be used with this expected value; use = or !=", op) +} + +func compareOrdered(got, want float64, op string) bool { + switch op { + case "=": + return got == want + case "!=": + return got != want + case "<": + return got < want + case "<=": + return got <= want + case ">": + return got > want + case ">=": + return got >= want + } + return false +} + +func compareOrderedStrings(got, want, op string) bool { + switch op { + case "=": + return got == want + case "!=": + return got != want + case "<": + return got < want + case "<=": + return got <= want + case ">": + return got > want + case ">=": + return got >= want + } + return false +} + +// numericValue coerces an OQL value to a float for comparison. JSON numbers +// arrive as float64 and, under numberHandling asString, as strings. +func numericValue(v any) (float64, bool) { + switch n := v.(type) { + case float64: + return n, true + case int: + return float64(n), true + case int64: + return float64(n), true + case string: + f, err := strconv.ParseFloat(strings.TrimSpace(n), 64) + return f, err == nil + } + return 0, false +} + +// renderOQLValue formats a returned value for a failure message. +func renderOQLValue(v any) string { + if v == nil { + return "NULL" + } + if s, ok := v.(string); ok { + return s + } + if f, ok := v.(float64); ok && f == float64(int64(f)) { + return strconv.FormatInt(int64(f), 10) + } + return fmt.Sprintf("%v", v) +} + +// runVerifies evaluates a test's @verify assertions against the running app and +// downgrades the result if any of them does not hold. +// +// The three outcomes are kept apart on purpose, because collapsing them is the +// defect this replaces: +// +// - the query ran and the comparison held — nothing changes; +// - the query ran and the comparison did not hold — FAIL, reporting the value +// that came back alongside the one that was wanted; +// - the query could not be run or its result cannot be compared — ERROR, which +// is counted with the failures, never a pass. +// +// The first failing assertion wins; later ones are not run, for the same reason +// @expect stops at the first failure — the first one is the informative one. +func runVerifies(res *TestResult, tc TestCase, admin docker.M2EEOptions, projectPath string) { + for _, v := range tc.Verify { + result, err := docker.ExecuteOQL(oqlOptionsFor(admin, projectPath), v.Query) + if err != nil { + res.Status = StatusError + res.Message = fmt.Sprintf("@verify %s: %v", v.Raw, err) + return + } + actual, err := v.scalarOf(result.Columns, result.Rows) + if err != nil { + res.Status = StatusError + res.Message = fmt.Sprintf("@verify %s: %v", v.Raw, err) + return + } + ok, shown, err := v.compare(actual) + if err != nil { + res.Status = StatusError + res.Message = fmt.Sprintf("@verify %s: %v", v.Raw, err) + return + } + if !ok { + res.Status = StatusFail + res.Message = fmt.Sprintf("expected %s, actual: %s", v.Raw, shown) + return + } + } +} + +// oqlOptionsFor adapts the admin connection the runner already holds into what +// ExecuteOQL wants. Direct is always set: the runner reaches the app over +// loopback in both the booted and attached cases, never through docker exec. +func oqlOptionsFor(admin docker.M2EEOptions, projectPath string) docker.OQLOptions { + return docker.OQLOptions{ + Host: admin.Host, + Port: admin.Port, + Token: admin.Token, + ProjectPath: projectPath, + Direct: true, + Stdout: io.Discard, + Stderr: io.Discard, + } +} diff --git a/cmd/mxcli/testrunner/verify_test.go b/cmd/mxcli/testrunner/verify_test.go new file mode 100644 index 000000000..e5d4610b4 --- /dev/null +++ b/cmd/mxcli/testrunner/verify_test.go @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: Apache-2.0 + +package testrunner + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/cmd/mxcli/docker" +) + +// TestVerifyCanariesFail is the regression test for FINDINGS #48: no OQL, well +// formed or not, true or not, against a real entity or an invented one, could +// make a @verify fail. Each row below is one the reporter measured as a wrongly +// green PASS. They must now either compare (and fail) or be rejected outright. +func TestVerifyCanariesFail(t *testing.T) { + // Comparable: these parse, and would run against the app. + for _, raw := range []string{ + "select count(*) from Sudoku.Game = 999999", + "select count(*) from Sudoku.Cell = 0", + "select count(*) from Sudoku.NoSuchEntity = 1", + } { + v, err := ParseVerify(raw) + if err != nil { + t.Errorf("ParseVerify(%q): %v", raw, err) + continue + } + if v.Query == "" || v.Operator == "" || v.Expected == "" { + t.Errorf("ParseVerify(%q) produced an incomplete assertion: %+v", raw, v) + } + } + + // Not comparable: these must be errors, not passes. + for _, raw := range []string{ + "select count(*) frm Sudoku.Cell = 1", // malformed: no FROM + "this is not a query", // not OQL at all + "select count(*) from Sudoku.Cell", // no assertion at all + } { + if v, err := ParseVerify(raw); err == nil { + t.Errorf("ParseVerify(%q) accepted it as %+v; want an error", raw, v) + } + } +} + +// TestParseVerifySplitsOnTheLastTopLevelComparison pins the parse rule. The +// expected value is a literal at the very end, so the split is the last +// comparison operator outside quotes and parentheses — which is what keeps a +// WHERE clause's own `=` out of it. +func TestParseVerifySplitsOnTheLastTopLevelComparison(t *testing.T) { + cases := []struct{ raw, query, op, expected string }{ + {"select count(*) from Mod.E = 81", "select count(*) from Mod.E", "=", "81"}, + {"select count(*) from Mod.E where Value = 5 = 81", + "select count(*) from Mod.E where Value = 5", "=", "81"}, + {"select count(*) from Mod.E where Name = 'a = b' = 2", + "select count(*) from Mod.E where Name = 'a = b'", "=", "2"}, + {"select count(*) from Mod.E where Id in (select Id from Mod.F where X = 1) = 3", + "select count(*) from Mod.E where Id in (select Id from Mod.F where X = 1)", "=", "3"}, + {"select count(*) from Mod.E > 0", "select count(*) from Mod.E", ">", "0"}, + {"select count(*) from Mod.E >= 1", "select count(*) from Mod.E", ">=", "1"}, + {"select count(*) from Mod.E <> 0", "select count(*) from Mod.E", "!=", "0"}, + {"select Name from Mod.E = 'Widget'", "select Name from Mod.E", "=", "'Widget'"}, + } + for _, c := range cases { + v, err := ParseVerify(c.raw) + if err != nil { + t.Errorf("ParseVerify(%q): %v", c.raw, err) + continue + } + if v.Query != c.query || v.Operator != c.op || v.Expected != c.expected { + t.Errorf("ParseVerify(%q) = {%q %q %q}, want {%q %q %q}", + c.raw, v.Query, v.Operator, v.Expected, c.query, c.op, c.expected) + } + } +} + +// TestParseVerifyRejectsANonLiteralExpectation. The right-hand side has to be +// something to compare against. Without this the split silently eats the last +// predicate of a WHERE clause and sends a truncated query to the runtime. +func TestParseVerifyRejectsANonLiteralExpectation(t *testing.T) { + for _, raw := range []string{ + "select count(*) from Mod.E where Value = SomeColumn", + "select count(*) from Mod.E = ", + } { + if _, err := ParseVerify(raw); err == nil { + t.Errorf("ParseVerify(%q) accepted a non-literal expectation", raw) + } + } +} + +// TestVerifyComparesScalars pins the comparison itself, including the thing that +// makes it subtle: the runtime is asked for numbers as strings, so "81" coming +// back has to compare numerically against 81 rather than by text. +func TestVerifyComparesScalars(t *testing.T) { + cases := []struct { + raw string + actual any + ok bool + }{ + {"select count(*) from Mod.E = 81", "81", true}, + {"select count(*) from Mod.E = 81", "27", false}, + {"select count(*) from Mod.E = 81", float64(81), true}, + {"select count(*) from Mod.E > 0", "1", true}, + {"select count(*) from Mod.E > 0", "0", false}, + {"select count(*) from Mod.E != 0", "5", true}, + {"select count(*) from Mod.E != 0", "0", false}, + {"select Name from Mod.E = 'Widget'", "Widget", true}, + {"select Name from Mod.E = 'Widget'", "Gadget", false}, + {"select IsDone from Mod.E = true", true, true}, + {"select IsDone from Mod.E = true", false, false}, + } + for _, c := range cases { + v, err := ParseVerify(c.raw) + if err != nil { + t.Fatalf("ParseVerify(%q): %v", c.raw, err) + } + got, _, err := v.compare(c.actual) + if err != nil { + t.Errorf("%q vs %v: %v", c.raw, c.actual, err) + continue + } + if got != c.ok { + t.Errorf("%q vs %v = %v, want %v", c.raw, c.actual, got, c.ok) + } + } +} + +// TestVerifyRequiresAScalarResult. A query returning a table cannot be compared +// against a literal, and guessing which cell was meant is exactly the kind of +// silent wrong answer this whole area is about. +func TestVerifyRequiresAScalarResult(t *testing.T) { + v, err := ParseVerify("select count(*) from Mod.E = 1") + if err != nil { + t.Fatal(err) + } + for _, res := range []*oqlScalarCase{ + {name: "no rows", cols: []string{"c"}, rows: nil}, + {name: "two rows", cols: []string{"c"}, rows: [][]any{{"1"}, {"2"}}}, + {name: "two columns", cols: []string{"a", "b"}, rows: [][]any{{"1", "2"}}}, + } { + if _, err := v.scalarOf(res.cols, res.rows); err == nil { + t.Errorf("%s: scalarOf accepted a non-scalar result", res.name) + } + } + got, err := v.scalarOf([]string{"c"}, [][]any{{"81"}}) + if err != nil { + t.Fatalf("scalar result rejected: %v", err) + } + if got != "81" { + t.Errorf("scalarOf = %v, want 81", got) + } +} + +type oqlScalarCase struct { + name string + cols []string + rows [][]any +} + +// TestVerifyIsRefusedUnderRollback. @verify asserts on rows the microflow wrote, +// and @cleanup rollback — the default — undoes them before anything can look. +// Running it anyway would compare against the pre-test state and report a +// confident, wrong answer. +func TestVerifyIsRefusedUnderRollback(t *testing.T) { + doc := `/** + * @test writes a row + * @verify select count(*) from Mod.E = 1 + */` + a := parseAnnotations(doc) + if a.Cleanup != "rollback" { + t.Fatalf("precondition: cleanup = %q, want the rollback default", a.Cleanup) + } + if len(a.AssertionErrors) != 1 { + t.Fatalf("AssertionErrors = %d, want 1 — @verify ran under rollback", len(a.AssertionErrors)) + } + if !strings.Contains(a.AssertionErrors[0], "@cleanup none") { + t.Errorf("AssertionErrors[0] = %q, want it to name the fix", a.AssertionErrors[0]) + } + + withNone := parseAnnotations(`/** + * @test writes a row + * @cleanup none + * @verify select count(*) from Mod.E = 1 + */`) + if len(withNone.AssertionErrors) != 0 { + t.Errorf("@cleanup none still refused: %v", withNone.AssertionErrors) + } + if len(withNone.Verify) != 1 { + t.Errorf("Verify = %d, want 1", len(withNone.Verify)) + } +} + +// TestVerifyCountsAsAnAssertion — it is one, now that it is evaluated. +func TestVerifyCountsAsAnAssertion(t *testing.T) { + tc := TestCase{Verify: []Verify{{Raw: "select count(*) from Mod.E = 1"}}} + if tc.AssertionCount() != 1 { + t.Errorf("AssertionCount() = %d, want 1", tc.AssertionCount()) + } +} + +// TestRunVerifiesDowngradesTheResult pins the three outcomes end to end against +// a stubbed OQL endpoint: a holding assertion leaves a PASS alone, a false one +// makes it FAIL with the observed value, and one that cannot be evaluated makes +// it ERROR — which FailCount counts, so the run exits non-zero. +func TestRunVerifiesDowngradesTheResult(t *testing.T) { + cases := []struct { + name string + verify string + reply string + want TestStatus + message string + }{ + {"holds", "select count(*) from Mod.E = 81", `{"data":[{"c":"81"}]}`, StatusPass, ""}, + {"does not hold", "select count(*) from Mod.E = 81", `{"data":[{"c":"27"}]}`, StatusFail, "actual: 27"}, + {"bad query", "select count(*) from Mod.NoSuch = 1", + `{"error":"Unknown entity Mod.NoSuch"}`, StatusError, "Unknown entity"}, + {"not a scalar", "select count(*) from Mod.E = 1", + `{"data":[{"a":"1","b":"2"}]}`, StatusError, "one row and one column"}, + {"no rows", "select count(*) from Mod.E = 1", `{"data":[]}`, StatusError, "no rows"}, + } + + for _, c := range cases { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, c.reply) + })) + + v, err := ParseVerify(c.verify) + if err != nil { + srv.Close() + t.Errorf("%s: ParseVerify: %v", c.name, err) + continue + } + tc := TestCase{ID: "test_1", Name: c.name, Cleanup: "none", Verify: []Verify{v}} + res := newResult(tc) + res.Status = StatusPass + runVerifies(&res, tc, adminOptionsForURL(srv.URL), "") + srv.Close() + + if res.Status != c.want { + t.Errorf("%s: status = %v (%q), want %v", c.name, res.Status, res.Message, c.want) + continue + } + if c.message != "" && !strings.Contains(res.Message, c.message) { + t.Errorf("%s: message = %q, want it to mention %q", c.name, res.Message, c.message) + } + } +} + +// adminOptionsForURL points the admin client at a stub server. +func adminOptionsForURL(rawURL string) docker.M2EEOptions { + u, err := url.Parse(rawURL) + if err != nil { + panic(err) + } + port, _ := strconv.Atoi(u.Port()) + return docker.M2EEOptions{Host: u.Hostname(), Port: port, Direct: true} +} + +// TestVerifyFailureAlwaysCarriesTheActualValue. A FAIL that reports only the +// expectation is the message this change set out to improve, so every comparison +// path has to thread the observed value through — including the boolean and null +// ones, which have no ordering and took a separate route. +func TestVerifyFailureAlwaysCarriesTheActualValue(t *testing.T) { + cases := []struct { + raw string + actual any + want string + }{ + {"select n as n from Mod.E = 81", "27", "27"}, + {"select f as f from Mod.E = true", false, "false"}, + {"select s as s from Mod.E = 'Widget'", "Gadget", "Gadget"}, + {"select s as s from Mod.E != empty", nil, "NULL"}, + } + for _, c := range cases { + v, err := ParseVerify(c.raw) + if err != nil { + t.Fatalf("ParseVerify(%q): %v", c.raw, err) + } + _, shown, err := v.compare(c.actual) + if err != nil { + t.Errorf("%q: %v", c.raw, err) + continue + } + if shown != c.want { + t.Errorf("%q vs %v: shown = %q, want %q", c.raw, c.actual, shown, c.want) + } + } +} diff --git a/cmd/mxcli/testrunner/watch.go b/cmd/mxcli/testrunner/watch.go index 2457f4aab..a3ba05129 100644 --- a/cmd/mxcli/testrunner/watch.go +++ b/cmd/mxcli/testrunner/watch.go @@ -74,7 +74,7 @@ func watchLoop(opts RunOptions, target testTarget, suite *TestSuite, w io.Writer for { gen++ - result, err := runSuite(target.endpoint(), injected, opts, w) + result, err := runSuite(target.endpoint(), target.adminOptions(), injected, opts, w) if err != nil { // The endpoint stopped answering — the runtime is probably gone, and // nothing further will work. Bail rather than spin. diff --git a/docs-site/src/tools/running-tests.md b/docs-site/src/tools/running-tests.md index 70048bfcc..6ddd362a0 100644 --- a/docs-site/src/tools/running-tests.md +++ b/docs-site/src/tools/running-tests.md @@ -182,16 +182,64 @@ write; what it may not do is look the same as a test with six assertions. `--require-assertions` makes every vacuous test an ERROR for projects that want CI to enforce it. -`@verify` was documented as an OQL post-condition and **is not implemented** — -it is parsed and evaluated by nothing, so a test whose only assertion was a -`@verify` asserted nothing. It is now rejected with an error pointing at -`@expect`. To check a database post-condition, return the value from the -microflow under test and assert on it, or query the app with `mxcli oql`. +`@verify` asserts on the database instead of the return value — see below. The JUnit report (`--junit`) carries the assertion count as a `` on each case, and `classname`/`file` identify the source test file so a failure in a multi-file run says where it lives. +### `@verify`: asserting on what the microflow wrote + +`@expect` sees only what a microflow returned. Most Mendix microflows are side +effects, so `@verify` is how you assert on the rows one left behind — an OQL +query, a comparison, and the value it must satisfy: + +```mdl +/** + * @test dealing a board writes 81 cells + * @cleanup none + * @expect $result = 'ok' + * @verify select count(*) as n from Sudoku.Cell = 81 + * @verify select count(*) as n from Sudoku.Cell where Value = 0 > 0 + */ +$result = CALL MICROFLOW Sudoku.ACT_DealGame(); +/ +``` + +The query runs after the microflow returns, over the same admin API `mxcli oql` +uses. Three rules follow, each enforced rather than left to trip you up: + +- **`@cleanup none` is required.** `rollback` is the default and undoes the + test's writes before the query could see them, so a `@verify` on a rollback + test is refused rather than run against the pre-test state. +- **The query must return exactly one row and one column** — aggregate it, or + select one attribute of one row. Picking a cell out of a table would be a + guess. +- **The expected value is a literal**: a number, a quoted string, `true`/`false` + or `empty`. It is split off at the last comparison operator outside quotes and + parentheses, so a `where Value = 5` inside the query is left alone. +- **Every selected column needs a name.** Mendix's OQL rejects a bare + `select count(*)` with *"All OQL select columns must have a name"* — write + `select count(*) as n`. + +Operators are `=`, `!=` (`<>` accepted), `<`, `<=`, `>`, `>=`; numbers compare +numerically even though the runtime returns them as strings. + +A `@verify` that cannot be evaluated — unknown entity, malformed OQL, a +non-scalar result, or something that was never a query — is an **ERROR**, never +a pass. A false one fails with the value that came back: + +``` +FAIL dealing a board writes 81 cells + expected select count(*) as n from Sudoku.Cell = 81, actual: 27 +ERROR cells exist for a game that does not + @verify select count(*) as n from Sudoku.NoSuch = 1: OQL error: Unknown entity +``` + +`@verify` needs the test endpoint, so it works under `--local` and `--attach`. +The Docker / `--legacy-runner` path refuses a suite that uses it: its tests run +during boot, so there is no point at which to query the app. + ### The app's own after-startup microflow Boot registers the endpoint and then runs the project's own after-startup diff --git a/mdl-examples/bug-tests/expect-vacuous-assertions.test.mdl b/mdl-examples/bug-tests/expect-vacuous-assertions.test.mdl index ccb0689b9..309aa5b99 100644 --- a/mdl-examples/bug-tests/expect-vacuous-assertions.test.mdl +++ b/mdl-examples/bug-tests/expect-vacuous-assertions.test.mdl @@ -18,8 +18,8 @@ -- test_2: asserts nothing at all -- (no assertions — this test can only report that the body did not throw) -- test_3: believes it asserts something --- ERROR: @verify ...: @verify is not implemented — no runner evaluates it, --- so it would assert nothing. ... +-- ERROR: @verify ...: this test uses @cleanup rollback (the default), so +-- its writes are undone before the query runs ... -- -- A run reports the same three states, and summarises the vacuous ones: -- @@ -31,9 +31,8 @@ -- -- test_2 still passes by default: a smoke test is a legitimate thing to write. -- --require-assertions turns it into an ERROR for a project that has decided --- otherwise. test_3 is an error either way — @verify was documented as an OQL --- post-condition and is evaluated by nothing, which is the same defect as a --- dropped @expect wearing a different annotation. +-- otherwise. test_3 is an error because its @verify sits on a test using the +-- rollback default — see the @verify section below. -- -- The microflows referenced below do not need to exist for --list. @@ -53,7 +52,64 @@ $result = CALL MICROFLOW MyModule.Deal(); /** * @test believes it asserts something - * @verify select count(*) from MyModule.Board where Code = 'X' = 1 + * @verify select count(*) as n from MyModule.Board where Code = 'X' = 1 + */ +$result = CALL MICROFLOW MyModule.Deal(); +/ + +-- --------------------------------------------------------------------------- +-- FINDINGS #48: `@verify` was the same defect in the other annotation. +-- +-- Before it was implemented, no OQL — well-formed or not, true or not, against a +-- real entity or an invented one — could make a @verify fail. It was parsed into +-- the TestCase and read by nothing but --list. +-- +-- The rows below are the reporter's canaries. `--list` alone shows which are +-- rejected before anything runs; a real run (needs a project and --local) shows +-- the rest fail or error against the app. +-- +-- mxcli test mdl-examples/bug-tests/expect-vacuous-assertions.test.mdl --list +-- +-- @cleanup none is not decoration: rollback is the default and would undo the +-- writes before the query could see them, so a @verify on a rollback test is +-- refused rather than quietly asserting against the pre-test state — which is +-- test_v5 below. + +/** + * @test verify asserts on rows the microflow wrote + * @cleanup none + * @verify select count(*) as n from MyModule.Cell = 81 + */ +$result = CALL MICROFLOW MyModule.Deal(); +/ + +/** + * @test verify must fail on a false claim + * @cleanup none + * @verify select count(*) as n from MyModule.Cell = 999999 + */ +$result = CALL MICROFLOW MyModule.Deal(); +/ + +/** + * @test verify must error on an unknown entity + * @cleanup none + * @verify select count(*) as n from MyModule.NoSuchEntity = 1 + */ +$result = CALL MICROFLOW MyModule.Deal(); +/ + +/** + * @test verify must error on something that was never a query + * @cleanup none + * @verify this is not a query + */ +$result = CALL MICROFLOW MyModule.Deal(); +/ + +/** + * @test verify under the rollback default is refused + * @verify select count(*) as n from MyModule.Cell = 81 */ $result = CALL MICROFLOW MyModule.Deal(); / From 622386c14f800bcf77f264f55b65e2515fc2fbc7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 20:37:12 +0000 Subject: [PATCH 17/35] fix(check): an action slot is authorable by its source, not its storage key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mapping the Combobox's `onChangeEvent` so that `OnChange:` reaches it had a side effect: `allowedWidgetProperties` adds both a mapping's PropertyKey and its Source, so `onChangeEvent:` became an accepted MDL property name. The engine never reads it — `resolveMapping` for an action mapping reads the fixed AST slot (`w.GetOnChange()`) — so the value would have been accepted by check and dropped on write. That is the same silent-drop class this branch set out to fix, one layer up. An `action` mapping is now authorable by its Source only (`Action`/`OnClick`/`OnChange`). Writing the storage key is an error that names the spelling that works, rather than a generic "unknown property". Caught by TestValidateProgram_ReportsWidgetPropertyError, which lands on main from #165 and uses `onChangeEvent` as its example of a property a widget does not have. It was right. Also fixes the OnChange repro script, which did not build: its data view sourced a microflow that takes a parameter and returns nothing (CE1571 + CE0552), and bound radiobuttons/combobox to a String (CE2421 + CE7247). Now 0 errors on mxbuild 11.13.0, with all six controls' actions stored — the page unit carries six Forms$MicroflowAction nodes. DESCRIBE shows five of them; the pluggable path does not read action slots, which is a separate pre-existing gap and is stated as such in the script. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017s476QkXr9CFMvKspVzcvu --- .claude/skills/fix-issue.md | 2 +- .../bug-tests/widget-onchange-dropped.mdl | 32 ++++++++-- mdl/executor/validate_widgets.go | 56 ++++++++++++++-- .../validate_widgets_action_key_test.go | 64 +++++++++++++++++++ 4 files changed, 144 insertions(+), 10 deletions(-) create mode 100644 mdl/executor/validate_widgets_action_key_test.go diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index 96b67ef00..90cf1f251 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -530,7 +530,7 @@ extracting `OffsetExpression`/`LimitExpression`. | After a `rename attribute`, `mx check` reports **CE0161 "Error(s) in XPath constraint."** at every retrieve, widget data source or entity access rule whose constraint named the attribute — and, if the attribute had a `READ *` access rule, **CE0066 "Entity access is out of date"** on the domain model | Two different causes with the same trigger. (a) XPath names an attribute as a **bare step** (`[Status = 'Open']`), which the qualified-name reference scanner cannot see — and which a scan for the bare name must not touch, since the same three letters are a string literal here, a function name there, and another entity's attribute somewhere else. (b) `UpdateEntity` re-derives an access rule's members from the attributes it can match, so renaming the attribute in the model left the old member orphaned; the reference scan then renamed the orphan into a **duplicate** of the new member | `mdl/xpathrefs/` (new: `rewrite.go`, `scan.go`), `mdl/executor/xpath_rename_model.go`, `mdl/executor/xpath_rename.go`, `mdl/executor/rename_attribute_entity_rules.go`, `mdl/executor/cmd_entities.go` | (a) **XPath needs no type inference** — a constraint's target entity is known *structurally* (a retrieve names its `Entity`, a widget data source its `EntityRef.QualifiedName`, an access rule the entity it sits inside), and every further hop is an association or entity named in the path. Resolve each bare step to its owning entity, then **edit one identifier token textually**; do NOT re-render the parsed tree, which would churn spacing in constraints the rename has no business touching and would corrupt any constraint the parser and renderer disagree about. **The load-bearing safety check is a count invariant**: `visitor.ParseXPathConstraint` runs with ANTLR's error listeners removed, so it recovers and can return a tree that silently omits part of its input (`[LastName = 'L' FirstName]` parses; the trailing step is simply absent) — require the lexical occurrence count and the walked occurrence count to agree, or refuse. Unresolvable → report and leave alone, never guess: for a *mutating* consumer the checking world's "unresolved → assume nothing → catch less" inverts into "unresolved → change nothing". (b) Fix the entity's own by-name references (`MemberAccess.AttributeName`, `ValidationRule.AttributeID`, range `Min/MaxAttributeQualifiedName`) **in the model before `UpdateEntity`**, not by leaving them to the scan afterwards. **Trap that cost an hour**: the stored `$Type` of a domain model entity node is `DomainModels$EntityImpl`, not `DomainModels$Entity` (that is the metamodel's name for a *reference target*) — checking the latter matched nothing and made every access rule look like a cautious refusal instead of a bug. Controls on Mendix 11.13.0 with `mdl-examples/bug-tests/910-rename-attribute-xpath-constraints.mdl`: same script without the rename = 0 errors, with the rename pre-fix = 3× CE0161 then 1× CE0066, post-fix = 0. mendixlabs/mxcli#910 (problem 2, XPath half) | | Authoring a pluggable widget with an **object list** (Accordion group, Pop-up menu item, chart series) raises **CE0463** "the definition of this widget has changed" on the widget itself, on a widget created seconds ago against the current package | An object-list item's **required TextTemplate** that the author left unset was serialized as `null`. `emptyClientTemplateRules` in `widgetobj/builder.go` is a hardcoded per-widget table covering only DataGrid columns, so every other object-list widget fell through it. Required-ness is genuinely absent from the widget XML for these properties — and the widget schema **defaults `required` to true** — so the Accordion's `headerText` is mandatory even though nothing says so | `mdl/backend/widgetobj/builder.go` (`isUnsetRequiredTextTemplate`, `buildDefaultTextClientTemplateProperty`), plus the entry plumbing in `mdl/types/widget_property_type.go`, `modelsdk/widgets/loader.go`, `sdk/pages/pages_widgets_advanced.go`, `mdl/backend/modelsdk/widget_pluggable_write.go` (`convertPropTypeIDs`) | Serialize a required unset TextTemplate with the widget's **shipped ``**, read off the template ValueType's `Translations` beside the `Required` flag. **Both weaker forms were measured and fail**: `null` is CE0463, and an *empty* `Forms$ClientTemplate` is **CE4899** "Property 'Groups/1/Text' is required" — so the intuitive "emit an empty template" fix only moves the error. Populating is what `mx update-widgets` itself writes (verified: mxcli and the reference now emit the same `'Header'`/`'Koptekst'`). Scope it to **required** properties: filling every TextTemplate is the documented way to take CE0463 from 33 to 127. Note `PropertyTypeIDEntry` exists in THREE places (`mdl/types` — canonical and aliased by `modelsdk/widgets`; `sdk/pages` — what the builder consumes; `sdk/widgets` — the legacy engine's own), so a field added for this must be carried through `convertPropTypeIDs` or it silently never arrives. Repro `mdl-examples/bug-tests/891-accordion-group-nested-widgets.mdl`. Issue #891 | | `ALTER ENTITY … MODIFY ATTRIBUTE X SET DEFAULT ` (or any unrecognised word in the type position, including a typo like `Integr`) silently retypes the attribute to `Enumeration()`, and the project then **cannot be loaded at all** — `mx check` dies before validating with `System.ArgumentNullException: Value cannot be null. (Parameter 'value') at EnumerationAttributeType.set_EnumerationId` | `MODIFY ATTRIBUTE` requires a `dataType`, whose last grammar alternative is a bare `qualifiedName` (entity reference), so any word matches and the visitor maps it to `TypeEnumeration`. The executor wrote it with no enumeration behind it. CREATE ENTITY had guarded this since #552; the guard was **inline in the create handler** and never applied to MODIFY | `mdl/executor/attribute_type_ref.go` (new), `mdl/executor/cmd_entities.go` (`AlterEntityModifyAttribute` case) | Extract the create-path guard to `validateAttributeTypeRef` and call it from the modify path **before** mutating the attribute. Add `DROP DEFAULT ON ATTRIBUTE ` so clearing a default has a spelling that does not go through the type slot, and point the refusal at it. Check `mxcli syntax` too: the topic documented `MODIFY ATTRIBUTE AttrName SET DEFAULT val` — the corrupting form — which is how the reporter got there. When a guard is added inline to one handler, grep for sibling handlers that take the same input. mendixlabs/mxcli#910 | -| An `OnChange:` on a `checkbox`, `radiobuttons`, `combobox`, `dropdown`, `textarea` or `datepicker` does nothing. The MDL parses, `mxcli check` passes, `exec` reports success, and `describe page` shows the widget with its Label + Attribute and **no OnChange**. In the browser the control changes state and issues **zero `/xas/` requests** — no server round-trip at all. `textbox` and `actionbutton` keep their actions, so it reads as a page problem rather than a per-widget one | Three independent drops, any one of which is sufficient: (1) the page builder read `w.GetOnChange()` in `buildTextBoxV3` **only**, so the property died between AST and model for the other five; (2) the legacy writer hardcoded `serializeClientAction(nil)` for TextArea/DatePicker/CheckBox/RadioButtons/DropDown; (3) for pluggable widgets `actionSourceForKey` matched only the bare keys `onClick`/`onChange`, but Mendix's Combobox names its slot **`onChangeEvent`**, so no action mapping was emitted — and the shipped `combobox.def.json`, which **overrides any .mpk-derived def**, had none either | `mdl/executor/cmd_pages_builder_v3_widgets.go` (`applyOnChangeV3`), `sdk/mpr/writer_widgets_input.go`, `mdl/executor/widget_defs.go` (`actionSourceForKey`), `sdk/widgets/definitions/combobox.def.json` + `modelsdk/widgets/definitions/combobox.def.json`, `mdl/executor/cmd_pages_describe_parse.go` + `_output.go` | Route every input widget's OnChange through one helper rather than repeating the block, so the next widget added cannot forget it. Strip **one** `Event`/`Action` suffix before matching an action slot — narrowly: a Combobox also carries `onChangeFilterInputEvent` and `onChangeDatabaseEvent`, which have no MDL surface and must stay unmapped or one `OnChange:` writes three actions. A hand-written built-in `.def.json` beats the generator, so fixing the generator alone changes nothing for COMBOBOX/GALLERY/DATAGRID/filters — patch both, and put the mapping in **every** mode (modes are exclusive). Wire DESCRIBE at the same time: DESCRIBE is how a dropped property gets *found*, and it could only see this one on `textbox`. `dropdown` has no DESCRIBE case at all (separate gap). Repro `mdl-examples/bug-tests/widget-onchange-dropped.mdl`. FINDINGS #14 | +| An `OnChange:` on a `checkbox`, `radiobuttons`, `combobox`, `dropdown`, `textarea` or `datepicker` does nothing. The MDL parses, `mxcli check` passes, `exec` reports success, and `describe page` shows the widget with its Label + Attribute and **no OnChange**. In the browser the control changes state and issues **zero `/xas/` requests** — no server round-trip at all. `textbox` and `actionbutton` keep their actions, so it reads as a page problem rather than a per-widget one | Three independent drops, any one of which is sufficient: (1) the page builder read `w.GetOnChange()` in `buildTextBoxV3` **only**, so the property died between AST and model for the other five; (2) the legacy writer hardcoded `serializeClientAction(nil)` for TextArea/DatePicker/CheckBox/RadioButtons/DropDown; (3) for pluggable widgets `actionSourceForKey` matched only the bare keys `onClick`/`onChange`, but Mendix's Combobox names its slot **`onChangeEvent`**, so no action mapping was emitted — and the shipped `combobox.def.json`, which **overrides any .mpk-derived def**, had none either | `mdl/executor/cmd_pages_builder_v3_widgets.go` (`applyOnChangeV3`), `sdk/mpr/writer_widgets_input.go`, `mdl/executor/widget_defs.go` (`actionSourceForKey`), `sdk/widgets/definitions/combobox.def.json` + `modelsdk/widgets/definitions/combobox.def.json`, `mdl/executor/cmd_pages_describe_parse.go` + `_output.go` | Route every input widget's OnChange through one helper rather than repeating the block, so the next widget added cannot forget it. Strip **one** `Event`/`Action` suffix before matching an action slot — narrowly: a Combobox also carries `onChangeFilterInputEvent` and `onChangeDatabaseEvent`, which have no MDL surface and must stay unmapped or one `OnChange:` writes three actions. A hand-written built-in `.def.json` beats the generator, so fixing the generator alone changes nothing for COMBOBOX/GALLERY/DATAGRID/filters — patch both, and put the mapping in **every** mode (modes are exclusive). Wire DESCRIBE at the same time: DESCRIBE is how a dropped property gets *found*, and it could only see this one on `textbox`. `dropdown` has no DESCRIBE case at all (separate gap). Repro `mdl-examples/bug-tests/widget-onchange-dropped.mdl`. **Sequel caught by CI:** mapping a slot also makes its storage key pass `allowedWidgetProperties`, so `onChangeEvent: …` became an accepted MDL keyword that `resolveMapping` never reads — the same silent drop one layer up. An `action` mapping is authorable by its **Source** only (`OnChange`/`OnClick`/`Action`), never by its PropertyKey; `addMappingNames` enforces that and `actionStorageKeys` reports the storage key with the spelling that works. FINDINGS #14 | | A skill claims a construct is unsupported that the grammar accepts (and documents the working form elsewhere in the same file) — an agent writes valid MDL, hits the contradicting claim, and undoes it. Seen for `case … end case` in `write-microflows.md` (supported at §CASE Statements, "not supported" under UNSUPPORTED Syntax), with `check-syntax.md`, `MDL_QUICK_REFERENCE.md`, three `docs-site` pages and the generated project CLAUDE.md (a third shape, `CASE $Var/Attr AS x WHEN Enum.Value`) all disagreeing | Nothing validates prose claims against the grammar. `scripts/check-skill-mdl.sh` extracts fenced blocks but deliberately **skips microflow bodies** (fragment-heavy), so every syntax claim about microflow statements is unguarded. The wrong "WRONG:" example also used string-literal case values, which fail for their own reason — making the claim look confirmed | Docs only: `.claude/skills/mendix/write-microflows.md` + `check-syntax.md`, `docs/01-project/MDL_QUICK_REFERENCE.md`, `docs-site/src/language/control-flow.md` + `appendixes/{quick-reference,common-mistakes}.md`, `cmd/mxcli/init_claudemd.go` (the generated table) | Settle it against the binary before editing, one variant per claim (`bin/mxcli check` on the documented form, the quoted form, the qualified form, the `AS` form, the `else` form) — a "WRONG" example that fails for an unrelated reason is not evidence. Then fix **every** surface: grep `end case\|END CASE` across `.claude/`, `docs/`, `docs-site/` and `cmd/mxcli/*.go`, not just the file in the report (7 surfaces, the issue named 3). Pin both directions in `mdl-examples/bug-tests/` — `907-case-enum-split-is-supported.mdl` (must pass) and `907-case-enum-split-invalid-forms.fail.mdl` (must fail) — so `make check-mdl` catches drift back. Issue #907 | | Docs disagree on whether an enumeration may be written as a string literal — the skills say "NEVER use a string literal", docs-site uses `Status = 'Draft'` in a dozen CREATE/CHANGE examples and calls it "plain strings when the context is unambiguous". Following either one wholesale is wrong, and `mxcli check` passes **both** the good and the bad form, so the build is where you find out | It is context-dependent and nobody had measured it. A string is accepted wherever the slot is **already enum-typed**; in an **expression** it is a String compared to an Enumeration | Docs only: `.claude/skills/mendix/write-microflows.md` (Enumeration Comparisons), `docs-site/src/language/{control-flow,expressions}.md`, `docs-site/src/reference/microflow/create-microflow.md` | Measure per context on a real project before touching a single example — one microflow per construct, `mxcli docker check` read per construct (`mxcli new EP --version 11.13.0 --skip-build`, then `exec` + `docker check`). Verified on 11.13.0: **CE0117** for `if $O/Status = 'Draft'` and for `'x' + $O/Status`; **clean** for `change`/`create` member values, attribute `DEFAULT 'Draft'`, XPath `[Status = 'Draft']`, and `getCaption()`/`toString()` in a concat. So fix **only** comparisons and concatenations — a sweep-and-replace over every `Status = '…'` would have "corrected" ~10 of 12 sites that are fine. No `.fail.mdl` is possible (check passes the failing form; the type checker is still a proposal — `docs/11-proposals/PROPOSAL_expression_type_checking.md`), so the verdicts live in `mdl-examples/bug-tests/enum-string-literal-contexts.mdl`, which pins the ACCEPTED forms and builds clean (control: adding it to a probe project left the error count unchanged at the 2 deliberate failures) | | `mxcli exec` applies a script that `mxcli check` rejects with an error — e.g. a page written with an invalid widget property, which is then silently dropped, so the widget renders but does nothing | `exec` ran no semantic checks at all. Parse errors stopped it; the ~24 `Validate*` checks lived inline in `cmd/mxcli/cmd_check.go` and had exactly one caller. "Run check before exec" was a convention nothing enforced, on a tool built for unattended agent use | `mdl/executor/validate_program.go` (the shared list), `cmd/mxcli/cmd_exec.go` (the gate), `cmd/mxcli/cmd_check.go` | `executor.ValidateProgram` is now the single definition of what `check` checks, and `exec` refuses to run when it reports an **error** (warnings print and do not block). `--no-check` opts out. **A check wired into nothing is invisible** — it reports no violations and reads as a clean project — so `TestValidateProgram_WiresEveryWholeProgramValidator` asserts structurally that every exported `Validate*(prog *ast.Program, …)` is called from `ValidateProgram`; per-statement helpers take a statement and are excluded. When adding a check, add it to `ValidateProgram` and both commands get it | diff --git a/mdl-examples/bug-tests/widget-onchange-dropped.mdl b/mdl-examples/bug-tests/widget-onchange-dropped.mdl index 45a05d050..4309c57ad 100644 --- a/mdl-examples/bug-tests/widget-onchange-dropped.mdl +++ b/mdl-examples/bug-tests/widget-onchange-dropped.mdl @@ -35,15 +35,28 @@ -- -- Verify: -- mxcli exec mdl-examples/bug-tests/widget-onchange-dropped.mdl -p app.mpr +-- mx check app.mpr # 0 errors -- mxcli -p app.mpr -c "describe page Ctrl.ControlPanel" --- -> every control below must show its `OnChange:`. +-- -> each of the five BUILT-IN controls shows its `OnChange:`. The combobox +-- does not: DESCRIBE renders pluggable widgets through a different path +-- that does not read action slots (a separate, pre-existing gap). For it, +-- count the stored actions instead — the page unit carries SIX +-- `Forms$MicroflowAction` nodes, one per control. -- ============================================================================ create module Ctrl; / +-- radiobuttons and combobox bind an ENUMERATION, not a String — CE2421 / +-- CE7247 otherwise. Only the widget's binding is picky; the OnChange behaviour +-- under test is the same either way. +create enumeration Ctrl.TopicKind ( + Income 'Income', + Health 'Health' +); +/ create non-persistent entity Ctrl.Filter ( Enabled: Boolean, - Topic: String, + Topic: Ctrl.TopicKind, Notes: String, AsOf: DateTime ); @@ -53,6 +66,15 @@ begin change $Filter (Notes = 'applied') refresh; end; / +-- The data view's own source: parameterless, returns the object the controls +-- bind to. ACT_Apply cannot serve here — it takes a parameter and returns +-- nothing (CE1571 + CE0552). +create microflow Ctrl.DS_Filter () returns Ctrl.Filter as $Filter +begin + $Filter = create Ctrl.Filter (Topic = Ctrl.TopicKind.Income, Enabled = false); + return $Filter; +end; +/ -- Each control below carries an OnChange. Before the fix only a `textbox` kept -- it; the other five wrote nothing and the page still built green. create or replace page Ctrl.ControlPanel ( @@ -61,9 +83,9 @@ create or replace page Ctrl.ControlPanel ( layoutgrid lgMain { row rowMain { column colMain (desktopwidth: autofill) { - dataview dvFilter (datasource: microflow Ctrl.ACT_Apply) { - textbox txtTopic ( - Label: 'Topic', Attribute: Topic, + dataview dvFilter (datasource: microflow Ctrl.DS_Filter) { + textbox txtNotesShort ( + Label: 'Notes (short)', Attribute: Notes, OnChange: MICROFLOW Ctrl.ACT_Apply(Filter: $currentObject) ) textarea taNotes ( diff --git a/mdl/executor/validate_widgets.go b/mdl/executor/validate_widgets.go index 088236a23..3ceb82dec 100644 --- a/mdl/executor/validate_widgets.go +++ b/mdl/executor/validate_widgets.go @@ -959,6 +959,7 @@ func validatePluggableWidgetProperties(w *ast.WidgetV3, registry *WidgetRegistry } allowed, knownKeys := allowedWidgetProperties(def) dsKeys := datasourceTypedKeys(def) + actionKeys := actionStorageKeys(def) knownUnmapped := knownUnmappedProperties(def, allowed) var out []linter.Violation @@ -990,6 +991,20 @@ func validatePluggableWidgetProperties(w *ast.WidgetV3, registry *WidgetRegistry continue } + // An action slot's storage key is not the MDL spelling — name the one + // that is, rather than leaving the author to guess from a fuzzy match. + if src, ok := actionKeys[lower]; ok { + out = append(out, linter.Violation{ + RuleID: "MDL-WIDGET01", + Severity: linter.SeverityError, + Message: fmt.Sprintf( + "%s: widget `%s` (%s) property `%s` is the widget's internal name for an action slot and is not written from MDL — use `%s:` instead (e.g. `%s: MICROFLOW Module.ACT_Name(...)`)", + locationPrefix, w.Name, def.MDLName, key, src, src, + ), + }) + continue + } + if allowed[lower] { continue } @@ -1028,6 +1043,41 @@ func validatePluggableWidgetProperties(w *ast.WidgetV3, registry *WidgetRegistry // datasourceTypedKeys returns the lowercased propertyKeys whose def.json mapping // has operation "datasource" (across the top-level mappings and every mode). +// addMappingNames records the MDL names a PropertyMapping is authorable under. +// +// For most operations that is both the widget's own storage key and the engine's +// source name. An `action` mapping is the exception: resolveMapping reads the +// fixed AST slot (`w.GetAction()` / `w.GetOnChange()`), so ONLY the source name +// (`Action`/`OnClick`/`OnChange`) reaches the writer. Allowing the storage key +// would accept `onChangeEvent: …` on a Combobox and drop it on write — the +// silent-drop class FINDINGS #14 was about. It is reported by +// actionStorageKeys() instead, with the spelling that works. +func addMappingNames(add func(string), m PropertyMapping) { + if m.Operation != "action" { + add(m.PropertyKey) + } + add(m.Source) +} + +// actionStorageKeys maps each action mapping's storage key to the MDL name that +// actually writes it, so the validator can say "use OnChange" rather than only +// "unknown property". +func actionStorageKeys(def *WidgetDefinition) map[string]string { + out := make(map[string]string) + collect := func(ms []PropertyMapping) { + for _, m := range ms { + if m.Operation == "action" && m.PropertyKey != "" && m.Source != "" { + out[strings.ToLower(m.PropertyKey)] = m.Source + } + } + } + collect(def.PropertyMappings) + for _, mode := range def.Modes { + collect(mode.PropertyMappings) + } + return out +} + // These must be authored via the widget `datasource:` clause, not by name. func datasourceTypedKeys(def *WidgetDefinition) map[string]bool { out := make(map[string]bool) @@ -1107,8 +1157,7 @@ func allowedWidgetProperties(def *WidgetDefinition) (map[string]bool, []string) } for _, m := range def.PropertyMappings { - add(m.PropertyKey) - add(m.Source) + addMappingNames(add, m) } for _, m := range def.ChildSlots { add(m.PropertyKey) @@ -1118,8 +1167,7 @@ func allowedWidgetProperties(def *WidgetDefinition) (map[string]bool, []string) } for _, mode := range def.Modes { for _, m := range mode.PropertyMappings { - add(m.PropertyKey) - add(m.Source) + addMappingNames(add, m) } for _, m := range mode.ChildSlots { add(m.PropertyKey) diff --git a/mdl/executor/validate_widgets_action_key_test.go b/mdl/executor/validate_widgets_action_key_test.go new file mode 100644 index 000000000..8e2e2e30b --- /dev/null +++ b/mdl/executor/validate_widgets_action_key_test.go @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/linter" +) + +// A widget's action slot is addressed in MDL by the engine's source name +// (`OnChange:` / `Action:` / `OnClick:`), never by the widget's own storage key. +// `resolveMapping` reads `w.GetOnChange()`, so `onChangeEvent: …` lands nowhere: +// it would be accepted by check and dropped on write — the exact failure class +// FINDINGS #14 was about, one layer over. +// +// Making `onChangeEvent` a mapped property (so `OnChange:` reaches the Combobox +// at all) put its storage key into the allowed set as a side effect. This pins +// the split: the source name is authorable, the storage key is not, and writing +// the storage key gets a message that names the spelling that works. +func TestValidatePluggableWidgetProperties_ActionStorageKeyIsNotAuthorable(t *testing.T) { + reg := LoadWidgetRegistry("") + if reg == nil { + t.Fatal("built-in widget registry not available") + } + + violationsFor := func(props map[string]any) []linter.Violation { + w := &ast.WidgetV3{Name: "cmbX", Type: "combobox", Properties: props} + return validatePluggableWidgetProperties(w, reg, "page M.P") + } + + // The storage key must be rejected, with a hint naming `OnChange`. + got := violationsFor(map[string]any{ + "Attribute": "Name", + "onChangeEvent": "nope", + }) + var found *linter.Violation + for i := range got { + if got[i].RuleID == "MDL-WIDGET01" { + found = &got[i] + } + } + if found == nil { + t.Fatalf("onChangeEvent must not be authorable — it is read from nowhere; got %v", got) + } + if found.Severity != linter.SeverityError { + t.Errorf("severity = %v, want error", found.Severity) + } + if !strings.Contains(found.Message, "OnChange") { + t.Errorf("message must name the MDL spelling `OnChange`:\n%s", found.Message) + } + + // The source name is the real spelling and must stay clean. + for _, v := range violationsFor(map[string]any{ + "Attribute": "Name", + "OnChange": &ast.ActionV3{Type: "close"}, + }) { + if v.Severity == linter.SeverityError { + t.Errorf("`OnChange:` on a combobox produced an error: %s %s", v.RuleID, v.Message) + } + } +} From 60ba4814cedb9e892634b665d57c4d31955d69e3 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 20:44:30 +0000 Subject: [PATCH 18/35] mxcli test: leave the project byte-identical after a run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FINDINGS #47: running the test suite dirties the project file. `git status` reports the `.mpr` modified after a run that changed nothing, so a "run the tests, then assert the tree is clean" CI step fails, and a pull request carries a diff nobody can read — a `.mpr` is a SQLite database, and there is no cheap way to tell a bookkeeping GUID from a real model edit. The runner injects an MxTest module, builds, runs, and takes it back out. That restore is very nearly perfect: `mprcontents/` comes back byte-identical. The `.mpr` does not, for two reasons rather than the one the report identified. Every unit write stamps a fresh UUID into `_Transaction.LastTransactionID`, and the insert/delete cycle also relays SQLite's pages. Restoring the row alone is measurably insufficient — three consecutive runs then hold the id stable and still produce a different file hash every time, which is how that first attempt was rejected. So the whole file is snapshotted before injection and written back after a successful cleanup: byte-exact by construction rather than by enumerating what might have changed. Two refusals are load-bearing. The restore is declined when cleanup failed, and when the `mprcontents/` tree changed during the run. In both cases the project is not in the state the snapshot describes, and putting the old file back would turn a visible, harmless discrepancy into an invisible, misleading one — a tree that reads as clean while the model is not. The write goes through a temp file and a rename, so an interrupted restore cannot truncate the `.mpr`; that would be far worse than the cosmetic diff being fixed. Verified with the reporter's own harness on Mendix 11.6.6 — three consecutive runs against an untouched copy: state = mpr-hash / mprcontents-hash initial 5c787c16e544 0f4a84f6b489 run 1 5c787c16e544 0f4a84f6b489 run 2 5c787c16e544 0f4a84f6b489 run 3 5c787c16e544 0f4a84f6b489 Before, the `.mpr` hash differed on every run. Stubbing restore() to return early puts the snapshot tests back to failing. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CqQLyxnppZGfqPYbUHvSWG --- .claude/skills/mendix/test-microflows.md | 19 +++ cmd/mxcli/testrunner/host.go | 3 + cmd/mxcli/testrunner/runner.go | 22 +++- cmd/mxcli/testrunner/snapshot.go | 153 +++++++++++++++++++++++ cmd/mxcli/testrunner/snapshot_test.go | 153 +++++++++++++++++++++++ docs-site/src/tools/running-tests.md | 16 +++ 6 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 cmd/mxcli/testrunner/snapshot.go create mode 100644 cmd/mxcli/testrunner/snapshot_test.go diff --git a/.claude/skills/mendix/test-microflows.md b/.claude/skills/mendix/test-microflows.md index 0e4b7d28a..1b1d54837 100644 --- a/.claude/skills/mendix/test-microflows.md +++ b/.claude/skills/mendix/test-microflows.md @@ -159,6 +159,25 @@ The markdown format turns your tests into living documentation. | `@throws` | Expect error | `@throws 'validation failed'` | | `@cleanup` | Rollback strategy | `@cleanup rollback` (default) or `@cleanup none` | +### A test run leaves the project byte-identical + +`mxcli test` injects an `MxTest` module, builds, runs, and takes the injection +back out. When cleanup succeeds the project file is restored **byte-for-byte**, +so `git status` is clean afterwards and a CI step of the form "run the tests, +then assert the tree is clean" holds. + +This needs saying because restoring the *model* is not enough. Every unit write +stamps a fresh UUID into the `.mpr`'s `_Transaction` bookkeeping row, and the +inject/remove cycle relays SQLite's pages, so the file differs even once its +content matches again. Version control compares bytes, and a `.mpr` diff is +opaque — there is no cheap way to tell a bookkeeping GUID from a real model edit, +which is what made the spurious modification expensive rather than merely untidy. + +The restore is declined, deliberately, when **cleanup failed** or when the +`mprcontents/` tree changed during the run. In both cases the project is not in +the state the snapshot describes, and putting the old file back would turn a +visible, harmless discrepancy into an invisible, misleading one. + ### `@expect` — any Mendix condition, and nothing it cannot evaluate An `@expect` is **a Mendix expression that must evaluate to true**, not a fixed diff --git a/cmd/mxcli/testrunner/host.go b/cmd/mxcli/testrunner/host.go index b70e0812a..f6eb966ad 100644 --- a/cmd/mxcli/testrunner/host.go +++ b/cmd/mxcli/testrunner/host.go @@ -115,5 +115,8 @@ func (h *HostedEndpoint) Remove() { return } removeGeneratedJavaSource(h.projectPath, h.out) + // Only past the error return above, so this never claims a project is + // unchanged while an injection is still in it. + restoreProjectFile(h.state, nil, h.out) fmt.Fprintln(h.out, " test endpoint removed; project restored") } diff --git a/cmd/mxcli/testrunner/runner.go b/cmd/mxcli/testrunner/runner.go index 4277c625f..f0f4555a5 100644 --- a/cmd/mxcli/testrunner/runner.go +++ b/cmd/mxcli/testrunner/runner.go @@ -263,6 +263,7 @@ func runEndpoint(opts RunOptions, suite *TestSuite, timeout time.Duration, w io. fmt.Fprintln(w, "Cleaning up...") cleanupErr := cleanupEndpoint(opts.ProjectPath, state, cleanupSuite, w) removeGeneratedJavaSource(opts.ProjectPath, w) + restoreProjectFile(state, cleanupErr, w) reportCleanup(w, cleanupErr) if cleanupErr == nil { fmt.Fprintln(w, " project restored") @@ -351,7 +352,9 @@ func runAfterStartup(opts RunOptions, suite *TestSuite, timeout time.Duration, w logOutput, err = runDockerAndCapture(opts, timeout, w) } if err != nil { - reportCleanup(w, cleanup(opts.ProjectPath, state, w)) + cleanupErr := cleanup(opts.ProjectPath, state, w) + restoreProjectFile(state, cleanupErr, w) + reportCleanup(w, cleanupErr) return nil, err } @@ -360,6 +363,7 @@ func runAfterStartup(opts RunOptions, suite *TestSuite, timeout time.Duration, w fmt.Fprintln(w, "Cleaning up...") cleanupErr := cleanup(opts.ProjectPath, state, w) + restoreProjectFile(state, cleanupErr, w) reportCleanup(w, cleanupErr) PrintResults(w, result, opts.Color) @@ -466,6 +470,11 @@ func parseTestFiles(paths []string) (*TestSuite, error) { // projectState records what Run changed in the project, captured before the first // mutation so cleanup can put things back exactly rather than guessing. type projectState struct { + // snapshot is the project file as it was before anything was injected, + // restored on the way out so a run that changes nothing leaves it + // byte-identical. + snapshot projectSnapshot + // afterStartup is the project's original after-startup microflow ("" = none). afterStartup string // createdMxTest reports whether Run created the MxTest module, i.e. it did not @@ -489,9 +498,20 @@ func captureProjectState(projectPath string) (projectState, error) { return st, fmt.Errorf("listing modules: %w", err) } st.createdMxTest = !exists + + st.snapshot = takeProjectSnapshot(projectPath) return st, nil } +// restoreProjectFile puts the .mpr back byte-for-byte once cleanup has genuinely +// undone every injection, so a test run is a no-op on disk. See projectSnapshot +// for why restoring the model's content is not enough on its own. +func restoreProjectFile(st projectState, cleanupErr error, w io.Writer) { + if err := st.snapshot.restore(cleanupErr == nil); err != nil { + fmt.Fprintf(w, " note: could not restore the project file: %v\n", err) + } +} + // getAfterStartup reads the current after-startup microflow setting. func getAfterStartup(projectPath string) (string, error) { mxcliPath, err := findMxcli() diff --git a/cmd/mxcli/testrunner/snapshot.go b/cmd/mxcli/testrunner/snapshot.go new file mode 100644 index 000000000..3a11398e7 --- /dev/null +++ b/cmd/mxcli/testrunner/snapshot.go @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: Apache-2.0 + +package testrunner + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sort" +) + +// projectSnapshot is the project file exactly as it was before anything was +// injected, plus a digest of the document tree beside it. +// +// A test run injects a module, builds, runs, and takes the injection back out. +// The taking-out is very nearly perfect: `mprcontents/` comes back +// byte-identical. The `.mpr` does not, and it is not the model's fault — every +// unit write stamps a fresh UUID into the `_Transaction` bookkeeping row, and +// the insert/delete cycle leaves SQLite's pages laid out differently even once +// the logical content matches again. +// +// Restoring the row alone therefore is not enough: measured over three +// consecutive runs it makes the id stable and leaves the file hash different +// every time. Version control compares bytes, so a run that changed nothing +// still shows as a modification — which breaks any "run the tests, then assert +// the tree is clean" CI step, puts a meaningless diff in a pull request, and +// costs whoever sees it an afternoon, because a `.mpr` diff is opaque and there +// is no cheap way to tell a bookkeeping GUID from a real model edit. +// +// So the whole file is put back. That is byte-exact by construction rather than +// by enumerating what might have changed. +type projectSnapshot struct { + path string + contents []byte + // treeDigest covers the mprcontents/ document tree (empty for MPR v1, which + // keeps everything in the .mpr). It is the safety interlock: the .mpr indexes + // those documents, so putting an old .mpr back over a tree that has moved on + // would leave the index pointing at documents that are no longer there. + treeDigest string +} + +// takeProjectSnapshot reads the project file and digests the document tree. +// +// A failure is not fatal to the run — the caller keeps the zero value and simply +// does not restore, which is the behaviour before this existed. Refusing to run +// tests over a cosmetic diff would be the wrong trade. +func takeProjectSnapshot(projectPath string) projectSnapshot { + data, err := os.ReadFile(projectPath) + if err != nil { + return projectSnapshot{} + } + digest, err := documentTreeDigest(projectPath) + if err != nil { + return projectSnapshot{} + } + return projectSnapshot{path: projectPath, contents: data, treeDigest: digest} +} + +// restore puts the project file back, and reports what it did for the caller to +// log. +// +// It refuses in two cases, both of which mean the project is not in the state +// the snapshot describes: cleanup failed, or the document tree has changed. +// Writing the old .mpr in either case would replace a visible, harmless +// discrepancy with an invisible, misleading one — the tree would read as clean +// while the model was not. +func (s projectSnapshot) restore(cleanupOK bool) error { + if s.path == "" || !cleanupOK { + return nil + } + digest, err := documentTreeDigest(s.path) + if err != nil { + return fmt.Errorf("re-reading the document tree: %w", err) + } + if digest != s.treeDigest { + return fmt.Errorf("the document tree changed during the run, so the original %s was left in place", + filepath.Base(s.path)) + } + + current, err := os.ReadFile(s.path) + if err != nil { + return fmt.Errorf("re-reading %s: %w", filepath.Base(s.path), err) + } + if string(current) == string(s.contents) { + return nil + } + // Write through a temp file in the same directory so an interrupted restore + // cannot leave a truncated .mpr — that would be a far worse outcome than the + // cosmetic diff this is fixing. + tmp, err := os.CreateTemp(filepath.Dir(s.path), ".mxcli-restore-*") + if err != nil { + return err + } + tmpName := tmp.Name() + defer os.Remove(tmpName) + + if _, err := tmp.Write(s.contents); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if info, err := os.Stat(s.path); err == nil { + _ = os.Chmod(tmpName, info.Mode()) + } + return os.Rename(tmpName, s.path) +} + +// documentTreeDigest hashes the mprcontents/ tree — every file's path and +// content — so a change anywhere in it is detectable. MPR v1 has no such +// directory and digests to a constant. +func documentTreeDigest(projectPath string) (string, error) { + dir := filepath.Join(filepath.Dir(projectPath), "mprcontents") + if _, err := os.Stat(dir); err != nil { + return "", nil + } + + var paths []string + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + paths = append(paths, path) + } + return nil + }) + if err != nil { + return "", err + } + sort.Strings(paths) + + sum := sha256.New() + for _, p := range paths { + rel, _ := filepath.Rel(dir, p) + fmt.Fprintf(sum, "%s\n", filepath.ToSlash(rel)) + f, err := os.Open(p) + if err != nil { + return "", err + } + if _, err := io.Copy(sum, f); err != nil { + f.Close() + return "", err + } + f.Close() + } + return hex.EncodeToString(sum.Sum(nil)), nil +} diff --git a/cmd/mxcli/testrunner/snapshot_test.go b/cmd/mxcli/testrunner/snapshot_test.go new file mode 100644 index 000000000..c32832056 --- /dev/null +++ b/cmd/mxcli/testrunner/snapshot_test.go @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: Apache-2.0 + +package testrunner + +import ( + "os" + "path/filepath" + "testing" +) + +// newProjectFixture writes a project file plus a document tree beside it. +func newProjectFixture(t *testing.T, mprBytes string, units map[string]string) string { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "App.mpr") + if err := os.WriteFile(path, []byte(mprBytes), 0o644); err != nil { + t.Fatal(err) + } + contents := filepath.Join(dir, "mprcontents") + if err := os.MkdirAll(contents, 0o755); err != nil { + t.Fatal(err) + } + for name, body := range units { + if err := os.WriteFile(filepath.Join(contents, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + return path +} + +func readFile(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +// TestSnapshotRestoresTheProjectFileByte is the regression test for the dirty +// .mpr: a run that injects and removes leaves the model identical but the file +// different, because every unit write stamps the _Transaction row and SQLite +// relays its pages. Version control compares bytes, so the run shows up as a +// modification. +func TestSnapshotRestoresTheProjectFileByte(t *testing.T) { + path := newProjectFixture(t, "original-bytes", map[string]string{"a.mxunit": "A"}) + snap := takeProjectSnapshot(path) + + // Stand in for the run: the file changes, the document tree ends up back + // where it started. + if err := os.WriteFile(path, []byte("churned-by-the-run"), 0o644); err != nil { + t.Fatal(err) + } + + if err := snap.restore(true); err != nil { + t.Fatalf("restore: %v", err) + } + if got := readFile(t, path); got != "original-bytes" { + t.Errorf("project file = %q, want the original bytes", got) + } +} + +// TestSnapshotRefusesWhenCleanupFailed. Restoring over a project that is still +// modified would replace a visible, harmless discrepancy with an invisible, +// misleading one — the tree would read as clean while the model was not. +func TestSnapshotRefusesWhenCleanupFailed(t *testing.T) { + path := newProjectFixture(t, "original-bytes", map[string]string{"a.mxunit": "A"}) + snap := takeProjectSnapshot(path) + if err := os.WriteFile(path, []byte("still-has-MxTest-in-it"), 0o644); err != nil { + t.Fatal(err) + } + + if err := snap.restore(false); err != nil { + t.Fatalf("restore returned an error rather than declining: %v", err) + } + if got := readFile(t, path); got != "still-has-MxTest-in-it" { + t.Errorf("project file = %q, want it left alone after a failed cleanup", got) + } +} + +// TestSnapshotRefusesWhenTheDocumentTreeMoved. The .mpr indexes the documents +// beside it, so putting an old one back over a changed tree would leave the +// index pointing at documents that are no longer there. +func TestSnapshotRefusesWhenTheDocumentTreeMoved(t *testing.T) { + path := newProjectFixture(t, "original-bytes", map[string]string{"a.mxunit": "A"}) + snap := takeProjectSnapshot(path) + + if err := os.WriteFile(path, []byte("churned"), 0o644); err != nil { + t.Fatal(err) + } + leftover := filepath.Join(filepath.Dir(path), "mprcontents", "leftover.mxunit") + if err := os.WriteFile(leftover, []byte("a unit cleanup missed"), 0o644); err != nil { + t.Fatal(err) + } + + err := snap.restore(true) + if err == nil { + t.Fatal("restore overwrote the .mpr while the document tree had changed") + } + if got := readFile(t, path); got != "churned" { + t.Errorf("project file = %q, want it left alone", got) + } +} + +// TestDocumentTreeDigestNoticesEveryKindOfChange — content, name and removal all +// have to register, or the interlock above is decorative. +func TestDocumentTreeDigestNoticesEveryKindOfChange(t *testing.T) { + path := newProjectFixture(t, "x", map[string]string{"a.mxunit": "A", "b.mxunit": "B"}) + contents := filepath.Join(filepath.Dir(path), "mprcontents") + + base, err := documentTreeDigest(path) + if err != nil { + t.Fatal(err) + } + if base == "" { + t.Fatal("a v2 project digested to the empty v1 value") + } + + cases := []struct { + name string + mutate func() + }{ + {"content", func() { os.WriteFile(filepath.Join(contents, "a.mxunit"), []byte("CHANGED"), 0o644) }}, + {"rename", func() { + os.Rename(filepath.Join(contents, "a.mxunit"), filepath.Join(contents, "renamed.mxunit")) + }}, + {"removal", func() { os.Remove(filepath.Join(contents, "b.mxunit")) }}, + {"addition", func() { os.WriteFile(filepath.Join(contents, "c.mxunit"), []byte("C"), 0o644) }}, + } + for _, c := range cases { + fresh := newProjectFixture(t, "x", map[string]string{"a.mxunit": "A", "b.mxunit": "B"}) + contents = filepath.Join(filepath.Dir(fresh), "mprcontents") + before, _ := documentTreeDigest(fresh) + c.mutate() + after, err := documentTreeDigest(fresh) + if err != nil { + t.Errorf("%s: %v", c.name, err) + continue + } + if after == before { + t.Errorf("%s: the digest did not change", c.name) + } + } +} + +// TestSnapshotOfAMissingProjectDeclines — a snapshot that could not be taken +// must do nothing rather than truncate the file it never read. +func TestSnapshotOfAMissingProjectDeclines(t *testing.T) { + snap := takeProjectSnapshot(filepath.Join(t.TempDir(), "nope.mpr")) + if err := snap.restore(true); err != nil { + t.Errorf("an empty snapshot errored instead of declining: %v", err) + } +} diff --git a/docs-site/src/tools/running-tests.md b/docs-site/src/tools/running-tests.md index 6ddd362a0..a6dcb183a 100644 --- a/docs-site/src/tools/running-tests.md +++ b/docs-site/src/tools/running-tests.md @@ -118,6 +118,22 @@ every request must present that token, non-loopback callers are refused, and it will only ever invoke the generated `MxTest.Test_*` microflows. The token is never written into your project. +### A run leaves the project byte-identical + +`mxcli test` injects an `MxTest` module, builds, runs, and removes it again. When +cleanup succeeds the `.mpr` is restored **byte-for-byte**, so `git status` is +clean after a run and a "run the tests, then assert the tree is clean" CI step +holds. + +Restoring the model is not enough on its own: every unit write stamps a fresh +UUID into the `.mpr`'s `_Transaction` row, and the inject/remove cycle relays +SQLite's pages, so the file differs even once its content matches. Version +control compares bytes, and a `.mpr` diff is opaque. + +The restore is declined when cleanup failed, or when the `mprcontents/` tree +changed during the run — in both cases the project is not in the state the +snapshot describes, and putting the old file back would hide that. + ### `@expect`: what an assertion may say, and what happens when it cannot An `@expect` is a **Mendix expression that must evaluate to true**. Any From 5c2cdcce5c236d7243bd12790b2612ae92cf0a8c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 20:44:44 +0000 Subject: [PATCH 19/35] run --local --watch: wait for a write to finish before building MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FINDINGS #45: the watch loop can deploy a half-applied model, and there is no way to recover from it. `watchAndApply` rebuilt on the first mtime bump. An `mxcli exec` of a real script rewrites the `.mpr` and many `mprcontents/*.mxunit` files over several seconds, so the build snapshots the tree mid-write. There was a settle for the web bundler and nothing for the model. On its own that was survivable — you noticed, re-ran the script, and the second build was complete. Byte-idempotent `exec` closed that escape hatch: re-running an already-applied script writes nothing, so nothing re-triggers the watcher and the stale build stands, with no way out the tool offers. Two behaviours each correct alone, composing into a state you cannot leave. The watcher now waits for the source mtime to stop advancing — two poll intervals of quiet — before it builds. A long exec produces one build of the finished model instead of a build of the first file it touched, and a single editor save costs one extra poll. The wait is unbounded on purpose: a long exec is the case it exists for, and building late beats building mid-write. It still selects on the interrupt channel, so Ctrl-C during a write is not swallowed. `touch` on the `.mpr` remains the force-rebuild hatch, now documented rather than folklore. The generalisable half, recorded in the symptom table: a change *signal* is not a change *event* — anything polling mtime samples a writer mid-flight unless it debounces — and an optimisation that skips work should prompt the question of what informal recovery procedure depended on that work happening. Note on the test: the first version asserted the written-file count after wg.Wait(), which holds against a settleSource that returns immediately. It reads the writer's state at the moment settleSource returned instead. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CqQLyxnppZGfqPYbUHvSWG --- .claude/skills/fix-issue.md | 2 + .claude/skills/mendix/run-local.md | 27 ++++++ cmd/mxcli/docker/runlocal.go | 50 ++++++++++ cmd/mxcli/docker/runlocal_settle_test.go | 112 +++++++++++++++++++++++ docs-site/src/tools/run-local.md | 12 +++ 5 files changed, 203 insertions(+) create mode 100644 cmd/mxcli/docker/runlocal_settle_test.go diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index 95a34a3d0..6516ddee0 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -522,6 +522,8 @@ extracting `OffsetExpression`/`LimitExpression`. | `mxcli test` reports **PASS for an assertion that must fail** — `@expect 1 = 2`, `@expect length($result) = 999`, `@expect find($result, 'Z') >= 0` with the needle absent. Nothing in the output distinguishes a real assertion from a vacuous one, so a suite certifies work as verified while asserting only that the microflow did not throw. Mutation testing is what exposes it: mutants that return an obviously wrong value survive the suite | The `@expect` annotation was matched with a regex for one shape — `@expect $var (=|<>) ` — and `FindStringSubmatch` returning nil produced **no assertion at all** rather than an error. A test with zero assertions passes if its body completes. So the narrow support was not the defect; the silence was | `cmd/mxcli/testrunner/parser.go` (`expectPattern`, `parseAnnotations`), `cmd/mxcli/testrunner/expect.go` (new — `ParseExpect`, the validating parser), `cmd/mxcli/testrunner/generator_endpoint.go` + `generator.go` (emit the condition, not a rebuilt equality), `cmd/mxcli/testrunner/results.go` (`expectErrorResult`) | Capture the **whole** annotation body and hand it to a validating parser; anything it cannot compile becomes an `ExpectErrors` entry, the test is not generated at all, and the runner reports `StatusError` (which `FailCount` counts, so the exit code is non-zero). The parser is a strict recursive-descent pass over `exprcheck.Lex` — **not** `mdl/exprcheck`'s own parser, which recovers and emits hints, exactly the wrong behaviour here. Two measurements pinned the emitted expression against mxbuild 11.6.6: `<>` really is CE0117 (so the rewrite to `!=` is load-bearing, not cosmetic) and a wrong-typed comparison really is caught (`$result = 3` → CE0117), which is what makes the 0-error run on the 11 generated shapes mean something. **Generalisable**: when a pattern-matching parser can match *less* than its input, the non-match branch is a silent-failure path — audit every `if m := re.FindStringSubmatch(...); m != nil` whose else-branch does nothing. Repro `mdl-examples/bug-tests/expect-vacuous-assertions.mdl`. mxcli-sudoku FINDINGS #46 | | A test suite's green is unreadable: a test that asserts **nothing** prints the same `PASS` as one with six assertions, and `@verify` — documented as an OQL post-condition — is parsed and evaluated by nothing at all. After @expect started failing closed, the cheapest way back to green is to delete the assertion, and the output cannot tell that apart from a repair | Two silent-absence paths rather than the silent-drop path fixed in the row above. `TestResult` carried no assertion count, so nothing downstream could report one; and `TestCase.Verify` was populated by `parseAnnotations` and read by nothing but `--list` — `grep -n '\.Verify' cmd/mxcli/testrunner/*.go` returns the parser and the lister, no runner | `cmd/mxcli/testrunner/results.go` (`TestResult.Assertions`/`SourceFile`, `newResult`, `vacuousResult`, `resultNote`, `VacuousCount`), `cmd/mxcli/testrunner/parser.go` (`AssertionCount`, `AssertionErrors`, the @verify rejection), `cmd/mxcli/testrunner/junit.go` (`junitClassName`, assertions property), `cmd/mxcli/main.go` + `cmd_test_run.go` (`--require-assertions`) | Count assertions on the test case and carry them onto every result through **one** constructor (`newResult`) — the previous code built `TestResult` literals at five sites, which is exactly how a new field gets populated in one path and silently missing in another. Report the count on the ordinary result line, not behind `--verbose`: the whole lesson of #46 is that the *default* output must distinguish a test that asserted from one that did not. Vacuous tests warn by default and error under `--require-assertions`, because a smoke test is legitimate but an indistinguishable one is not. **Generalisable**: when auditing an annotation/config field for dead ends, grep for *readers*, not writers — a field with a parser and no consumer is a feature the docs promise and the code does not deliver, and it fails silently by construction. mxcli-sudoku FINDINGS #46 (follow-up) | | `@verify` cannot fail. No OQL — well-formed or not, true or not, against a real entity or an invented one — makes a test fail: `@verify select count(*) as n from Mod.Game = 999999` reports PASS on a table with one row, and so does `this is not a query` | The annotation was parsed into `TestCase.Verify` and read by nothing but `--list`. Same silent-absence class as the dropped `@expect`, in the annotation covering the *harder* half of Mendix testing: most microflows are side effects, so asserting on rows written is the only way to test them | `cmd/mxcli/testrunner/verify.go` (new — `ParseVerify`, `scalarOf`, `compare`, `runVerifies`), `cmd/mxcli/testrunner/parser.go` (`checkVerifyCleanup`), `runner_endpoint.go` + `runner_attach.go` + `watch.go` (`adminOptions` on `testTarget`), `runner.go` (`rejectVerifyOnLegacyRunner`) | Run the OQL over the admin API after the microflow returns, compare against a literal, and keep three outcomes apart: holds → unchanged, does not hold → FAIL with the observed value, cannot be evaluated → ERROR. Three things are enforced rather than left to trip you up, each learned by running it: **`@cleanup rollback` is refused** (the default undoes the writes before the query could see them, so it would assert against the pre-test state); **the result must be one row × one column** (picking a cell out of a table is a guess); and the **legacy after-startup runner refuses the whole suite**, since its tests run during boot with no seam to query at. Split the expected value off at the **last** comparison operator outside quotes and parens, then require the RHS to be a literal — without that check the split silently eats a `where x = 1` and sends a truncated query. **Generalisable**: unit tests with a stubbed endpoint proved the logic and would have shipped it broken — the first real run against a booted app found the OQL endpoint unreachable on that Mendix and the alias requirement, neither of which any stub could show. mxcli-sudoku FINDINGS #48 | +| Running `mxcli test` dirties the project: `git status` reports the `.mpr` modified after a run that changed nothing, so a "run the tests, assert the tree is clean" CI step fails and a pull request carries a meaningless diff. `mprcontents/` is byte-identical; only the `.mpr` differs, and a `.mpr` diff is opaque | The runner injects an `MxTest` module and takes it back out. Cleanup restores the model, but every unit write stamps a fresh UUID into the `.mpr`'s `_Transaction.LastTransactionID` (`updateTransactionID`, in both engines' writers) **and** the insert/delete cycle relays SQLite's pages. Restoring the row alone is measurably insufficient: three consecutive runs then hold the id stable and still produce a different file hash each time | `cmd/mxcli/testrunner/snapshot.go` (new — `projectSnapshot`, `documentTreeDigest`), `cmd/mxcli/testrunner/runner.go` (`captureProjectState`, `restoreProjectFile` at all three cleanup sites), `cmd/mxcli/testrunner/host.go` (`HostedEndpoint.Remove`) | Snapshot the whole `.mpr` before injecting and write it back after a successful cleanup — byte-exact by construction rather than by enumerating what might have changed. Two refusals are load-bearing: **cleanup failed** and **the `mprcontents/` tree moved**. In either case the project is not in the state the snapshot describes, and restoring would turn a visible harmless discrepancy into an invisible misleading one — a tree that reads as clean while the model is not. Write through a temp file + rename so an interrupted restore cannot truncate the `.mpr`. **Generalisable**: a tool that mutates a project to do its work owes a restore that is byte-exact, not semantically equivalent — every consumer downstream of it compares bytes. And when the obvious cause is a single row, measure the file hash before concluding it is the only one. Verified with the reporter's own harness: 3 runs, `.mpr` and `mprcontents` hashes unchanged throughout. mxcli-sudoku FINDINGS #47 | +| `run --local --watch` deploys a **half-applied model** when an `mxcli exec` is running, and there is no way to recover: re-running the script writes nothing (it is byte-idempotent), so nothing re-triggers the watcher and the stale build stands | `watchAndApply` (`cmd/mxcli/docker/runlocal.go`) rebuilt on the **first** mtime bump. An exec rewrites the `.mpr` and many `mprcontents/*.mxunit` over seconds, so the build snapshots the tree mid-write. There was a settle for the web bundler and nothing for the model. Two behaviours each correct alone: idempotency removed a *recovery path* nobody had written down as one — "just run it again" was load-bearing | `cmd/mxcli/docker/runlocal.go` (`settleSource`, `sourceSettleWindow`, the `case <-ticker.C` branch) | Wait for the source mtime to stop advancing (two poll intervals of quiet) before building. The wait is unbounded on purpose — a long exec is the case it exists for, and building late beats building mid-write — but it still honours the interrupt channel so Ctrl-C is not swallowed. `touch` on the `.mpr` remains the force-rebuild hatch, now documented. **Generalisable, twice over**: (a) a change *signal* is not a change *event* — anything polling mtime must debounce or it samples a writer mid-flight; (b) when adding an optimisation that skips work, ask what informal recovery procedure depended on that work happening. **Test trap hit while fixing it**: the first version asserted the file count *after* `wg.Wait()`, which holds against a `settleSource` that returns immediately — the assertion has to read the writer's state at the moment it returned. mxcli-sudoku FINDINGS #45 | | `mxcli oql` fails on every Mendix older than 11.11 with *"Action not found ... upgrade mxcli"*, and separately reports a **rejected query as `0 rows`** rather than as an error | Two independent misreadings of the runtime's replies. (1) The 11.11+ REST route `/dev/preview_execute_oql` does not exist on older runtimes, and the admin API does not 404 it — it dispatches the POST as an ordinary admin request, finds no `action` field, and answers **HTTP 200** with `{"result":1,"message":"Action not found"}`. The fallback to the legacy M2EE action keyed on the 404 alone, so the legacy action — which works fine there — was never tried. (2) That legacy action reports a bad query as `{"feedback":{"error":"..."},"result":0}` — inside the feedback, with a **success** result code — so `M2EEError()` (which keys off the result) says nothing and the error body parses as an empty result | `cmd/mxcli/docker/oql.go` (`legacyOQL`, `oqlDevErrorKind`, the `error` field in `parseOQLFeedback`'s envelope) | Fall back to the legacy action on *either* absence signal, and surface `feedback.error` regardless of the result code. Measured on 11.6.6: before, `mxcli oql` could not run any query; after, `select count(*) as n from Mod.E` returns a row, and a bad query is an error instead of `0 rows`. **Generalisable**: when a fallback is keyed on one specific failure signal, check what the *other* end actually sends — an HTTP-level 404 and an application-level "not found" are different wires, and a success code next to an error message is common enough to assume it happens. Found while wiring @verify (FINDINGS #48); related to #39 | | Windows Defender flags the mxcli **Windows** release binary as `Trojan:Script/Sabsik.EN.A!ml`; enterprise EDR (Defender for Endpoint, CrowdStrike, SentinelOne) blocks it harder. Not the generic unsigned-Go-binary false positive of #185 | The binary genuinely embedded **chisel**, a dual-use tunnelling/pivoting tool (SSH over WebSocket), on every platform — although the tunnel only ever runs inside a Linux container. `run --hub` linked `chisel/client`, `tunnel-hub` linked `chisel/server`, so windows/darwin carried 32 packages incl. the whole `x/crypto/ssh` stack for a feature they cannot use | `cmd/mxcli/docker/tunnel_linux.go` + `tunnel_other.go` (client seam), `cmd/mxcli/tunnelhub/control_linux.go` + `control_other.go` (server seam), `scripts/check-tunnel-deps.sh` (guard) | **Never obfuscate, pack or rename to dodge the scanner** — attacker tradecraft, and it makes the binary less trustworthy, not more. **Code signing does not fix this class**: a signed binary containing chisel is still flagged behaviourally; signing only addresses #185's generic false positive. The fix is to stop shipping the capability where it is unused: one interface per seam, `_linux.go` impl + `!linux` stub, commands still registered everywhere but failing with an actionable message. **Prove absence three ways, and know that `go tool nm` is not one of them** — release ldflags `-s -w` strip the symbol table, so nm reports "no symbols" whether or not the code is linked and would give a false pass; use `go list -deps`, `go version -m`, and `strings` (nm only on a deliberately unstripped build). **Guard against the transitive path, not the name**: match the module list (`x/crypto/ssh`, `gorilla/websocket`, `armon/go-socks5`, `jpillora/*`) so re-entry without the word "chisel" still trips it, and assert a **positive control** (chisel IS in the linux graph) so the check cannot pass vacuously. Verified by re-adding the import and watching the guard fail on all four windows/darwin targets. Result: -13.5 MB (-14.7%) on windows+darwin, linux unchanged. See ADR-0009 | | CE7375 "must be published and be the key when associations are exposed as an associated object id" on a service publishing no associations | `PublishAssociations` is the representation, not a yes/no — `No` selects "as an associated object id", which needs the system ID as key | `mdl/executor/validate_odata_service_shape.go` | Set `PublishAssociations: Yes` ("as a link", and the default when omitted). MDL-ODATA06 warns at check time | diff --git a/.claude/skills/mendix/run-local.md b/.claude/skills/mendix/run-local.md index 536e856d6..60ea7830a 100644 --- a/.claude/skills/mendix/run-local.md +++ b/.claude/skills/mendix/run-local.md @@ -101,6 +101,33 @@ $cf-over: rgb(168, 50, 30); background-color: rgba($cf-over, 0.1); ``` +## `--watch` waits for a write to finish before it builds + +The watcher polls the model source and rebuilds when it changes — but it does +**not** rebuild the instant it sees the first change. It waits for the source to +stop moving first. + +That matters because an `mxcli exec` of a real script rewrites the `.mpr` and +many `mprcontents/*.mxunit` files over several seconds. Building on the first +change deploys whatever is on disk at that instant: a **half-applied model**, +which looks like an ordinary stale build until you notice the app is missing +things the script definitely created. + +The escape hatch that used to cover this — "just run the script again" — stopped +working when `exec` became byte-idempotent. A re-run of an already-applied script +writes nothing, so nothing re-triggers the watcher, and the stale build is what +you are left with. Waiting for the write to settle is what makes that +unreachable rather than merely unlikely. + +Practical consequences: + +- A rebuild starts a couple of poll intervals after your last change, not + immediately. A single editor save is unaffected in practice. +- A long `exec` produces **one** build, of the finished model, instead of a + build of the first file it happened to touch. +- If you ever do need to force a rebuild without changing anything, `touch` the + `.mpr` — the watcher keys on mtime, so that re-triggers it. + ## "My edit didn't show up" — stale process, not stale cache `run --local` refuses to boot when its ports (8080/8090/6543) are already answering, diff --git a/cmd/mxcli/docker/runlocal.go b/cmd/mxcli/docker/runlocal.go index 455db38e7..63dfd8f84 100644 --- a/cmd/mxcli/docker/runlocal.go +++ b/cmd/mxcli/docker/runlocal.go @@ -1094,6 +1094,39 @@ func ensureClientServed(deployDir, appURL, mxbuildPath string, out io.Writer) er return nil } +// sourceSettleWindow is how long the model source must stop changing before a +// rebuild starts. Two poll intervals: long enough that the gaps between an +// exec's individual unit writes do not read as "finished", short enough that a +// single editor save still rebuilds promptly. +const sourceSettleWindow = 2 + +// settleSource waits until the model source stops changing, and returns the +// mtime it settled at (or the zero time if interrupted). +// +// The wait is unbounded on purpose: a long exec is exactly the case this exists +// for, and rebuilding a project mid-write is worse than rebuilding it late. It +// returns as soon as the source has been quiet for sourceSettleWindow polls, so +// an ordinary single-file save costs one extra poll. +func settleSource(projectPath string, seen time.Time, poll time.Duration, sigCh <-chan os.Signal) time.Time { + quiet := 0 + for { + select { + case <-sigCh: + return time.Time{} + case <-time.After(poll): + } + now := sourceMTime(projectPath) + if now.After(seen) { + seen = now + quiet = 0 + continue + } + if quiet++; quiet >= sourceSettleWindow { + return seen + } + } +} + // watchAndApply polls the project for changes and applies each rebuild until the // user interrupts (Ctrl-C). StartLocalRuntime already resolved the JVM; here we // only rebuild via serve and let the RuntimeController decide reload vs restart. @@ -1123,6 +1156,23 @@ func watchAndApply(opts LocalRunOptions, serve *ServeServer, rt *LocalRuntime, w if !now.After(last) { continue } + // Wait for the writer to finish before reading the model. + // + // An `mxcli exec` of a real script rewrites the .mpr and many + // mprcontents/*.mxunit files over several seconds. Building on the + // first mtime bump deploys whatever happens to be on disk at that + // instant — a half-applied model — and the escape hatch used to be + // "run the script again". Byte-idempotent exec closed that: the + // second run writes nothing, so nothing re-triggers the watcher and + // the stale build is what you are left with, with no way out the tool + // offers. Settling first means the watcher can only ever observe a + // model that has stopped changing. + now = settleSource(opts.ProjectPath, now, opts.PollInterval, sigCh) + if now.IsZero() { + // Interrupted while settling. + fmt.Fprintln(w, "\nShutting down...") + return nil + } last = now gen++ fmt.Fprintf(w, "Change detected, rebuilding (build #%d)...\n", gen) diff --git a/cmd/mxcli/docker/runlocal_settle_test.go b/cmd/mxcli/docker/runlocal_settle_test.go new file mode 100644 index 000000000..257db5034 --- /dev/null +++ b/cmd/mxcli/docker/runlocal_settle_test.go @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" +) + +// writeUnits simulates what an `mxcli exec` does to the model on disk: many +// files rewritten over a stretch of time, not one atomic change. +func writeUnits(t *testing.T, dir string, n int, gap time.Duration) { + t.Helper() + for i := 0; i < n; i++ { + path := filepath.Join(dir, "unit"+string(rune('a'+i))+".mxunit") + if err := os.WriteFile(path, []byte{byte(i)}, 0o644); err != nil { + t.Error(err) + return + } + time.Sleep(gap) + } +} + +// TestSettleSourceWaitsForAMultiFileWrite is the regression test for the stale +// build. The watcher used to rebuild on the first mtime bump, so a multi-second +// exec was deployed half-applied — and once exec became byte-idempotent, +// re-running it wrote nothing and there was no way to re-trigger a build. +func TestSettleSourceWaitsForAMultiFileWrite(t *testing.T) { + dir := t.TempDir() + mpr := filepath.Join(dir, "App.mpr") + if err := os.WriteFile(mpr, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + contents := filepath.Join(dir, "mprcontents") + if err := os.MkdirAll(contents, 0o755); err != nil { + t.Fatal(err) + } + + // The gap between unit writes is well inside one poll, which is what an exec + // actually does — it rewrites units as fast as the disk takes them, over a + // span of seconds. A writer that paused longer than the settle window between + // files would be indistinguishable from a finished one, and no timeout-based + // watcher can tell those apart. + const poll = 20 * time.Millisecond + var done atomic.Bool + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + writeUnits(t, contents, 12, poll/4) + done.Store(true) + }() + + settled := settleSource(mpr, sourceMTime(mpr), poll, nil) + // Read the writer's state at the moment settleSource returned — after + // wg.Wait() every unit is on disk regardless, so that assertion would hold + // against a settleSource that returned immediately. + finishedFirst := done.Load() + wg.Wait() + + if settled.IsZero() { + t.Fatal("settleSource reported an interrupt that never happened") + } + if !finishedFirst { + t.Error("settleSource returned while the model was still being written — the build would be half-applied") + } + if final := sourceMTime(mpr); settled.Before(final) { + t.Errorf("settled at %v but the source is newer (%v)", settled, final) + } +} + +// TestSettleSourceReturnsPromptlyForOneChange guards the other direction: a +// single editor save must not pay a long wait. +func TestSettleSourceReturnsPromptlyForOneChange(t *testing.T) { + dir := t.TempDir() + mpr := filepath.Join(dir, "App.mpr") + if err := os.WriteFile(mpr, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + + const poll = 20 * time.Millisecond + start := time.Now() + settled := settleSource(mpr, sourceMTime(mpr), poll, nil) + elapsed := time.Since(start) + + if settled.IsZero() { + t.Fatal("settleSource reported an interrupt that never happened") + } + if max := poll * (sourceSettleWindow + 3); elapsed > max { + t.Errorf("a quiet source took %v to settle, want under %v", elapsed, max) + } +} + +// TestSettleSourceHonoursInterrupt — Ctrl-C during a long exec must still stop +// the loop rather than wait the write out. +func TestSettleSourceHonoursInterrupt(t *testing.T) { + dir := t.TempDir() + mpr := filepath.Join(dir, "App.mpr") + if err := os.WriteFile(mpr, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + sigCh := make(chan os.Signal, 1) + sigCh <- os.Interrupt + + if settled := settleSource(mpr, sourceMTime(mpr), 20*time.Millisecond, sigCh); !settled.IsZero() { + t.Errorf("settleSource ignored the interrupt, returned %v", settled) + } +} diff --git a/docs-site/src/tools/run-local.md b/docs-site/src/tools/run-local.md index 524eca890..6b13f2b77 100644 --- a/docs-site/src/tools/run-local.md +++ b/docs-site/src/tools/run-local.md @@ -274,6 +274,18 @@ the log instead of guessed. The intended cycle: an agent (or you) edits the model with `mxcli exec`/MDL — or edits a theme `.scss` — and the running `run --local` picks it up and hot-applies it. +**A rebuild starts once the source stops changing, not on the first change.** An +`mxcli exec` of a real script rewrites the `.mpr` and many `mprcontents/*.mxunit` files +over several seconds; building on the first change would deploy whatever was on disk at +that instant — a half-applied model. The watcher therefore waits for a couple of quiet +polls before it builds, so a long `exec` produces one build of the finished model +rather than a build of the first file it touched. + +This matters more than it used to. The old escape hatch was "run the script again", and +byte-idempotent `exec` closed it: re-running an already-applied script writes nothing, +so nothing re-triggers the watcher and the stale build has no way out. If you do need to +force a rebuild without changing anything, `touch` the `.mpr` — the signal is mtime. + ## Editing themes (SCSS): rebuild, don't clear caches A theme edit (e.g. `theme/web/main.scss`) needs a **rebuild**, not a cache-clear. From 72560621299e73bbbd0b57febda3e7cab7f15253 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 20:53:34 +0000 Subject: [PATCH 20/35] fix(pages): read a pluggable widget's action back, and refuse the association storage key Two follow-ups on top of the action-slot fix, both measured against a real Mendix 11.13 project. 1. DESCRIBE could not see a ComboBox's OnChange, so describe -> exec DELETED it. The write was already correct -- the action reaches BSON as Forms$MicroflowAction -- but DESCRIBE read only the built-in OnChangeAction slot, which a pluggable widget does not use. It emitted `combobox cbo (Label: 'X')`, and re-executing that output wrote a widget with no action: 1 Forms$MicroflowAction before the round trip, 0 after, while the datepicker beside it survived. Read via customWidgetPropertyActionMap + renderClientActionMDL, the shape DataGrid2's onClick already used. All six controls now survive the round trip and the page's own Verify recipe can be completed. 2. `attributeAssociation:` is the same silent drop as `onChangeEvent:` -- the association operation also resolves from a fixed AST accessor -- so the storage-key exclusion covers it too. The tempting generalisation is wrong and worth recording: excluding every storage key whose mapping has a Source rejects working syntax, because `optionsSourceAssociationCaptionAttribute:` DOES persist. The list is two operations because that is what the written documents show; a test pins it so the next addition has to be measured rather than reasoned. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01KTdvGwVewkdgNQGDCxpQPZ --- .../cmd_pages_describe_onchange_test.go | 60 +++++++++++++++++++ mdl/executor/cmd_pages_describe_output.go | 6 ++ mdl/executor/cmd_pages_describe_parse.go | 10 ++++ mdl/executor/validate_widgets.go | 27 +++++++-- mdl/executor/widget_onchange_event_test.go | 24 ++++++++ 5 files changed, 123 insertions(+), 4 deletions(-) diff --git a/mdl/executor/cmd_pages_describe_onchange_test.go b/mdl/executor/cmd_pages_describe_onchange_test.go index e9f10a530..ef6e3676d 100644 --- a/mdl/executor/cmd_pages_describe_onchange_test.go +++ b/mdl/executor/cmd_pages_describe_onchange_test.go @@ -53,3 +53,63 @@ func TestParseRawWidget_OnChangeOnEveryInputWidget(t *testing.T) { }) } } + +// TestParseRawWidget_ComboBoxOnChangeEvent is the pluggable half, and it is +// about data loss rather than visibility. +// +// The ComboBox stores its action under the widget property `onChangeEvent`, so +// the built-in OnChangeAction path above does not see it. The write path was +// correct — the action reached BSON as a Forms$MicroflowAction — but DESCRIBE +// emitted `combobox cbo (Label: 'X')` with no OnChange, so a +// describe → edit → exec cycle *deleted* it. Measured on a real 11.13 project: +// 1 Forms$MicroflowAction before the round trip, 0 after, while the datepicker +// beside it survived. +func TestParseRawWidget_ComboBoxOnChangeEvent(t *testing.T) { + ctx, _ := newMockCtx(t) + + // Shape mirrors a stored ComboBox: the property key lives in + // Type.PropertyTypes and the value points back at it by TypePointer, so the + // action is only reachable through that indirection — which is why the + // built-in OnChangeAction reader never saw it. + raw := map[string]any{ + "$Type": "CustomWidgets$CustomWidget", + "Name": "cboKind", + "Type": map[string]any{ + "$Type": "CustomWidgets$WidgetType", + "WidgetId": "com.mendix.widget.web.combobox.Combobox", + "PropertyTypes": []any{int32(3), map[string]any{ + "$ID": "pt-onchange", + "$Type": "CustomWidgets$WidgetPropertyType", + "PropertyKey": "onChangeEvent", + }}, + }, + "Object": map[string]any{ + "$Type": "CustomWidgets$WidgetObject", + "Properties": []any{int32(3), map[string]any{ + "$Type": "CustomWidgets$WidgetProperty", + "TypePointer": "pt-onchange", + "Value": map[string]any{ + "$Type": "CustomWidgets$WidgetValue", + "Action": map[string]any{ + "$Type": "Forms$MicroflowAction", + "MicroflowSettings": map[string]any{ + "$Type": "Forms$MicroflowSettings", + "Microflow": "MyFirstModule.ACT_Apply", + }, + }, + }, + }}, + }, + } + + got := parseRawWidget(ctx, raw) + if len(got) != 1 { + t.Fatalf("expected 1 widget, got %d", len(got)) + } + if got[0].OnChange == "" { + t.Fatal("ComboBox onChangeEvent was not read back — a describe→exec round trip deletes it") + } + if !strings.Contains(got[0].OnChange, "ACT_Apply") { + t.Errorf("OnChange = %q, want it to name the microflow", got[0].OnChange) + } +} diff --git a/mdl/executor/cmd_pages_describe_output.go b/mdl/executor/cmd_pages_describe_output.go index fb84a79d0..f30dca6c3 100644 --- a/mdl/executor/cmd_pages_describe_output.go +++ b/mdl/executor/cmd_pages_describe_output.go @@ -783,6 +783,12 @@ func outputWidgetMDLV3(ctx *ExecContext, w rawWidget, indent int) { props = append(props, fmt.Sprintf("CaptionAttribute: %s", w.CaptionAttribute)) } } + // A pluggable widget's on-change action (ComboBox `onChangeEvent`). + // Emitted for the same reason as the built-in inputs above: without + // it a describe→edit→exec cycle silently drops the action. + if w.OnChange != "" { + props = append(props, fmt.Sprintf("OnChange: %s", w.OnChange)) + } // Show filter attributes for filter widgets if len(w.FilterAttributes) > 0 { props = append(props, fmt.Sprintf("Attributes: [%s]", strings.Join(w.FilterAttributes, ", "))) diff --git a/mdl/executor/cmd_pages_describe_parse.go b/mdl/executor/cmd_pages_describe_parse.go index 0722287d0..3f24c728b 100644 --- a/mdl/executor/cmd_pages_describe_parse.go +++ b/mdl/executor/cmd_pages_describe_parse.go @@ -307,6 +307,16 @@ func parseRawWidget(ctx *ExecContext, w map[string]any, parentEntityContext ...s widget.Content = extractCustomWidgetPropertyAssociation(ctx, w, "attributeAssociation") widget.CaptionAttribute = extractCustomWidgetPropertyAttributeRef(ctx, w, "optionsSourceAssociationCaptionAttribute") } + // The on-change action, in BOTH modes — the def maps `onChangeEvent` + // in each, and modes are exclusive. Read outside the DataSource + // branch so an enumeration-mode ComboBox keeps its action too. + // + // Without this the write path stored the action correctly and + // describe simply never emitted it, so a describe→edit→exec cycle + // deleted it: measured 1 Forms$MicroflowAction → 0, while the + // datepicker beside it survived. Same shape as the DataGrid2 + // `onClick` round trip below (ledger #67). + widget.OnChange = renderClientActionMDL(ctx, customWidgetPropertyActionMap(ctx, w, "onChangeEvent")) } // The drop-down filter's association mode is the same shape as the // ComboBox's, on differently-named properties: `baseType` selects it and diff --git a/mdl/executor/validate_widgets.go b/mdl/executor/validate_widgets.go index 3ceb82dec..5a07ab0ba 100644 --- a/mdl/executor/validate_widgets.go +++ b/mdl/executor/validate_widgets.go @@ -998,8 +998,8 @@ func validatePluggableWidgetProperties(w *ast.WidgetV3, registry *WidgetRegistry RuleID: "MDL-WIDGET01", Severity: linter.SeverityError, Message: fmt.Sprintf( - "%s: widget `%s` (%s) property `%s` is the widget's internal name for an action slot and is not written from MDL — use `%s:` instead (e.g. `%s: MICROFLOW Module.ACT_Name(...)`)", - locationPrefix, w.Name, def.MDLName, key, src, src, + "%s: widget `%s` (%s) property `%s` is the widget's internal storage name and is not written from MDL — use `%s:` instead", + locationPrefix, w.Name, def.MDLName, key, src, ), }) continue @@ -1053,12 +1053,31 @@ func validatePluggableWidgetProperties(w *ast.WidgetV3, registry *WidgetRegistry // silent-drop class FINDINGS #14 was about. It is reported by // actionStorageKeys() instead, with the spelling that works. func addMappingNames(add func(string), m PropertyMapping) { - if m.Operation != "action" { + if !readsFixedASTSlot(m.Operation) { add(m.PropertyKey) } add(m.Source) } +// readsFixedASTSlot reports whether an operation's value is resolved from a +// dedicated AST accessor rather than from a property looked up by name. +// +// resolveMapping switches on the mapping's Source, and for these operations it +// reads a fixed slot — `w.GetOnChange()` for an action, the association binding +// for an association — so a script naming the widget's own storage key is +// accepted by check and written by nothing. +// +// Measured on a Combobox against Mendix 11.13, which is why this is a list of +// two rather than "every operation with a Source": `attributeAssociation:` does +// not persist while `Association:` does, and `onChangeEvent:` does not persist +// while `OnChange:` does — but `optionsSourceAssociationCaptionAttribute:` DOES +// persist, so excluding every storage key would reject working syntax. Add an +// operation here only after checking the written document, not from the shape of +// the mapping. +func readsFixedASTSlot(op string) bool { + return op == "action" || op == "association" +} + // actionStorageKeys maps each action mapping's storage key to the MDL name that // actually writes it, so the validator can say "use OnChange" rather than only // "unknown property". @@ -1066,7 +1085,7 @@ func actionStorageKeys(def *WidgetDefinition) map[string]string { out := make(map[string]string) collect := func(ms []PropertyMapping) { for _, m := range ms { - if m.Operation == "action" && m.PropertyKey != "" && m.Source != "" { + if readsFixedASTSlot(m.Operation) && m.PropertyKey != "" && m.Source != "" { out[strings.ToLower(m.PropertyKey)] = m.Source } } diff --git a/mdl/executor/widget_onchange_event_test.go b/mdl/executor/widget_onchange_event_test.go index 64146857e..8bd99167f 100644 --- a/mdl/executor/widget_onchange_event_test.go +++ b/mdl/executor/widget_onchange_event_test.go @@ -129,3 +129,27 @@ func TestCombobox_OnChangeMappedInBothModes(t *testing.T) { }) } } + +// TestReadsFixedASTSlot_MeasuredOperations pins the exclusion list to what was +// measured on a real Combobox, not to the shape of the mapping. +// +// The tempting rule — "any mapping whose PropertyKey differs from its Source" — +// is wrong and would reject working syntax: on Mendix 11.13, +// `optionsSourceAssociationCaptionAttribute:` persists even though its MDL name +// is `CaptionAttribute`, while `attributeAssociation:` and `onChangeEvent:` do +// not. Only the two operations that resolve from a dedicated AST accessor are +// excluded. +func TestReadsFixedASTSlot_MeasuredOperations(t *testing.T) { + excluded := map[string]bool{"action": true, "association": true} + for _, op := range []string{ + "action", "association", + "attribute", "primitive", "datasource", "texttemplate", + "selection", "widgets", "attributeObjects", "expression", + } { + if got := readsFixedASTSlot(op); got != excluded[op] { + t.Errorf("readsFixedASTSlot(%q) = %v, want %v — "+ + "an operation belongs here only once the written document shows "+ + "its storage key is dropped", op, got, excluded[op]) + } + } +} From c1d590c182a963e5c3f42aa92306f4c666f894a4 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 21:46:42 +0000 Subject: [PATCH 21/35] docs: correct the capability docs, and read a query's table mapping back MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FINDINGS #20: a reader grepped MISSING_CAPABILITIES.md and MDL_FEATURE_MATRIX.md, found the External Database Connector recorded as unsupported, and designed a whole workaround around a blocker that CREATE DATABASE CONNECTION had already removed. The docs were wrong about far more than that one row. Verified each claim against the grammar's statement rules and `mxcli syntax` rather than against memory. Of the 13 document types MISSING_CAPABILITIES.md lists as unsupported, 11 now ship. The feature matrix's "Not Yet Implemented" table listed nine implemented features, several with a full row of N. - MDL_FEATURE_MATRIX.md: the section now means what its heading says. Genuinely unimplemented types (microflow rules — not to be confused with the supported CREATE VALIDATION RULE — message definitions, XML schemas, SOAP, data importer, building blocks, extensions, widget packages, system texts) are listed on their own; partial ones are separated out; the fifteen implemented ones move into Core Document Types rather than being deleted. - MISSING_CAPABILITIES.md keeps its measurement — how many documents of each type real apps contain, which has not changed — and gains a Status column, a per-section status line, and a header saying plainly that it is a dated survey and not the current status. Its execution plan is marked done. A `database-connection` syntax topic now exists. Its absence is what made the gap look real: `mxcli syntax` had SHOW DATABASE CONNECTIONS and nothing about CREATE. It documents three traps, all measured on mxbuild 11.13.0, and its example runs to 0 errors on a fresh project: 1. connection string / username / password must reference constants, not literals (a literal makes the project unopenable; mxbuild stays green); 2. username and password are required even when the driver needs neither; 3. a named type needs its JDBC driver declared via ALTER MODULE ... ADD JAR DEPENDENCY, or the build fails CE5278. BYOD skips that check. Two real bugs found while verifying, rather than assumed: - The modelsdk (default) engine never read a query's TableMappings, so `describe database connection` emitted a query with no `returns`/`map (…)` and a describe → exec round trip dropped the entity binding and the column mapping. The renderer was dead code on that engine. The round-trip test existed but was pinned to the legacy backend, so it stayed green while the default engine was lossy; it now runs on both, and fails on modelsdk without this fix. - `CREATE NON_PERSISTENT ENTITY` does not parse — the token is hyphenated (`NON-PERSISTENT`). It was documented in two syntax topics, a skill, and an LSP completion snippet, which inserted unparseable MDL on selection. A drift guard pairs each capability with the syntax topic that proves it ships, so the matrix cannot re-assert a filled gap; controls confirm both guards fail when the exact #20 claim is reintroduced. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017s476QkXr9CFMvKspVzcvu --- .claude/skills/mendix/rest-call-from-json.md | 56 ++++---- cmd/mxcli/lsp_completion.go | 2 +- .../syntax/capability_docs_drift_test.go | 124 ++++++++++++++++++ cmd/mxcli/syntax/features_domain_model.go | 4 +- cmd/mxcli/syntax/features_integration.go | 106 +++++++++++++++ docs/01-project/MDL_FEATURE_MATRIX.md | 70 ++++++---- docs/01-project/MISSING_CAPABILITIES.md | 110 +++++++++++----- mdl/backend/modelsdk/integration_read.go | 41 ++++++ mdl/executor/roundtrip_dbconnection_test.go | 14 +- 9 files changed, 443 insertions(+), 84 deletions(-) create mode 100644 cmd/mxcli/syntax/capability_docs_drift_test.go diff --git a/.claude/skills/mendix/rest-call-from-json.md b/.claude/skills/mendix/rest-call-from-json.md index 3f5754a73..e961f60c8 100644 --- a/.claude/skills/mendix/rest-call-from-json.md +++ b/.claude/skills/mendix/rest-call-from-json.md @@ -42,15 +42,17 @@ describe json structure Module.JSON_MyStructure; Derive one entity per JSON object type. Name them after what they represent (not after JSON keys). ```sql -create entity Module.MyRootObject (NON_PERSISTENT) - stringField : string - intField : integer - decimalField : decimal - boolField : boolean default false; - -create entity Module.MyNestedObject (NON_PERSISTENT) - name : string - code : string; +create non-persistent entity Module.MyRootObject ( + stringField : string, + intField : integer, + decimalField : decimal, + boolField : boolean default false +); + +create non-persistent entity Module.MyNestedObject ( + name : string, + code : string +); create association Module.MyRootObject_MyNestedObject from Module.MyRootObject @@ -61,7 +63,9 @@ create association Module.MyRootObject_MyNestedObject - All string fields: bare `string` (no length — unlimited) - All number fields: `integer`, `decimal`, or `long` — remove defaults for optional fields - Boolean fields **require** `default true|false` -- `NON_PERSISTENT` — these entities are not stored in the database +- `non-persistent` — these entities are not stored in the database. The keyword is + **hyphenated and goes before `entity`**: `create non-persistent entity Mod.X (...)`. + `NON_PERSISTENT`, and a `(NON_PERSISTENT)` inside the body, are both parse errors - One association per parent→child relationship; name it `Parent_Child` --- @@ -188,21 +192,23 @@ create json structure Integrations.JSON_BibleVerse snippet '{"translation":{"identifier":"web","name":"World English Bible","language":"English","language_code":"eng","license":"Public Domain"},"random_verse":{"book_id":"1SA","book":"1 Samuel","chapter":17,"verse":49,"text":"David put his hand in his bag, took a stone, and slung it."}}'; -- Step 2: Entities -create entity Integrations.BibleApiResponse (NON_PERSISTENT); - -create entity Integrations.BibleTranslation (NON_PERSISTENT) - identifier : string - name : string - language : string - language_code : string - license : string; - -create entity Integrations.BibleVerse (NON_PERSISTENT) - book_id : string - book : string - chapter : integer - verse : integer - text : string; +create non-persistent entity Integrations.BibleApiResponse (); + +create non-persistent entity Integrations.BibleTranslation ( + identifier : string, + name : string, + language : string, + language_code : string, + license : string +); + +create non-persistent entity Integrations.BibleVerse ( + book_id : string, + book : string, + chapter : integer, + verse : integer, + text : string +); create association Integrations.BibleApiResponse_BibleTranslation from Integrations.BibleApiResponse diff --git a/cmd/mxcli/lsp_completion.go b/cmd/mxcli/lsp_completion.go index 8e16d5e8b..d06f9f1fa 100644 --- a/cmd/mxcli/lsp_completion.go +++ b/cmd/mxcli/lsp_completion.go @@ -197,7 +197,7 @@ func snippet(label, insertText, detail string) protocol.CompletionItem { var mdlCreateSnippets = []protocol.CompletionItem{ snippet("CREATE ENTITY", "CREATE ENTITY ${1:Module}.${2:EntityName}\n(\n\t${3:AttributeName} : ${4:String}\n);", "Create a new entity"), snippet("CREATE PERSISTENT ENTITY", "CREATE PERSISTENT ENTITY ${1:Module}.${2:EntityName}\n(\n\t${3:AttributeName} : ${4:String}\n);", "Create a persistent entity"), - snippet("CREATE NON_PERSISTENT ENTITY", "CREATE NON_PERSISTENT ENTITY ${1:Module}.${2:EntityName}\n(\n\t${3:AttributeName} : ${4:String}\n);", "Create a non-persistent entity"), + snippet("CREATE NON-PERSISTENT ENTITY", "CREATE NON-PERSISTENT ENTITY ${1:Module}.${2:EntityName}\n(\n\t${3:AttributeName} : ${4:String}\n);", "Create a non-persistent entity"), snippet("CREATE MICROFLOW", "CREATE MICROFLOW ${1:Module}.${2:MicroflowName}\nBEGIN\n\t$0\nEND;", "Create a new microflow"), snippet("CREATE MICROFLOW (with params)", "CREATE MICROFLOW ${1:Module}.${2:MicroflowName}\n(\n\t$$${3:Param}: ${4:Module.Entity}\n)\nRETURNS ${5:Boolean} AS $$${6:Result}\nBEGIN\n\t$0\nEND;", "Create microflow with parameters"), snippet("CREATE NANOFLOW", "CREATE NANOFLOW ${1:Module}.${2:NanoflowName}\nBEGIN\n\t$0\nEND;", "Create a new nanoflow"), diff --git a/cmd/mxcli/syntax/capability_docs_drift_test.go b/cmd/mxcli/syntax/capability_docs_drift_test.go new file mode 100644 index 000000000..2c78ffe9d --- /dev/null +++ b/cmd/mxcli/syntax/capability_docs_drift_test.go @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: Apache-2.0 + +package syntax + +import ( + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +// docsRoot resolves docs/01-project/ from this package's directory. +func docsRoot(t *testing.T) string { + t.Helper() + root := filepath.Join("..", "..", "..", "docs", "01-project") + if _, err := os.Stat(root); err != nil { + t.Fatalf("docs/01-project not found from %s: %v", mustWD(t), err) + } + return root +} + +func mustWD(t *testing.T) string { + t.Helper() + wd, _ := os.Getwd() + return wd +} + +func readDoc(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join(docsRoot(t), name)) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + return string(b) +} + +// TestCapabilityDocsDoNotClaimShippedFeaturesAreMissing is the drift guard for +// FINDINGS #20. +// +// The failure it catches is not a typo — it is a document that keeps asserting a +// gap after the gap was filled. That is worse than no document: a reader greps +// it, believes the feature is unavailable, and designs an elaborate workaround +// around a blocker that no longer exists. The report that prompted this named +// the External Database Connector, which had been listed as unsupported long +// after CREATE DATABASE CONNECTION shipped. +// +// The check is deliberately indirect: rather than re-encoding a list of features +// (which would itself drift), it asserts that every capability with a registered +// `mxcli syntax` topic is NOT sitting in the docs' "no MDL surface at all" +// table. The syntax registry is populated from the code, so it cannot claim a +// topic for something that does not exist. +func TestCapabilityDocsDoNotClaimShippedFeaturesAreMissing(t *testing.T) { + matrix := readDoc(t, "MDL_FEATURE_MATRIX.md") + + start := strings.Index(matrix, "### Not Yet Implemented") + if start < 0 { + t.Fatal(`MDL_FEATURE_MATRIX.md has no "### Not Yet Implemented" section — ` + + `if it was renamed, update this guard rather than deleting it`) + } + section := matrix[start:] + if end := strings.Index(section, "\n## "); end > 0 { + section = section[:end] + } + + // Each capability that must not appear as unimplemented, keyed by the syntax + // topic that proves it ships. A topic is registered from Go code, so this + // pairing cannot go stale in the direction that matters. + for _, c := range []struct{ topic, claim string }{ + {"database-connection", "Ext. DB connector"}, + {"queue", "Task queue"}, + {"scheduled-event", "Scheduled events"}, + {"regular-expression", "Regular expressions"}, + {"image-collection", "Image collection"}, + {"navigation.menu-document", "Menus"}, + {"workflow", "Workflows"}, + } { + if ByPath(c.topic) == nil { + t.Errorf("no `mxcli syntax %s` topic — either the feature was removed "+ + "(then drop this row) or the topic is missing (then add it)", c.topic) + continue + } + if strings.Contains(section, c.claim) { + t.Errorf("MDL_FEATURE_MATRIX.md still lists %q under \"Not Yet Implemented\", "+ + "but `mxcli syntax %s` exists — the doc sends readers to Studio Pro "+ + "for work mxcli can already do", c.claim, c.topic) + } + } +} + +// TestMissingCapabilitiesIsMarkedAsDated pins the header that stops the older +// survey being read as current status. Its per-type counts are still worth +// keeping; its conclusions are from Mendix 11.6.3 and mostly overtaken. +func TestMissingCapabilitiesIsMarkedAsDated(t *testing.T) { + doc := readDoc(t, "MISSING_CAPABILITIES.md") + + // The warning must come before any per-type analysis, or a reader who lands + // mid-document via grep (which is how #20 happened) never sees it. + warn := strings.Index(doc, "Dated survey") + if warn < 0 { + t.Fatal("MISSING_CAPABILITIES.md no longer warns that it is a dated survey — " + + "without that, grepping it for a document type reads as current status") + } + if first := strings.Index(doc, "## Summary"); first >= 0 && warn > first { + t.Error("the dated-survey warning must precede the summary table") + } + if !strings.Contains(doc, "MDL_FEATURE_MATRIX.md") { + t.Error("MISSING_CAPABILITIES.md must point at the canonical current status") + } + + // Every row of the summary table needs a status cell, so no type can be read + // as unsupported by omission. + row := regexp.MustCompile(`(?m)^\| ` + "`" + `[A-Za-z]+\$?[A-Za-z]*` + "`" + ` \|.*$`) + rows := row.FindAllString(doc, -1) + if len(rows) < 10 { + t.Fatalf("only matched %d summary rows; the guard is not looking at the table", len(rows)) + } + for _, r := range rows { + if !strings.Contains(r, "Supported") && !strings.Contains(r, "Still missing") && + !strings.Contains(r, "Read-only") { + t.Errorf("summary row has no status cell:\n%s", r) + } + } +} diff --git a/cmd/mxcli/syntax/features_domain_model.go b/cmd/mxcli/syntax/features_domain_model.go index eaeb3508f..ea814b916 100644 --- a/cmd/mxcli/syntax/features_domain_model.go +++ b/cmd/mxcli/syntax/features_domain_model.go @@ -24,7 +24,7 @@ func init() { "entity", "create entity", "persistent", "non-persistent", "generalization", "extends", "event handler", "attribute", }, - Syntax: "CREATE PERSISTENT ENTITY Module.Name (\n Attr: Type [constraints],\n ...\n) [INDEX (attr1)] [COMMENT 'text'];\n\nCREATE NON_PERSISTENT ENTITY Module.Name (...);\n\nCREATE PERSISTENT ENTITY Module.Name EXTENDS Module.Parent (...);", + Syntax: "CREATE PERSISTENT ENTITY Module.Name (\n Attr: Type [constraints],\n ...\n) [INDEX (attr1)] [COMMENT 'text'];\n\nCREATE NON-PERSISTENT ENTITY Module.Name (...);\n\nCREATE PERSISTENT ENTITY Module.Name EXTENDS Module.Parent (...);", Example: "CREATE PERSISTENT ENTITY MyModule.Customer (\n Name: String(100) NOT NULL ERROR 'Name is required',\n Email: String(200) UNIQUE,\n Balance: Decimal DEFAULT 0,\n IsActive: Boolean DEFAULT true,\n Status: Enumeration(MyModule.CustomerType)\n)\nINDEX (Email)\nCOMMENT 'Stores customer information';", SeeAlso: []string{"domain-model.entity.create", "domain-model.entity.alter", "domain-model.entity.attributes"}, }) @@ -37,7 +37,7 @@ func init() { "non-persistent", "extends", "generalization", "index", "event handler", "before commit", "after commit", }, - Syntax: "CREATE PERSISTENT ENTITY Module.Name (\n Attr: Type [NOT NULL [ERROR 'msg']] [UNIQUE [ERROR 'msg']] [DEFAULT val],\n ...\n)\n[INDEX (attr1, attr2)]\n[ON BEFORE|AFTER CREATE|COMMIT|DELETE|ROLLBACK CALL Module.MF [RAISE ERROR]]\n[COMMENT 'text'];\n\nCREATE NON_PERSISTENT ENTITY Module.Name (...);\nCREATE PERSISTENT ENTITY Module.Name EXTENDS Module.Parent (...);", + Syntax: "CREATE PERSISTENT ENTITY Module.Name (\n Attr: Type [NOT NULL [ERROR 'msg']] [UNIQUE [ERROR 'msg']] [DEFAULT val],\n ...\n)\n[INDEX (attr1, attr2)]\n[ON BEFORE|AFTER CREATE|COMMIT|DELETE|ROLLBACK CALL Module.MF [RAISE ERROR]]\n[COMMENT 'text'];\n\nCREATE NON-PERSISTENT ENTITY Module.Name (...);\nCREATE PERSISTENT ENTITY Module.Name EXTENDS Module.Parent (...);", Example: "-- Persistent with constraints and index\nCREATE PERSISTENT ENTITY Shop.Order (\n OrderNumber: String(20) NOT NULL,\n Total: Decimal DEFAULT 0,\n CreatedAt: DateTime\n)\nINDEX (OrderNumber)\nON BEFORE COMMIT CALL Shop.ValidateOrder($currentObject) RAISE ERROR;\n\n-- With generalization\nCREATE PERSISTENT ENTITY Shop.ProductImage EXTENDS System.Image (\n Caption: String(200)\n);", SeeAlso: []string{"domain-model.entity.alter", "domain-model.entity.attributes"}, }) diff --git a/cmd/mxcli/syntax/features_integration.go b/cmd/mxcli/syntax/features_integration.go index 7f09f44d9..12783c706 100644 --- a/cmd/mxcli/syntax/features_integration.go +++ b/cmd/mxcli/syntax/features_integration.go @@ -301,6 +301,112 @@ func init() { SeeAlso: []string{"sql"}, }) + // ── External database connector ─────────────────────────────────── + + Register(SyntaxFeature{ + Path: "database-connection", + Summary: "External Database Connector — query another database from a microflow", + Keywords: []string{ + "database connection", "database connections", "external database", + "create database connection", "drop database connection", + "describe database connection", "show database connections", + "jdbc", "byod", "database connector", "execute database query", + "postgresql", "mysql", "oracle", "snowflake", "sql server", + }, + Syntax: `CREATE [OR MODIFY] DATABASE CONNECTION Module.Name + TYPE '' + CONNECTION STRING @Module.UrlConstant + USERNAME @Module.UserConstant + PASSWORD @Module.PasswordConstant +[BEGIN + QUERY + SQL $$$$ + [PARAMETER : [DEFAULT '' | NULL]] + [RETURNS Module.Entity + [MAP ( AS , ... )]] + ; +END]; + +SHOW DATABASE CONNECTIONS [IN ]; +DESCRIBE DATABASE CONNECTION Module.Name; +DROP DATABASE CONNECTION Module.Name; + +Calling a query from a microflow: + EXECUTE DATABASE QUERY Module.Connection.QueryName (...); + +TYPE is one of Studio Pro's entries — 'MSSQL', 'MySQL', 'Oracle', +'PostgreSQL', 'Snowflake' — or 'BYOD' ("bring your own driver"), which skips +the driver-presence check and takes the connection string verbatim. Use BYOD +for any JDBC driver Mendix has no entry for; put the driver on the classpath +with ALTER MODULE ... ADD JAR DEPENDENCY + mxcli sync-java-deps. An +unrecognised type is reported by MDL-DB01 — mxbuild does not check it, so the +build stays green and the connection simply does not work. + +Three traps: + + 1. CONNECTION STRING / USERNAME / PASSWORD must reference CONSTANTS + (@Module.Name), not literals. A literal produces a project Studio Pro + cannot open at all: StorageLoadException "is not a valid + ConstantIdentifier". mxcli catches it (MDL058); mxbuild does not, so the + build is green. + 2. USERNAME and PASSWORD must be given even when the driver needs neither. + Omitting them writes an empty constant reference, the build stays green, + and the query fails only at run time with "Could not find value for + constant ''". + 3. A named type needs its JDBC driver declared on the module, or the build + fails with CE5278 ("The PostgreSQL JDBC driver (org.postgresql:postgresql) + is missing from the module settings"). Unlike the first two, this one is + caught at build time: + + ALTER MODULE Ops ADD JAR DEPENDENCY ( + group = 'org.postgresql', artifact = 'postgresql', + version = '42.7.4', included = true + ); + + then run: mxcli sync-java-deps -p app.mpr — to put it on the classpath — + declaring without syncing gives a green build and a + ClassNotFoundException. BYOD skips this check, which is the point of it. + +The connector runtime is part of the platform — no marketplace module is +needed in the project.`, + Example: `-- The three constants the connection points at. +CREATE CONSTANT Ops.DbUrl TYPE String DEFAULT 'jdbc:postgresql://localhost:5432/erp'; +CREATE CONSTANT Ops.DbUser TYPE String DEFAULT 'reader'; +CREATE CONSTANT Ops.DbPass TYPE String DEFAULT ''; + +CREATE NON-PERSISTENT ENTITY Ops.EmployeeRow ( + EmployeeId: Integer, + Name: String(100) +); + +CREATE DATABASE CONNECTION Ops.Erp + TYPE 'PostgreSQL' + CONNECTION STRING @Ops.DbUrl + USERNAME @Ops.DbUser + PASSWORD @Ops.DbPass +BEGIN + QUERY GetEmployees + SQL $$SELECT id, name FROM employees WHERE dept = {dept}$$ + PARAMETER dept: String DEFAULT 'sales' + RETURNS Ops.EmployeeRow + MAP ( + id AS EmployeeId, + name AS Name + ) + ; +END; + +-- A named type needs its driver declared, or the build fails with CE5278. +ALTER MODULE Ops ADD JAR DEPENDENCY ( + group = 'org.postgresql', artifact = 'postgresql', + version = '42.7.4', included = true +); + +SHOW DATABASE CONNECTIONS IN Ops; +DESCRIBE DATABASE CONNECTION Ops.Erp;`, + SeeAlso: []string{"integration", "sql", "microflow.call"}, + }) + // ── Business Events ─────────────────────────────────────────────── Register(SyntaxFeature{ diff --git a/docs/01-project/MDL_FEATURE_MATRIX.md b/docs/01-project/MDL_FEATURE_MATRIX.md index 80c631624..94e29ee83 100644 --- a/docs/01-project/MDL_FEATURE_MATRIX.md +++ b/docs/01-project/MDL_FEATURE_MATRIX.md @@ -180,6 +180,21 @@ live distinction is **MPR vs MCP**. | **Business Events** | Y | Y | Y | N | Y | N | 13 | N | Y | N | Y | N | Y | N | Y | Y | N | | **Project Settings** | Y | Y | - | - | - | Y | N | N | Y | Y | Y | N | Y | N | N | Y | P | | **Task Queues** | Y | Y | Y | Y | Y | N | 21 | Y | Y | N | N | Y | Y | N | Y | Y | N | +| **Scheduled Events** | Y | Y | Y | Y | Y | N | 21 | Y | Y | Y | N | Y | Y | N | Y | Y | N | +| **Database Connections** | Y | Y | Y | Y | Y | N | 05 | Y | Y | N | N | Y | Y | N | Y | Y | N | +| **Regular Expressions** | Y | Y | Y | Y | Y | N | N | Y | Y | Y | N | N | Y | N | Y | Y | N | +| **Validation Rules** | - | Y | Y | - | - | Y | N | Y | N | Y | N | N | Y | N | N | Y | N | +| **Menus** | - | Y | Y | Y | Y | N | N | Y | N | N | N | N | Y | N | N | Y | N | +| **Image Collections** | Y | Y | Y | N | Y | N | N | Y | N | N | N | N | Y | N | Y | Y | N | +| **JavaScript Actions** | Y | Y | Y | N | Y | N | N | Y | Y | Y | N | N | Y | N | Y | N | N | +| **Published REST Services** | Y | Y | Y | Y | Y | N | N | N | Y | N | P | N | Y | N | N | Y | N | +| **REST Clients** | Y | Y | Y | Y | Y | Y | 06 | Y | Y | P | Y | Y | Y | N | Y | Y | N | +| **Import Mappings** | Y | Y | Y | N | Y | N | 06 | Y | N | N | P | Y | N | N | N | Y | N | +| **Export Mappings** | Y | Y | Y | N | Y | N | 06 | Y | N | N | P | Y | N | N | N | Y | N | +| **JSON Structures** | Y | Y | Y | Y | Y | N | 20 | Y | N | N | P | N | N | N | N | N | N | +| **Workflows** | Y | Y | Y | N | Y | Y | N | Y | Y | Y | N | Y | Y | N | N | Y | N | +| **AI Agent documents** | Y | Y | Y | N | Y | N | N | Y | N | N | N | Y | Y | N | Y | Y | N | +| **Pluggable widgets** | Y | Y | Y | - | Y | Y | 03 | Y | N | N | P | Y | Y | N | N | Y | N | ## Security Features @@ -333,36 +348,41 @@ Not yet implemented: - **Call graphs** — `show context of` / `show callers of` as directed graphs - **Module overview** — Combined ER + dependency diagram -### Not Yet Implemented +### Partially Implemented -Document types that exist in Mendix but have no MDL support: +Features with an MDL surface that does not yet cover the whole document type. | Feature | SHOW | DESCRIBE | CREATE | OR MODIFY | DROP | ALTER | Examples | Tests | Catalog | REFS | LSP | Skills | Help | Viz | REPL | Syntax | Starlark | Notes | |---------|------|----------|--------|-----------|------|-------|----------|-------|---------|------|-----|--------|------|-----|------|--------|----------|-------| | **Microflow activities** | - | - | P | - | - | - | 02 | Y | P | P | P | Y | Y | - | - | - | P | 60+ activities supported; some edge cases missing | -| **Building blocks** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | Reusable page building blocks | -| **Styling** | P | Y | P | N | N | Y | Y | Y | N | N | N | P | N | P | N | N | N | Class/Style/DesignProperties on widgets via ALTER STYLING (#631); full theme system not yet | -| **Extensions** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | Mendix extensions / add-ons | -| **Custom JS actions** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | JavaScript actions for nanoflows | -| **Custom widgets** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | Pluggable widget packages | -| **REST publish** | Y | Y | Y | N | Y | N | N | N | Y | N | P | N | Y | N | N | Y | N | Published REST services (CREATE/DROP/SHOW/DESCRIBE) | -| **REST consume (v2)** | N | N | N | N | N | N | 06 | N | N | N | N | Y | N | N | N | N | N | Consumed REST services; partial grammar exists | -| **Web service publish** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | Published SOAP web services | -| **Web service consume** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | Consumed SOAP web services | -| **Ext. DB connector** | N | N | N | N | N | N | 05 | N | N | N | N | Y | N | N | N | N | N | External database connections | -| **Import mappings** | Y | Y | Y | N | Y | N | 06 | Y | N | N | P | Y | N | N | N | Y | N | JSON import mappings with v2 syntax | -| **Export mappings** | Y | Y | Y | N | Y | N | 06 | Y | N | N | P | Y | N | N | N | Y | N | JSON export mappings with v2 syntax | -| **JSON transformations** | Y | Y | Y | Y | Y | N | 20 | Y | N | N | P | N | N | N | N | N | N | JSON structure definitions | -| **Message definitions** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | Message definition documents | -| **XML schemas** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | Imported XML schema documents | -| **Workflows** | Y | Y | Y | N | Y | N | N | N | Y | Y | N | N | Y | N | N | N | N | SHOW/DESCRIBE/CREATE/DROP implemented; GRANT/REVOKE removed (workflows lack AllowedModuleRoles) | -| **Module settings** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | Module-level configuration | -| **Image collection** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | Image document collections | -| **Icon collection** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | Icon/glyph collections | -| **Rules** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | Microflow rules (decision logic) | -| **Regular expressions** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | Reusable regex definitions | -| **Scheduled events** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | Timer-triggered microflows | -| **Data importer** | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | N | Excel/CSV data import documents | +| **Styling** | P | Y | P | N | N | Y | Y | Y | N | N | N | P | N | P | N | N | N | Class/Style/DesignProperties on widgets via ALTER STYLING (#631); `mxcli theme` covers the file side, the full theme system is not modelled | +| **Icon collections** | Y | Y | N | N | N | N | N | Y | N | N | N | Y | Y | N | Y | Y | N | **Read-only by design** — collections ship with Atlas/the theme; DESCRIBE lists each icon's reference form for use in a widget's `icon:` | +| **Module settings** | - | - | - | - | - | Y | N | Y | N | N | N | Y | Y | N | N | Y | N | `ALTER MODULE … ADD JAR DEPENDENCY` + `mxcli sync-java-deps`; other module-level settings are not modelled | +| **REST consume** | Y | Y | Y | Y | Y | Y | 06 | Y | Y | P | Y | Y | Y | N | Y | Y | N | Consumed REST services via `CREATE REST CLIENT`; OpenAPI-driven generation is partial | + +### Not Yet Implemented + +Document types that exist in Mendix and have **no** MDL surface at all. + +| Feature | Notes | +|---------|-------| +| **Microflow rules** (`Microflows$Rule`) | Reusable decision logic called from a microflow. Not to be confused with `CREATE VALIDATION RULE`, which is an attribute constraint and *is* supported | +| **Message definitions** (`MessageDefinitions$MessageDefinitionCollection`) | Message definition documents | +| **XML schemas** | Imported XSD documents | +| **Web service publish / consume** | SOAP. `CALL WEB SERVICE` exists in microflows for a stored service; the service documents themselves are not authorable | +| **Data importer** | Excel/CSV import documents | +| **Building blocks** | Reusable page building blocks | +| **Extensions** | Mendix extensions / add-ons | +| **Custom widget packages** | Authoring a `.mpk`. Note this is NOT "using a pluggable widget on a page", which is supported — see **Pluggable widgets** above | +| **System text collections** (`Texts$SystemTextCollection`) | Translatable system text | + +> **Keeping this honest.** Every row above was checked against the grammar's +> statement rules and `mxcli syntax`, not against memory: a row claiming a gap +> that has since been filled is worse than no table, because it sends people to +> Studio Pro for work mxcli can do. FINDINGS #20 in `ako/mxcli-owid` is the +> worked example — the External Database Connector was recorded here as +> unsupported long after `CREATE DATABASE CONNECTION` shipped, and a reader +> designed around a blocker that no longer existed. ## Checklist for New Features diff --git a/docs/01-project/MISSING_CAPABILITIES.md b/docs/01-project/MISSING_CAPABILITIES.md index c8d89d996..72f17095e 100644 --- a/docs/01-project/MISSING_CAPABILITIES.md +++ b/docs/01-project/MISSING_CAPABILITIES.md @@ -1,5 +1,22 @@ # Missing Capabilities Analysis +> **Dated survey — not the current status.** This is a point-in-time gap +> analysis run against **Mendix 11.6.3** projects. Its *measurement* — how many +> documents of each type real apps contain — is what makes it worth keeping, and +> that has not changed. Its *conclusions* have: **11 of the 13 document types +> below are now supported.** +> +> For what mxcli can do today, the canonical source is +> [MDL_FEATURE_MATRIX.md](MDL_FEATURE_MATRIX.md), and the authority behind both +> is the grammar's statement rules plus `mxcli syntax`. Do not answer "can mxcli +> do X?" from this file. FINDINGS #20 in `ako/mxcli-owid` is what that costs: a +> reader grepped this document, found `DatabaseConnector$DatabaseConnection` +> listed as "listing only", and designed a whole workaround around a blocker +> that `CREATE DATABASE CONNECTION` had already removed. Every claim in it was +> checkable in about five minutes. +> +> Status column added and verified against the grammar; last checked 2026-08-17. + Based on investigation of three real-world Mendix 11.6.3 projects: - **EnquiriesManagement** (28 modules, AI agent app with workflows) - **Evora-FactoryManagement** (39 modules, industrial IoT app with REST/OData/workflows) @@ -7,22 +24,22 @@ Based on investigation of three real-world Mendix 11.6.3 projects: ## Summary: Unsupported Document Types Across All Projects -| Document Type ($Type) | EM | FM | LPI | Total | Priority | -|----------------------|----|----|-----|-------|----------| -| `workflows$workflow` | 12 | 1 | 0 | **13** | **High** | -| `JsonStructures$JsonStructure` | 23 | 42 | 38 | **103** | Medium | -| `ImportMappings$ImportMapping` | 22 | 35 | 33 | **90** | Medium | -| `ExportMappings$ExportMapping` | 19 | 31 | 24 | **74** | Medium | -| `microflows$rule` | 15 | 28 | 12 | **55** | Medium | -| `MessageDefinitions$MessageDefinitionCollection` | 12 | 11 | 10 | **33** | Low | -| `rest$PublishedRestService` | 2 | 7 | 8 | **17** | **High** | -| `RegularExpressions$RegularExpression` | 8 | 4 | 4 | **16** | Low | -| `CustomIcons$CustomIconCollection` | 5 | 2 | 3 | **10** | Low | -| `Menus$MenuDocument` | 3 | 2 | 2 | **7** | Low | -| `Queues$Queue` | 2 | 2 | 1 | **5** | Low | -| `Texts$SystemTextCollection` | 1 | 1 | 1 | **3** | Low | -| `rest$ConsumedRestService` | 0 | 2 | 0 | **2** | Medium | -| `DatabaseConnector$DatabaseConnection` | 0 | 1 | 0 | **1** | Low | +| Document Type ($Type) | EM | FM | LPI | Total | Priority (then) | Status today | +|----------------------|----|----|-----|-------|-----------------|--------------| +| `workflows$workflow` | 12 | 1 | 0 | **13** | **High** | **Supported** | +| `JsonStructures$JsonStructure` | 23 | 42 | 38 | **103** | Medium | **Supported** | +| `ImportMappings$ImportMapping` | 22 | 35 | 33 | **90** | Medium | **Supported** | +| `ExportMappings$ExportMapping` | 19 | 31 | 24 | **74** | Medium | **Supported** | +| `microflows$rule` | 15 | 28 | 12 | **55** | Medium | Still missing | +| `MessageDefinitions$MessageDefinitionCollection` | 12 | 11 | 10 | **33** | Low | Still missing | +| `rest$PublishedRestService` | 2 | 7 | 8 | **17** | **High** | **Supported** | +| `RegularExpressions$RegularExpression` | 8 | 4 | 4 | **16** | Low | **Supported** | +| `CustomIcons$CustomIconCollection` | 5 | 2 | 3 | **10** | Low | Read-only (by design) | +| `Menus$MenuDocument` | 3 | 2 | 2 | **7** | Low | **Supported** | +| `Queues$Queue` | 2 | 2 | 1 | **5** | Low | **Supported** (+ `IN QUEUE` on calls) | +| `Texts$SystemTextCollection` | 1 | 1 | 1 | **3** | Low | Still missing | +| `rest$ConsumedRestService` | 0 | 2 | 0 | **2** | Medium | **Supported** (REST clients) | +| `DatabaseConnector$DatabaseConnection` | 0 | 1 | 0 | **1** | Low | **Supported** | EM = EnquiriesManagement, FM = FactoryManagement, LPI = LatoProductInventory @@ -32,6 +49,8 @@ EM = EnquiriesManagement, FM = FactoryManagement, LPI = LatoProductInventory #### 1. Workflows (`workflows$workflow`) - 13 documents +> **Status: supported.** `CREATE`/`ALTER`/`DROP`/`DESCRIBE WORKFLOW`, all activity types. See `mxcli syntax workflow`. + **Impact**: Core Mendix feature for business process automation. Used heavily in the EnquiriesManagement project for agent-orchestrated enquiry handling. **What's needed**: Full proposal in [PROPOSAL_workflow_support.md](PROPOSAL_workflow_support.md). @@ -40,6 +59,8 @@ EM = EnquiriesManagement, FM = FactoryManagement, LPI = LatoProductInventory #### 2. Published REST Services (`rest$PublishedRestService`) - 17 documents +> **Status: supported.** `CREATE [OR REPLACE]`/`DROP`/`SHOW`/`DESCRIBE PUBLISHED REST SERVICE`, with resources, operations and path params. + **Impact**: Published REST services are a primary integration mechanism in Mendix. FactoryManagement exposes 7 REST services (oauth2, discovery, TCSSO, ViewerService, etc.), LatoProductInventory exposes 8. **What's needed**: @@ -55,6 +76,8 @@ EM = EnquiriesManagement, FM = FactoryManagement, LPI = LatoProductInventory #### 3. JSON Structures (`JsonStructures$JsonStructure`) - 103 documents +> **Status: supported.** `CREATE JSON STRUCTURE … SNIPPET ''` infers the structure. + **Impact**: JSON structures define schemas used by import/export mappings for REST/JSON data transformations. Very common in integration-heavy apps. **What's needed**: @@ -67,6 +90,8 @@ EM = EnquiriesManagement, FM = FactoryManagement, LPI = LatoProductInventory #### 4. Import/Export Mappings (90 + 74 = 164 documents) +> **Status: supported.** `CREATE IMPORT MAPPING` / `CREATE EXPORT MAPPING` (v2 syntax). + **Impact**: Mappings define how JSON/XML data is transformed to/from Mendix objects. Critical for REST integration. Always paired with JSON structures. **What's needed**: @@ -80,6 +105,8 @@ EM = EnquiriesManagement, FM = FactoryManagement, LPI = LatoProductInventory #### 5. Rules (`microflows$rule`) - 55 documents +> **Status: still missing.** Note `CREATE VALIDATION RULE` *is* supported — that is an attribute constraint, a different document type from `Microflows$Rule`. + **Impact**: Rules are a special microflow subtype used for entity validation. Currently parsed as regular microflows but not distinguished. **What's needed**: @@ -92,6 +119,8 @@ EM = EnquiriesManagement, FM = FactoryManagement, LPI = LatoProductInventory #### 6. Consumed REST Services (`rest$ConsumedRestService`) - 2 documents +> **Status: supported** via `CREATE REST CLIENT` (see `mxcli syntax rest`). OpenAPI-driven generation is partial. + **Impact**: Different from consumed OData services (which are already supported). These are plain REST API integrations. **What's needed**: @@ -106,6 +135,8 @@ EM = EnquiriesManagement, FM = FactoryManagement, LPI = LatoProductInventory #### 7. Message Definitions (`MessageDefinitions$MessageDefinitionCollection`) - 33 documents +> **Status: still missing.** + **Impact**: Message definitions describe the structure of messages used by published services. Related to REST/SOAP operations. **What's needed**: BSON parser, Reader methods, SHOW command. @@ -114,6 +145,8 @@ EM = EnquiriesManagement, FM = FactoryManagement, LPI = LatoProductInventory #### 8. Regular Expressions (`RegularExpressions$RegularExpression`) - 16 documents +> **Status: supported.** `CREATE [OR MODIFY] REGULAR EXPRESSION`, and `CREATE VALIDATION RULE … REGEX` binds one to an attribute. + **Impact**: Named regex patterns referenced by validation rules. Minimal. **What's needed**: Simple BSON parser (Name, Expression fields), Reader methods, SHOW command. @@ -122,6 +155,8 @@ EM = EnquiriesManagement, FM = FactoryManagement, LPI = LatoProductInventory #### 9. Custom Icons (`CustomIcons$CustomIconCollection`) - 10 documents +> **Status: read-only, by design.** Icon collections ship with Atlas/the theme; `SHOW`/`DESCRIBE ICON COLLECTION` lists each icon's reference form for a widget's `icon:`. Authoring one is not planned. + **Impact**: Icon collections for use in the UI. Binary content, not useful for code analysis. **What's needed**: Reader listing only (for completeness). No describe/MDL needed. @@ -130,6 +165,8 @@ EM = EnquiriesManagement, FM = FactoryManagement, LPI = LatoProductInventory #### 10. Menus (`Menus$MenuDocument`) - 7 documents +> **Status: supported.** `CREATE OR MODIFY`/`DESCRIBE`/`DROP MENU`, round-trippable. + **Impact**: Menu configurations (separate from navigation). Rarely used directly. **What's needed**: Reader listing, basic SHOW command. @@ -138,6 +175,8 @@ EM = EnquiriesManagement, FM = FactoryManagement, LPI = LatoProductInventory #### 11. Queues (`Queues$Queue`) - 5 documents +> **Status: supported.** `CREATE [OR MODIFY] QUEUE`, and a call is bound to one with `CALL MICROFLOW … IN QUEUE Module.Queue` (same clause on `CALL JAVA ACTION`). + **Impact**: Task queue definitions for asynchronous processing. Lightweight documents. **What's needed**: BSON parser (Name, Module), Reader methods, SHOW command, catalog table. @@ -146,6 +185,8 @@ EM = EnquiriesManagement, FM = FactoryManagement, LPI = LatoProductInventory #### 12. System Texts (`Texts$SystemTextCollection`) - 3 documents +> **Status: still missing.** + **Impact**: Translation/localization texts. One per project. **What's needed**: Reader listing only. @@ -154,6 +195,8 @@ EM = EnquiriesManagement, FM = FactoryManagement, LPI = LatoProductInventory #### 13. Database Connections (`DatabaseConnector$DatabaseConnection`) - 1 document +> **Status: supported.** `CREATE [OR MODIFY] DATABASE CONNECTION` with queries, parameters and `RETURNS … MAP (…)`; `EXECUTE DATABASE QUERY` calls one from a microflow. Use type `'BYOD'` for a JDBC driver Mendix has no entry for. See `mxcli syntax database-connection`. + **Impact**: External database connection configuration. Only in FactoryManagement. **What's needed**: Reader listing, basic SHOW command. @@ -162,25 +205,32 @@ EM = EnquiriesManagement, FM = FactoryManagement, LPI = LatoProductInventory ## Recommended Implementation Order +> **This plan has been executed.** Kept for the record; ~~struck~~ items shipped. +> What is left is items 2, 10 and 13 — see the summary table's Status column. + ### Sprint 1: High-Impact Read-Only -1. **Workflows** (Phase 1 from workflow proposal) - highest value, complex -2. **Rules** - low effort, completes microflow domain -3. **Published REST Services** - high value for integration projects +1. ~~**Workflows**~~ — shipped (authorable, not read-only: CREATE/ALTER/DROP) +2. **Rules** — *still open*, low effort, completes the microflow domain +3. ~~**Published REST Services**~~ — shipped ### Sprint 2: Integration Support -4. **JSON Structures** - prerequisite for mappings -5. **Import Mappings** - completes REST integration chain -6. **Export Mappings** - completes REST integration chain -7. **Consumed REST Services** - completes REST support +4. ~~**JSON Structures**~~ — shipped +5. ~~**Import Mappings**~~ — shipped +6. ~~**Export Mappings**~~ — shipped +7. ~~**Consumed REST Services**~~ — shipped as REST clients ### Sprint 3: Completeness -8. **Queues** - simple, useful for async patterns -9. **Regular Expressions** - simple -10. **Message Definitions** - completes service descriptions -11. **Menus** - listing only -12. **Custom Icons** - listing only -13. **System Texts** - listing only -14. **Database Connections** - listing only +8. ~~**Queues**~~ — shipped, and a call binds to one with `IN QUEUE` +9. ~~**Regular Expressions**~~ — shipped, plus `CREATE VALIDATION RULE` +10. **Message Definitions** — *still open* +11. ~~**Menus**~~ — shipped as full authoring, not listing only +12. ~~**Custom Icons**~~ — shipped as SHOW/DESCRIBE; authoring is not planned +13. **System Texts** — *still open* +14. ~~**Database Connections**~~ — shipped as full authoring, not listing only + +Note how often "listing only" in the original plan became full authoring. That +is the specific way this document goes stale: the estimate of *how much* would +be built was as wrong as the timing, and always in the same direction. ## Impact on Catalog Coverage diff --git a/mdl/backend/modelsdk/integration_read.go b/mdl/backend/modelsdk/integration_read.go index 262947733..01d7292f9 100644 --- a/mdl/backend/modelsdk/integration_read.go +++ b/mdl/backend/modelsdk/integration_read.go @@ -615,6 +615,7 @@ func (b *Backend) ListDatabaseConnections() ([]*model.DatabaseConnection, error) } dq.ID = model.ID(q.ID()) dq.TypeName = "DatabaseConnector$DatabaseQuery" + dq.TableMappings = tableMappingsFromGen(q) conn.Queries = append(conn.Queries, dq) } out = append(out, conn) @@ -622,6 +623,46 @@ func (b *Backend) ListDatabaseConnections() ([]*model.DatabaseConnection, error) return out, nil } +// tableMappingsFromGen reads a query's TableMappings back into the semantic +// model — the entity a query returns and its column→attribute mapping. +// +// Without this the DESCRIBE renderer's `returns`/`map (…)` block was dead code +// on this engine (it only ever saw an empty slice), so `describe database +// connection` emitted a query with no return entity and no mapping, and a +// describe → exec round trip dropped both. The legacy reader has always +// populated it, which is why the round-trip test — pinned to legacy — stayed +// green. +func tableMappingsFromGen(q *genDb.DatabaseQuery) []*model.DatabaseTableMapping { + var out []*model.DatabaseTableMapping + for _, el := range q.TableMappingsItems() { + tm, ok := el.(*genDb.TableMapping) + if !ok { + continue + } + m := &model.DatabaseTableMapping{ + Entity: tm.EntityQualifiedName(), + TableName: tm.TableName(), + } + m.ID = model.ID(tm.ID()) + m.TypeName = "DatabaseConnector$TableMapping" + for _, cEl := range tm.ColumnsItems() { + c, ok := cEl.(*genDb.ColumnMapping) + if !ok { + continue + } + cm := &model.DatabaseColumnMapping{ + Attribute: c.AttributeQualifiedName(), + ColumnName: c.ColumnName(), + } + cm.ID = model.ID(c.ID()) + cm.TypeName = "DatabaseConnector$ColumnMapping" + m.Columns = append(m.Columns, cm) + } + out = append(out, m) + } + return out +} + // firstNonEmpty returns the first non-empty string in vals, or "". func firstNonEmpty(vals ...string) string { for _, v := range vals { diff --git a/mdl/executor/roundtrip_dbconnection_test.go b/mdl/executor/roundtrip_dbconnection_test.go index 3c8e49874..7dd2eaf5f 100644 --- a/mdl/executor/roundtrip_dbconnection_test.go +++ b/mdl/executor/roundtrip_dbconnection_test.go @@ -8,8 +8,20 @@ import ( "testing" ) +// The round trip runs on BOTH engines. It used to run only on legacy (the +// setupTestEnv default), which is why the modelsdk reader never reading +// TableMappings stayed invisible: `describe database connection` on the DEFAULT +// engine emitted a query with no `returns`/`map`, so a describe → exec cycle +// silently dropped the entity binding and the column mapping, while this test +// stayed green. Measured on 11.13: legacy renders the clause, modelsdk did not. func TestRoundtripDatabaseConnection_Simple(t *testing.T) { - env := setupTestEnv(t) + for _, eng := range gateEngines { + t.Run(eng.name, func(t *testing.T) { testRoundtripDatabaseConnectionSimple(t, eng) }) + } +} + +func testRoundtripDatabaseConnectionSimple(t *testing.T, eng gateEngine) { + env := setupTestEnvWithBackend(t, eng.factory) defer env.teardown() connName := testModule + ".TestDatabase" From 678b3fdf21dd7ce2246205ce71387623a2fc0594 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 18 Aug 2026 01:16:48 +0000 Subject: [PATCH 22/35] docs(matrix): audit against Studio Pro's document-type list, not against mxcli MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous pass enumerated the feature matrix by inverting what mxcli supports. That can only ever rediscover gaps someone already wrote down. Enumerating Studio Pro's own `Add other` menu instead — the full set of document types the product offers — is the first check able to find a type nobody has thought about, and it found four: data sets, page templates, ML model mappings, change data capture (beta) None appeared in any mxcli document before. All 37 types were probed against the grammar's statement rules, `mxcli syntax`, and a live project. Two corrections in the other direction: - Building blocks are read-only (`show building blocks`), not absent as the table claimed. - Data transformers are fully authorable (createDataTransformerStatement, `mxcli syntax data-transformer`) and were listed nowhere at all — neither as supported nor as a gap. Counted from the menu: 24 of 37 types are authorable, 4 are read-only (layouts, icon collections, building blocks, plus pluggable widgets on the page side), and 12 have no MDL surface. The note now records where the row list came from, because that is the part that decays: values can be re-derived from the code, but a missing row is invisible until someone enumerates the product again. Re-run the pass when onboarding a new Mendix major. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017s476QkXr9CFMvKspVzcvu --- docs/01-project/MDL_FEATURE_MATRIX.md | 45 ++++++++++----------------- 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/docs/01-project/MDL_FEATURE_MATRIX.md b/docs/01-project/MDL_FEATURE_MATRIX.md index 94e29ee83..ecb0b0d6b 100644 --- a/docs/01-project/MDL_FEATURE_MATRIX.md +++ b/docs/01-project/MDL_FEATURE_MATRIX.md @@ -195,6 +195,7 @@ live distinction is **MPR vs MCP**. | **Workflows** | Y | Y | Y | N | Y | Y | N | Y | Y | Y | N | Y | Y | N | N | Y | N | | **AI Agent documents** | Y | Y | Y | N | Y | N | N | Y | N | N | N | Y | Y | N | Y | Y | N | | **Pluggable widgets** | Y | Y | Y | - | Y | Y | 03 | Y | N | N | P | Y | Y | N | N | Y | N | +| **Data Transformers** | Y | Y | Y | N | Y | N | N | Y | N | N | N | N | Y | N | Y | Y | N | ## Security Features @@ -359,6 +360,7 @@ Features with an MDL surface that does not yet cover the whole document type. | **Icon collections** | Y | Y | N | N | N | N | N | Y | N | N | N | Y | Y | N | Y | Y | N | **Read-only by design** — collections ship with Atlas/the theme; DESCRIBE lists each icon's reference form for use in a widget's `icon:` | | **Module settings** | - | - | - | - | - | Y | N | Y | N | N | N | Y | Y | N | N | Y | N | `ALTER MODULE … ADD JAR DEPENDENCY` + `mxcli sync-java-deps`; other module-level settings are not modelled | | **REST consume** | Y | Y | Y | Y | Y | Y | 06 | Y | Y | P | Y | Y | Y | N | Y | Y | N | Consumed REST services via `CREATE REST CLIENT`; OpenAPI-driven generation is partial | +| **Building blocks** | Y | N | N | N | N | N | N | Y | N | N | N | N | Y | N | Y | Y | N | **Read-only** — `show building blocks`; authoring one is not supported | ### Not Yet Implemented @@ -371,10 +373,13 @@ Document types that exist in Mendix and have **no** MDL surface at all. | **XML schemas** | Imported XSD documents | | **Web service publish / consume** | SOAP. `CALL WEB SERVICE` exists in microflows for a stored service; the service documents themselves are not authorable | | **Data importer** | Excel/CSV import documents | -| **Building blocks** | Reusable page building blocks | | **Extensions** | Mendix extensions / add-ons | | **Custom widget packages** | Authoring a `.mpk`. Note this is NOT "using a pluggable widget on a page", which is supported — see **Pluggable widgets** above | | **System text collections** (`Texts$SystemTextCollection`) | Translatable system text | +| **Data sets** | Dataset documents (reporting) | +| **Page templates** | Reusable page starting points offered by Studio Pro's New Page dialog | +| **ML model mappings** | Mapping a model to entities for the ML Kit | +| **Change data capture services** (beta) | CDC service documents | > **Keeping this honest.** Every row above was checked against the grammar's > statement rules and `mxcli syntax`, not against memory: a row claiming a gap @@ -383,31 +388,13 @@ Document types that exist in Mendix and have **no** MDL surface at all. > worked example — the External Database Connector was recorded here as > unsupported long after `CREATE DATABASE CONNECTION` shipped, and a reader > designed around a blocker that no longer existed. - -## Checklist for New Features - -When adding a new MDL document type, ensure all dimensions are covered: - -- [ ] **Grammar** — Add tokens to `MDLLexer.g4`, rules to `MDLParser.g4`, regenerate parser -- [ ] **AST** — Add statement types in `mdl/ast/` (Show, Describe, Create, Drop) -- [ ] **Visitor** — Add listener methods in `mdl/visitor/` to build AST from parse tree -- [ ] **Executor** — Add execution handlers in `mdl/executor/` - - [ ] SHOW handler (list all, filter by module) - - [ ] DESCRIBE handler (output MDL format) - - [ ] CREATE handler (with OR MODIFY support) - - [ ] DROP handler - - [ ] ALTER handler (if applicable) -- [ ] **Catalog** — Add table in `mdl/catalog/tables.go` and builder in `builder_modules.go` -- [ ] **REFS** — Track cross-references in `refs` table for impact analysis -- [ ] **LSP** — Add completions in `cmd/mxcli/lsp_completions_gen.go`, hover/definition in `lsp.go` -- [ ] **REPL** — Add autocomplete entries in `mdl/repl/repl.go` (prefix completer) and `mdl/executor/autocomplete.go` (dynamic name completions) -- [ ] **Syntax** — Add help topic file in `cmd/mxcli/help_topics/.txt` and register in `cmd/mxcli/help.go` -- [ ] **Starlark** — Expose query function in `mdl/linter/starlark.go` (e.g., `my_types()`) and conversion in `context.go` -- [ ] **Help** — Document in `cmd/mxcli/help.go` -- [ ] **CLAUDE.md** — Add to syntax quick reference -- [ ] **Examples** — Create `mdl-examples/doctype-tests/NN--examples.mdl` -- [ ] **Tests** — Add roundtrip tests in `mdl/executor/roundtrip_test.go` -- [ ] **Skills** — Create or update skill file in `cmd/mxcli/skills/` -- [ ] **VS Code** — Ensure syntax highlighting covers new keywords in `vscode-mdl/` -- [ ] **Viz** — Add Mermaid diagram generator in `mdl/executor/cmd_mermaid.go` (if visual representation is useful) -- [ ] **Init docs** — Update generated CLAUDE.md template in `cmd/mxcli/init.go` +> +> **Where the list of rows comes from matters as much as their values.** This +> table is enumerated from **Studio Pro's own `Add other` menu** — the full set +> of document types the product offers — and not from inverting what mxcli +> happens to support. Inverting mxcli can only rediscover gaps someone already +> wrote down; enumerating the product finds the ones nobody has thought about. +> Doing that pass is what surfaced **data sets, page templates, ML model +> mappings and change-data-capture services**, none of which appeared in any +> mxcli document before. Re-run it against the menu when onboarding a new +> Mendix major. From b7b82ff1d3459210cc7dd41e1de0f621fa9b53a6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 18 Aug 2026 01:33:20 +0000 Subject: [PATCH 23/35] fix(show): a grammar alternative with no visitor branch exited 0 in silence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `mxcli -c "show page Mod.Home"` exited 0, printed nothing, and did nothing. That is the worst available failure shape: it is indistinguishable from "the page is empty", so it reads as a fact about the project rather than as an unimplemented command. A parse error would have been strictly better. The cause is not a handler bug. `showOrList PAGE qualifiedName` exists in the grammar, but ExitShowStatement has no branch for it, so the visitor appends no statement and the program runs zero statements. Probing every showStatement alternative through the real parser found three in that state, not one: show page Mod.Name -> now an alias for DESCRIBE PAGE, matching how SHOW ENTITY / SHOW ASSOCIATION read as their describe. A missing page now reports "page not found". show connections -> now lists the external SQL connections open in this session (sql.Manager.List, which already existed). Distinct from SHOW DATABASE CONNECTIONS, which lists stored DatabaseConnector documents. show notebooks -> grammar alternative removed. The notebook grammar has no AST, visitor, executor or backend behind it, so there is nothing to wire; it now fails loudly like any other unsupported noun. A structural grep cannot find this class. `ctx.PAGE()` DOES appear in ExitShowStatement — for the unrelated SHOW ACCESS ON PAGE branch — so searching for the token reports the broken alternative as covered. The guard therefore synthesizes a probe from each grammar alternative (dropping optional groups, which nest) and asserts a statement comes out. It covers 81 of 82 alternatives and fails if that coverage drops. The guard immediately earned itself: the first version of the SHOW CONNECTIONS branch swallowed SHOW DATABASE CONNECTIONS, which also matches CONNECTIONS(), and answered a stored-document query with live session state. Guarded with ctx.DATABASE() == nil. Control: reverting the visitor branch makes the guard fail on `show page`. Still open, reported rather than swept: `create notebook X begin end` is the same silent no-op, and the whole notebook grammar (createNotebookStatement, notebookOptions, notebookPage, alterNotebookAction) is dead — no Go code references any of it. Deleting a subsystem someone may be part-way through building is the maintainer's call, not a side effect of this fix. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017s476QkXr9CFMvKspVzcvu --- .claude/skills/fix-issue.md | 1 + mdl/ast/ast_query.go | 3 + mdl/executor/cmd_sql.go | 24 +++ mdl/executor/executor_query.go | 2 + mdl/grammar/domains/MDLCatalog.g4 | 7 +- mdl/visitor/show_statement_coverage_test.go | 211 ++++++++++++++++++++ mdl/visitor/visitor_query.go | 22 ++ 7 files changed, 269 insertions(+), 1 deletion(-) create mode 100644 mdl/visitor/show_statement_coverage_test.go diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index a776dea66..5fa38d06a 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -536,3 +536,4 @@ extracting `OffsetExpression`/`LimitExpression`. | A skill claims a construct is unsupported that the grammar accepts (and documents the working form elsewhere in the same file) — an agent writes valid MDL, hits the contradicting claim, and undoes it. Seen for `case … end case` in `write-microflows.md` (supported at §CASE Statements, "not supported" under UNSUPPORTED Syntax), with `check-syntax.md`, `MDL_QUICK_REFERENCE.md`, three `docs-site` pages and the generated project CLAUDE.md (a third shape, `CASE $Var/Attr AS x WHEN Enum.Value`) all disagreeing | Nothing validates prose claims against the grammar. `scripts/check-skill-mdl.sh` extracts fenced blocks but deliberately **skips microflow bodies** (fragment-heavy), so every syntax claim about microflow statements is unguarded. The wrong "WRONG:" example also used string-literal case values, which fail for their own reason — making the claim look confirmed | Docs only: `.claude/skills/mendix/write-microflows.md` + `check-syntax.md`, `docs/01-project/MDL_QUICK_REFERENCE.md`, `docs-site/src/language/control-flow.md` + `appendixes/{quick-reference,common-mistakes}.md`, `cmd/mxcli/init_claudemd.go` (the generated table) | Settle it against the binary before editing, one variant per claim (`bin/mxcli check` on the documented form, the quoted form, the qualified form, the `AS` form, the `else` form) — a "WRONG" example that fails for an unrelated reason is not evidence. Then fix **every** surface: grep `end case\|END CASE` across `.claude/`, `docs/`, `docs-site/` and `cmd/mxcli/*.go`, not just the file in the report (7 surfaces, the issue named 3). Pin both directions in `mdl-examples/bug-tests/` — `907-case-enum-split-is-supported.mdl` (must pass) and `907-case-enum-split-invalid-forms.fail.mdl` (must fail) — so `make check-mdl` catches drift back. Issue #907 | | Docs disagree on whether an enumeration may be written as a string literal — the skills say "NEVER use a string literal", docs-site uses `Status = 'Draft'` in a dozen CREATE/CHANGE examples and calls it "plain strings when the context is unambiguous". Following either one wholesale is wrong, and `mxcli check` passes **both** the good and the bad form, so the build is where you find out | It is context-dependent and nobody had measured it. A string is accepted wherever the slot is **already enum-typed**; in an **expression** it is a String compared to an Enumeration | Docs only: `.claude/skills/mendix/write-microflows.md` (Enumeration Comparisons), `docs-site/src/language/{control-flow,expressions}.md`, `docs-site/src/reference/microflow/create-microflow.md` | Measure per context on a real project before touching a single example — one microflow per construct, `mxcli docker check` read per construct (`mxcli new EP --version 11.13.0 --skip-build`, then `exec` + `docker check`). Verified on 11.13.0: **CE0117** for `if $O/Status = 'Draft'` and for `'x' + $O/Status`; **clean** for `change`/`create` member values, attribute `DEFAULT 'Draft'`, XPath `[Status = 'Draft']`, and `getCaption()`/`toString()` in a concat. So fix **only** comparisons and concatenations — a sweep-and-replace over every `Status = '…'` would have "corrected" ~10 of 12 sites that are fine. No `.fail.mdl` is possible (check passes the failing form; the type checker is still a proposal — `docs/11-proposals/PROPOSAL_expression_type_checking.md`), so the verdicts live in `mdl-examples/bug-tests/enum-string-literal-contexts.mdl`, which pins the ACCEPTED forms and builds clean (control: adding it to a probe project left the error count unchanged at the 2 deliberate failures) | | `mxcli exec` applies a script that `mxcli check` rejects with an error — e.g. a page written with an invalid widget property, which is then silently dropped, so the widget renders but does nothing | `exec` ran no semantic checks at all. Parse errors stopped it; the ~24 `Validate*` checks lived inline in `cmd/mxcli/cmd_check.go` and had exactly one caller. "Run check before exec" was a convention nothing enforced, on a tool built for unattended agent use | `mdl/executor/validate_program.go` (the shared list), `cmd/mxcli/cmd_exec.go` (the gate), `cmd/mxcli/cmd_check.go` | `executor.ValidateProgram` is now the single definition of what `check` checks, and `exec` refuses to run when it reports an **error** (warnings print and do not block). `--no-check` opts out. **A check wired into nothing is invisible** — it reports no violations and reads as a clean project — so `TestValidateProgram_WiresEveryWholeProgramValidator` asserts structurally that every exported `Validate*(prog *ast.Program, …)` is called from `ValidateProgram`; per-statement helpers take a statement and are excluded. When adding a check, add it to `ValidateProgram` and both commands get it | +| A `show`/`list` command **exits 0, prints nothing, and does nothing** — `mxcli -c "show page Mod.Home"` looks like a page with no content, `show connections` looks like no connections are open. No error, no parse failure, no output. Indistinguishable from a legitimately empty result, which is why it survives: the answer is plausible | The grammar alternative exists (`showOrList PAGE qualifiedName`, `showOrList CONNECTIONS`, `showOrList NOTEBOOKS`) but `ExitShowStatement` has **no `else if` branch for it**, so the visitor appends no statement and the program runs zero statements. Not a handler bug — the AST node is never created. `create notebook … begin end` is the same shape (grammar with no AST/visitor/executor anywhere) | `mdl/visitor/visitor_query.go` (`ExitShowStatement`), `mdl/grammar/domains/MDLCatalog.g4`, `mdl/ast/ast_query.go`, `mdl/executor/executor_query.go` | Either wire a visitor branch or delete the grammar alternative — an unimplemented command must fail loudly. `SHOW PAGE X` became an alias for `DESCRIBE PAGE` (matching `SHOW ENTITY`), `SHOW CONNECTIONS` wired to `sql.Manager.List()`, `SHOW NOTEBOOKS` removed. **A structural grep cannot find these**: `ctx.PAGE()` *does* appear in `ExitShowStatement` — for the unrelated `SHOW ACCESS ON PAGE` branch — so a token search reports the broken alternative as covered. The guard (`TestEveryShowAlternativeProducesAStatement`) instead synthesizes a probe from each grammar alternative and asserts a statement comes out. **Ordering trap when adding a branch**: `SHOW DATABASE CONNECTIONS` also matches `CONNECTIONS()`, so a new `CONNECTIONS` branch placed first swallows it and answers a stored-document query with live session state — guard with `ctx.DATABASE() == nil`. Found while auditing mxcli against Studio Pro's document-type list | diff --git a/mdl/ast/ast_query.go b/mdl/ast/ast_query.go index 728bbe708..6760c0cdb 100644 --- a/mdl/ast/ast_query.go +++ b/mdl/ast/ast_query.go @@ -102,6 +102,7 @@ const ( ShowConsumedMCPServices // SHOW CONSUMED MCP SERVICES [IN module] (agent-editor MCP documents) ShowJarDependencies // LIST JAR DEPENDENCIES [IN module] ShowBuildingBlocks // SHOW BUILDING BLOCKS [IN module] + ShowConnections // SHOW CONNECTIONS (open external SQL connections in this session) ) // String returns the human-readable name of the show object type. @@ -245,6 +246,8 @@ func (t ShowObjectType) String() string { return "JAR DEPENDENCIES" case ShowBuildingBlocks: return "BUILDING BLOCKS" + case ShowConnections: + return "CONNECTIONS" default: return "UNKNOWN" } diff --git a/mdl/executor/cmd_sql.go b/mdl/executor/cmd_sql.go index 61293e2a5..5f00f1be3 100644 --- a/mdl/executor/cmd_sql.go +++ b/mdl/executor/cmd_sql.go @@ -269,3 +269,27 @@ func execSQLDescribeTable(ctx *ExecContext, s *ast.SQLDescribeTableStmt) error { } // Executor wrappers for unmigrated callers. + +// listOpenSQLConnections handles SHOW CONNECTIONS: the external SQL connections +// open in this session, as opened by SQL CONNECT. +// +// Distinct from SHOW DATABASE CONNECTIONS, which lists the DatabaseConnector +// documents stored in the project. These are live session state and vanish with +// the process. +func listOpenSQLConnections(ctx *ExecContext) error { + conns := ensureSQLManager(ctx).List() + + if len(conns) == 0 && ctx.Format != FormatJSON { + fmt.Fprintln(ctx.Output, "No open SQL connections. Open one with: sql connect '' as ") + return nil + } + + tr := &TableResult{ + Columns: []string{"Alias", "Driver"}, + Summary: fmt.Sprintf("(%d open connection(s))", len(conns)), + } + for _, c := range conns { + tr.Rows = append(tr.Rows, []any{c.Alias, string(c.Driver)}) + } + return writeResult(ctx, tr) +} diff --git a/mdl/executor/executor_query.go b/mdl/executor/executor_query.go index af95a3e55..ed7653212 100644 --- a/mdl/executor/executor_query.go +++ b/mdl/executor/executor_query.go @@ -121,6 +121,8 @@ func execShow(ctx *ExecContext, s *ast.ShowStmt) error { return listFragments(ctx) case ast.ShowDatabaseConnections: return listDatabaseConnections(ctx, s.InModule) + case ast.ShowConnections: + return listOpenSQLConnections(ctx) case ast.ShowImageCollections: return listImageCollections(ctx, s.InModule) case ast.ShowIconCollections: diff --git a/mdl/grammar/domains/MDLCatalog.g4 b/mdl/grammar/domains/MDLCatalog.g4 index 7090e4694..142598887 100644 --- a/mdl/grammar/domains/MDLCatalog.g4 +++ b/mdl/grammar/domains/MDLCatalog.g4 @@ -34,7 +34,12 @@ showStatement | showOrList CONSTANTS (IN (qualifiedName | IDENTIFIER))? | showOrList CONSTANT VALUES (IN (qualifiedName | IDENTIFIER))? | showOrList LAYOUTS (IN (qualifiedName | IDENTIFIER))? - | showOrList NOTEBOOKS (IN (qualifiedName | IDENTIFIER))? + // NOTEBOOKS deliberately absent: the notebook grammar (createNotebookStatement, + // notebookPage, alterNotebookAction) has no AST, visitor, executor or backend + // behind it, so `show notebooks` parsed to nothing and exited 0 printing + // nothing — which reads as "this project has no notebooks". An unimplemented + // command must fail loudly; restore this alternative together with a visitor + // branch, not before. | showOrList QUEUES (IN (qualifiedName | IDENTIFIER))? | showOrList SCHEDULED EVENTS (IN (qualifiedName | IDENTIFIER))? | showOrList REGULAR EXPRESSIONS (IN (qualifiedName | IDENTIFIER))? diff --git a/mdl/visitor/show_statement_coverage_test.go b/mdl/visitor/show_statement_coverage_test.go new file mode 100644 index 000000000..7fcf69cf7 --- /dev/null +++ b/mdl/visitor/show_statement_coverage_test.go @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: Apache-2.0 + +package visitor + +import ( + mdlast "github.com/mendixlabs/mxcli/mdl/ast" + + "os" + "regexp" + "strings" + "testing" +) + +// TestEveryShowAlternativeProducesAStatement is the drift guard for a whole +// class of silent failure: a grammar alternative that parses cleanly while the +// visitor has no branch for it, so it produces NO AST statement at all. +// +// The result is the worst possible shape for a CLI — `mxcli -c "show page X"` +// exits 0, prints nothing, and writes nothing. It is indistinguishable from +// "the query ran and there was nothing to show", so it reads as an empty +// project rather than as an unimplemented command. A parse error would have +// been strictly better. Four commands were in this state when the guard was +// written (`show page`, `show connections`, `show notebooks`, and +// `create notebook`). +// +// A structural check on the visitor source cannot catch this: `ctx.PAGE()` DOES +// appear in ExitShowStatement — for the unrelated `SHOW ACCESS ON PAGE` +// alternative — so grepping for the token says "covered" while the singular +// PAGE alternative is unhandled. Only running the text through the real parser +// and visitor distinguishes them. +func TestEveryShowAlternativeProducesAStatement(t *testing.T) { + probes, skipped := showProbesFromGrammar(t) + if len(probes) < 60 { + t.Fatalf("only synthesized %d probes; the grammar reader is broken, "+ + "so a pass here would prove nothing", len(probes)) + } + + for _, p := range probes { + prog, errs := Build(p) + if len(errs) > 0 { + // A probe the synthesizer got wrong is not a product bug. It must + // stay rare, or this guard quietly stops covering the grammar. + skipped = append(skipped, p) + continue + } + if prog == nil || len(prog.Statements) == 0 { + t.Errorf("`%s` parses but produces no AST statement — it will exit 0 "+ + "and print nothing, which reads as an empty result rather than an "+ + "unimplemented command. Add a visitor branch, or remove the "+ + "grammar alternative so it fails loudly.", p) + } + } + + if len(skipped) > len(probes)/4 { + t.Errorf("%d of %d probes were skipped; the guard is no longer covering "+ + "most of the grammar:\n %s", len(skipped), len(probes)+len(skipped), + strings.Join(skipped, "\n ")) + } +} + +var ( + reGrammarComment = regexp.MustCompile(`//.*$`) + reOptionalToken = regexp.MustCompile(`\b([A-Z_]+)\?`) + reAllCaps = regexp.MustCompile(`^[A-Z_]+$`) +) + +// showProbesFromGrammar turns each `showOrList …` alternative of the +// showStatement rule into a concrete command string. Optional groups are +// dropped (the shortest legal form is the one worth probing) and the few +// alternatives whose tail is a sub-rule are reported as skipped rather than +// guessed at. +func showProbesFromGrammar(t *testing.T) (probes, skipped []string) { + t.Helper() + src, err := os.ReadFile("../grammar/domains/MDLCatalog.g4") + if err != nil { + t.Fatalf("read grammar: %v", err) + } + body := string(src) + start := strings.Index(body, "showStatement") + if start < 0 { + t.Fatal("no showStatement rule in MDLCatalog.g4") + } + body = body[start:] + if end := strings.Index(body, "\n ;"); end > 0 { + body = body[:end] + } + + for _, line := range strings.Split(body, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "|") { + continue + } + alt := strings.TrimSpace(reGrammarComment.ReplaceAllString(strings.TrimLeft(line, "| "), "")) + if !strings.HasPrefix(alt, "showOrList") { + continue + } + alt = strings.TrimSpace(strings.TrimPrefix(alt, "showOrList")) + alt = dropOptionalGroups(alt) + alt = reOptionalToken.ReplaceAllString(alt, "") + + words := []string{"show"} + unsynthesizable := false + for _, tok := range strings.Fields(alt) { + switch { + case tok == "qualifiedName": + words = append(words, "Mod.Name") + case tok == "NUMBER_LITERAL": + words = append(words, "11") + case tok == "MENU_KW": + words = append(words, "menu") + case reAllCaps.MatchString(tok): + words = append(words, strings.ToLower(strings.TrimSuffix(tok, "_KW"))) + default: + unsynthesizable = true // a sub-rule tail; don't guess + } + if unsynthesizable { + break + } + } + if unsynthesizable { + skipped = append(skipped, alt) + continue + } + probes = append(probes, strings.Join(words, " ")) + } + return probes, skipped +} + +// dropOptionalGroups removes every balanced `( … )?` group. Nesting is real in +// this grammar — `(IN (qualifiedName | IDENTIFIER))?` — so a regex is not +// enough. +func dropOptionalGroups(s string) string { + var out strings.Builder + for i := 0; i < len(s); { + if s[i] != '(' { + out.WriteByte(s[i]) + i++ + continue + } + depth, j := 0, i + for ; j < len(s); j++ { + if s[j] == '(' { + depth++ + } else if s[j] == ')' { + if depth--; depth == 0 { + break + } + } + } + if j+1 < len(s) && s[j+1] == '?' { + i = j + 2 // drop the optional group entirely + continue + } + out.WriteString(s[i : j+1]) + i = j + 1 + } + return out.String() +} + +// TestShowPageIsDescribePageAlias pins the shape, not just the presence, of the +// singular SHOW PAGE branch: it must produce a DescribeStmt, so `show page X` +// and `describe page X` agree — including reporting a missing page instead of +// exiting 0 in silence. +func TestShowPageIsDescribePageAlias(t *testing.T) { + prog, errs := Build("show page Mod.Home") + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + if len(prog.Statements) != 1 { + t.Fatalf("got %d statements, want 1", len(prog.Statements)) + } + d, ok := prog.Statements[0].(*mdlast.DescribeStmt) + if !ok { + t.Fatalf("got %T, want *ast.DescribeStmt — SHOW PAGE must reuse the "+ + "describe path so both spellings behave identically", prog.Statements[0]) + } + if d.ObjectType != mdlast.DescribePage { + t.Errorf("ObjectType = %v, want DescribePage", d.ObjectType) + } + if d.Name.String() != "Mod.Home" { + t.Errorf("Name = %q, want Mod.Home", d.Name.String()) + } +} + +// TestShowConnectionsProducesShowStmt covers the other silent no-op. SHOW +// CONNECTIONS lists live SQL sessions; SHOW DATABASE CONNECTIONS lists stored +// DatabaseConnector documents. They must not collapse into one another. +func TestShowConnectionsProducesShowStmt(t *testing.T) { + for _, tc := range []struct { + src string + want mdlast.ShowObjectType + }{ + {"show connections", mdlast.ShowConnections}, + {"show database connections", mdlast.ShowDatabaseConnections}, + } { + prog, errs := Build(tc.src) + if len(errs) > 0 { + t.Fatalf("parse %q: %v", tc.src, errs) + } + if len(prog.Statements) != 1 { + t.Fatalf("%q: got %d statements, want 1", tc.src, len(prog.Statements)) + } + s, ok := prog.Statements[0].(*mdlast.ShowStmt) + if !ok { + t.Fatalf("%q: got %T, want *ast.ShowStmt", tc.src, prog.Statements[0]) + } + if s.ObjectType != tc.want { + t.Errorf("%q: ObjectType = %v, want %v", tc.src, s.ObjectType, tc.want) + } + } +} diff --git a/mdl/visitor/visitor_query.go b/mdl/visitor/visitor_query.go index 8285321a6..19b819d9f 100644 --- a/mdl/visitor/visitor_query.go +++ b/mdl/visitor/visitor_query.go @@ -184,6 +184,28 @@ func (b *Builder) ExitShowStatement(ctx *parser.ShowStatementContext) { } } b.statements = append(b.statements, stmt) + } else if ctx.PAGE() != nil { + // `SHOW PAGE Module.Page` (singular) is an alias for DESCRIBE PAGE, the + // same way SHOW ENTITY / SHOW ASSOCIATION read as their describe. The + // grammar alternative existed with no visitor branch, so the statement + // parsed to nothing and the command exited 0 printing nothing — which + // reads as "this page is empty", not "this command does nothing". + if qn := ctx.QualifiedName(); qn != nil { + b.statements = append(b.statements, &ast.DescribeStmt{ + ObjectType: ast.DescribePage, + Name: buildQualifiedName(qn), + }) + } + } else if ctx.CONNECTIONS() != nil && ctx.DATABASE() == nil { + // SHOW CONNECTIONS lists the external SQL connections open in this + // session (sql.Manager). The grammar alternative had no visitor branch, + // so it parsed to nothing and printed nothing — indistinguishable from + // "no connections are open", which is a plausible and wrong answer. + // + // The DATABASE() guard is load-bearing: `SHOW DATABASE CONNECTIONS` + // also matches CONNECTIONS(), so without it this branch swallows the + // stored-document listing and answers it with live session state. + b.statements = append(b.statements, &ast.ShowStmt{ObjectType: ast.ShowConnections}) } else if ctx.SNIPPETS() != nil { stmt := &ast.ShowStmt{ObjectType: ast.ShowSnippets} if ctx.IN() != nil { From 5cf3b26ce73541c43abcbe278d751d322fbca87e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 18 Aug 2026 01:44:51 +0000 Subject: [PATCH 24/35] refactor(grammar): remove the dead notebook grammar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All four notebook statements were silent no-ops: `create notebook X begin end`, `alter notebook X add page Y`, `drop notebook X` and `show notebooks` each parsed cleanly, exited 0, printed nothing and wrote nothing. No AST node, visitor branch, executor handler or backend method existed for any of them — `grep -ri notebook` over mdl/ast, mdl/visitor, mdl/executor, mdl/backend, sdk and modelsdk returns nothing. Dead grammar is not inert. It accepts input and discards it, which is the one outcome worse than rejecting it: `create notebook` looked like it worked. Removed the statements (createNotebookStatement, ALTER NOTEBOOK, DROP NOTEBOOK), their sub-rules (notebookOptions, notebookOption, notebookPage, alterNotebookAction), and the NOTEBOOK/NOTEBOOKS lexer tokens plus their identifierOrKeyword entries. All four now fail as parse errors. Dropping the tokens also frees `notebook` as an ordinary identifier — one fewer reserved word, verified with an entity and attribute of that name. Control for a grammar change this broad: every mdl-examples script was swept through `mxcli check` before and after and the failure SETS diffed. 15 fail either way — all pre-existing — so the count alone would have proved nothing; the set difference is empty. Neighbouring commands (`show pages`, `show page X`, `show connections`, `show database connections`) verified unchanged against a real project. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017s476QkXr9CFMvKspVzcvu --- .claude/skills/fix-issue.md | 2 +- mdl/grammar/MDLLexer.g4 | 2 -- mdl/grammar/MDLParser.g4 | 3 --- mdl/grammar/domains/MDLCatalog.g4 | 6 ------ mdl/grammar/domains/MDLDomainModel.g4 | 8 +------- mdl/grammar/domains/MDLPage.g4 | 23 +---------------------- mdl/grammar/domains/MDLSettings.g4 | 2 +- 7 files changed, 4 insertions(+), 42 deletions(-) diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index 5fa38d06a..2d133ce87 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -536,4 +536,4 @@ extracting `OffsetExpression`/`LimitExpression`. | A skill claims a construct is unsupported that the grammar accepts (and documents the working form elsewhere in the same file) — an agent writes valid MDL, hits the contradicting claim, and undoes it. Seen for `case … end case` in `write-microflows.md` (supported at §CASE Statements, "not supported" under UNSUPPORTED Syntax), with `check-syntax.md`, `MDL_QUICK_REFERENCE.md`, three `docs-site` pages and the generated project CLAUDE.md (a third shape, `CASE $Var/Attr AS x WHEN Enum.Value`) all disagreeing | Nothing validates prose claims against the grammar. `scripts/check-skill-mdl.sh` extracts fenced blocks but deliberately **skips microflow bodies** (fragment-heavy), so every syntax claim about microflow statements is unguarded. The wrong "WRONG:" example also used string-literal case values, which fail for their own reason — making the claim look confirmed | Docs only: `.claude/skills/mendix/write-microflows.md` + `check-syntax.md`, `docs/01-project/MDL_QUICK_REFERENCE.md`, `docs-site/src/language/control-flow.md` + `appendixes/{quick-reference,common-mistakes}.md`, `cmd/mxcli/init_claudemd.go` (the generated table) | Settle it against the binary before editing, one variant per claim (`bin/mxcli check` on the documented form, the quoted form, the qualified form, the `AS` form, the `else` form) — a "WRONG" example that fails for an unrelated reason is not evidence. Then fix **every** surface: grep `end case\|END CASE` across `.claude/`, `docs/`, `docs-site/` and `cmd/mxcli/*.go`, not just the file in the report (7 surfaces, the issue named 3). Pin both directions in `mdl-examples/bug-tests/` — `907-case-enum-split-is-supported.mdl` (must pass) and `907-case-enum-split-invalid-forms.fail.mdl` (must fail) — so `make check-mdl` catches drift back. Issue #907 | | Docs disagree on whether an enumeration may be written as a string literal — the skills say "NEVER use a string literal", docs-site uses `Status = 'Draft'` in a dozen CREATE/CHANGE examples and calls it "plain strings when the context is unambiguous". Following either one wholesale is wrong, and `mxcli check` passes **both** the good and the bad form, so the build is where you find out | It is context-dependent and nobody had measured it. A string is accepted wherever the slot is **already enum-typed**; in an **expression** it is a String compared to an Enumeration | Docs only: `.claude/skills/mendix/write-microflows.md` (Enumeration Comparisons), `docs-site/src/language/{control-flow,expressions}.md`, `docs-site/src/reference/microflow/create-microflow.md` | Measure per context on a real project before touching a single example — one microflow per construct, `mxcli docker check` read per construct (`mxcli new EP --version 11.13.0 --skip-build`, then `exec` + `docker check`). Verified on 11.13.0: **CE0117** for `if $O/Status = 'Draft'` and for `'x' + $O/Status`; **clean** for `change`/`create` member values, attribute `DEFAULT 'Draft'`, XPath `[Status = 'Draft']`, and `getCaption()`/`toString()` in a concat. So fix **only** comparisons and concatenations — a sweep-and-replace over every `Status = '…'` would have "corrected" ~10 of 12 sites that are fine. No `.fail.mdl` is possible (check passes the failing form; the type checker is still a proposal — `docs/11-proposals/PROPOSAL_expression_type_checking.md`), so the verdicts live in `mdl-examples/bug-tests/enum-string-literal-contexts.mdl`, which pins the ACCEPTED forms and builds clean (control: adding it to a probe project left the error count unchanged at the 2 deliberate failures) | | `mxcli exec` applies a script that `mxcli check` rejects with an error — e.g. a page written with an invalid widget property, which is then silently dropped, so the widget renders but does nothing | `exec` ran no semantic checks at all. Parse errors stopped it; the ~24 `Validate*` checks lived inline in `cmd/mxcli/cmd_check.go` and had exactly one caller. "Run check before exec" was a convention nothing enforced, on a tool built for unattended agent use | `mdl/executor/validate_program.go` (the shared list), `cmd/mxcli/cmd_exec.go` (the gate), `cmd/mxcli/cmd_check.go` | `executor.ValidateProgram` is now the single definition of what `check` checks, and `exec` refuses to run when it reports an **error** (warnings print and do not block). `--no-check` opts out. **A check wired into nothing is invisible** — it reports no violations and reads as a clean project — so `TestValidateProgram_WiresEveryWholeProgramValidator` asserts structurally that every exported `Validate*(prog *ast.Program, …)` is called from `ValidateProgram`; per-statement helpers take a statement and are excluded. When adding a check, add it to `ValidateProgram` and both commands get it | -| A `show`/`list` command **exits 0, prints nothing, and does nothing** — `mxcli -c "show page Mod.Home"` looks like a page with no content, `show connections` looks like no connections are open. No error, no parse failure, no output. Indistinguishable from a legitimately empty result, which is why it survives: the answer is plausible | The grammar alternative exists (`showOrList PAGE qualifiedName`, `showOrList CONNECTIONS`, `showOrList NOTEBOOKS`) but `ExitShowStatement` has **no `else if` branch for it**, so the visitor appends no statement and the program runs zero statements. Not a handler bug — the AST node is never created. `create notebook … begin end` is the same shape (grammar with no AST/visitor/executor anywhere) | `mdl/visitor/visitor_query.go` (`ExitShowStatement`), `mdl/grammar/domains/MDLCatalog.g4`, `mdl/ast/ast_query.go`, `mdl/executor/executor_query.go` | Either wire a visitor branch or delete the grammar alternative — an unimplemented command must fail loudly. `SHOW PAGE X` became an alias for `DESCRIBE PAGE` (matching `SHOW ENTITY`), `SHOW CONNECTIONS` wired to `sql.Manager.List()`, `SHOW NOTEBOOKS` removed. **A structural grep cannot find these**: `ctx.PAGE()` *does* appear in `ExitShowStatement` — for the unrelated `SHOW ACCESS ON PAGE` branch — so a token search reports the broken alternative as covered. The guard (`TestEveryShowAlternativeProducesAStatement`) instead synthesizes a probe from each grammar alternative and asserts a statement comes out. **Ordering trap when adding a branch**: `SHOW DATABASE CONNECTIONS` also matches `CONNECTIONS()`, so a new `CONNECTIONS` branch placed first swallows it and answers a stored-document query with live session state — guard with `ctx.DATABASE() == nil`. Found while auditing mxcli against Studio Pro's document-type list | +| A `show`/`list` command **exits 0, prints nothing, and does nothing** — `mxcli -c "show page Mod.Home"` looks like a page with no content, `show connections` looks like no connections are open. No error, no parse failure, no output. Indistinguishable from a legitimately empty result, which is why it survives: the answer is plausible | The grammar alternative exists (`showOrList PAGE qualifiedName`, `showOrList CONNECTIONS`, `showOrList NOTEBOOKS`) but `ExitShowStatement` has **no `else if` branch for it**, so the visitor appends no statement and the program runs zero statements. Not a handler bug — the AST node is never created. `create notebook … begin end` is the same shape (grammar with no AST/visitor/executor anywhere) | `mdl/visitor/visitor_query.go` (`ExitShowStatement`), `mdl/grammar/domains/MDLCatalog.g4`, `mdl/ast/ast_query.go`, `mdl/executor/executor_query.go` | Either wire a visitor branch or delete the grammar alternative — an unimplemented command must fail loudly. `SHOW PAGE X` became an alias for `DESCRIBE PAGE` (matching `SHOW ENTITY`), `SHOW CONNECTIONS` wired to `sql.Manager.List()`, `SHOW NOTEBOOKS` removed. **A structural grep cannot find these**: `ctx.PAGE()` *does* appear in `ExitShowStatement` — for the unrelated `SHOW ACCESS ON PAGE` branch — so a token search reports the broken alternative as covered. The guard (`TestEveryShowAlternativeProducesAStatement`) instead synthesizes a probe from each grammar alternative and asserts a statement comes out. **Ordering trap when adding a branch**: `SHOW DATABASE CONNECTIONS` also matches `CONNECTIONS()`, so a new `CONNECTIONS` branch placed first swallows it and answers a stored-document query with live session state — guard with `ctx.DATABASE() == nil`. Found while auditing mxcli against Studio Pro's document-type list. **Follow-up**: the notebook grammar (`createNotebookStatement`, `notebookOptions`, `notebookPage`, `alterNotebookAction`, the `SHOW NOTEBOOKS` alternative, and the `NOTEBOOK`/`NOTEBOOKS` lexer tokens) was removed outright — all four of `create`/`alter`/`drop`/`show notebook` were silent no-ops with no Go code behind any of them. Dead grammar is not inert: it *accepts input and discards it*. Removing the tokens also frees `notebook` as an ordinary identifier. **Control for a broad grammar change**: sweep every `mdl-examples/**/*.mdl` through `mxcli check` before and after and diff the failure sets — 15 scripts fail either way, so the raw count proves nothing and only the set difference does | diff --git a/mdl/grammar/MDLLexer.g4 b/mdl/grammar/MDLLexer.g4 index 69600518b..83637172d 100644 --- a/mdl/grammar/MDLLexer.g4 +++ b/mdl/grammar/MDLLexer.g4 @@ -70,7 +70,6 @@ SNIPPET: S N I P P E T; BUILDING: B U I L D I N G; BLOCK: B L O C K; LAYOUT: L A Y O U T; -NOTEBOOK: N O T E B O O K; CONSTANT: C O N S T A N T; ATTRIBUTE: A T T R I B U T E; @@ -240,7 +239,6 @@ PAGES: P A G E S; LAYOUTS: L A Y O U T S; SNIPPETS: S N I P P E T S; BLOCKS: B L O C K S; -NOTEBOOKS: N O T E B O O K S; PLACEHOLDER: P L A C E H O L D E R; SNIPPETCALL: S N I P P E T C A L L; diff --git a/mdl/grammar/MDLParser.g4 b/mdl/grammar/MDLParser.g4 index 964dafc59..5d9714215 100644 --- a/mdl/grammar/MDLParser.g4 +++ b/mdl/grammar/MDLParser.g4 @@ -100,7 +100,6 @@ createStatement | createSnippetStatement | createEnumerationStatement | createValidationRuleStatement - | createNotebookStatement | createDatabaseConnectionStatement | createConstantStatement | createRestClientStatement @@ -137,7 +136,6 @@ alterStatement : ALTER ENTITY qualifiedName alterEntityAction (COMMA? alterEntityAction)* | ALTER ASSOCIATION qualifiedName alterAssociationAction+ | ALTER ENUMERATION qualifiedName alterEnumerationAction+ - | ALTER NOTEBOOK qualifiedName alterNotebookAction+ | ALTER ODATA CLIENT qualifiedName SET odataAlterAssignment (COMMA odataAlterAssignment)* | ALTER ODATA SERVICE qualifiedName SET odataAlterAssignment (COMMA odataAlterAssignment)* | ALTER STYLING ON (PAGE | SNIPPET) qualifiedName WIDGET IDENTIFIER alterStylingAction+ @@ -321,7 +319,6 @@ dropStatement | DROP SNIPPET qualifiedName | DROP MENU_KW qualifiedName | DROP MODULE qualifiedName - | DROP NOTEBOOK qualifiedName | DROP QUEUE qualifiedName | DROP SCHEDULED EVENT qualifiedName | DROP REGULAR EXPRESSION qualifiedName diff --git a/mdl/grammar/domains/MDLCatalog.g4 b/mdl/grammar/domains/MDLCatalog.g4 index 142598887..815f5e99e 100644 --- a/mdl/grammar/domains/MDLCatalog.g4 +++ b/mdl/grammar/domains/MDLCatalog.g4 @@ -34,12 +34,6 @@ showStatement | showOrList CONSTANTS (IN (qualifiedName | IDENTIFIER))? | showOrList CONSTANT VALUES (IN (qualifiedName | IDENTIFIER))? | showOrList LAYOUTS (IN (qualifiedName | IDENTIFIER))? - // NOTEBOOKS deliberately absent: the notebook grammar (createNotebookStatement, - // notebookPage, alterNotebookAction) has no AST, visitor, executor or backend - // behind it, so `show notebooks` parsed to nothing and exited 0 printing - // nothing — which reads as "this project has no notebooks". An unimplemented - // command must fail loudly; restore this alternative together with a visitor - // branch, not before. | showOrList QUEUES (IN (qualifiedName | IDENTIFIER))? | showOrList SCHEDULED EVENTS (IN (qualifiedName | IDENTIFIER))? | showOrList REGULAR EXPRESSIONS (IN (qualifiedName | IDENTIFIER))? diff --git a/mdl/grammar/domains/MDLDomainModel.g4 b/mdl/grammar/domains/MDLDomainModel.g4 index 4dd9d1d9f..0c83076e9 100644 --- a/mdl/grammar/domains/MDLDomainModel.g4 +++ b/mdl/grammar/domains/MDLDomainModel.g4 @@ -193,7 +193,7 @@ deleteBehavior ; // ============================================================================= -// ALTER ENTITY / ASSOCIATION / ENUMERATION / NOTEBOOK ACTIONS +// ALTER ENTITY / ASSOCIATION / ENUMERATION ACTIONS // ============================================================================= alterEntityAction @@ -254,12 +254,6 @@ alterEnumerationAction | SET COMMENT STRING_LITERAL ; -alterNotebookAction - : ADD PAGE qualifiedName (POSITION NUMBER_LITERAL)? - | DROP PAGE qualifiedName - | SET COMMENT STRING_LITERAL - ; - // ============================================================================= // MODULE CREATION // ============================================================================= diff --git a/mdl/grammar/domains/MDLPage.g4 b/mdl/grammar/domains/MDLPage.g4 index a8522f2b5..77e9633ee 100644 --- a/mdl/grammar/domains/MDLPage.g4 +++ b/mdl/grammar/domains/MDLPage.g4 @@ -1,6 +1,6 @@ /** * MDL Page Grammar — pages, snippets, shared page/snippet rules, xpath expressions, - * page V3 syntax, notebooks. + * page V3 syntax. */ parser grammar MDLPage; @@ -575,24 +575,3 @@ widgetBodyV3 : LBRACE pageBodyV3 RBRACE ; -// ============================================================================= -// NOTEBOOK CREATION -// ============================================================================= - -createNotebookStatement - : NOTEBOOK qualifiedName - notebookOptions? - BEGIN notebookPage* END - ; - -notebookOptions - : notebookOption+ - ; - -notebookOption - : COMMENT STRING_LITERAL - ; - -notebookPage - : PAGE qualifiedName (CAPTION STRING_LITERAL)? - ; diff --git a/mdl/grammar/domains/MDLSettings.g4 b/mdl/grammar/domains/MDLSettings.g4 index fcf8f914e..c3509cb33 100644 --- a/mdl/grammar/domains/MDLSettings.g4 +++ b/mdl/grammar/domains/MDLSettings.g4 @@ -509,7 +509,7 @@ keyword // Module / project structure | ACTIONS | ARTIFACT | COLLECTION | DEPENDENCIES | DEPENDENCY | EXCLUSION | FOLDER | FOLDERS | INCLUDED | JAR | LAYOUT | LAYOUTS | LOCAL | MODEL | MODELS | MODULE | MODULES - | NOTEBOOK | NOTEBOOKS | PAGE | PAGES | PROJECT | SNIPPET | SNIPPETS + | PAGE | PAGES | PROJECT | SNIPPET | SNIPPETS | BUILDING | BLOCK | BLOCKS | STORE | STRUCTURE | STRUCTURES | VIEW From 256e32e2cb6d63e1ce1cfb81cb04e68440e5b0fd Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 18 Aug 2026 02:09:02 +0000 Subject: [PATCH 25/35] fix(check): MDL003 no longer demands a RETURN that the builder synthesizes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `RETURNS T AS $Var` names the return variable, and buildFlowGraph sets the final EndEvent's ReturnValue to "$"+Var whenever the AS clause is present — so the return exists whether or not the body spells one out. That is the point of the clause. MDL003 demanded an explicit RETURN on top of it and flagged the documented idiom as broken. It stopped being cosmetic when `exec` began refusing any script whose checks report an error: MDL003 is an error, so seven shipped mdl-examples scripts could no longer run at all without --no-check. The check now skips when returnType.Variable != "", mirroring the builder's own condition rather than inventing a second rule. A companion test keeps the no-AS-clause case failing, so simply disabling the rule would not pass. Evidence the check was wrong rather than the scripts, all on mxbuild 11.13.0: the flagged microflow builds with 0 errors, `describe microflow` renders the synthesized `return $Found;`, and the stored EndEvent carries a ReturnValue. End to end, 343-list-attribute-find-filter.mdl now execs without --no-check and the project checks clean. Control: reverting the one-line condition fails the new test with the reported symptom. The example sweep goes 15 -> 5. The remaining five are not this bug and are left alone: 552-npe-reserved-words.mdl fails on purpose (it is the regression test for reserved-word detection), and four scripts have real defects of their own — three declare a CreatedDate attribute where the AutoCreatedDate pseudo-type is required, and 116-datagrid2 uses a `SET WIDGET x.y (...)` form that has never existed in the grammar. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017s476QkXr9CFMvKspVzcvu --- .claude/skills/fix-issue.md | 1 + mdl/executor/validate_microflow.go | 12 +++- ...validate_microflow_return_variable_test.go | 63 +++++++++++++++++++ 3 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 mdl/executor/validate_microflow_return_variable_test.go diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index 2d133ce87..be2401897 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -537,3 +537,4 @@ extracting `OffsetExpression`/`LimitExpression`. | Docs disagree on whether an enumeration may be written as a string literal — the skills say "NEVER use a string literal", docs-site uses `Status = 'Draft'` in a dozen CREATE/CHANGE examples and calls it "plain strings when the context is unambiguous". Following either one wholesale is wrong, and `mxcli check` passes **both** the good and the bad form, so the build is where you find out | It is context-dependent and nobody had measured it. A string is accepted wherever the slot is **already enum-typed**; in an **expression** it is a String compared to an Enumeration | Docs only: `.claude/skills/mendix/write-microflows.md` (Enumeration Comparisons), `docs-site/src/language/{control-flow,expressions}.md`, `docs-site/src/reference/microflow/create-microflow.md` | Measure per context on a real project before touching a single example — one microflow per construct, `mxcli docker check` read per construct (`mxcli new EP --version 11.13.0 --skip-build`, then `exec` + `docker check`). Verified on 11.13.0: **CE0117** for `if $O/Status = 'Draft'` and for `'x' + $O/Status`; **clean** for `change`/`create` member values, attribute `DEFAULT 'Draft'`, XPath `[Status = 'Draft']`, and `getCaption()`/`toString()` in a concat. So fix **only** comparisons and concatenations — a sweep-and-replace over every `Status = '…'` would have "corrected" ~10 of 12 sites that are fine. No `.fail.mdl` is possible (check passes the failing form; the type checker is still a proposal — `docs/11-proposals/PROPOSAL_expression_type_checking.md`), so the verdicts live in `mdl-examples/bug-tests/enum-string-literal-contexts.mdl`, which pins the ACCEPTED forms and builds clean (control: adding it to a probe project left the error count unchanged at the 2 deliberate failures) | | `mxcli exec` applies a script that `mxcli check` rejects with an error — e.g. a page written with an invalid widget property, which is then silently dropped, so the widget renders but does nothing | `exec` ran no semantic checks at all. Parse errors stopped it; the ~24 `Validate*` checks lived inline in `cmd/mxcli/cmd_check.go` and had exactly one caller. "Run check before exec" was a convention nothing enforced, on a tool built for unattended agent use | `mdl/executor/validate_program.go` (the shared list), `cmd/mxcli/cmd_exec.go` (the gate), `cmd/mxcli/cmd_check.go` | `executor.ValidateProgram` is now the single definition of what `check` checks, and `exec` refuses to run when it reports an **error** (warnings print and do not block). `--no-check` opts out. **A check wired into nothing is invisible** — it reports no violations and reads as a clean project — so `TestValidateProgram_WiresEveryWholeProgramValidator` asserts structurally that every exported `Validate*(prog *ast.Program, …)` is called from `ValidateProgram`; per-statement helpers take a statement and are excluded. When adding a check, add it to `ValidateProgram` and both commands get it | | A `show`/`list` command **exits 0, prints nothing, and does nothing** — `mxcli -c "show page Mod.Home"` looks like a page with no content, `show connections` looks like no connections are open. No error, no parse failure, no output. Indistinguishable from a legitimately empty result, which is why it survives: the answer is plausible | The grammar alternative exists (`showOrList PAGE qualifiedName`, `showOrList CONNECTIONS`, `showOrList NOTEBOOKS`) but `ExitShowStatement` has **no `else if` branch for it**, so the visitor appends no statement and the program runs zero statements. Not a handler bug — the AST node is never created. `create notebook … begin end` is the same shape (grammar with no AST/visitor/executor anywhere) | `mdl/visitor/visitor_query.go` (`ExitShowStatement`), `mdl/grammar/domains/MDLCatalog.g4`, `mdl/ast/ast_query.go`, `mdl/executor/executor_query.go` | Either wire a visitor branch or delete the grammar alternative — an unimplemented command must fail loudly. `SHOW PAGE X` became an alias for `DESCRIBE PAGE` (matching `SHOW ENTITY`), `SHOW CONNECTIONS` wired to `sql.Manager.List()`, `SHOW NOTEBOOKS` removed. **A structural grep cannot find these**: `ctx.PAGE()` *does* appear in `ExitShowStatement` — for the unrelated `SHOW ACCESS ON PAGE` branch — so a token search reports the broken alternative as covered. The guard (`TestEveryShowAlternativeProducesAStatement`) instead synthesizes a probe from each grammar alternative and asserts a statement comes out. **Ordering trap when adding a branch**: `SHOW DATABASE CONNECTIONS` also matches `CONNECTIONS()`, so a new `CONNECTIONS` branch placed first swallows it and answers a stored-document query with live session state — guard with `ctx.DATABASE() == nil`. Found while auditing mxcli against Studio Pro's document-type list. **Follow-up**: the notebook grammar (`createNotebookStatement`, `notebookOptions`, `notebookPage`, `alterNotebookAction`, the `SHOW NOTEBOOKS` alternative, and the `NOTEBOOK`/`NOTEBOOKS` lexer tokens) was removed outright — all four of `create`/`alter`/`drop`/`show notebook` were silent no-ops with no Go code behind any of them. Dead grammar is not inert: it *accepts input and discards it*. Removing the tokens also frees `notebook` as an ordinary identifier. **Control for a broad grammar change**: sweep every `mdl-examples/**/*.mdl` through `mxcli check` before and after and diff the failure sets — 15 scripts fail either way, so the raw count proves nothing and only the set difference does | +| `mxcli check` reports **MDL003** *"microflow returns X but not all code paths have a return statement"* on a microflow that is correct and builds cleanly. Since `exec` began refusing scripts whose checks report an error, the script also will not run without `--no-check` — so a false positive became a hard block. Seven shipped `mdl-examples` scripts were in this state | The microflow uses the documented `RETURNS T AS $Var` form and assigns `$Var` instead of writing an explicit `RETURN`. `buildFlowGraph` sets the final EndEvent's `ReturnValue` to `"$"+Var` **whenever the AS clause is present** (`cmd_microflows_builder_graph.go`), so the return is synthesized either way — that is what the clause is for. The check demanded a `RETURN` statement in the source on top of it | `mdl/executor/validate_microflow.go` (Check 5 in `microflowValidator.validate`) | Skip MDL003 when `returnType.Variable != ""`, mirroring the builder's own condition exactly rather than inventing a second rule. **Verify against mxbuild, not intuition**: the flagged microflow builds with **0 errors** on 11.13.0, `describe microflow` renders the synthesized `return $Found;`, and the stored EndEvent carries a `ReturnValue` — three independent confirmations the check was wrong, not the script. Keep a companion test for the no-AS-clause case, or a fix that simply disables the rule passes. **Sweeping examples as a control**: exclude `*.test.mdl` (those are `mxcli test` files with `@test`/`@expect` blocks, unparseable by `check` by design) and `*.fail.mdl`, and diff failure SETS rather than counts | diff --git a/mdl/executor/validate_microflow.go b/mdl/executor/validate_microflow.go index 04a03bc14..9e96ff01d 100644 --- a/mdl/executor/validate_microflow.go +++ b/mdl/executor/validate_microflow.go @@ -72,8 +72,16 @@ func (v *microflowValidator) validate(body []ast.MicroflowStatement) { v.emptyListVars = make(map[string]bool) v.walkBody(body) - // Check 5: missing RETURN on non-void microflow paths - if v.returnType != nil && v.returnType.Type.Kind != ast.TypeVoid { + // Check 5: missing RETURN on non-void microflow paths. + // + // `RETURNS T AS $Var` is exempt: buildFlowGraph sets the final EndEvent's + // ReturnValue to "$"+Var whenever the AS clause is present, so the return is + // synthesized whether or not the body spells one out — the whole point of the + // clause. Demanding an explicit RETURN on top of it flagged the documented + // idiom as broken, and once `exec` began refusing scripts whose checks report + // an error, that false positive blocked seven shipped examples from running. + // Verified on mxbuild 11.13.0: such a microflow builds with 0 errors. + if v.returnType != nil && v.returnType.Type.Kind != ast.TypeVoid && v.returnType.Variable == "" { if !bodyReturns(body) { v.addViolation("MDL003", linter.SeverityError, fmt.Sprintf("microflow returns %s but not all code paths have a return statement", diff --git a/mdl/executor/validate_microflow_return_variable_test.go b/mdl/executor/validate_microflow_return_variable_test.go new file mode 100644 index 000000000..30dcbede5 --- /dev/null +++ b/mdl/executor/validate_microflow_return_variable_test.go @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" +) + +// TestValidateMicroflow_ReturnsAsVariableNeedsNoExplicitReturn covers the +// MDL003 false positive. +// +// `RETURNS T AS $Var` names the return variable, and buildFlowGraph then sets +// the final EndEvent's ReturnValue to "$"+Var unconditionally — so the return +// is synthesized whether or not the body spells one out. Requiring an explicit +// RETURN on top of that flags the documented idiom as broken. +// +// It stopped being cosmetic when `mxcli exec` began refusing any script whose +// checks report an error: MDL003 is an error, so seven shipped example scripts +// could no longer run without --no-check. Measured on mxbuild 11.13.0, the very +// microflow below builds with 0 errors and `describe microflow` renders the +// synthesized `return $Found;`. +func TestValidateMicroflow_ReturnsAsVariableNeedsNoExplicitReturn(t *testing.T) { + stmt := &ast.CreateMicroflowStmt{ + Name: ast.QualifiedName{Module: "Sample", Name: "MF_FindByAttribute"}, + ReturnType: &ast.MicroflowReturnType{ + Type: ast.DataType{Kind: ast.TypeBoolean}, + Variable: "Found", // RETURNS Boolean AS $Found + }, + Body: []ast.MicroflowStatement{ + &ast.MfSetStmt{Target: "Found", Value: &ast.LiteralExpr{Kind: ast.LiteralBoolean, Value: true}}, + }, + } + + for _, v := range ValidateMicroflow(stmt) { + if v.RuleID == "MDL003" { + t.Fatalf("`returns … as $Found` synthesizes the return; MDL003 must not fire: %#v", v) + } + } +} + +// The check must still fire without the AS clause — that is the case it exists +// for, and a fix that simply disabled MDL003 would pass the test above. +func TestValidateMicroflow_ReturnsWithoutVariableStillNeedsReturn(t *testing.T) { + stmt := &ast.CreateMicroflowStmt{ + Name: ast.QualifiedName{Module: "Sample", Name: "MF_NoReturn"}, + ReturnType: &ast.MicroflowReturnType{Type: ast.DataType{Kind: ast.TypeBoolean}}, + Body: []ast.MicroflowStatement{ + &ast.LogStmt{Level: ast.LogInfo, Message: &ast.LiteralExpr{Kind: ast.LiteralString, Value: "x"}}, + }, + } + + found := false + for _, v := range ValidateMicroflow(stmt) { + if v.RuleID == "MDL003" { + found = true + } + } + if !found { + t.Fatal("a non-void microflow with no AS clause and no return must still be flagged") + } +} From 13bf70c32f4537f6a930f2b9ba4b6fec9f8ebf32 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 18 Aug 2026 02:25:19 +0000 Subject: [PATCH 26/35] fix(queues): the rewrite guard was a no-op on the legacy engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FINDINGS #25 reported the queue-binding data loss as fixed. It was fixed on one engine. Under `--engine legacy`, `CREATE OR REPLACE MICROFLOW` still silently destroyed a task-queue binding — the exact loss checkNoQueuedCalls exists to prevent, and with the same misleading signature: `mx check` goes quiet afterwards because the configuration its CE1613 referred to is gone. The cause is a shape difference, not a logic error. The two readers return different Go types for the same BSON: modelsdk yields []interface{} for arrays, the legacy mpr reader yields bson.A. bson.A is a NAMED slice type, so `case []any:` does not match it, and the walk never descended into ObjectCollection.Objects — it found no queued calls and allowed the rewrite. Both walks now handle bson.A. Also read QueueSettings back in the legacy parser. It wrote the binding correctly but never parsed it, so on that engine `describe microflow` rendered a queued call as an ordinary one and a describe → exec cycle dropped it. The engines now agree on the same document, verified side by side. Pre-existing, not a regression from the IN QUEUE work: the binary built at 4f893ce (before that merge) drops the binding under legacy too. But my commit claimed the guard protected rewrites without qualifying by engine, and it did not. Verified live on 11.13, seeded binding then rewritten: before legacy dropping rewrite -> "Replaced microflow", binding gone after legacy dropping rewrite -> refused, binding intact legacy restating rewrite -> succeeds, binding intact describe agrees across both engines The same []any-only assumption remains in mdl/backend/mcp/page.go, page_mutator.go and widget_sync_apply.go; noted in the symptom table rather than changed here, since each needs its own reachability check. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017s476QkXr9CFMvKspVzcvu --- .claude/skills/fix-issue.md | 1 + mdl/executor/validate_queued_calls.go | 18 +++++++ mdl/executor/validate_queued_calls_test.go | 59 ++++++++++++++++++++++ sdk/mpr/parser_microflow_actions.go | 20 ++++++++ sdk/mpr/parser_queued_call_test.go | 54 ++++++++++++++++++++ 5 files changed, 152 insertions(+) create mode 100644 sdk/mpr/parser_queued_call_test.go diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index be2401897..1fc9a3c4c 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -538,3 +538,4 @@ extracting `OffsetExpression`/`LimitExpression`. | `mxcli exec` applies a script that `mxcli check` rejects with an error — e.g. a page written with an invalid widget property, which is then silently dropped, so the widget renders but does nothing | `exec` ran no semantic checks at all. Parse errors stopped it; the ~24 `Validate*` checks lived inline in `cmd/mxcli/cmd_check.go` and had exactly one caller. "Run check before exec" was a convention nothing enforced, on a tool built for unattended agent use | `mdl/executor/validate_program.go` (the shared list), `cmd/mxcli/cmd_exec.go` (the gate), `cmd/mxcli/cmd_check.go` | `executor.ValidateProgram` is now the single definition of what `check` checks, and `exec` refuses to run when it reports an **error** (warnings print and do not block). `--no-check` opts out. **A check wired into nothing is invisible** — it reports no violations and reads as a clean project — so `TestValidateProgram_WiresEveryWholeProgramValidator` asserts structurally that every exported `Validate*(prog *ast.Program, …)` is called from `ValidateProgram`; per-statement helpers take a statement and are excluded. When adding a check, add it to `ValidateProgram` and both commands get it | | A `show`/`list` command **exits 0, prints nothing, and does nothing** — `mxcli -c "show page Mod.Home"` looks like a page with no content, `show connections` looks like no connections are open. No error, no parse failure, no output. Indistinguishable from a legitimately empty result, which is why it survives: the answer is plausible | The grammar alternative exists (`showOrList PAGE qualifiedName`, `showOrList CONNECTIONS`, `showOrList NOTEBOOKS`) but `ExitShowStatement` has **no `else if` branch for it**, so the visitor appends no statement and the program runs zero statements. Not a handler bug — the AST node is never created. `create notebook … begin end` is the same shape (grammar with no AST/visitor/executor anywhere) | `mdl/visitor/visitor_query.go` (`ExitShowStatement`), `mdl/grammar/domains/MDLCatalog.g4`, `mdl/ast/ast_query.go`, `mdl/executor/executor_query.go` | Either wire a visitor branch or delete the grammar alternative — an unimplemented command must fail loudly. `SHOW PAGE X` became an alias for `DESCRIBE PAGE` (matching `SHOW ENTITY`), `SHOW CONNECTIONS` wired to `sql.Manager.List()`, `SHOW NOTEBOOKS` removed. **A structural grep cannot find these**: `ctx.PAGE()` *does* appear in `ExitShowStatement` — for the unrelated `SHOW ACCESS ON PAGE` branch — so a token search reports the broken alternative as covered. The guard (`TestEveryShowAlternativeProducesAStatement`) instead synthesizes a probe from each grammar alternative and asserts a statement comes out. **Ordering trap when adding a branch**: `SHOW DATABASE CONNECTIONS` also matches `CONNECTIONS()`, so a new `CONNECTIONS` branch placed first swallows it and answers a stored-document query with live session state — guard with `ctx.DATABASE() == nil`. Found while auditing mxcli against Studio Pro's document-type list. **Follow-up**: the notebook grammar (`createNotebookStatement`, `notebookOptions`, `notebookPage`, `alterNotebookAction`, the `SHOW NOTEBOOKS` alternative, and the `NOTEBOOK`/`NOTEBOOKS` lexer tokens) was removed outright — all four of `create`/`alter`/`drop`/`show notebook` were silent no-ops with no Go code behind any of them. Dead grammar is not inert: it *accepts input and discards it*. Removing the tokens also frees `notebook` as an ordinary identifier. **Control for a broad grammar change**: sweep every `mdl-examples/**/*.mdl` through `mxcli check` before and after and diff the failure sets — 15 scripts fail either way, so the raw count proves nothing and only the set difference does | | `mxcli check` reports **MDL003** *"microflow returns X but not all code paths have a return statement"* on a microflow that is correct and builds cleanly. Since `exec` began refusing scripts whose checks report an error, the script also will not run without `--no-check` — so a false positive became a hard block. Seven shipped `mdl-examples` scripts were in this state | The microflow uses the documented `RETURNS T AS $Var` form and assigns `$Var` instead of writing an explicit `RETURN`. `buildFlowGraph` sets the final EndEvent's `ReturnValue` to `"$"+Var` **whenever the AS clause is present** (`cmd_microflows_builder_graph.go`), so the return is synthesized either way — that is what the clause is for. The check demanded a `RETURN` statement in the source on top of it | `mdl/executor/validate_microflow.go` (Check 5 in `microflowValidator.validate`) | Skip MDL003 when `returnType.Variable != ""`, mirroring the builder's own condition exactly rather than inventing a second rule. **Verify against mxbuild, not intuition**: the flagged microflow builds with **0 errors** on 11.13.0, `describe microflow` renders the synthesized `return $Found;`, and the stored EndEvent carries a `ReturnValue` — three independent confirmations the check was wrong, not the script. Keep a companion test for the no-AS-clause case, or a fix that simply disables the rule passes. **Sweeping examples as a control**: exclude `*.test.mdl` (those are `mxcli test` files with `@test`/`@expect` blocks, unparseable by `check` by design) and `*.fail.mdl`, and diff failure SETS rather than counts | +| A guard that walks a stored BSON document works on the **modelsdk** engine and is a **silent no-op on legacy** (or vice versa) — same project, same document, opposite behaviour. Here `checkNoQueuedCalls` refused a rewrite that would drop a task-queue binding under modelsdk, and let it through under `--engine legacy`, destroying the binding exactly as the guard existed to prevent | The two readers return **different Go shapes for the same BSON**: `modelsdk` yields `[]interface{}` for arrays, the legacy `mpr` reader yields **`bson.A`**. `bson.A` is a NAMED slice type (`type A []interface{}`), so a type switch `case []any:` does **not** match it and the walk never descends into `ObjectCollection.Objects` | `mdl/executor/validate_queued_calls.go` (`queuedCallTargets`, `storedQueueRetries`); the same blind spot exists in `mdl/backend/mcp/page.go`, `mdl/backend/mcp/page_mutator.go`, `mdl/executor/widget_sync_apply.go` | Add `case bson.A:` alongside `case []any:` in every raw-BSON walk. The house pattern already does this (`mdl/backend/bsonnav`, `mdl/catalog/builder_pages.go`, `mdl/xpathrefs/scan.go` and ~7 more) — the ones missing it are the outliers: `for f in $(grep -rln "case \[\]any:" --include=*.go mdl/); do grep -q "bson.A" $f \|\| echo $f; done`. **Test both shapes of the same document in one test**, not just the one your engine returns. **A unit test alone is not enough to find this**: it only surfaced by running the same command under `MXCLI_ENGINE=legacy` and diffing against modelsdk — engine parity has to be exercised, not assumed. Reported as FINDINGS #25 | diff --git a/mdl/executor/validate_queued_calls.go b/mdl/executor/validate_queued_calls.go index 09f16785e..6901f6ee4 100644 --- a/mdl/executor/validate_queued_calls.go +++ b/mdl/executor/validate_queued_calls.go @@ -8,6 +8,8 @@ import ( "sort" "strings" + "go.mongodb.org/mongo-driver/bson" + "github.com/mendixlabs/mxcli/mdl/ast" mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" "github.com/mendixlabs/mxcli/model" @@ -130,6 +132,14 @@ func addQueueName(q *ast.QualifiedName, out map[string]bool) { // queuedCallTargets walks a stored microflow document and returns the queue // bound to each call that has one. // +// Arrays are matched as BOTH []any and bson.A. The two engines' readers return +// different shapes for the same document — modelsdk yields []interface{}, +// the legacy mpr reader yields bson.A — and bson.A is a NAMED slice type, so +// `case []any` silently does not match it. Missing that case made this guard a +// no-op on `--engine legacy`: the walk never descended into +// ObjectCollection.Objects, found no queued calls, and let the rewrite drop the +// binding (see TestQueuedCallTargets_HandlesBsonArrays). +// // The binding that matters is QueueSettings — a Queues$QueueSettings node whose // own Queue property names the queue. The call's top-level Queue property is // also read, because it is in the metamodel, but on its own it is inert: @@ -157,6 +167,10 @@ func queuedCallTargets(v any) []string { for _, el := range t { out = append(out, queuedCallTargets(el)...) } + case bson.A: + for _, el := range t { + out = append(out, queuedCallTargets(el)...) + } } return dedupeStrings(out) } @@ -184,6 +198,10 @@ func storedQueueRetries(v any) []string { for _, el := range t { out = append(out, storedQueueRetries(el)...) } + case bson.A: + for _, el := range t { + out = append(out, storedQueueRetries(el)...) + } } return dedupeStrings(out) } diff --git a/mdl/executor/validate_queued_calls_test.go b/mdl/executor/validate_queued_calls_test.go index dc9203292..d76ebe8c1 100644 --- a/mdl/executor/validate_queued_calls_test.go +++ b/mdl/executor/validate_queued_calls_test.go @@ -6,6 +6,8 @@ import ( "strings" "testing" + "go.mongodb.org/mongo-driver/bson" + "github.com/mendixlabs/mxcli/mdl/ast" "github.com/mendixlabs/mxcli/mdl/backend/mock" "github.com/mendixlabs/mxcli/model" @@ -262,3 +264,60 @@ func TestCheckNoQueuedCalls_RefusesStoredRetry(t *testing.T) { t.Errorf("error should name the retry:\n%s", err.Error()) } } + +// TestQueuedCallTargets_HandlesBsonArrays is the engine-parity guard. +// +// The two backends hand GetRawUnit back in different shapes: the modelsdk +// reader yields `[]interface{}` for arrays, the legacy (mpr) reader yields +// `bson.A`. `bson.A` is a NAMED slice type, so `case []any:` does not match it +// — the walk never descended into ObjectCollection.Objects under legacy, found +// no queued calls, and let the rewrite through. +// +// The consequence was the exact data loss this guard exists to prevent, still +// live on `--engine legacy`: measured on 11.13, a Studio-Pro-made queue binding +// was silently dropped by `CREATE OR REPLACE MICROFLOW`, and `mx check` went +// quiet because the configuration its CE1613 referred to had been deleted. +func TestQueuedCallTargets_HandlesBsonArrays(t *testing.T) { + call := map[string]any{ + "$Type": "Microflows$MicroflowCall", + "Microflow": "Q.Target", + "QueueSettings": map[string]any{ + "$Type": "Queues$QueueSettings", + "Queue": "Q.MyQueue", + }, + } + activity := map[string]any{"$Type": "Microflows$ActionActivity", "Action": call} + + // Same document, the two shapes the two readers produce. + shapes := map[string]any{ + "modelsdk ([]any)": map[string]any{ + "ObjectCollection": map[string]any{"Objects": []any{activity}}, + }, + "legacy (bson.A)": map[string]any{ + "ObjectCollection": map[string]any{"Objects": bson.A{activity}}, + }, + } + + for name, doc := range shapes { + t.Run(name, func(t *testing.T) { + got := queuedCallTargets(doc) + if len(got) != 1 || got[0] != "Q.MyQueue" { + t.Fatalf("queuedCallTargets = %v, want [Q.MyQueue] — a rewrite would "+ + "silently drop the binding on this engine", got) + } + }) + } +} + +// The retry walk shares the traversal and so shares the blind spot. +func TestStoredQueueRetries_HandlesBsonArrays(t *testing.T) { + doc := map[string]any{"Objects": bson.A{map[string]any{ + "QueueSettings": map[string]any{ + "Queue": "Q.MyQueue", + "Retry": map[string]any{"$Type": "Queues$QueueFixedRetry", "Retries": 3}, + }, + }}} + if got := storedQueueRetries(doc); len(got) != 1 { + t.Fatalf("storedQueueRetries = %v, want [Q.MyQueue]", got) + } +} diff --git a/sdk/mpr/parser_microflow_actions.go b/sdk/mpr/parser_microflow_actions.go index c740ac7ec..c375c6f68 100644 --- a/sdk/mpr/parser_microflow_actions.go +++ b/sdk/mpr/parser_microflow_actions.go @@ -61,6 +61,7 @@ func parseMicroflowCallAction(raw map[string]any) *microflows.MicroflowCallActio } } } + call.QueueSettings = parseQueueSettings(mfCall) action.MicroflowCall = call } @@ -102,6 +103,7 @@ func parseJavaActionCallAction(raw map[string]any) *microflows.JavaActionCallAct action.ID = model.ID(extractBsonID(raw["$ID"])) action.ErrorHandlingType = microflows.ErrorHandlingType(extractString(raw["ErrorHandlingType"])) action.JavaAction = extractString(raw["JavaAction"]) + action.QueueSettings = parseQueueSettings(raw) action.ResultVariableName = extractString(raw["ResultVariableName"]) action.UseReturnVariable = extractBool(raw["UseReturnVariable"], false) @@ -836,3 +838,21 @@ func parseExportXmlAction(raw map[string]any) *microflows.ExportXmlAction { return action } + +// parseQueueSettings reads a call's Queues$QueueSettings child — the binding to +// a task queue. Without it the legacy engine's DESCRIBE rendered a queued call +// as an ordinary one, so a describe → exec round trip dropped the binding and +// nothing on this engine could see it (FINDINGS #25's "describe showing nothing +// is not evidence of nothing"). +func parseQueueSettings(raw map[string]any) *microflows.QueueSettings { + qs, ok := raw["QueueSettings"].(map[string]any) + if !ok || qs == nil { + return nil + } + out := µflows.QueueSettings{Queue: extractString(qs["Queue"])} + out.ID = model.ID(extractBsonID(qs["$ID"])) + if retry, ok := qs["Retry"]; ok && retry != nil { + out.Retry = retry + } + return out +} diff --git a/sdk/mpr/parser_queued_call_test.go b/sdk/mpr/parser_queued_call_test.go new file mode 100644 index 000000000..adbdcf828 --- /dev/null +++ b/sdk/mpr/parser_queued_call_test.go @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: Apache-2.0 + +package mpr + +import "testing" + +// TestParseQueueSettings covers the legacy engine's half of the queued-call +// round trip (FINDINGS #25). +// +// The legacy writer stored the binding correctly, but the legacy PARSER never +// read it back — so on `--engine legacy` a queued call described as an ordinary +// one, and a describe → exec cycle dropped the binding. The two engines +// disagreed about the same stored document, which is the shape of bug that +// survives longest: each looks self-consistent. +func TestParseQueueSettings(t *testing.T) { + call := map[string]any{ + "$Type": "Microflows$MicroflowCall", + "Microflow": "Q.Target", + "QueueSettings": map[string]any{ + "$Type": "Queues$QueueSettings", + "Queue": "Q.MyQueue", + "Retry": nil, + }, + } + + qs := parseQueueSettings(call) + if qs == nil { + t.Fatal("QueueSettings not read back — a describe→exec round trip drops the binding") + } + if qs.Queue != "Q.MyQueue" { + t.Errorf("Queue = %q, want Q.MyQueue", qs.Queue) + } + if qs.Retry != nil { + t.Errorf("Retry = %v, want nil for an explicit BSON null", qs.Retry) + } + + // An unqueued call must stay unqueued — the common case by far. + if got := parseQueueSettings(map[string]any{"QueueSettings": nil}); got != nil { + t.Errorf("unqueued call produced %+v, want nil", got) + } + if got := parseQueueSettings(map[string]any{}); got != nil { + t.Errorf("absent QueueSettings produced %+v, want nil", got) + } + + // A stored retry must survive the read, because checkNoQueuedCalls refuses + // the rewrite on its presence — losing it here would re-enable the reset. + withRetry := parseQueueSettings(map[string]any{"QueueSettings": map[string]any{ + "Queue": "Q.MyQueue", + "Retry": map[string]any{"$Type": "Queues$QueueFixedRetry"}, + }}) + if withRetry == nil || withRetry.Retry == nil { + t.Fatal("a stored Retry must be carried, or the guard that refuses resetting it goes blind") + } +} From cde2b8a00cb738011ca7d36de0d74380be434955 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 18 Aug 2026 06:07:42 +0000 Subject: [PATCH 27/35] chore(lsp): drop NOTEBOOK from the generated completions NOTEBOOK/NOTEBOOKS appear nowhere in the grammar -- not the lexer, not the parser, not the domain grammars -- so the committed completion list offers two keywords the parser would reject. `make build` regenerates the file and removes them, which left a clean checkout dirty. Same class as the FIRST drift fixed earlier, in the other direction: the keyword left the grammar and the generated list was never refreshed. Regeneration is stable -- running it twice produces no further change. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013uQvFDd5R4eNqqita59jM8 --- cmd/mxcli/lsp_completions_gen.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/mxcli/lsp_completions_gen.go b/cmd/mxcli/lsp_completions_gen.go index 51df6704e..7b07da072 100644 --- a/cmd/mxcli/lsp_completions_gen.go +++ b/cmd/mxcli/lsp_completions_gen.go @@ -42,7 +42,6 @@ var mdlGeneratedKeywords = []protocol.CompletionItem{ {Label: "BUILDING", Kind: protocol.CompletionItemKindKeyword, Detail: "DDL keyword"}, {Label: "BLOCK", Kind: protocol.CompletionItemKindKeyword, Detail: "DDL keyword"}, {Label: "LAYOUT", Kind: protocol.CompletionItemKindKeyword, Detail: "DDL keyword"}, - {Label: "NOTEBOOK", Kind: protocol.CompletionItemKindKeyword, Detail: "DDL keyword"}, {Label: "CONSTANT", Kind: protocol.CompletionItemKindKeyword, Detail: "DDL keyword"}, {Label: "ATTRIBUTE", Kind: protocol.CompletionItemKindKeyword, Detail: "DDL keyword"}, {Label: "COLUMN", Kind: protocol.CompletionItemKindKeyword, Detail: "DDL keyword"}, @@ -175,7 +174,6 @@ var mdlGeneratedKeywords = []protocol.CompletionItem{ {Label: "LAYOUTS", Kind: protocol.CompletionItemKindKeyword, Detail: "Widget keyword"}, {Label: "SNIPPETS", Kind: protocol.CompletionItemKindKeyword, Detail: "Widget keyword"}, {Label: "BLOCKS", Kind: protocol.CompletionItemKindKeyword, Detail: "Widget keyword"}, - {Label: "NOTEBOOKS", Kind: protocol.CompletionItemKindKeyword, Detail: "Widget keyword"}, {Label: "PLACEHOLDER", Kind: protocol.CompletionItemKindKeyword, Detail: "Widget keyword"}, {Label: "SNIPPETCALL", Kind: protocol.CompletionItemKindKeyword, Detail: "Widget keyword"}, {Label: "LAYOUTGRID", Kind: protocol.CompletionItemKindKeyword, Detail: "Widget keyword"}, From dada96cca672cddecddf5b2d1870d8bcc07a4497 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 18 Aug 2026 06:08:45 +0000 Subject: [PATCH 28/35] fix(dbconnection): SQL replaced by a parameter default, and a lossy read on the default engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs found by asking whether the External Database Connector is fully supported on the default engine, and testing the full surface on both rather than answering from what was last changed. It was not. 1. A query written with a $$…$$ dollar-quoted SQL AND a `parameter … default '…'` stored the DEFAULT AS THE QUERY BODY. `describe` rendered `sql 'sales'`, the SELECT text was absent from the BSON entirely, and mx check reported 0 errors on both engines — nothing surfaced it. buildDatabaseConnection took STRING_LITERAL(0) as the SQL before falling back to DOLLAR_STRING. When the SQL is a dollar string, STRING_LITERAL(0) is the first parameter's DEFAULT. The parameter-index logic ten lines below already compensates for exactly this; the SQL extraction did not. Checking DOLLAR_STRING first fixes it. Isolated by bisecting the clause combination: `parameter d: String` alone keeps the SQL, `parameter d: String default 'x'` destroys it. 2. The modelsdk (default) engine read back neither a query's Parameters nor — until #171 — its TableMappings, so `describe database connection` dropped the parameters, the return entity and the column mapping, and a describe → exec round trip silently produced a query without them. The renderer handles all three; it only ever saw empty slices. #171 fixed TableMappings and missed its sibling Parameters, which is the characteristic version of this mistake. Verified with a cross-engine DESCRIBE matrix — write with engine A, read with engine B, all four combinations — because a gap on one engine is invisible from inside that engine. All four now agree on sql/returns/map/parameter, a describe → exec round trip is lossless, and mx check is 0 errors on 11.13.0. Controls: reverting the visitor ordering fails the new test with `SQL = "sales"`; the round-trip test runs on both engines and fails on modelsdk without the reader change. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017s476QkXr9CFMvKspVzcvu --- .claude/skills/fix-issue.md | 2 + mdl/backend/modelsdk/integration_read.go | 31 +++++++++ mdl/visitor/dbconnection_sql_dollar_test.go | 70 +++++++++++++++++++++ mdl/visitor/visitor_dbconnection.go | 13 ++-- 4 files changed, 112 insertions(+), 4 deletions(-) create mode 100644 mdl/visitor/dbconnection_sql_dollar_test.go diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index 1fc9a3c4c..3d7f24f06 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -539,3 +539,5 @@ extracting `OffsetExpression`/`LimitExpression`. | A `show`/`list` command **exits 0, prints nothing, and does nothing** — `mxcli -c "show page Mod.Home"` looks like a page with no content, `show connections` looks like no connections are open. No error, no parse failure, no output. Indistinguishable from a legitimately empty result, which is why it survives: the answer is plausible | The grammar alternative exists (`showOrList PAGE qualifiedName`, `showOrList CONNECTIONS`, `showOrList NOTEBOOKS`) but `ExitShowStatement` has **no `else if` branch for it**, so the visitor appends no statement and the program runs zero statements. Not a handler bug — the AST node is never created. `create notebook … begin end` is the same shape (grammar with no AST/visitor/executor anywhere) | `mdl/visitor/visitor_query.go` (`ExitShowStatement`), `mdl/grammar/domains/MDLCatalog.g4`, `mdl/ast/ast_query.go`, `mdl/executor/executor_query.go` | Either wire a visitor branch or delete the grammar alternative — an unimplemented command must fail loudly. `SHOW PAGE X` became an alias for `DESCRIBE PAGE` (matching `SHOW ENTITY`), `SHOW CONNECTIONS` wired to `sql.Manager.List()`, `SHOW NOTEBOOKS` removed. **A structural grep cannot find these**: `ctx.PAGE()` *does* appear in `ExitShowStatement` — for the unrelated `SHOW ACCESS ON PAGE` branch — so a token search reports the broken alternative as covered. The guard (`TestEveryShowAlternativeProducesAStatement`) instead synthesizes a probe from each grammar alternative and asserts a statement comes out. **Ordering trap when adding a branch**: `SHOW DATABASE CONNECTIONS` also matches `CONNECTIONS()`, so a new `CONNECTIONS` branch placed first swallows it and answers a stored-document query with live session state — guard with `ctx.DATABASE() == nil`. Found while auditing mxcli against Studio Pro's document-type list. **Follow-up**: the notebook grammar (`createNotebookStatement`, `notebookOptions`, `notebookPage`, `alterNotebookAction`, the `SHOW NOTEBOOKS` alternative, and the `NOTEBOOK`/`NOTEBOOKS` lexer tokens) was removed outright — all four of `create`/`alter`/`drop`/`show notebook` were silent no-ops with no Go code behind any of them. Dead grammar is not inert: it *accepts input and discards it*. Removing the tokens also frees `notebook` as an ordinary identifier. **Control for a broad grammar change**: sweep every `mdl-examples/**/*.mdl` through `mxcli check` before and after and diff the failure sets — 15 scripts fail either way, so the raw count proves nothing and only the set difference does | | `mxcli check` reports **MDL003** *"microflow returns X but not all code paths have a return statement"* on a microflow that is correct and builds cleanly. Since `exec` began refusing scripts whose checks report an error, the script also will not run without `--no-check` — so a false positive became a hard block. Seven shipped `mdl-examples` scripts were in this state | The microflow uses the documented `RETURNS T AS $Var` form and assigns `$Var` instead of writing an explicit `RETURN`. `buildFlowGraph` sets the final EndEvent's `ReturnValue` to `"$"+Var` **whenever the AS clause is present** (`cmd_microflows_builder_graph.go`), so the return is synthesized either way — that is what the clause is for. The check demanded a `RETURN` statement in the source on top of it | `mdl/executor/validate_microflow.go` (Check 5 in `microflowValidator.validate`) | Skip MDL003 when `returnType.Variable != ""`, mirroring the builder's own condition exactly rather than inventing a second rule. **Verify against mxbuild, not intuition**: the flagged microflow builds with **0 errors** on 11.13.0, `describe microflow` renders the synthesized `return $Found;`, and the stored EndEvent carries a `ReturnValue` — three independent confirmations the check was wrong, not the script. Keep a companion test for the no-AS-clause case, or a fix that simply disables the rule passes. **Sweeping examples as a control**: exclude `*.test.mdl` (those are `mxcli test` files with `@test`/`@expect` blocks, unparseable by `check` by design) and `*.fail.mdl`, and diff failure SETS rather than counts | | A guard that walks a stored BSON document works on the **modelsdk** engine and is a **silent no-op on legacy** (or vice versa) — same project, same document, opposite behaviour. Here `checkNoQueuedCalls` refused a rewrite that would drop a task-queue binding under modelsdk, and let it through under `--engine legacy`, destroying the binding exactly as the guard existed to prevent | The two readers return **different Go shapes for the same BSON**: `modelsdk` yields `[]interface{}` for arrays, the legacy `mpr` reader yields **`bson.A`**. `bson.A` is a NAMED slice type (`type A []interface{}`), so a type switch `case []any:` does **not** match it and the walk never descends into `ObjectCollection.Objects` | `mdl/executor/validate_queued_calls.go` (`queuedCallTargets`, `storedQueueRetries`); the same blind spot exists in `mdl/backend/mcp/page.go`, `mdl/backend/mcp/page_mutator.go`, `mdl/executor/widget_sync_apply.go` | Add `case bson.A:` alongside `case []any:` in every raw-BSON walk. The house pattern already does this (`mdl/backend/bsonnav`, `mdl/catalog/builder_pages.go`, `mdl/xpathrefs/scan.go` and ~7 more) — the ones missing it are the outliers: `for f in $(grep -rln "case \[\]any:" --include=*.go mdl/); do grep -q "bson.A" $f \|\| echo $f; done`. **Test both shapes of the same document in one test**, not just the one your engine returns. **A unit test alone is not enough to find this**: it only surfaced by running the same command under `MXCLI_ENGINE=legacy` and diffing against modelsdk — engine parity has to be exercised, not assumed. Reported as FINDINGS #25 | +| A `CREATE DATABASE CONNECTION` whose query uses a `$$…$$` dollar-quoted SQL **and** a `parameter … default '…'` stores the **parameter's default as the query body**. `describe` shows `sql 'sales'` where the SELECT should be, the SQL text is absent from the BSON entirely, and `mx check` reports 0 errors on both engines | `buildDatabaseConnection` took `STRING_LITERAL(0)` as the SQL before falling back to `DOLLAR_STRING`. When the SQL is a dollar string, `STRING_LITERAL(0)` is the first parameter's DEFAULT — so the query body was overwritten by it. The parameter-index logic ten lines below already compensates (`if qc.DOLLAR_STRING() != nil { slIdx = defaultIdx }`); the SQL extraction did not | `mdl/visitor/visitor_dbconnection.go` | Check `DOLLAR_STRING()` FIRST, then fall back to `STRING_LITERAL(0)`. **Isolate by bisecting the clause combination**, not by reading: `parameter d: String` alone kept the SQL, `parameter d: String default 'x'` destroyed it — that one-clause difference names the cause immediately. When two sites in one function index the same token stream, they must agree about the offsets; a comment on one is not enough | +| `describe database connection` omits `returns` / `map (…)` / `parameter` on the **default (modelsdk) engine** while the legacy engine renders them, so a describe → exec round trip drops the return entity, the column mapping and every query parameter | The modelsdk reader populated neither `TableMappings` nor `Parameters`, so the DESCRIBE renderer (which does handle them) only ever saw empty slices — dead code on that engine. The round-trip test existed but was pinned to the legacy backend via `setupTestEnv`, so it passed while the default engine was lossy | `mdl/backend/modelsdk/integration_read.go` (`tableMappingsFromGen`, `queryParametersFromGen`); test `mdl/executor/roundtrip_dbconnection_test.go` | Mirror the legacy reader field for field, and run the round-trip test over `gateEngines` rather than the default backend. **The cross-engine DESCRIBE matrix is the check that finds these**: write with engine A, read with engine B, all four combinations — a gap on one engine is invisible from inside that engine. Fixing one field (TableMappings) and not its sibling (Parameters) is the characteristic mistake; enumerate every child collection of the document before calling it done | diff --git a/mdl/backend/modelsdk/integration_read.go b/mdl/backend/modelsdk/integration_read.go index 01d7292f9..a361a432b 100644 --- a/mdl/backend/modelsdk/integration_read.go +++ b/mdl/backend/modelsdk/integration_read.go @@ -616,6 +616,7 @@ func (b *Backend) ListDatabaseConnections() ([]*model.DatabaseConnection, error) dq.ID = model.ID(q.ID()) dq.TypeName = "DatabaseConnector$DatabaseQuery" dq.TableMappings = tableMappingsFromGen(q) + dq.Parameters = queryParametersFromGen(q) conn.Queries = append(conn.Queries, dq) } out = append(out, conn) @@ -623,6 +624,36 @@ func (b *Backend) ListDatabaseConnections() ([]*model.DatabaseConnection, error) return out, nil } +// queryParametersFromGen reads a query's Parameters back into the semantic +// model. Sibling of tableMappingsFromGen and missed for the same reason: the +// DESCRIBE renderer emits `parameter : [default …]` from this +// slice, so leaving it empty made `describe database connection` drop every +// parameter on this engine while the legacy reader showed them. A describe → +// exec round trip then produced a query with no parameters at all. +func queryParametersFromGen(q *genDb.DatabaseQuery) []*model.DatabaseQueryParameter { + var out []*model.DatabaseQueryParameter + for _, el := range q.ParametersItems() { + p, ok := el.(*genDb.QueryParameter) + if !ok { + continue + } + param := &model.DatabaseQueryParameter{ + ParameterName: p.ParameterName(), + DefaultValue: p.DefaultValue(), + EmptyValueBecomesNull: p.EmptyValueBecomesNull(), + } + param.ID = model.ID(p.ID()) + param.TypeName = "DatabaseConnector$QueryParameter" + // DataType is a child element whose $Type carries the type + // (DataTypes$StringType, …) — the same string the legacy reader lifts. + if dt := p.DataType(); dt != nil { + param.DataType = dt.TypeName() + } + out = append(out, param) + } + return out +} + // tableMappingsFromGen reads a query's TableMappings back into the semantic // model — the entity a query returns and its column→attribute mapping. // diff --git a/mdl/visitor/dbconnection_sql_dollar_test.go b/mdl/visitor/dbconnection_sql_dollar_test.go new file mode 100644 index 000000000..380a5626b --- /dev/null +++ b/mdl/visitor/dbconnection_sql_dollar_test.go @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: Apache-2.0 + +package visitor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" +) + +// TestDatabaseQuery_DollarSQLWithParameterDefault covers a silent data-loss bug: +// a query whose SQL is a $$…$$ dollar string AND whose parameter carries a +// DEFAULT '…' lost the SQL entirely — q.SQL was set from STRING_LITERAL(0), +// which in that combination is the parameter's default, not the query. +// +// The result stored a connection whose query body was the string "sales", and +// `mx check` reported 0 errors either way, so nothing surfaced it. The +// parameter-index logic further down the same function already compensates for +// the dollar-string case; the SQL extraction did not. +func TestDatabaseQuery_DollarSQLWithParameterDefault(t *testing.T) { + const wantSQL = "SELECT id FROM t WHERE d = {d}" + + cases := map[string]struct { + src string + wantDefault string // "" when the case writes no DEFAULT clause + }{ + "dollar SQL + parameter default": {wantDefault: "sales", src: `create database connection M.C + type 'BYOD' connection string @M.U username @M.N password @M.P +begin + query Q sql $$` + wantSQL + `$$ parameter d: String default 'sales' returns M.E map ( id as Id ); +end;`}, + // The forms that already worked must keep working. + "dollar SQL, no default": {src: `create database connection M.C + type 'BYOD' connection string @M.U username @M.N password @M.P +begin + query Q sql $$` + wantSQL + `$$ parameter d: String returns M.E map ( id as Id ); +end;`}, + "quoted SQL + parameter default": {wantDefault: "sales", src: `create database connection M.C + type 'BYOD' connection string @M.U username @M.N password @M.P +begin + query Q sql 'SELECT id FROM t WHERE d = {d}' parameter d: String default 'sales' returns M.E map ( id as Id ); +end;`}, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + prog, errs := Build(tc.src) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + stmt, ok := prog.Statements[0].(*ast.CreateDatabaseConnectionStmt) + if !ok { + t.Fatalf("got %T, want *ast.CreateDatabaseConnectionStmt", prog.Statements[0]) + } + if len(stmt.Queries) != 1 { + t.Fatalf("got %d queries, want 1", len(stmt.Queries)) + } + q := stmt.Queries[0] + if q.SQL != wantSQL { + t.Errorf("SQL = %q, want %q — the query body was replaced", q.SQL, wantSQL) + } + if len(q.Parameters) != 1 { + t.Fatalf("got %d parameters, want 1", len(q.Parameters)) + } + if got := q.Parameters[0].DefaultValue; got != tc.wantDefault { + t.Errorf("parameter default = %q, want %q", got, tc.wantDefault) + } + }) + } +} diff --git a/mdl/visitor/visitor_dbconnection.go b/mdl/visitor/visitor_dbconnection.go index 234bc14e4..e6e8d7367 100644 --- a/mdl/visitor/visitor_dbconnection.go +++ b/mdl/visitor/visitor_dbconnection.go @@ -74,11 +74,16 @@ func (b *Builder) ExitCreateDatabaseConnectionStatement(ctx *parser.CreateDataba q.Name = identifierOrKeywordText(iok) } - // SQL string (STRING_LITERAL(0) is the SQL query) - if sl := qc.STRING_LITERAL(0); sl != nil { - q.SQL = unquoteString(sl.GetText()) - } else if ds := qc.DOLLAR_STRING(); ds != nil { + // SQL string. The DOLLAR_STRING form is checked FIRST: when the SQL is + // written as $$…$$, STRING_LITERAL(0) is not the query at all — it is the + // first parameter's DEFAULT '…'. Taking it produced a connection whose + // query body was silently replaced by that default, with mx check + // reporting 0 errors either way. The parameter-default indexing below + // already compensates for the dollar-string case; this did not. + if ds := qc.DOLLAR_STRING(); ds != nil { q.SQL = unquoteDollarString(ds.GetText()) + } else if sl := qc.STRING_LITERAL(0); sl != nil { + q.SQL = unquoteString(sl.GetText()) } // RETURNS entity From fcfbe7b8f573aeb0f39982fc010e53a3b2365ce4 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 18 Aug 2026 06:41:19 +0000 Subject: [PATCH 29/35] fix(layout): size a loop box from its contents, not a statement count (#884) LOOP and WHILE containers took their Size from measureStatementsSpan, a pre-pass over the AST run BEFORE the body was built, so the box was a function of statement COUNT alone. Varying only the children's positions changed nothing: children at x=150/310 -> 480;160 children at x=1500/2000 -> 480;160 (both children OUTSIDE the box) children at x=160/170 -> 480;160 four children, default -> 800;160 The model carries no geometry rules, so mx check reports nothing and only the eye catches it. An explicit @position on a loop child moved the child but not the container. The box is now sized after the body exists, from the real child bounding box (Position +/- Size/2). Same four cases now measure 420;160, 2110;140, 280;140 and 740;160 -- each tracking its contents, and case B grown to contain children it previously excluded. Nesting needed no extra code: an inner loop is sized when its own addLoopStatement returns, so the outer container measures a correct inner box (verified: inner 580;160 inside outer 1320;260). The proposal called for build-first / size-after / TRANSLATE-once. The translation step is deliberately dropped. A child's position round-trips through DESCRIBE as @position, so translating the body would make a describe->exec cycle store different coordinates than it read, and under ADR-0008 that turns an otherwise-quiet re-run into a write. Growing the box achieves the same containment without moving anything the author placed -- which is what a hand-laid-out loop needs. The proposal records this divergence. Verified: a hand-laid-out loop round-trips byte-identically; re-running a script is a no-op, with the MXCLI_ALWAYS_WRITE=1 control confirming the comparison detects writes; mx check clean on both fixtures. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013uQvFDd5R4eNqqita59jM8 --- .claude/skills/fix-issue.md | 1 + CHANGELOG.md | 4 + .../PROPOSAL_container_autosize.md | 39 ++++- .../cmd_microflows_builder_control.go | 18 ++ mdl/executor/layout.go | 59 +++++++ mdl/executor/loop_containment_test.go | 155 ++++++++++++++++++ 6 files changed, 275 insertions(+), 1 deletion(-) create mode 100644 mdl/executor/loop_containment_test.go diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index 1fc9a3c4c..ac6da4774 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -539,3 +539,4 @@ extracting `OffsetExpression`/`LimitExpression`. | A `show`/`list` command **exits 0, prints nothing, and does nothing** — `mxcli -c "show page Mod.Home"` looks like a page with no content, `show connections` looks like no connections are open. No error, no parse failure, no output. Indistinguishable from a legitimately empty result, which is why it survives: the answer is plausible | The grammar alternative exists (`showOrList PAGE qualifiedName`, `showOrList CONNECTIONS`, `showOrList NOTEBOOKS`) but `ExitShowStatement` has **no `else if` branch for it**, so the visitor appends no statement and the program runs zero statements. Not a handler bug — the AST node is never created. `create notebook … begin end` is the same shape (grammar with no AST/visitor/executor anywhere) | `mdl/visitor/visitor_query.go` (`ExitShowStatement`), `mdl/grammar/domains/MDLCatalog.g4`, `mdl/ast/ast_query.go`, `mdl/executor/executor_query.go` | Either wire a visitor branch or delete the grammar alternative — an unimplemented command must fail loudly. `SHOW PAGE X` became an alias for `DESCRIBE PAGE` (matching `SHOW ENTITY`), `SHOW CONNECTIONS` wired to `sql.Manager.List()`, `SHOW NOTEBOOKS` removed. **A structural grep cannot find these**: `ctx.PAGE()` *does* appear in `ExitShowStatement` — for the unrelated `SHOW ACCESS ON PAGE` branch — so a token search reports the broken alternative as covered. The guard (`TestEveryShowAlternativeProducesAStatement`) instead synthesizes a probe from each grammar alternative and asserts a statement comes out. **Ordering trap when adding a branch**: `SHOW DATABASE CONNECTIONS` also matches `CONNECTIONS()`, so a new `CONNECTIONS` branch placed first swallows it and answers a stored-document query with live session state — guard with `ctx.DATABASE() == nil`. Found while auditing mxcli against Studio Pro's document-type list. **Follow-up**: the notebook grammar (`createNotebookStatement`, `notebookOptions`, `notebookPage`, `alterNotebookAction`, the `SHOW NOTEBOOKS` alternative, and the `NOTEBOOK`/`NOTEBOOKS` lexer tokens) was removed outright — all four of `create`/`alter`/`drop`/`show notebook` were silent no-ops with no Go code behind any of them. Dead grammar is not inert: it *accepts input and discards it*. Removing the tokens also frees `notebook` as an ordinary identifier. **Control for a broad grammar change**: sweep every `mdl-examples/**/*.mdl` through `mxcli check` before and after and diff the failure sets — 15 scripts fail either way, so the raw count proves nothing and only the set difference does | | `mxcli check` reports **MDL003** *"microflow returns X but not all code paths have a return statement"* on a microflow that is correct and builds cleanly. Since `exec` began refusing scripts whose checks report an error, the script also will not run without `--no-check` — so a false positive became a hard block. Seven shipped `mdl-examples` scripts were in this state | The microflow uses the documented `RETURNS T AS $Var` form and assigns `$Var` instead of writing an explicit `RETURN`. `buildFlowGraph` sets the final EndEvent's `ReturnValue` to `"$"+Var` **whenever the AS clause is present** (`cmd_microflows_builder_graph.go`), so the return is synthesized either way — that is what the clause is for. The check demanded a `RETURN` statement in the source on top of it | `mdl/executor/validate_microflow.go` (Check 5 in `microflowValidator.validate`) | Skip MDL003 when `returnType.Variable != ""`, mirroring the builder's own condition exactly rather than inventing a second rule. **Verify against mxbuild, not intuition**: the flagged microflow builds with **0 errors** on 11.13.0, `describe microflow` renders the synthesized `return $Found;`, and the stored EndEvent carries a `ReturnValue` — three independent confirmations the check was wrong, not the script. Keep a companion test for the no-AS-clause case, or a fix that simply disables the rule passes. **Sweeping examples as a control**: exclude `*.test.mdl` (those are `mxcli test` files with `@test`/`@expect` blocks, unparseable by `check` by design) and `*.fail.mdl`, and diff failure SETS rather than counts | | A guard that walks a stored BSON document works on the **modelsdk** engine and is a **silent no-op on legacy** (or vice versa) — same project, same document, opposite behaviour. Here `checkNoQueuedCalls` refused a rewrite that would drop a task-queue binding under modelsdk, and let it through under `--engine legacy`, destroying the binding exactly as the guard existed to prevent | The two readers return **different Go shapes for the same BSON**: `modelsdk` yields `[]interface{}` for arrays, the legacy `mpr` reader yields **`bson.A`**. `bson.A` is a NAMED slice type (`type A []interface{}`), so a type switch `case []any:` does **not** match it and the walk never descends into `ObjectCollection.Objects` | `mdl/executor/validate_queued_calls.go` (`queuedCallTargets`, `storedQueueRetries`); the same blind spot exists in `mdl/backend/mcp/page.go`, `mdl/backend/mcp/page_mutator.go`, `mdl/executor/widget_sync_apply.go` | Add `case bson.A:` alongside `case []any:` in every raw-BSON walk. The house pattern already does this (`mdl/backend/bsonnav`, `mdl/catalog/builder_pages.go`, `mdl/xpathrefs/scan.go` and ~7 more) — the ones missing it are the outliers: `for f in $(grep -rln "case \[\]any:" --include=*.go mdl/); do grep -q "bson.A" $f \|\| echo $f; done`. **Test both shapes of the same document in one test**, not just the one your engine returns. **A unit test alone is not enough to find this**: it only surfaced by running the same command under `MXCLI_ENGINE=legacy` and diffing against modelsdk — engine parity has to be exercised, not assumed. Reported as FINDINGS #25 | +| A generated loop/while box does not fit its contents: activities sit outside the container, or the box is far wider than what is in it. An explicit `@position` on a loop child moves the child but not the box | The container's `Size` came from `measureStatementsSpan`, a pre-pass over the AST run BEFORE the body was built, so it was a function of statement COUNT alone. Varying only the children's positions changed nothing: 2 activities at x=150/310, at x=1500/2000 and at x=160/170 all produced `480;160`, and in the second case both children sat entirely outside their own container. Nothing catches it — the model carries no geometry rules, so `mx check` is silent | `mdl/executor/layout.go` (`containerBounds`, `fitContainerSize`), called from `addLoopStatement` / `addWhileStatement` in `cmd_microflows_builder_control.go` | Size the box AFTER the body is built, from the real child bounding box (`Position ± Size/2`). Nesting needs no extra code — an inner loop is sized when its own `addLoopStatement` returns, so the outer measures a correct inner box. **Do NOT translate the children to fit**, however tempting: their positions round-trip through DESCRIBE as `@position`, so moving them makes a describe→exec cycle store different coordinates than it read, which under ADR-0008 turns a quiet re-run into a write. Grow the box instead. **Two measurement traps hit while verifying this.** A helper that sets an annotation via `ast.StatementAnnotations` is a silent no-op when the field is nil (it returns nil rather than allocating) — the explicit-`@position` test passed while testing nothing until the field was assigned directly. And an idempotence check over `mprcontents` proved blind because the re-run script began with `create module`/`create entity` and aborted at "already exists" before reaching the microflow; the `MXCLI_ALWAYS_WRITE=1` control is what exposed it, so never assert "nothing changed" without it. Repro `mdl-examples/bug-tests/container-autosize-884.mdl`. Issue #884 problem 1 | diff --git a/CHANGELOG.md b/CHANGELOG.md index 439614566..8fef00e58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **A loop's box is sized from its contents, not from a statement count** (#884 problem 1) — `LOOP`/`WHILE` containers took their `Size` from a pre-pass over the AST run before the body was built, so it depended only on how many statements were inside. Two activities at x=150/310, at x=1500/2000 and at x=160/170 all produced `480;160`, and in the second case both children sat entirely outside their own container with `mx check` reporting nothing. The box is now derived from the real child bounding box after the body exists, which also makes an explicit `@position` on a loop child effective. Nested loops size bottom-up. Children are not moved: their positions round-trip through `DESCRIBE`, so the box grows to fit them rather than the contents being translated to fit the box. + +### Fixed + - **Authoring a pluggable widget with an object list no longer raises CE0463** (#891) — an object-list item's *required* TextTemplate that the author left unset was written as `null`, so a freshly authored Accordion failed "the definition of this widget has changed" against the very package it was built from. The empty-ClientTemplate convention was a hardcoded table covering only DataGrid columns; required-ness now comes from the widget's own PropertyTypes, and the property is serialized with the widget's shipped translations — the same text `mx update-widgets` writes. Both weaker forms were measured and rejected: `null` is CE0463 and an empty template is CE4899. Optional TextTemplates keep their null, which is what Studio Pro stores. - **`DESCRIBE PAGE` no longer renders an Accordion group empty** (#891) — an object-list item's child widgets (the group's `content` slot) were never read, and the emitter had no body to put them in, so a group holding a DataGrid2 described as a bare `group group1 (…)` and a describe→exec round-trip silently deleted the grid. Both halves are fixed, and the description now re-parses with the nested widgets intact. Applies to any pluggable widget's object-list items, not just Accordion. diff --git a/docs/11-proposals/PROPOSAL_container_autosize.md b/docs/11-proposals/PROPOSAL_container_autosize.md index a5e4f06c2..9032f0421 100644 --- a/docs/11-proposals/PROPOSAL_container_autosize.md +++ b/docs/11-proposals/PROPOSAL_container_autosize.md @@ -1,6 +1,6 @@ --- title: Size containers from their contents, not from a statement count -status: draft +status: accepted --- # Proposal: Size containers from their contents, not from a statement count @@ -127,6 +127,43 @@ adjacent children obeying different origins, which is unexplainable in a doc and unpredictable in a diff. Whichever is chosen, it belongs in `.claude/skills/mendix/` alongside the `@position` reference. +## Outcome — shipped + +Implemented for `LOOP` and `WHILE`: the box is sized from the children that were +actually built (`fitContainerSize`), after the body exists rather than from a +pre-pass over the AST. Measured on the four cases above, which previously all +reported `480;160` except D: + +| | children (X) | before | after | +|---|---|---|---| +| A | 150, 310 | `480;160` | `420;160` | +| B | `@position` 1500, 2000 | `480;160` | `2110;140` | +| C | `@position` 160, 170 | `480;160` | `280;140` | +| D | 150 … 630 | `800;160` | `740;160` | + +Nesting needed no extra work: an inner loop is sized when its own +`addLoopStatement` returns, so the outer container measures a correct inner box +(verified — inner `580;160`, outer `1320;260`). + +### One deliberate divergence: the body is NOT translated + +The plan above called for "build-first / size-after / **translate-once**". The +translation step was dropped, and the recommendation on `@position` semantics +(container-relative, translated) is superseded by it. + +A child's position round-trips through `DESCRIBE` as `@position`. Translating the +body would therefore make a describe→exec cycle store *different* coordinates +than it read — and under [ADR-0008](../13-decisions/0008-identity-and-idempotence.md) +that converts an otherwise-quiet re-run into a write. Growing the box to fit the +contents achieves the same containment without moving anything the author placed, +which is also what a hand-laid-out loop needs. Verified: a loop with +`@position(300,90)` / `@position(700,90)` round-trips byte-identically, and +re-running a script is a no-op (with the `MXCLI_ALWAYS_WRITE=1` control confirming +the comparison detects writes). + +Still open from this proposal: the **MPR009** containment lint rule, and the +`@size` escape hatch (still rejected by MDL059). + ## Non-goals - **Splits.** `IF` / enum split / inheritance split have no `Size` property in diff --git a/mdl/executor/cmd_microflows_builder_control.go b/mdl/executor/cmd_microflows_builder_control.go index 7c8a437b4..a62ee954f 100644 --- a/mdl/executor/cmd_microflows_builder_control.go +++ b/mdl/executor/cmd_microflows_builder_control.go @@ -656,6 +656,16 @@ func (fb *flowBuilder) addLoopStatement(s *ast.LoopStmt) model.ID { loopBuilder.flows = append(loopBuilder.flows, newHorizontalFlowWithCase(lastBodyID, continueID, pendingCase)) } + // Size the box from the children that were actually built, not from the + // statement count measured before the body existed (#884 problem 1). Recompute + // the centre too: the width just changed under it. + loopWidth, loopHeight = fitContainerSize(loopBuilder.objects, innerStartX, MinLoopWidth, MinLoopHeight) + loopCenterX = loopLeftX + loopWidth/2 + if s.Annotations != nil && s.Annotations.Position != nil { + loopCenterX = s.Annotations.Position.X + loopLeftX = loopCenterX - loopWidth/2 + } + // Create LoopedActivity with calculated size // Position is the CENTER point (RelativeMiddlePoint in Mendix) loop := µflows.LoopedActivity{ @@ -941,6 +951,14 @@ func (fb *flowBuilder) addWhileStatement(s *ast.WhileStmt) model.ID { whileExpr := fb.exprToString(s.Condition) + // Size from the built children, as addLoopStatement does (#884 problem 1). + loopWidth, loopHeight = fitContainerSize(loopBuilder.objects, innerStartX, MinLoopWidth, MinLoopHeight) + loopCenterX = loopLeftX + loopWidth/2 + if s.Annotations != nil && s.Annotations.Position != nil { + loopCenterX = s.Annotations.Position.X + loopLeftX = loopCenterX - loopWidth/2 + } + loop := µflows.LoopedActivity{ BaseMicroflowObject: microflows.BaseMicroflowObject{ BaseElement: model.BaseElement{ID: model.ID(types.GenerateID())}, diff --git a/mdl/executor/layout.go b/mdl/executor/layout.go index b2c694aa2..b63cf693d 100644 --- a/mdl/executor/layout.go +++ b/mdl/executor/layout.go @@ -11,6 +11,8 @@ package executor import ( "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/microflows" ) // Layout constants @@ -284,3 +286,60 @@ const ( AnchorBottom = 2 AnchorLeft = 3 ) + +// containerBounds returns the bounding box actually occupied by a container's +// children, in the container's own coordinate space. Each object contributes +// Position ± Size/2, since a Mendix position is the element's centre. +func containerBounds(objects []microflows.MicroflowObject) (minX, minY, maxX, maxY int, n int) { + for _, o := range objects { + if o == nil { + continue + } + p := o.GetPosition() + var sz model.Size + if withSize, ok := o.(interface{ GetSize() model.Size }); ok { + sz = withSize.GetSize() + } + l, t := p.X-sz.Width/2, p.Y-sz.Height/2 + r, b := p.X+sz.Width/2, p.Y+sz.Height/2 + if n == 0 { + minX, minY, maxX, maxY = l, t, r, b + } else { + minX, minY = min(minX, l), min(minY, t) + maxX, maxY = max(maxX, r), max(maxY, b) + } + n++ + } + return +} + +// fitContainerSize sizes a container box to the children it actually holds +// (mendixlabs/mxcli#884 problem 1). +// +// The box used to come from measureStatementsSpan — a pre-pass over the AST run +// BEFORE the body was built — so it was a function of statement COUNT alone. +// Varying only the children's positions left it unchanged: two activities at +// x=150/310, at x=1500/2000 and at x=160/170 all produced 480;160, and in the +// second case both children sat entirely outside their own container. The model +// carries no geometry rules, so `mx check` reports nothing. +// +// Sizing happens AFTER the body is built, which is what makes an explicit +// `@position` on a child effective. Nesting falls out of the recursion: an inner +// loop is sized when its own addLoopStatement returns, so by the time the outer +// container measures it, its Size is already correct. +// +// The children are NOT translated. Their positions round-trip through DESCRIBE +// as `@position`, so moving them would make a describe→exec cycle produce +// different coordinates than it read — and under ADR-0008 that turns an +// otherwise-quiet re-run into a write. The box grows to fit the contents +// instead, which is also what a hand-laid-out loop needs. +func fitContainerSize(objects []microflows.MicroflowObject, leftInset, minWidth, minHeight int) (width, height int) { + _, _, maxX, maxY, n := containerBounds(objects) + if n == 0 { + return minWidth, minHeight + } + width = max(maxX+LoopPadding, minWidth) + height = max(maxY+LoopPadding, minHeight) + _ = leftInset + return width, height +} diff --git a/mdl/executor/loop_containment_test.go b/mdl/executor/loop_containment_test.go new file mode 100644 index 000000000..27710d7de --- /dev/null +++ b/mdl/executor/loop_containment_test.go @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: Apache-2.0 + +// mendixlabs/mxcli#884 problem 1: a LoopedActivity's Size was computed from a +// pre-pass over the AST, before the body was built, so it was a function of +// statement COUNT alone. Child positions — including an explicit @position — +// had no effect on the box meant to contain them. +// +// Measured before the fix, on the same body with only the children's positions +// varied: 2 activities at x=150/310, at x=1500/2000 and at x=160/170 all +// produced Size 480;160. In the second case both children sat entirely outside +// their own container, and `mx check` reported nothing — the model has no +// geometry rules, so only the eye catches it. +// +// The invariant these tests pin is containment, not specific pixels: every +// child of a LoopedActivity must lie inside its parent's box. +package executor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// loopChildBounds returns the bounding box of a LoopedActivity's own children, +// in the container's coordinate space. +func loopChildBounds(loop *microflows.LoopedActivity) (minX, minY, maxX, maxY int, n int) { + if loop.ObjectCollection == nil { + return 0, 0, 0, 0, 0 + } + first := true + for _, o := range loop.ObjectCollection.Objects { + p := o.GetPosition() + sz := model.Size{} + if withSize, ok := o.(interface{ GetSize() model.Size }); ok { + sz = withSize.GetSize() + } + l, t := p.X-sz.Width/2, p.Y-sz.Height/2 + r, bt := p.X+sz.Width/2, p.Y+sz.Height/2 + if first { + minX, minY, maxX, maxY = l, t, r, bt + first = false + n++ + continue + } + minX = min(minX, l) + minY = min(minY, t) + maxX = max(maxX, r) + maxY = max(maxY, bt) + n++ + } + return +} + +// assertLoopContainsItsChildren is the invariant. Pixel values are deliberately +// not asserted — the box must fit the contents, whatever the layout engine +// chooses to do with them. +func assertLoopContainsItsChildren(t *testing.T, objects []microflows.MicroflowObject) { + t.Helper() + if got := checkLoops(t, objects); got == 0 { + t.Fatal("no LoopedActivity in the built flow — fixture is wrong") + } +} + +// checkLoops walks every LoopedActivity at any depth and returns how many it +// checked, so nested loops are covered without demanding one at every level. +func checkLoops(t *testing.T, objects []microflows.MicroflowObject) int { + t.Helper() + seen := 0 + for _, o := range objects { + loop, ok := o.(*microflows.LoopedActivity) + if !ok { + continue + } + seen++ + minX, minY, maxX, maxY, n := loopChildBounds(loop) + if n > 0 { + w, h := loop.Size.Width, loop.Size.Height + if minX < 0 || minY < 0 || maxX > w || maxY > h { + t.Errorf("loop children escape the box: children span x[%d,%d] y[%d,%d], box is %dx%d", + minX, maxX, minY, maxY, w, h) + } + } + if loop.ObjectCollection != nil { + seen += checkLoops(t, loop.ObjectCollection.Objects) + } + } + return seen +} + +func logStmt(msg string) *ast.LogStmt { + return &ast.LogStmt{Message: &ast.LiteralExpr{Kind: ast.LiteralString, Value: msg}} +} + +// positioned sets an explicit @position. It assigns the field directly rather +// than going through ast.StatementAnnotations, which returns nil when the field +// is nil and would make this a silent no-op — the test would then pass without +// testing anything. +func positioned(s *ast.LogStmt, x, y int) ast.MicroflowStatement { + s.Annotations = &ast.ActivityAnnotations{Position: &ast.Position{X: x, Y: y}} + return s +} + +func buildLoopFlow(t *testing.T, body []ast.MicroflowStatement) []microflows.MicroflowObject { + t.Helper() + fb := &flowBuilder{ + posX: 100, + posY: 200, + spacing: HorizontalSpacing, + varTypes: map[string]string{"Items": "List of Test.Item", "Inner": "List of Test.Item"}, + declaredVars: map[string]string{}, + measurer: &layoutMeasurer{varTypes: map[string]string{"Items": "List of Test.Item"}}, + } + fb.addLoopStatement(&ast.LoopStmt{ + ListVariable: "Items", + LoopVariable: "Item", + Body: body, + }) + return fb.objects +} + +// The default case: the box must fit its contents rather than a count. +func TestLoopBox_ContainsDefaultLaidOutChildren(t *testing.T) { + for _, n := range []int{1, 2, 4, 7} { + body := make([]ast.MicroflowStatement, 0, n) + for i := 0; i < n; i++ { + body = append(body, logStmt("x")) + } + assertLoopContainsItsChildren(t, buildLoopFlow(t, body)) + } +} + +// The reported case: an explicit @position must not be able to put a child +// outside its own container. +func TestLoopBox_GrowsToContainExplicitlyPositionedChildren(t *testing.T) { + body := []ast.MicroflowStatement{ + positioned(logStmt("one"), 1500, 60), + positioned(logStmt("two"), 2000, 60), + } + assertLoopContainsItsChildren(t, buildLoopFlow(t, body)) +} + +// Loops inside loops: the inner box must be sized before the outer measures it, +// or the outer fits a stale inner size. +func TestLoopBox_NestedLoopsEachContainTheirChildren(t *testing.T) { + inner := &ast.LoopStmt{ + ListVariable: "Inner", + LoopVariable: "I2", + Body: []ast.MicroflowStatement{logStmt("a"), logStmt("b"), logStmt("c")}, + } + assertLoopContainsItsChildren(t, buildLoopFlow(t, []ast.MicroflowStatement{ + logStmt("before"), inner, logStmt("after"), + })) +} From c13425104fa06e457f55c948cd9aa264c5534c87 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 15:00:20 +0000 Subject: [PATCH 30/35] fix(lint): stop CONV010 and QUAL004 reporting correct code as violations Both bundled rules matched on vocabulary the catalog never emits, so both produced false positives on ordinary, correct Mendix patterns. CONV010's ALLOWED_ACTIONS held the Mendix storage names (ShowFormAction, CloseFormAction) while the catalog labels an action with its SDK name, derived from the parsed Go type in getMicroflowActionType -- ShowPageAction, ClosePageAction, MicroflowCallAction. The allowlist matched nothing, so every ACT_ microflow that showed a page, closed one, or called a sub-microflow was flagged. In the banking-app report that was 11 false positives out of 13 findings, which buried the 2 real ones. QUAL004 counted only the 'call' and 'schedule' reference kinds. The builder also emits 'datasource', 'action' and 'calculate' for microflows and 'home_page', 'login_page', 'menu_item' for pages, so a microflow used as a page data source was reported as 'not called from anywhere'. The page half was masked by ENTRY_PAGE_PATTERNS, which happens to cover the pages most likely to be navigation targets. The guard is the point: mdl/catalog/lint_rule_vocabulary_test.go pins CONV010's allowlist to what getMicroflowActionType actually returns, and QUAL004's two kind lists to the RefKind constants, so a rule cannot drift from the vocabulary it matches on again. Both tests fail against the shipped rules with the reported symptom. Verified on a real 11.13 project: an ACT_ microflow doing show-page plus a sub-call and a DS_ microflow behind a datagrid are no longer flagged, while a retrieve in an ACT_ microflow and a genuinely uncalled microflow still are. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01KTdvGwVewkdgNQGDCxpQPZ --- .../conv010_act_microflow_content.star | 29 +++- .claude/lint-rules/orphaned_elements.star | 33 +++- .claude/skills/fix-issue.md | 2 + mdl/catalog/lint_rule_vocabulary_test.go | 149 ++++++++++++++++++ 4 files changed, 200 insertions(+), 13 deletions(-) create mode 100644 mdl/catalog/lint_rule_vocabulary_test.go diff --git a/.claude/lint-rules/conv010_act_microflow_content.star b/.claude/lint-rules/conv010_act_microflow_content.star index 05e5f0517..ecd97bac6 100644 --- a/.claude/lint-rules/conv010_act_microflow_content.star +++ b/.claude/lint-rules/conv010_act_microflow_content.star @@ -2,15 +2,25 @@ # # Microflows prefixed with ACT_ are page action microflows. They should only # contain UI-related activities: -# - ShowFormAction (show page) -# - CloseFormAction (close page) -# - ShowHomeFormAction (show home page) +# - ShowPageAction (show page) +# - ClosePageAction (close page) +# - ShowHomePageAction (show home page) # - ShowMessageAction (show message) # - DownloadFileAction (download file) -# - SubMicroflow (call sub-microflow for logic delegation) +# - MicroflowCallAction (call sub-microflow for logic delegation) # # Business logic should be delegated to SUB_ microflows. # Requires FULL catalog (REFRESH CATALOG FULL). +# +# NOTE ON NAMES: the catalog labels an action with its *SDK* type name, derived +# from the parsed action's Go type (catalog.getMicroflowActionType). That is not +# always the name Mendix uses in BSON: ShowPageAction is stored as +# "Microflows$ShowFormAction", ClosePageAction as "CloseFormAction", and so on +# (see the storage-name table in CLAUDE.md). This rule matches what the linter +# sees, so it must use the SDK names — it previously used the storage names and +# therefore matched nothing, flagging every ACT_ microflow that showed a page, +# closed one, or called a sub-microflow. Both spellings are listed so the rule +# keeps working if the catalog's vocabulary is ever changed to the storage names. RULE_ID = "CONV010" RULE_NAME = "ACTMicroflowContent" @@ -20,16 +30,23 @@ SEVERITY = "warning" # Allowed action types in ACT_ microflows ALLOWED_ACTIONS = ( + # SDK names — what the catalog actually reports. + "ShowPageAction", + "ClosePageAction", + "ShowHomePageAction", + "ShowMessageAction", + "DownloadFileAction", + "MicroflowCallAction", + # Storage names — belt and braces; see the note above. "ShowFormAction", "CloseFormAction", "ShowHomeFormAction", - "ShowMessageAction", - "DownloadFileAction", ) # Allowed activity types (non-action activities) ALLOWED_ACTIVITY_TYPES = ( "SubMicroflow", + "MicroflowCallAction", "StartEvent", "EndEvent", "ExclusiveSplit", diff --git a/.claude/lint-rules/orphaned_elements.star b/.claude/lint-rules/orphaned_elements.star index 1aa790cb2..47a608ccd 100644 --- a/.claude/lint-rules/orphaned_elements.star +++ b/.claude/lint-rules/orphaned_elements.star @@ -25,6 +25,14 @@ ENTRY_POINT_PREFIXES = ["ACT_", "SCH_", "WS_", "REST_", "OData_"] # Page name patterns that are likely entry points ENTRY_PAGE_PATTERNS = ["Home", "Login", "Index", "Dashboard"] +# Reference kinds that mean "something causes this microflow to run". These are +# catalog RefKind values (mdl/catalog/builder_references.go); a kind missing here +# turns a live document into a false "not called from anywhere" finding. +MICROFLOW_ENTRY_KINDS = ["call", "schedule", "datasource", "action", "calculate"] + +# Reference kinds that mean "something opens this page". +PAGE_ENTRY_KINDS = ["show_page", "home_page", "login_page", "menu_item", "action"] + def is_entry_point_microflow(name): """Check if a microflow name suggests it's a UI/scheduled entry point.""" for prefix in ENTRY_POINT_PREFIXES: @@ -54,13 +62,20 @@ def check(): # Get references to this microflow refs = refs_to(mf.qualified_name) - # A scheduled event is an entry point: it runs the microflow without - # anything "calling" it, so a 'schedule' edge counts as a caller. Without - # this, a microflow that runs nightly in production was reported as - # orphaned — with the suggestion "Remove if unused". + # Anything that causes the microflow to run counts as a caller, not just + # a literal "call" edge. A microflow reached only through one of the other + # kinds was reported as orphaned with the suggestion "Remove if unused": + # + # schedule a scheduled event runs it (Mendix's cron) + # datasource a page or widget uses it as a data source + # action a widget button calls it + # calculate a calculated attribute computes with it + # + # The banking-app report hit the 'datasource' case: DS_CurrentCustomer and + # DS_MyAccounts are both page data sources and both were flagged. has_callers = False for ref in refs: - if ref.ref_kind == "call" or ref.ref_kind == "schedule": + if ref.ref_kind in MICROFLOW_ENTRY_KINDS: has_callers = True break @@ -86,10 +101,14 @@ def check(): # Get references to this page refs = refs_to(page.qualified_name) - # Check if any reference shows this page + # A page is reachable if anything opens it. Navigation counts: a page that + # is only a home page, a login page or a menu item is reached by the + # client, not by a microflow. Counting only 'show_page' reported those as + # orphaned — masked until now by ENTRY_PAGE_PATTERNS, which happens to + # cover the pages most likely to be navigation targets. is_shown = False for ref in refs: - if ref.ref_kind == "show_page": + if ref.ref_kind in PAGE_ENTRY_KINDS: is_shown = True break diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index 3d7f24f06..b4bbaad4c 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -541,3 +541,5 @@ extracting `OffsetExpression`/`LimitExpression`. | A guard that walks a stored BSON document works on the **modelsdk** engine and is a **silent no-op on legacy** (or vice versa) — same project, same document, opposite behaviour. Here `checkNoQueuedCalls` refused a rewrite that would drop a task-queue binding under modelsdk, and let it through under `--engine legacy`, destroying the binding exactly as the guard existed to prevent | The two readers return **different Go shapes for the same BSON**: `modelsdk` yields `[]interface{}` for arrays, the legacy `mpr` reader yields **`bson.A`**. `bson.A` is a NAMED slice type (`type A []interface{}`), so a type switch `case []any:` does **not** match it and the walk never descends into `ObjectCollection.Objects` | `mdl/executor/validate_queued_calls.go` (`queuedCallTargets`, `storedQueueRetries`); the same blind spot exists in `mdl/backend/mcp/page.go`, `mdl/backend/mcp/page_mutator.go`, `mdl/executor/widget_sync_apply.go` | Add `case bson.A:` alongside `case []any:` in every raw-BSON walk. The house pattern already does this (`mdl/backend/bsonnav`, `mdl/catalog/builder_pages.go`, `mdl/xpathrefs/scan.go` and ~7 more) — the ones missing it are the outliers: `for f in $(grep -rln "case \[\]any:" --include=*.go mdl/); do grep -q "bson.A" $f \|\| echo $f; done`. **Test both shapes of the same document in one test**, not just the one your engine returns. **A unit test alone is not enough to find this**: it only surfaced by running the same command under `MXCLI_ENGINE=legacy` and diffing against modelsdk — engine parity has to be exercised, not assumed. Reported as FINDINGS #25 | | A `CREATE DATABASE CONNECTION` whose query uses a `$$…$$` dollar-quoted SQL **and** a `parameter … default '…'` stores the **parameter's default as the query body**. `describe` shows `sql 'sales'` where the SELECT should be, the SQL text is absent from the BSON entirely, and `mx check` reports 0 errors on both engines | `buildDatabaseConnection` took `STRING_LITERAL(0)` as the SQL before falling back to `DOLLAR_STRING`. When the SQL is a dollar string, `STRING_LITERAL(0)` is the first parameter's DEFAULT — so the query body was overwritten by it. The parameter-index logic ten lines below already compensates (`if qc.DOLLAR_STRING() != nil { slIdx = defaultIdx }`); the SQL extraction did not | `mdl/visitor/visitor_dbconnection.go` | Check `DOLLAR_STRING()` FIRST, then fall back to `STRING_LITERAL(0)`. **Isolate by bisecting the clause combination**, not by reading: `parameter d: String` alone kept the SQL, `parameter d: String default 'x'` destroyed it — that one-clause difference names the cause immediately. When two sites in one function index the same token stream, they must agree about the offsets; a comment on one is not enough | | `describe database connection` omits `returns` / `map (…)` / `parameter` on the **default (modelsdk) engine** while the legacy engine renders them, so a describe → exec round trip drops the return entity, the column mapping and every query parameter | The modelsdk reader populated neither `TableMappings` nor `Parameters`, so the DESCRIBE renderer (which does handle them) only ever saw empty slices — dead code on that engine. The round-trip test existed but was pinned to the legacy backend via `setupTestEnv`, so it passed while the default engine was lossy | `mdl/backend/modelsdk/integration_read.go` (`tableMappingsFromGen`, `queryParametersFromGen`); test `mdl/executor/roundtrip_dbconnection_test.go` | Mirror the legacy reader field for field, and run the round-trip test over `gateEngines` rather than the default backend. **The cross-engine DESCRIBE matrix is the check that finds these**: write with engine A, read with engine B, all four combinations — a gap on one engine is invisible from inside that engine. Fixing one field (TableMappings) and not its sibling (Parameters) is the characteristic mistake; enumerate every child collection of the document before calling it done | +| Lint CONV010 flags **every** `ACT_` microflow that shows a page, closes one, or calls a sub-microflow — 11 false positives out of 13 findings, burying the real ones | The rule's `ALLOWED_ACTIONS` held the Mendix **storage** names (`ShowFormAction`, `CloseFormAction`); the catalog labels an action with its **SDK** name, derived from the parsed Go type in `getMicroflowActionType` (`ShowPageAction`, `ClosePageAction`, `MicroflowCallAction`). The allowlist matched nothing | `.claude/lint-rules/conv010_act_microflow_content.star`, `mdl/catalog/builder_microflows.go` | List the SDK names (both spellings is cheap insurance). The storage-name split is the same one in CLAUDE.md's `$Type` table — it bites rule authors because the catalog deliberately does **not** use storage names. `mdl/catalog/lint_rule_vocabulary_test.go` pins the allowlist to what `getMicroflowActionType` actually returns, so the rule cannot drift from the labeller again. Copy the rule into the target project's `.claude/lint-rules/` when testing: `mxcli lint -p ` prefers the **project's** copy over the embedded one, so editing the repo's copy alone changes nothing | +| Lint QUAL004 reports a live microflow as "not called from anywhere" (page datasource, widget button, calculated attribute), or a navigation-only page as orphaned | The rule counted only the `call` and `schedule` reference kinds. The builder emits `datasource`, `action` and `calculate` for microflows, and `home_page` / `login_page` / `menu_item` for pages — all ignored. The page half was masked by `ENTRY_PAGE_PATTERNS`, which happens to cover the pages most likely to be navigation targets | `.claude/lint-rules/orphaned_elements.star`, `mdl/catalog/builder_references.go` | Count every kind that means "this runs" / "this opens", via the `MICROFLOW_ENTRY_KINDS` / `PAGE_ENTRY_KINDS` lists. `TestQUAL004CountsEveryEntryPointKind` fails when one goes missing and `TestQUAL004EntryKindsAreRealRefKinds` when one is misspelled. Adding a new `RefKind` that means reachability means adding it to the right list | diff --git a/mdl/catalog/lint_rule_vocabulary_test.go b/mdl/catalog/lint_rule_vocabulary_test.go new file mode 100644 index 000000000..a5ca29a6f --- /dev/null +++ b/mdl/catalog/lint_rule_vocabulary_test.go @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// The bundled Starlark rules match on strings this package produces: action +// labels from getMicroflowActionType and reference kinds from the RefKind +// constants. Nothing connected the two, so a rule could name something the +// catalog never emits and simply match nothing — which is not a visible failure, +// it is a rule that silently flags everything or nothing. +// +// CONV010 shipped that way. Its ALLOWED_ACTIONS held the Mendix *storage* names +// (ShowFormAction, CloseFormAction) while the catalog reports the *SDK* names +// (ShowPageAction, ClosePageAction) — see the storage-name table in CLAUDE.md. +// The allowlist matched nothing, so every ACT_ microflow that showed a page, +// closed one, or called a sub-microflow was flagged: 11 false positives out of 13 +// findings in the banking-app report, which buried the 2 real ones. +// +// QUAL004 had the sibling bug in the other vocabulary: it counted only the "call" +// and "schedule" reference kinds, so a microflow reached through a page data +// source, a widget action or a calculated attribute was reported as +// "not called from anywhere". + +func lintRulesDir(t *testing.T) string { + t.Helper() + dir := filepath.Join("..", "..", ".claude", "lint-rules") + if _, err := os.Stat(dir); err != nil { + t.Skipf("bundled lint rules not present: %v", err) + } + return dir +} + +func readRule(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join(lintRulesDir(t), name)) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + return string(b) +} + +// TestCONV010AllowsWhatTheCatalogCallsUIActions pins the rule's allowlist to the +// labels this package actually produces, by asking the labeller rather than +// hardcoding a second copy of the names. +func TestCONV010AllowsWhatTheCatalogCallsUIActions(t *testing.T) { + src := readRule(t, "conv010_act_microflow_content.star") + + // The activities CONV010 documents as permitted in an ACT_ microflow. + permitted := []microflows.MicroflowAction{ + µflows.ShowPageAction{}, + µflows.ClosePageAction{}, + µflows.MicroflowCallAction{}, + } + + for _, action := range permitted { + label := getMicroflowActionType(action) + t.Run(label, func(t *testing.T) { + if !strings.Contains(src, `"`+label+`"`) { + t.Errorf("CONV010 does not allow %q, which is what the catalog labels this action.\n"+ + "Every ACT_ microflow using it will be flagged.", label) + } + }) + } +} + +// starListItems returns the quoted strings of a top-level `NAME = [...]` or +// `NAME = (...)` literal in a Starlark source file. +func starListItems(src, name string) []string { + re := regexp.MustCompile(`(?s)\b` + regexp.QuoteMeta(name) + `\s*=\s*[\[(](.*?)[\])]`) + m := re.FindStringSubmatch(src) + if m == nil { + return nil + } + var out []string + for _, q := range regexp.MustCompile(`"([^"]*)"`).FindAllStringSubmatch(m[1], -1) { + out = append(out, q[1]) + } + return out +} + +// TestQUAL004EntryKindsAreRealRefKinds keeps the orphan rule's two kind lists +// spelled the way the reference builder emits them. A typo or an invented kind +// silently narrows the rule into false positives. +func TestQUAL004EntryKindsAreRealRefKinds(t *testing.T) { + known := map[string]bool{} + for _, k := range []string{ + RefKindCall, RefKindCreate, RefKindRetrieve, RefKindShowPage, + RefKindGeneralize, RefKindAssociate, RefKindLayout, RefKindDatasource, + RefKindParameter, RefKindAction, RefKindHomePage, RefKindLoginPage, + RefKindMenuItem, RefKindChange, RefKindDelete, RefKindCalculate, + RefKindReturn, RefKindSchedule, RefKindValidate, + } { + known[k] = true + } + + src := readRule(t, "orphaned_elements.star") + for _, listName := range []string{"MICROFLOW_ENTRY_KINDS", "PAGE_ENTRY_KINDS"} { + items := starListItems(src, listName) + if len(items) == 0 { + t.Fatalf("%s not found in orphaned_elements.star, or is empty", listName) + } + for _, kind := range items { + if !known[kind] { + t.Errorf("%s lists %q, which the reference builder never emits", listName, kind) + } + } + } +} + +// The kinds that mean "this runs" / "this opens" must actually be listed. Losing +// one is how the rule regresses into reporting live documents as dead. +func TestQUAL004CountsEveryEntryPointKind(t *testing.T) { + src := readRule(t, "orphaned_elements.star") + + for _, want := range []string{ + RefKindCall, RefKindSchedule, RefKindDatasource, RefKindAction, RefKindCalculate, + } { + if !contains(starListItems(src, "MICROFLOW_ENTRY_KINDS"), want) { + t.Errorf("MICROFLOW_ENTRY_KINDS is missing %q — a microflow reached only that way "+ + "is reported as 'not called from anywhere'", want) + } + } + for _, want := range []string{ + RefKindShowPage, RefKindHomePage, RefKindLoginPage, RefKindMenuItem, + } { + if !contains(starListItems(src, "PAGE_ENTRY_KINDS"), want) { + t.Errorf("PAGE_ENTRY_KINDS is missing %q — a page reached only that way "+ + "is reported as orphaned", want) + } + } +} + +func contains(hay []string, needle string) bool { + for _, h := range hay { + if h == needle { + return true + } + } + return false +} From 7a3bf7a08aeea45ab051a4014e9ec51e4319f64d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 18 Aug 2026 06:55:26 +0000 Subject: [PATCH 31/35] feat(lint): add MPR011, flagging activities outside their loop container The Mendix model carries no geometry rules, so a loop whose children sit outside its box passes mx check with zero errors and builds and runs normally -- it is simply drawn wrong when the flow is opened. That is what let #884 problem 1 survive unnoticed: the container's Size came from a statement count, so children at x=1500/2000 lived in a 480-wide box and nothing said so. The sizing fix stops mxcli producing that, but the condition can still arrive from a project written by an older mxcli, hand-edited in Studio Pro, or a future layout regression. MPR011 catches it either way. Checked in each container's OWN coordinate space, so a nested loop is never compared against an ancestor's canvas -- the same distinction MPR008 needed after it was found comparing a loop child against absolute coordinates. An unpositioned child at the origin is skipped rather than flagged, matching MPR008 and keeping the real cases visible. Verified with a pre-fix control binary: on a project it built, MPR011 reports both escaped children while mx check reports nothing beyond the blank app's pre-existing error. Silent on the same script through the fixed binary, on the stock blank app, and on a nested-loop project. The ID is MPR011, not the MPR009 the proposal named -- MPR009 and MPR010 were already taken (gallery selection listener, dataview layout grid). The proposal is corrected. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013uQvFDd5R4eNqqita59jM8 --- CHANGELOG.md | 4 + cmd/mxcli/cmd_lint.go | 1 + cmd/mxcli/cmd_report.go | 1 + .../PROPOSAL_container_autosize.md | 13 +- .../rules/mpr011_loop_child_containment.go | 153 ++++++++++++++++++ .../mpr011_loop_child_containment_test.go | 92 +++++++++++ 6 files changed, 260 insertions(+), 4 deletions(-) create mode 100644 mdl/linter/rules/mpr011_loop_child_containment.go create mode 100644 mdl/linter/rules/mpr011_loop_child_containment_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fef00e58..5a7ad0816 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +### Added + +- **`MPR011` — loop child containment** — a new lint rule flagging a microflow activity positioned outside the loop container that holds it. This is the only automated check for the condition: the Mendix model carries no geometry rules, so such a flow passes `mx check` with zero errors and builds and runs normally, it is just drawn wrong when opened. It catches the condition however it arrives — a project written by an older mxcli, hand-edited in Studio Pro, or a future layout regression. Nested loops are checked in their own coordinate space, and an unpositioned child at the origin is skipped rather than flagged. + ### Fixed - **A loop's box is sized from its contents, not from a statement count** (#884 problem 1) — `LOOP`/`WHILE` containers took their `Size` from a pre-pass over the AST run before the body was built, so it depended only on how many statements were inside. Two activities at x=150/310, at x=1500/2000 and at x=160/170 all produced `480;160`, and in the second case both children sat entirely outside their own container with `mx check` reporting nothing. The box is now derived from the real child bounding box after the body exists, which also makes an explicit `@position` on a loop child effective. Nested loops size bottom-up. Children are not moved: their positions round-trip through `DESCRIBE`, so the box grows to fit them rather than the contents being translated to fit the box. diff --git a/cmd/mxcli/cmd_lint.go b/cmd/mxcli/cmd_lint.go index 56c137e99..919703506 100644 --- a/cmd/mxcli/cmd_lint.go +++ b/cmd/mxcli/cmd_lint.go @@ -134,6 +134,7 @@ Examples: rules.NewWeakPasswordPolicyRule(), rules.NewDemoUsersActiveRule(), rules.NewOverlappingActivitiesRule(), // MPR008 - requires BSON inspection + rules.NewLoopChildContainmentRule(), // MPR011 - requires BSON inspection rules.NewNoCommitInLoopRule(), // CONV011-CONV014 - require BSON inspection rules.NewExclusiveSplitCaptionRule(), rules.NewErrorHandlingOnCallsRule(), diff --git a/cmd/mxcli/cmd_report.go b/cmd/mxcli/cmd_report.go index 7d5f06668..3926ae42b 100644 --- a/cmd/mxcli/cmd_report.go +++ b/cmd/mxcli/cmd_report.go @@ -100,6 +100,7 @@ Examples: // MPR008 - requires BSON inspection lint.AddRule(rules.NewOverlappingActivitiesRule()) + lint.AddRule(rules.NewLoopChildContainmentRule()) // Convention rules (CONV011-CONV014) lint.AddRule(rules.NewNoCommitInLoopRule()) diff --git a/docs/11-proposals/PROPOSAL_container_autosize.md b/docs/11-proposals/PROPOSAL_container_autosize.md index 9032f0421..7e78bd03f 100644 --- a/docs/11-proposals/PROPOSAL_container_autosize.md +++ b/docs/11-proposals/PROPOSAL_container_autosize.md @@ -161,8 +161,11 @@ which is also what a hand-laid-out loop needs. Verified: a loop with re-running a script is a no-op (with the `MXCLI_ALWAYS_WRITE=1` control confirming the comparison detects writes). -Still open from this proposal: the **MPR009** containment lint rule, and the -`@size` escape hatch (still rejected by MDL059). +Also shipped: the containment lint rule, as **MPR011** — MPR009 and MPR010 were +already taken (gallery selection listener, dataview layout grid), so the ID named +below was wrong. + +Still open from this proposal: the `@size` escape hatch (still rejected by MDL059). ## Non-goals @@ -215,11 +218,13 @@ project is exactly the kind of thing that gets reported as a new bug. contents — and not merely pass against fixed code. - **`mx check`** on every fixture, both engines. -### Candidate lint rule: MPR009 +### Candidate lint rule: MPR011 (shipped) The measurement above shows `mx check` is blind to this. A rule *"every child of a `LoopedActivity` lies within its parent's box"* would have caught the entire class from the outside, and would keep catching it if a future layout change reintroduces it. It sits naturally beside MPR008 (which, after the #884 work, already partitions objects by canvas and therefore has the container -geometry in hand). Proposed as a follow-up, not part of this change. +geometry in hand). Shipped as MPR011: verified to fire on a project built by a +pre-fix binary — where `mx check` reports nothing — and silent on the fixed +output, the stock blank app and a nested-loop project. diff --git a/mdl/linter/rules/mpr011_loop_child_containment.go b/mdl/linter/rules/mpr011_loop_child_containment.go new file mode 100644 index 000000000..412c5179e --- /dev/null +++ b/mdl/linter/rules/mpr011_loop_child_containment.go @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rules + +import ( + "fmt" + + "github.com/mendixlabs/mxcli/mdl/linter" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// LoopChildContainmentRule flags a microflow activity that lies outside the loop +// container holding it. +// +// This is the only automated check for the condition. The Mendix model carries no +// geometry rules, so a loop whose children sit far outside its box passes +// `mx check` with zero errors and builds and runs normally — it is simply drawn +// wrong when the flow is opened. That is what let upstream #884 problem 1 survive: +// the container's Size was computed from a statement COUNT before the body was +// built, so children at x=1500/2000 lived in a 480-wide box and nothing said so. +// +// mxcli no longer produces that (the box is now sized from the real child bounding +// box), but the rule catches it however it arrives — a project written by an older +// mxcli, hand-edited in Studio Pro, or a future layout change that reintroduces it. +type LoopChildContainmentRule struct{} + +func NewLoopChildContainmentRule() *LoopChildContainmentRule { + return &LoopChildContainmentRule{} +} + +func (r *LoopChildContainmentRule) ID() string { return "MPR011" } +func (r *LoopChildContainmentRule) Name() string { return "LoopChildContainment" } +func (r *LoopChildContainmentRule) Category() string { return "correctness" } +func (r *LoopChildContainmentRule) DefaultSeverity() linter.Severity { return linter.SeverityWarning } +func (r *LoopChildContainmentRule) Description() string { + return "Microflow activities positioned outside the loop container that holds them, which renders wrong in Studio Pro but passes mx check" +} + +func (r *LoopChildContainmentRule) Check(ctx *linter.LintContext) []linter.Violation { + reader := ctx.Reader() + if reader == nil { + return nil + } + + var violations []linter.Violation + for mf := range ctx.Microflows() { + if ctx.IsExcluded(mf.ModuleName) { + continue + } + fullMF, err := ctx.FullMicroflow(model.ID(mf.ID)) + if err != nil || fullMF == nil || fullMF.ObjectCollection == nil { + continue + } + for _, e := range escapedLoopChildren(fullMF.ObjectCollection.Objects) { + violations = append(violations, linter.Violation{ + RuleID: r.ID(), + Severity: r.DefaultSeverity(), + Message: fmt.Sprintf( + "Activity '%s' at (%d,%d) lies outside the loop '%s' that contains it (box %dx%d) "+ + "in microflow '%s.%s'. The flow renders wrong in Studio Pro; mx check does not detect this.", + e.Child, e.ChildX, e.ChildY, e.Loop, e.BoxW, e.BoxH, mf.ModuleName, mf.Name), + Location: linter.Location{ + Module: mf.ModuleName, + DocumentType: "microflow", + DocumentName: mf.Name, + DocumentID: mf.ID, + }, + Suggestion: "Give the activity an @position inside the container, or remove a hand-set @position so the layout engine places it. " + + "A loop's box is sized from its children, so a contained @position widens the box rather than escaping it.", + }) + } + } + return violations +} + +// escapee is one child positioned outside the container that holds it. +type escapee struct { + Loop string + Child string + ChildX, ChildY int + BoxW, BoxH int +} + +// escapedLoopChildren walks every LoopedActivity at any depth and returns the +// children whose boxes are not fully inside their container. +// +// A child's coordinates are relative to its OWN container, so each loop is +// checked in its own space and never against an ancestor's — the same +// distinction MPR008 needed after it was found comparing a loop child against +// the microflow's absolute canvas (#884). +func escapedLoopChildren(objects []microflows.MicroflowObject) []escapee { + var out []escapee + for _, obj := range objects { + loop, ok := obj.(*microflows.LoopedActivity) + if !ok || loop.ObjectCollection == nil { + continue + } + boxW, boxH := loop.Size.Width, loop.Size.Height + for _, child := range loop.ObjectCollection.Objects { + if child == nil { + continue + } + p := child.GetPosition() + // (0,0) is unpositioned, not escaped — MPR008 skips these for the + // same reason, and flagging them would bury the real cases. + if p.X == 0 && p.Y == 0 { + continue + } + var sz model.Size + if withSize, ok := child.(interface{ GetSize() model.Size }); ok { + sz = withSize.GetSize() + } + if boxW <= 0 || boxH <= 0 { + continue + } + left, top := p.X-sz.Width/2, p.Y-sz.Height/2 + right, bottom := p.X+sz.Width/2, p.Y+sz.Height/2 + if left < 0 || top < 0 || right > boxW || bottom > boxH { + out = append(out, escapee{ + Loop: captionOf(loop, "loop"), + Child: captionOf(child, "(unnamed)"), + ChildX: p.X, ChildY: p.Y, + BoxW: boxW, BoxH: boxH, + }) + } + } + // Nested containers are checked in their own coordinate space. + out = append(out, escapedLoopChildren(loop.ObjectCollection.Objects)...) + } + return out +} + +// captionOf names a canvas object for the message. Caption lives on the concrete +// types rather than the MicroflowObject interface, so this switches the way +// MPR008 does. +func captionOf(o microflows.MicroflowObject, fallback string) string { + var c string + switch act := o.(type) { + case *microflows.ActionActivity: + c = act.Caption + case *microflows.LoopedActivity: + c = act.Caption + case *microflows.ExclusiveSplit: + c = act.Caption + case *microflows.ExclusiveMerge: + c = "(merge)" + } + if c == "" { + return fallback + } + return c +} diff --git a/mdl/linter/rules/mpr011_loop_child_containment_test.go b/mdl/linter/rules/mpr011_loop_child_containment_test.go new file mode 100644 index 000000000..eb6f00f1a --- /dev/null +++ b/mdl/linter/rules/mpr011_loop_child_containment_test.go @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rules + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/linter" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +func loopChild(caption string, x, y int) *microflows.ActionActivity { + a := µflows.ActionActivity{} + a.Position = model.Point{X: x, Y: y} + a.Size = model.Size{Width: 120, Height: 60} + a.Caption = caption + return a +} + +func loopWith(w, h int, children ...microflows.MicroflowObject) *microflows.LoopedActivity { + loop := µflows.LoopedActivity{ + ObjectCollection: µflows.MicroflowObjectCollection{Objects: children}, + } + loop.Position = model.Point{X: 500, Y: 200} + loop.Size = model.Size{Width: w, Height: h} + loop.Caption = "loop" + return loop +} + +func TestLoopChildContainment_FlagsAChildOutsideItsBox(t *testing.T) { + // The #884 case: the box is 480 wide, the child sits at x=2000. + loop := loopWith(480, 160, loopChild("two", 2000, 60)) + got := escapedLoopChildren([]microflows.MicroflowObject{loop}) + if len(got) != 1 { + t.Fatalf("expected 1 escapee, got %d: %+v", len(got), got) + } + if got[0].Child != "two" { + t.Errorf("child = %q, want %q", got[0].Child, "two") + } +} + +func TestLoopChildContainment_AcceptsAContainedChild(t *testing.T) { + // x=310 with a 120-wide box spans [250,370], inside a 480-wide container. + loop := loopWith(480, 160, loopChild("one", 310, 80)) + if got := escapedLoopChildren([]microflows.MicroflowObject{loop}); len(got) != 0 { + t.Errorf("a contained child must not be flagged, got %+v", got) + } +} + +// A child at the origin is unpositioned rather than escaped — MPR008 skips +// those for the same reason, and flagging them would bury the real cases. +func TestLoopChildContainment_IgnoresUnpositionedChildren(t *testing.T) { + loop := loopWith(480, 160, loopChild("unpositioned", 0, 0)) + if got := escapedLoopChildren([]microflows.MicroflowObject{loop}); len(got) != 0 { + t.Errorf("an unpositioned child must not be flagged, got %+v", got) + } +} + +// Loops inside loops: the inner container is checked against its own box, in +// its own coordinate space — the same distinction MPR008 needed (#884). +func TestLoopChildContainment_ChecksNestedLoops(t *testing.T) { + inner := loopWith(200, 100, loopChild("deep", 900, 50)) + inner.Caption = "inner" + inner.Position = model.Point{X: 150, Y: 80} + outer := loopWith(600, 200, inner) + + got := escapedLoopChildren([]microflows.MicroflowObject{outer}) + if len(got) != 1 { + t.Fatalf("expected the nested escapee, got %d: %+v", len(got), got) + } + if got[0].Child != "deep" || got[0].Loop != "inner" { + t.Errorf("got child=%q loop=%q, want child=deep loop=inner", got[0].Child, got[0].Loop) + } +} + +func TestLoopChildContainmentRule_Metadata(t *testing.T) { + r := NewLoopChildContainmentRule() + if r.ID() != "MPR011" { + t.Errorf("ID = %q, want MPR011", r.ID()) + } + if r.Category() != "correctness" { + t.Errorf("Category = %q, want correctness", r.Category()) + } +} + +func TestLoopChildContainmentRule_NilReader(t *testing.T) { + r := NewLoopChildContainmentRule() + if v := r.Check(linter.NewLintContextFromDB(nil)); v != nil { + t.Errorf("expected nil with nil reader, got %v", v) + } +} From 44d6b6023022d024da22ac907fffb138f14b2844 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 18 Aug 2026 06:56:25 +0000 Subject: [PATCH 32/35] docs(oql): ORDER BY DESC on a nullable column puts the nulls first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FINDINGS #39 (second half) reports that `ORDER BY` on a DateTime is ignored by the Mendix runtime: `order by DealtAt desc limit 5` returns old rows while `order by Id desc` returns genuinely new ones, stably across runs. The recommended workaround is to order by `Id` for recency. Measured on Mendix 11.13.0 against PostgreSQL with statement logging on, that diagnosis is wrong — in a way worth writing down, because the evidence for it is genuinely convincing. With four rows carrying distinct non-null timestamps, the SQL the runtime sends is: SELECT "myfirstmodule$probe"."label" AS "label" FROM "myfirstmodule$probe" ORDER BY "myfirstmodule$probe"."dealtat" DESC LIMIT $1 The ordering is present, on the right column, before the LIMIT, and the result comes back newest-first. It is not dropped, and mxcli is not in the path either: ExecuteOQL passes the query verbatim and parseOQLFeedback preserves row order. Adding two rows whose DealtAt was never set reproduces the report exactly. Mendix emits no null placement, so the database default applies, and on PostgreSQL DESC means NULLS FIRST: order by DealtAt desc limit 3 -> null-a, null-b, newest Running Mendix's own emitted SQL by hand confirms the mechanism: identical with the default, correct with NULLS LAST. The trap is that the wrong answer is *stable*, so it reads as "the engine is ignoring my ORDER BY" rather than "my sort key is empty for some rows". The report anticipates the tie objection and answers it with "stable across runs, so it is not a tie-break artifact" — but stability rules out randomness, not ties. A degenerate sort key is exactly as repeatable as a correct one. Documented in the OQL page and the query-writing skill: use NULLS LAST on an optional attribute rather than falling back to `order by id`, which answers a different question (insertion order, matching recency only when nothing backdates a row). Null placement is database-specific, so being explicit is also the portable choice. No code change — mxcli is not in the path. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CqQLyxnppZGfqPYbUHvSWG --- .claude/skills/fix-issue.md | 1 + .claude/skills/mendix/write-oql-queries.md | 32 ++++++++++++++++++++ docs-site/src/tools/oql.md | 35 ++++++++++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index 1fc9a3c4c..a6a399bcb 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -524,6 +524,7 @@ extracting `OffsetExpression`/`LimitExpression`. | `@verify` cannot fail. No OQL — well-formed or not, true or not, against a real entity or an invented one — makes a test fail: `@verify select count(*) as n from Mod.Game = 999999` reports PASS on a table with one row, and so does `this is not a query` | The annotation was parsed into `TestCase.Verify` and read by nothing but `--list`. Same silent-absence class as the dropped `@expect`, in the annotation covering the *harder* half of Mendix testing: most microflows are side effects, so asserting on rows written is the only way to test them | `cmd/mxcli/testrunner/verify.go` (new — `ParseVerify`, `scalarOf`, `compare`, `runVerifies`), `cmd/mxcli/testrunner/parser.go` (`checkVerifyCleanup`), `runner_endpoint.go` + `runner_attach.go` + `watch.go` (`adminOptions` on `testTarget`), `runner.go` (`rejectVerifyOnLegacyRunner`) | Run the OQL over the admin API after the microflow returns, compare against a literal, and keep three outcomes apart: holds → unchanged, does not hold → FAIL with the observed value, cannot be evaluated → ERROR. Three things are enforced rather than left to trip you up, each learned by running it: **`@cleanup rollback` is refused** (the default undoes the writes before the query could see them, so it would assert against the pre-test state); **the result must be one row × one column** (picking a cell out of a table is a guess); and the **legacy after-startup runner refuses the whole suite**, since its tests run during boot with no seam to query at. Split the expected value off at the **last** comparison operator outside quotes and parens, then require the RHS to be a literal — without that check the split silently eats a `where x = 1` and sends a truncated query. **Generalisable**: unit tests with a stubbed endpoint proved the logic and would have shipped it broken — the first real run against a booted app found the OQL endpoint unreachable on that Mendix and the alias requirement, neither of which any stub could show. mxcli-sudoku FINDINGS #48 | | Running `mxcli test` dirties the project: `git status` reports the `.mpr` modified after a run that changed nothing, so a "run the tests, assert the tree is clean" CI step fails and a pull request carries a meaningless diff. `mprcontents/` is byte-identical; only the `.mpr` differs, and a `.mpr` diff is opaque | The runner injects an `MxTest` module and takes it back out. Cleanup restores the model, but every unit write stamps a fresh UUID into the `.mpr`'s `_Transaction.LastTransactionID` (`updateTransactionID`, in both engines' writers) **and** the insert/delete cycle relays SQLite's pages. Restoring the row alone is measurably insufficient: three consecutive runs then hold the id stable and still produce a different file hash each time | `cmd/mxcli/testrunner/snapshot.go` (new — `projectSnapshot`, `documentTreeDigest`), `cmd/mxcli/testrunner/runner.go` (`captureProjectState`, `restoreProjectFile` at all three cleanup sites), `cmd/mxcli/testrunner/host.go` (`HostedEndpoint.Remove`) | Snapshot the whole `.mpr` before injecting and write it back after a successful cleanup — byte-exact by construction rather than by enumerating what might have changed. Two refusals are load-bearing: **cleanup failed** and **the `mprcontents/` tree moved**. In either case the project is not in the state the snapshot describes, and restoring would turn a visible harmless discrepancy into an invisible misleading one — a tree that reads as clean while the model is not. Write through a temp file + rename so an interrupted restore cannot truncate the `.mpr`. **Generalisable**: a tool that mutates a project to do its work owes a restore that is byte-exact, not semantically equivalent — every consumer downstream of it compares bytes. And when the obvious cause is a single row, measure the file hash before concluding it is the only one. Verified with the reporter's own harness: 3 runs, `.mpr` and `mprcontents` hashes unchanged throughout. mxcli-sudoku FINDINGS #47 | | `run --local --watch` deploys a **half-applied model** when an `mxcli exec` is running, and there is no way to recover: re-running the script writes nothing (it is byte-idempotent), so nothing re-triggers the watcher and the stale build stands | `watchAndApply` (`cmd/mxcli/docker/runlocal.go`) rebuilt on the **first** mtime bump. An exec rewrites the `.mpr` and many `mprcontents/*.mxunit` over seconds, so the build snapshots the tree mid-write. There was a settle for the web bundler and nothing for the model. Two behaviours each correct alone: idempotency removed a *recovery path* nobody had written down as one — "just run it again" was load-bearing | `cmd/mxcli/docker/runlocal.go` (`settleSource`, `sourceSettleWindow`, the `case <-ticker.C` branch) | Wait for the source mtime to stop advancing (two poll intervals of quiet) before building. The wait is unbounded on purpose — a long exec is the case it exists for, and building late beats building mid-write — but it still honours the interrupt channel so Ctrl-C is not swallowed. `touch` on the `.mpr` remains the force-rebuild hatch, now documented. **Generalisable, twice over**: (a) a change *signal* is not a change *event* — anything polling mtime must debounce or it samples a writer mid-flight; (b) when adding an optimisation that skips work, ask what informal recovery procedure depended on that work happening. **Test trap hit while fixing it**: the first version asserted the file count *after* `wg.Wait()`, which holds against a `settleSource` that returns immediately — the assertion has to read the writer's state at the moment it returned. mxcli-sudoku FINDINGS #45 | +| An OQL `ORDER BY` on a **DateTime** looks ignored: `order by DealtAt desc limit 5` returns old rows while `order by Id desc` returns genuinely new ones, and the wrong answer is **stable across runs** — so it reads as a platform bug rather than a data problem, and the natural workaround becomes "order by Id for recency" | **Not an mxcli bug, and not a Mendix bug.** Mendix emits the ordering with no null placement, so the database default applies: on PostgreSQL `DESC` means **NULLS FIRST**, and rows whose attribute was never set head the result. Stability is what makes it convincing and wrong — a degenerate sort key is exactly as repeatable as a correct one, so "stable, therefore not a tie-break artifact" does not discriminate | Nothing to change in mxcli — `ExecuteOQL` passes the query verbatim and `parseOQLFeedback` preserves row order. Docs only: `docs-site/src/tools/oql.md`, `.claude/skills/mendix/write-oql-queries.md` | Measured on **Mendix 11.13.0** with PostgreSQL statement logging on, which is the method to reuse: `ALTER SYSTEM SET log_statement='all'`, run the query, read the SQL the runtime actually sent. With four distinct non-null timestamps the emitted SQL is `… ORDER BY "x"."dealtat" DESC LIMIT $1` and the result is correctly ordered — the ordering is **not** dropped. Adding two rows with an empty DealtAt reproduces the reported symptom exactly, and running Mendix's own emitted SQL by hand with `NULLS LAST` fixes it. **Generalisable**: when a query returns a wrong-but-stable answer, suspect a degenerate sort key before suspecting the engine; and when a bug report blames a layer, check whether that layer is even in the path — here the same finding also reported the column being *dropped from the projection*, whose known mechanism is a null in the first row, which was direct evidence for the real cause sitting unread in the report. mxcli-sudoku FINDINGS #39 (second half) | | `mxcli oql` fails on every Mendix older than 11.11 with *"Action not found ... upgrade mxcli"*, and separately reports a **rejected query as `0 rows`** rather than as an error | Two independent misreadings of the runtime's replies. (1) The 11.11+ REST route `/dev/preview_execute_oql` does not exist on older runtimes, and the admin API does not 404 it — it dispatches the POST as an ordinary admin request, finds no `action` field, and answers **HTTP 200** with `{"result":1,"message":"Action not found"}`. The fallback to the legacy M2EE action keyed on the 404 alone, so the legacy action — which works fine there — was never tried. (2) That legacy action reports a bad query as `{"feedback":{"error":"..."},"result":0}` — inside the feedback, with a **success** result code — so `M2EEError()` (which keys off the result) says nothing and the error body parses as an empty result | `cmd/mxcli/docker/oql.go` (`legacyOQL`, `oqlDevErrorKind`, the `error` field in `parseOQLFeedback`'s envelope) | Fall back to the legacy action on *either* absence signal, and surface `feedback.error` regardless of the result code. Measured on 11.6.6: before, `mxcli oql` could not run any query; after, `select count(*) as n from Mod.E` returns a row, and a bad query is an error instead of `0 rows`. **Generalisable**: when a fallback is keyed on one specific failure signal, check what the *other* end actually sends — an HTTP-level 404 and an application-level "not found" are different wires, and a success code next to an error message is common enough to assume it happens. Found while wiring @verify (FINDINGS #48); related to #39 | | Windows Defender flags the mxcli **Windows** release binary as `Trojan:Script/Sabsik.EN.A!ml`; enterprise EDR (Defender for Endpoint, CrowdStrike, SentinelOne) blocks it harder. Not the generic unsigned-Go-binary false positive of #185 | The binary genuinely embedded **chisel**, a dual-use tunnelling/pivoting tool (SSH over WebSocket), on every platform — although the tunnel only ever runs inside a Linux container. `run --hub` linked `chisel/client`, `tunnel-hub` linked `chisel/server`, so windows/darwin carried 32 packages incl. the whole `x/crypto/ssh` stack for a feature they cannot use | `cmd/mxcli/docker/tunnel_linux.go` + `tunnel_other.go` (client seam), `cmd/mxcli/tunnelhub/control_linux.go` + `control_other.go` (server seam), `scripts/check-tunnel-deps.sh` (guard) | **Never obfuscate, pack or rename to dodge the scanner** — attacker tradecraft, and it makes the binary less trustworthy, not more. **Code signing does not fix this class**: a signed binary containing chisel is still flagged behaviourally; signing only addresses #185's generic false positive. The fix is to stop shipping the capability where it is unused: one interface per seam, `_linux.go` impl + `!linux` stub, commands still registered everywhere but failing with an actionable message. **Prove absence three ways, and know that `go tool nm` is not one of them** — release ldflags `-s -w` strip the symbol table, so nm reports "no symbols" whether or not the code is linked and would give a false pass; use `go list -deps`, `go version -m`, and `strings` (nm only on a deliberately unstripped build). **Guard against the transitive path, not the name**: match the module list (`x/crypto/ssh`, `gorilla/websocket`, `armon/go-socks5`, `jpillora/*`) so re-entry without the word "chisel" still trips it, and assert a **positive control** (chisel IS in the linux graph) so the check cannot pass vacuously. Verified by re-adding the import and watching the guard fail on all four windows/darwin targets. Result: -13.5 MB (-14.7%) on windows+darwin, linux unchanged. See ADR-0009 | | CE7375 "must be published and be the key when associations are exposed as an associated object id" on a service publishing no associations | `PublishAssociations` is the representation, not a yes/no — `No` selects "as an associated object id", which needs the system ID as key | `mdl/executor/validate_odata_service_shape.go` | Set `PublishAssociations: Yes` ("as a link", and the default when omitted). MDL-ODATA06 warns at check time | diff --git a/.claude/skills/mendix/write-oql-queries.md b/.claude/skills/mendix/write-oql-queries.md index 87a900d0c..30da3732b 100644 --- a/.claude/skills/mendix/write-oql-queries.md +++ b/.claude/skills/mendix/write-oql-queries.md @@ -194,6 +194,38 @@ from Finance.Transaction as t ORDER by OrderYear desc ``` +### 6b. ORDER BY DESC on a nullable column puts the nulls FIRST + +`ORDER BY DESC` does **not** give you the newest rows when some rows +leave that attribute empty. Mendix emits the ordering with no null placement, so +the database default applies — on PostgreSQL, `DESC` means **NULLS FIRST**. + +```sql +-- ❌ MISLEADING - the empty rows come back first, so a top-N is not the top N +select g.Label as Label from Sudoku.Game as g +order by g.DealtAt desc +limit 5 + +-- ✅ CORRECT when the attribute is optional +select g.Label as Label from Sudoku.Game as g +order by g.DealtAt desc nulls last +limit 5 +``` + +This is worth knowing because it does not look like a null problem. The result is +**stable across runs**, so it reads as "the ordering is being ignored" rather than +"the sort key is empty for some rows" — and the natural next step, falling back to +`order by id desc`, answers a different question (insertion order, which only +matches recency when nothing backdates a row). + +Check before concluding anything: +`select count(*) as n from Sudoku.Game where DealtAt = empty`. + +Measured on Mendix 11.13.0 / PostgreSQL: with values present, `ORDER BY` on a +DateTime is emitted to the database correctly and orders correctly. Null placement +is database-specific (SQL Server and Oracle differ), so being explicit is also the +portable choice. + ### 7. Operators (Use != not <>) ```sql -- ❌ WRONG - <> causes errors in Mendix diff --git a/docs-site/src/tools/oql.md b/docs-site/src/tools/oql.md index 58fa9fbe5..3dd5b3262 100644 --- a/docs-site/src/tools/oql.md +++ b/docs-site/src/tools/oql.md @@ -35,6 +35,41 @@ mxcli oql -p app.mpr "SELECT o.OrderNumber, c.Name FROM Sales.Order o JOIN Sales mxcli oql -p app.mpr "SELECT COUNT(*) FROM Sales.Customer" ``` +## `ORDER BY` and null values + +`ORDER BY DESC` on an attribute that some rows leave empty does **not** +return the newest rows first. Mendix emits the ordering to the database without a +null placement, so the database's default applies — and on PostgreSQL, `DESC` +means **NULLS FIRST**: + +```sql +-- what Mendix sends for `ORDER BY DealtAt DESC LIMIT 3` +SELECT "sales$game"."label" AS "label" +FROM "sales$game" +ORDER BY "sales$game"."dealtat" DESC +LIMIT $1 +``` + +With two null `DealtAt` rows in the table, that returns both nulls and then one +real row — so "give me the most recent 3" silently answers with rows that are not +recent, **stably** across runs. The stability is what makes it convincing and +wrong: a degenerate sort key is every bit as repeatable as a correct one. + +Two things follow: + +- **Check for nulls before concluding the ordering is broken.** + `SELECT count(*) AS n FROM Sales.Game WHERE DealtAt = empty` settles it in one + query. Measured on Mendix 11.13.0 against PostgreSQL, `ORDER BY` on a DateTime + is passed through correctly and orders correctly when the values are present — + it is not ignored. +- **Add `NULLS LAST` when the attribute is optional**, rather than falling back to + ordering by `id`. Ordering by `id` answers a different question (insertion + order), which happens to agree with recency only when nothing backdates a row. + +Null placement is database-specific — SQL Server and Oracle differ from +PostgreSQL — so a query relying on the default behaves differently per +environment. Being explicit is the portable choice. + ## Difference from Catalog Queries | Feature | OQL (`mxcli oql`) | Catalog (`SELECT FROM CATALOG.*`) | From d70598b450e13df3b704dc14e7ddd2ea0c0adb50 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 18 Aug 2026 06:59:03 +0000 Subject: [PATCH 33/35] chore(lsp): regenerate the completion keywords from the current lexer cmd/mxcli/lsp_completions_gen.go is generated by `make build` from mdl/grammar/MDLLexer.g4 and is tracked, but the committed copy still carried NOTEBOOK and NOTEBOOKS, which the lexer no longer defines. Every `make build` therefore regenerated the file and left the working tree dirty with a change nobody made. No behaviour change: the two entries were completions the LSP offered for keywords the parser would reject. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01KTdvGwVewkdgNQGDCxpQPZ --- cmd/mxcli/lsp_completions_gen.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/mxcli/lsp_completions_gen.go b/cmd/mxcli/lsp_completions_gen.go index 51df6704e..7b07da072 100644 --- a/cmd/mxcli/lsp_completions_gen.go +++ b/cmd/mxcli/lsp_completions_gen.go @@ -42,7 +42,6 @@ var mdlGeneratedKeywords = []protocol.CompletionItem{ {Label: "BUILDING", Kind: protocol.CompletionItemKindKeyword, Detail: "DDL keyword"}, {Label: "BLOCK", Kind: protocol.CompletionItemKindKeyword, Detail: "DDL keyword"}, {Label: "LAYOUT", Kind: protocol.CompletionItemKindKeyword, Detail: "DDL keyword"}, - {Label: "NOTEBOOK", Kind: protocol.CompletionItemKindKeyword, Detail: "DDL keyword"}, {Label: "CONSTANT", Kind: protocol.CompletionItemKindKeyword, Detail: "DDL keyword"}, {Label: "ATTRIBUTE", Kind: protocol.CompletionItemKindKeyword, Detail: "DDL keyword"}, {Label: "COLUMN", Kind: protocol.CompletionItemKindKeyword, Detail: "DDL keyword"}, @@ -175,7 +174,6 @@ var mdlGeneratedKeywords = []protocol.CompletionItem{ {Label: "LAYOUTS", Kind: protocol.CompletionItemKindKeyword, Detail: "Widget keyword"}, {Label: "SNIPPETS", Kind: protocol.CompletionItemKindKeyword, Detail: "Widget keyword"}, {Label: "BLOCKS", Kind: protocol.CompletionItemKindKeyword, Detail: "Widget keyword"}, - {Label: "NOTEBOOKS", Kind: protocol.CompletionItemKindKeyword, Detail: "Widget keyword"}, {Label: "PLACEHOLDER", Kind: protocol.CompletionItemKindKeyword, Detail: "Widget keyword"}, {Label: "SNIPPETCALL", Kind: protocol.CompletionItemKindKeyword, Detail: "Widget keyword"}, {Label: "LAYOUTGRID", Kind: protocol.CompletionItemKindKeyword, Detail: "Widget keyword"}, From 751c739270a30d9f61db23c8fa592b08925ca7be Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 18 Aug 2026 06:50:47 +0000 Subject: [PATCH 34/35] fix(executor): keep excluded documents excluded, and target the live twin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Studio Pro can "Exclude from project" any document, and Mendix then allows a second document of the SAME NAME in the same module. Measured on 11.13.0 with two microflows named MyFirstModule.Calc: one of them Excluded -> mx check: 0 errors both active -> [error] CE0122 "Duplicate document name" So exclusion is exactly what makes the duplicate legal, and mxcli got both consequences wrong. Wrong target: every by-name lookup took the first match, so which document it resolved to depended on enumeration order. `create or replace` edited the excluded twin while the live microflow silently kept its old body, and `describe` printed a body the app does not run. Dropped flag: every rebuild wrote Excluded from the AST default (false) rather than carrying the stored value, so the excluded twin became active. One `create or replace` took the probe project from 0 errors to CE0122 — unbuildable, from a statement that never mentioned exclusion. Fixed for every document type that carries the flag. Lookups route through a single selector (mdl/executor/excluded_docs.go: pickLive — live match wins, an all-excluded set still resolves so lookups do not start reporting "not found"), and each create path carries the stored flag next to the ID and roles it already preserved. `@excluded` already existed and round-trips through DESCRIBE, so an absent annotation now means "the script does not say", never "make it active". Four types had no Excluded field to carry at all — model.Enumeration, types.JavaAction, types.ImageCollection, pages.Snippet — so the field is added and populated in both engines' readers, and the backends that hardcoded SetExcluded(false) (enumeration, snippet, image collection) now write the stored value. Pages and snippets additionally collected ALL name matches and deleted the extras, destroying the excluded twin outright; the replace set is now the live documents only. Measured, not read: queues looked correct — reader and writer both carry the flag — and still dropped it, because the executor built a fresh struct. A matrix run over nine authored document types found it. All nine now survive a re-run of an unchanged script, and the original repro rewrites the live microflow at 0 errors. Tests fail with the reported symptoms when the fix is reverted. --- .claude/skills/fix-issue.md | 1 + .claude/skills/mendix/write-microflows.md | 33 ++++ .../914-excluded-document-preserved.mdl | 71 ++++++++ mdl/backend/modelsdk/enumeration.go | 1 + mdl/backend/modelsdk/enumeration_write.go | 9 +- .../modelsdk/image_collection_write.go | 3 +- mdl/backend/modelsdk/java.go | 1 + mdl/backend/modelsdk/page.go | 2 +- mdl/backend/modelsdk/snippet_write.go | 4 +- mdl/backend/mpr/convert.go | 2 + mdl/backend/mpr/convert_roundtrip_test.go | 10 +- mdl/executor/cmd_agenteditor_models.go | 2 + mdl/executor/cmd_agenteditor_write.go | 6 + mdl/executor/cmd_businessevents.go | 23 ++- mdl/executor/cmd_datatransformer.go | 19 ++- mdl/executor/cmd_dbconnection.go | 24 ++- mdl/executor/cmd_enumerations.go | 18 +- mdl/executor/cmd_export_mappings.go | 4 + mdl/executor/cmd_imagecollections.go | 2 + mdl/executor/cmd_import_mappings.go | 4 + mdl/executor/cmd_javaactions.go | 22 ++- mdl/executor/cmd_javascript_actions_write.go | 21 ++- mdl/executor/cmd_jsonstructures.go | 4 + mdl/executor/cmd_microflows_create.go | 31 ++-- mdl/executor/cmd_microflows_show.go | 66 ++++---- mdl/executor/cmd_nanoflows_create.go | 30 ++-- mdl/executor/cmd_pages_create_v3.go | 71 ++++++-- mdl/executor/cmd_published_rest.go | 16 +- mdl/executor/cmd_queues.go | 22 ++- mdl/executor/cmd_workflows_write.go | 24 ++- mdl/executor/excluded_docs.go | 67 ++++++++ mdl/executor/excluded_docs_test.go | 160 ++++++++++++++++++ mdl/types/java.go | 3 + mdl/types/mapping.go | 5 +- model/types.go | 12 +- sdk/mpr/parser_enumeration.go | 5 + sdk/mpr/parser_misc.go | 9 + sdk/mpr/writer_enumeration.go | 2 +- sdk/pages/pages.go | 17 +- 39 files changed, 660 insertions(+), 166 deletions(-) create mode 100644 mdl-examples/bug-tests/914-excluded-document-preserved.mdl create mode 100644 mdl/executor/excluded_docs.go create mode 100644 mdl/executor/excluded_docs_test.go diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index 3d7f24f06..6160b3c65 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -541,3 +541,4 @@ extracting `OffsetExpression`/`LimitExpression`. | A guard that walks a stored BSON document works on the **modelsdk** engine and is a **silent no-op on legacy** (or vice versa) — same project, same document, opposite behaviour. Here `checkNoQueuedCalls` refused a rewrite that would drop a task-queue binding under modelsdk, and let it through under `--engine legacy`, destroying the binding exactly as the guard existed to prevent | The two readers return **different Go shapes for the same BSON**: `modelsdk` yields `[]interface{}` for arrays, the legacy `mpr` reader yields **`bson.A`**. `bson.A` is a NAMED slice type (`type A []interface{}`), so a type switch `case []any:` does **not** match it and the walk never descends into `ObjectCollection.Objects` | `mdl/executor/validate_queued_calls.go` (`queuedCallTargets`, `storedQueueRetries`); the same blind spot exists in `mdl/backend/mcp/page.go`, `mdl/backend/mcp/page_mutator.go`, `mdl/executor/widget_sync_apply.go` | Add `case bson.A:` alongside `case []any:` in every raw-BSON walk. The house pattern already does this (`mdl/backend/bsonnav`, `mdl/catalog/builder_pages.go`, `mdl/xpathrefs/scan.go` and ~7 more) — the ones missing it are the outliers: `for f in $(grep -rln "case \[\]any:" --include=*.go mdl/); do grep -q "bson.A" $f \|\| echo $f; done`. **Test both shapes of the same document in one test**, not just the one your engine returns. **A unit test alone is not enough to find this**: it only surfaced by running the same command under `MXCLI_ENGINE=legacy` and diffing against modelsdk — engine parity has to be exercised, not assumed. Reported as FINDINGS #25 | | A `CREATE DATABASE CONNECTION` whose query uses a `$$…$$` dollar-quoted SQL **and** a `parameter … default '…'` stores the **parameter's default as the query body**. `describe` shows `sql 'sales'` where the SELECT should be, the SQL text is absent from the BSON entirely, and `mx check` reports 0 errors on both engines | `buildDatabaseConnection` took `STRING_LITERAL(0)` as the SQL before falling back to `DOLLAR_STRING`. When the SQL is a dollar string, `STRING_LITERAL(0)` is the first parameter's DEFAULT — so the query body was overwritten by it. The parameter-index logic ten lines below already compensates (`if qc.DOLLAR_STRING() != nil { slIdx = defaultIdx }`); the SQL extraction did not | `mdl/visitor/visitor_dbconnection.go` | Check `DOLLAR_STRING()` FIRST, then fall back to `STRING_LITERAL(0)`. **Isolate by bisecting the clause combination**, not by reading: `parameter d: String` alone kept the SQL, `parameter d: String default 'x'` destroyed it — that one-clause difference names the cause immediately. When two sites in one function index the same token stream, they must agree about the offsets; a comment on one is not enough | | `describe database connection` omits `returns` / `map (…)` / `parameter` on the **default (modelsdk) engine** while the legacy engine renders them, so a describe → exec round trip drops the return entity, the column mapping and every query parameter | The modelsdk reader populated neither `TableMappings` nor `Parameters`, so the DESCRIBE renderer (which does handle them) only ever saw empty slices — dead code on that engine. The round-trip test existed but was pinned to the legacy backend via `setupTestEnv`, so it passed while the default engine was lossy | `mdl/backend/modelsdk/integration_read.go` (`tableMappingsFromGen`, `queryParametersFromGen`); test `mdl/executor/roundtrip_dbconnection_test.go` | Mirror the legacy reader field for field, and run the round-trip test over `gateEngines` rather than the default backend. **The cross-engine DESCRIBE matrix is the check that finds these**: write with engine A, read with engine B, all four combinations — a gap on one engine is invisible from inside that engine. Fixing one field (TableMappings) and not its sibling (Parameters) is the characteristic mistake; enumerate every child collection of the document before calling it done | +| `create or replace` on a document whose module holds a same-named **excluded** twin edits the WRONG one — the live document keeps its old body and the script looks like a no-op — and the rewrite also clears the twin's `Excluded` flag, so a project that built at 0 errors fails **CE0122** "Duplicate document name". Also `describe` returns the excluded document's body | Studio Pro's "Exclude from project" makes a document name non-unique: Mendix allows the duplicate as long as at most one is active (measured on 11.13.0 — excluded pair = 0 errors, both active = CE0122). Every by-name lookup took the FIRST match, so which document it hit depended on enumeration order; and every rebuild wrote `Excluded` from the AST default (`false`) instead of carrying the stored value. `@excluded` exists and round-trips through DESCRIBE, so absence of the annotation must mean "the script does not say", never "make it active" | `mdl/executor/excluded_docs.go` (`pickLive`) + the create paths: `cmd_microflows_create.go`, `cmd_nanoflows_create.go`, `cmd_pages_create_v3.go` (pages **and** snippets), `cmd_enumerations.go`, `cmd_queues.go`, `cmd_workflows_write.go`, `cmd_javaactions.go`, `cmd_javascript_actions_write.go`, `cmd_businessevents.go`, `cmd_published_rest.go`, `cmd_dbconnection.go`, `cmd_datatransformer.go`, `cmd_import_mappings.go`, `cmd_export_mappings.go`, `cmd_jsonstructures.go`, `cmd_imagecollections.go`, `cmd_agenteditor_*.go`, plus `cmd_microflows_show.go` for DESCRIBE | Route every by-name lookup through `pickLive` (live match wins; an all-excluded set still resolves to the first, so lookups do not start reporting "not found") and carry the stored flag next to the ID/roles each path already preserves. Four types had **no** `Excluded` field to carry — `model.Enumeration`, `types.JavaAction`, `types.ImageCollection`, `pages.Snippet` — so the field had to be added and populated in BOTH engines' readers (`TestFieldCountDrift` catches the `mdl/types` half and requires `convert.go` + the expected counts to be updated). Backends that hardcoded `SetExcluded(false)` (enumeration, snippet, image collection) are the same bug wearing a different hat. **Measure, do not read**: queues looked correct (their reader and writer both carry the flag) and still dropped it, because the executor built a fresh struct — the matrix run is what found it. Pages/snippets additionally collected ALL name matches and DELETED the extras, which destroyed the excluded twin outright. Tests `TestPickLive`, `TestCreateOrModifyMicroflow_PreservesStoredExclusion`, `TestCreateOrModifyMicroflow_TargetsLiveTwin` (both fail with the reported symptoms when the fix is reverted); fixture `mdl-examples/bug-tests/914-excluded-document-preserved.mdl`. Issue #914 | diff --git a/.claude/skills/mendix/write-microflows.md b/.claude/skills/mendix/write-microflows.md index 22c85d95a..77d0f8e57 100644 --- a/.claude/skills/mendix/write-microflows.md +++ b/.claude/skills/mendix/write-microflows.md @@ -77,6 +77,39 @@ begin end; ``` +### `@excluded` — documents excluded from the project + +`@excluded` before a `create microflow` marks the document **"Exclude from project"** +(the same checkbox Studio Pro offers). The document stays in the `.mpr`, does not +build, and `show microflows` reports it in the `Excluded` column. + +```mdl +@excluded +create microflow MyModule.LegacyCalc () +returns Integer +begin + return 7; +end; +``` + +Two rules follow, and both are enforced rather than documented-and-hoped: + +- **An absent `@excluded` never un-excludes.** It means "the script does not say", + not "make this active" — so re-running a `create or modify` you wrote before the + document was excluded leaves the exclusion alone. Un-exclude in Studio Pro. + (Before #914 the rewrite cleared it, which is how a valid project ended up + failing **CE0122** — see the next rule.) +- **A name is not unique when a twin is excluded.** Mendix allows two documents of + the same name in one module as long as at most one is active — verified on + 11.13.0: the excluded pair builds at 0 errors, the same pair both active is + `[error] CE0122 "Duplicate document name"`. `create or modify`, `describe` and + the other by-name lookups therefore target the **live** document; the excluded + twin is neither rewritten nor deleted. + +The same applies to every document type that carries the flag — nanoflows, pages, +snippets, enumerations, queues, workflows, Java/JavaScript actions, mappings, JSON +structures, REST/OData services, image collections and the agent documents. + ### FOLDER Option Place microflows in folders for organization: diff --git a/mdl-examples/bug-tests/914-excluded-document-preserved.mdl b/mdl-examples/bug-tests/914-excluded-document-preserved.mdl new file mode 100644 index 000000000..a0e71d806 --- /dev/null +++ b/mdl-examples/bug-tests/914-excluded-document-preserved.mdl @@ -0,0 +1,71 @@ +-- ============================================================================ +-- #914 — an excluded document survives a rewrite, and a name is not unique +-- ============================================================================ +-- +-- POSITIVE TEST: `mxcli check` MUST accept this file. +-- +-- Studio Pro can "Exclude from project" any document, and Mendix then allows a +-- second document of the SAME NAME in the same module. Measured on 11.13.0: +-- +-- two microflows named MyFirstModule.Calc, one Excluded -> mx check 0 errors +-- the same pair with both active -> [error] CE0122 +-- "Duplicate document name 'MyFirstModule.Calc'." +-- +-- So exclusion is exactly what makes the duplicate legal, and two things follow +-- for every by-name lookup and every rewrite: +-- +-- 1. A name is not a unique key. `create or replace` used to edit whichever +-- document came first, which could be the excluded one — the live +-- microflow kept its old body and the script appeared to do nothing. +-- 2. `Excluded` is model state, not script state. The rebuild wrote the AST +-- default (false), so the excluded twin became active and the project went +-- from 0 errors to CE0122 — unbuildable, from a statement that never +-- mentioned exclusion. +-- +-- Both are fixed for every document type that carries the flag. The regression +-- guards live in Go (mdl/executor/excluded_docs_test.go), because `mxcli check` +-- cannot see project state and the duplicate only exists inside a real .mpr. +-- +-- What this fixture pins is the SYNTAX half: @excluded is an annotation mxcli +-- both writes and round-trips through DESCRIBE, which is what makes an excluded +-- document expressible in MDL at all. Verified end to end against a real +-- project (exec -> show microflows reports Excluded=true -> describe re-emits +-- the annotation). + +CREATE OR MODIFY ENUMERATION Bug914.Status ( Open 'Open', Closed 'Closed' ); + +-- An excluded microflow: still part of the .mpr, not part of the app. +@excluded +CREATE OR MODIFY MICROFLOW Bug914.LegacyCalc () +RETURNS Integer +BEGIN + return 7; +END; + +-- The live microflow. In a real project these two may share a name; MDL cannot +-- author that pair (it would be ambiguous by name), which is why the ordering +-- and preservation rules are enforced in the executor rather than here. +CREATE OR MODIFY MICROFLOW Bug914.Calc () +RETURNS Integer +BEGIN + return 1; +END; + +-- Re-running this statement must not clear the exclusion above: an absent +-- @excluded means "the script does not say", never "make it active". +@excluded +CREATE OR MODIFY MICROFLOW Bug914.LegacyCalc () +RETURNS Integer +BEGIN + return 8; +END; + +@excluded +create or modify page Bug914.LegacyPage +( + title: 'Legacy', + layout: Atlas_Core.Atlas_Default +) +{ + -- empty +} diff --git a/mdl/backend/modelsdk/enumeration.go b/mdl/backend/modelsdk/enumeration.go index 233b81655..6b8303435 100644 --- a/mdl/backend/modelsdk/enumeration.go +++ b/mdl/backend/modelsdk/enumeration.go @@ -43,6 +43,7 @@ func enumToModel(e *genEnum.Enumeration, containerID model.ID) *model.Enumeratio ContainerID: containerID, Name: e.Name(), Documentation: e.Documentation(), + Excluded: e.Excluded(), } out.ID = model.ID(e.ID()) out.TypeName = "Enumerations$Enumeration" diff --git a/mdl/backend/modelsdk/enumeration_write.go b/mdl/backend/modelsdk/enumeration_write.go index 86d46c8f6..e7f176d55 100644 --- a/mdl/backend/modelsdk/enumeration_write.go +++ b/mdl/backend/modelsdk/enumeration_write.go @@ -75,14 +75,15 @@ func (b *Backend) DeleteEnumeration(id model.ID) error { return b.writer.DeleteUnit(string(id)) } -// enumToGen builds a gen Enumeration from the model. Excluded=false and -// ExportLevel="Hidden" mirror the legacy serializer; RemoteSource (null) comes -// from the registered default. +// enumToGen builds a gen Enumeration from the model. ExportLevel="Hidden" +// mirrors the legacy serializer; RemoteSource (null) comes from the registered +// default. Excluded is carried from the model — hardcoding false here made a +// CREATE OR MODIFY silently un-exclude the document (#914). func enumToGen(enum *model.Enumeration) *genEnum.Enumeration { out := genEnum.NewEnumeration() out.SetName(enum.Name) out.SetDocumentation(enum.Documentation) - out.SetExcluded(false) + out.SetExcluded(enum.Excluded) out.SetExportLevel("Hidden") for _, v := range enum.Values { out.AddValues(enumValueToGen(v)) diff --git a/mdl/backend/modelsdk/image_collection_write.go b/mdl/backend/modelsdk/image_collection_write.go index 1098952d7..bb7b61615 100644 --- a/mdl/backend/modelsdk/image_collection_write.go +++ b/mdl/backend/modelsdk/image_collection_write.go @@ -36,6 +36,7 @@ func (b *Backend) ListImageCollections() ([]*types.ImageCollection, error) { ic.Name, _ = doc["Name"].(string) ic.Documentation, _ = doc["Documentation"].(string) ic.ExportLevel, _ = doc["ExportLevel"].(string) + ic.Excluded, _ = doc["Excluded"].(bool) if arr, ok := doc["Images"].(bson.A); ok { for _, el := range arr { imgDoc, ok := el.(bson.M) @@ -129,7 +130,7 @@ func serializeImageCollection(ic *types.ImageCollection) ([]byte, error) { {Key: "$ID", Value: bsonutil.IDToBsonBinary(string(ic.ID))}, {Key: "$Type", Value: "Images$ImageCollection"}, {Key: "Documentation", Value: ic.Documentation}, - {Key: "Excluded", Value: false}, + {Key: "Excluded", Value: ic.Excluded}, {Key: "ExportLevel", Value: ic.ExportLevel}, {Key: "Images", Value: images}, {Key: "Name", Value: ic.Name}, diff --git a/mdl/backend/modelsdk/java.go b/mdl/backend/modelsdk/java.go index e596479ee..9eb709455 100644 --- a/mdl/backend/modelsdk/java.go +++ b/mdl/backend/modelsdk/java.go @@ -25,6 +25,7 @@ func (b *Backend) ListJavaActions() ([]*types.JavaAction, error) { ContainerID: u.ContainerID, Name: u.Element.Name(), Documentation: u.Element.Documentation(), + Excluded: u.Element.Excluded(), } ja.ID = model.ID(u.Element.ID()) out = append(out, ja) diff --git a/mdl/backend/modelsdk/page.go b/mdl/backend/modelsdk/page.go index f9e4d7f31..c59dc79db 100644 --- a/mdl/backend/modelsdk/page.go +++ b/mdl/backend/modelsdk/page.go @@ -84,7 +84,7 @@ func (b *Backend) ListSnippets() ([]*pages.Snippet, error) { } out := make([]*pages.Snippet, 0, len(units)) for _, u := range units { - s := &pages.Snippet{ContainerID: u.ContainerID, Name: u.Element.Name()} + s := &pages.Snippet{ContainerID: u.ContainerID, Name: u.Element.Name(), Excluded: u.Element.Excluded()} s.ID = model.ID(u.Element.ID()) // Populate declared parameters — the page builder reads these to validate // and wire SNIPPETCALL argument mappings (without them every parameterised diff --git a/mdl/backend/modelsdk/snippet_write.go b/mdl/backend/modelsdk/snippet_write.go index b27be6d7a..cd2c99a9a 100644 --- a/mdl/backend/modelsdk/snippet_write.go +++ b/mdl/backend/modelsdk/snippet_write.go @@ -96,7 +96,9 @@ func snippetToGen(s *pages.Snippet) (*genPg.Snippet, error) { out := genPg.NewSnippet() out.SetName(s.Name) out.SetDocumentation(s.Documentation) - out.SetExcluded(false) + // Carry the stored exclusion: hardcoding false silently un-excluded the + // document on every rewrite (#914). + out.SetExcluded(s.Excluded) out.SetExportLevel("Hidden") out.SetCanvasWidth(800) out.SetCanvasHeight(600) diff --git a/mdl/backend/mpr/convert.go b/mdl/backend/mpr/convert.go index 4807166f9..658a82c7d 100644 --- a/mdl/backend/mpr/convert.go +++ b/mdl/backend/mpr/convert.go @@ -121,6 +121,7 @@ func convertJavaActionSlice(in []*mpr.JavaAction, err error) ([]*types.JavaActio ContainerID: ja.ContainerID, Name: ja.Name, Documentation: ja.Documentation, + Excluded: ja.Excluded, } } return out, nil @@ -388,6 +389,7 @@ func convertImageCollection(in *mpr.ImageCollection) *types.ImageCollection { Name: in.Name, ExportLevel: in.ExportLevel, Documentation: in.Documentation, + Excluded: in.Excluded, } if in.Images != nil { ic.Images = make([]types.Image, len(in.Images)) diff --git a/mdl/backend/mpr/convert_roundtrip_test.go b/mdl/backend/mpr/convert_roundtrip_test.go index 1292f6eee..d6f85a8cf 100644 --- a/mdl/backend/mpr/convert_roundtrip_test.go +++ b/mdl/backend/mpr/convert_roundtrip_test.go @@ -610,8 +610,10 @@ func TestFieldCountDrift(t *testing.T) { assertFieldCount(t, "types.RawUnitInfo", types.RawUnitInfo{}, 5) assertFieldCount(t, "mpr.RawCustomWidgetType", mpr.RawCustomWidgetType{}, 6) assertFieldCount(t, "types.RawCustomWidgetType", types.RawCustomWidgetType{}, 6) - assertFieldCount(t, "mpr.JavaAction", mpr.JavaAction{}, 4) - assertFieldCount(t, "types.JavaAction", types.JavaAction{}, 4) + // +1 each for Excluded (#914): reads must carry it so a rewrite does not + // clear the document's "Exclude from project" flag. + assertFieldCount(t, "mpr.JavaAction", mpr.JavaAction{}, 5) + assertFieldCount(t, "types.JavaAction", types.JavaAction{}, 5) assertFieldCount(t, "mpr.JavaScriptAction", mpr.JavaScriptAction{}, 12) assertFieldCount(t, "types.JavaScriptAction", types.JavaScriptAction{}, 12) assertFieldCount(t, "mpr.NavigationDocument", mpr.NavigationDocument{}, 4) @@ -620,8 +622,8 @@ func TestFieldCountDrift(t *testing.T) { assertFieldCount(t, "types.JsonStructure", types.JsonStructure{}, 8) assertFieldCount(t, "mpr.JsonElement", mpr.JsonElement{}, 14) assertFieldCount(t, "types.JsonElement", types.JsonElement{}, 14) - assertFieldCount(t, "mpr.ImageCollection", mpr.ImageCollection{}, 6) - assertFieldCount(t, "types.ImageCollection", types.ImageCollection{}, 6) + assertFieldCount(t, "mpr.ImageCollection", mpr.ImageCollection{}, 7) + assertFieldCount(t, "types.ImageCollection", types.ImageCollection{}, 7) assertFieldCount(t, "mpr.EntityMemberAccess", mpr.EntityMemberAccess{}, 3) assertFieldCount(t, "types.EntityMemberAccess", types.EntityMemberAccess{}, 3) assertFieldCount(t, "mpr.EntityAccessRevocation", mpr.EntityAccessRevocation{}, 6) diff --git a/mdl/executor/cmd_agenteditor_models.go b/mdl/executor/cmd_agenteditor_models.go index 46ce245af..adde30b4e 100644 --- a/mdl/executor/cmd_agenteditor_models.go +++ b/mdl/executor/cmd_agenteditor_models.go @@ -185,6 +185,8 @@ func execCreateAgentEditorModel(ctx *ExecContext, s *ast.CreateModelStmt) error if existing != nil { m.ID = existing.ID + // Excluded is model state, not script state (#914). + m.Excluded = existing.Excluded if err := ctx.Backend.UpdateAgentEditorModel(m); err != nil { return mdlerrors.NewBackend("update model", err) } diff --git a/mdl/executor/cmd_agenteditor_write.go b/mdl/executor/cmd_agenteditor_write.go index 42f3454b2..8b4a16f6f 100644 --- a/mdl/executor/cmd_agenteditor_write.go +++ b/mdl/executor/cmd_agenteditor_write.go @@ -53,6 +53,8 @@ func execCreateConsumedMCPService(ctx *ExecContext, s *ast.CreateConsumedMCPServ if existing != nil { c.ID = existing.ID + // Excluded is model state, not script state (#914). + c.Excluded = existing.Excluded if err := ctx.Backend.UpdateAgentEditorConsumedMCPService(c); err != nil { return mdlerrors.NewBackend("update consumed mcp service", err) } @@ -142,6 +144,8 @@ func execCreateKnowledgeBase(ctx *ExecContext, s *ast.CreateKnowledgeBaseStmt) e if existing != nil { k.ID = existing.ID + // Excluded is model state, not script state (#914). + k.Excluded = existing.Excluded if err := ctx.Backend.UpdateAgentEditorKnowledgeBase(k); err != nil { return mdlerrors.NewBackend("update knowledge base", err) } @@ -291,6 +295,8 @@ func execCreateAgent(ctx *ExecContext, s *ast.CreateAgentStmt) error { if existingAgent != nil { a.ID = existingAgent.ID + // Excluded is model state, not script state (#914). + a.Excluded = existingAgent.Excluded if err := ctx.Backend.UpdateAgentEditorAgent(a); err != nil { return mdlerrors.NewBackend("update agent", err) } diff --git a/mdl/executor/cmd_businessevents.go b/mdl/executor/cmd_businessevents.go index f3a3562d8..8cf133bfe 100644 --- a/mdl/executor/cmd_businessevents.go +++ b/mdl/executor/cmd_businessevents.go @@ -279,16 +279,20 @@ func createBusinessEventService(ctx *ExecContext, stmt *ast.CreateBusinessEventS } var existingID model.ID - for _, existing := range existingServices { - existModID := h.FindModuleID(existing.ContainerID) - existModName := h.GetModuleName(existModID) - if strings.EqualFold(existModName, moduleName) && strings.EqualFold(existing.Name, stmt.Name.Name) { - if !stmt.CreateOrModify { - return mdlerrors.NewAlreadyExistsMsg("business event service", moduleName+"."+stmt.Name.Name, fmt.Sprintf("business event service already exists: %s.%s (use create or modify to update)", moduleName, stmt.Name.Name)) - } - existingID = existing.ID - break + // Target the live service and carry its exclusion forward (#914). + existingExcluded := false + if existing, ok := pickLive(existingServices, + func(svc *model.BusinessEventService) bool { + return strings.EqualFold(h.GetModuleName(h.FindModuleID(svc.ContainerID)), moduleName) && + strings.EqualFold(svc.Name, stmt.Name.Name) + }, + func(svc *model.BusinessEventService) bool { return svc.Excluded }, + ); ok { + if !stmt.CreateOrModify { + return mdlerrors.NewAlreadyExistsMsg("business event service", moduleName+"."+stmt.Name.Name, fmt.Sprintf("business event service already exists: %s.%s (use create or modify to update)", moduleName, stmt.Name.Name)) } + existingID = existing.ID + existingExcluded = existing.Excluded } // Resolve folder if specified @@ -307,6 +311,7 @@ func createBusinessEventService(ctx *ExecContext, stmt *ast.CreateBusinessEventS Name: stmt.Name.Name, Documentation: stmt.Documentation, ExportLevel: "Hidden", + Excluded: existingExcluded, } if existingID != "" { svc.ID = existingID diff --git a/mdl/executor/cmd_datatransformer.go b/mdl/executor/cmd_datatransformer.go index ae04b9a62..37476d1fc 100644 --- a/mdl/executor/cmd_datatransformer.go +++ b/mdl/executor/cmd_datatransformer.go @@ -131,6 +131,10 @@ func execCreateDataTransformer(ctx *ExecContext, s *ast.CreateDataTransformerStm SourceType: s.SourceType, SourceJSON: s.SourceJSON, } + if existing != nil { + // Excluded is model state, not script state (#914). + dt.Excluded = existing.Excluded + } for _, step := range s.Steps { dt.Steps = append(dt.Steps, &model.DataTransformerStep{ @@ -172,12 +176,15 @@ func findDataTransformer(ctx *ExecContext, moduleName, name string) (*model.Data if err != nil { return nil, "" } - for _, dt := range transformers { - modID := h.FindModuleID(dt.ContainerID) - modName := h.GetModuleName(modID) - if strings.EqualFold(modName, moduleName) && strings.EqualFold(dt.Name, name) { - return dt, dt.ID - } + // Prefer the live transformer over an excluded twin of the same name (#914). + if dt, ok := pickLive(transformers, + func(dt *model.DataTransformer) bool { + return strings.EqualFold(h.GetModuleName(h.FindModuleID(dt.ContainerID)), moduleName) && + strings.EqualFold(dt.Name, name) + }, + func(dt *model.DataTransformer) bool { return dt.Excluded }, + ); ok { + return dt, dt.ID } return nil, "" } diff --git a/mdl/executor/cmd_dbconnection.go b/mdl/executor/cmd_dbconnection.go index 33a493781..ad18740d1 100644 --- a/mdl/executor/cmd_dbconnection.go +++ b/mdl/executor/cmd_dbconnection.go @@ -32,16 +32,21 @@ func createDatabaseConnection(ctx *ExecContext, stmt *ast.CreateDatabaseConnecti h, _ := getHierarchy(ctx) var existingConnID model.ID - for _, ex := range existing { - modID := h.FindModuleID(ex.ContainerID) - modName := h.GetModuleName(modID) - if strings.EqualFold(modName, stmt.Name.Module) && strings.EqualFold(ex.Name, stmt.Name.Name) { - if stmt.CreateOrModify { - existingConnID = ex.ID - } else { - return mdlerrors.NewAlreadyExistsMsg("database connection", modName+"."+ex.Name, fmt.Sprintf("database connection already exists: %s.%s (use create or modify to update)", modName, ex.Name)) - } + // Target the live connection and carry its exclusion forward (#914). + existingExcluded := false + if ex, ok := pickLive(existing, + func(c *model.DatabaseConnection) bool { + return strings.EqualFold(h.GetModuleName(h.FindModuleID(c.ContainerID)), stmt.Name.Module) && + strings.EqualFold(c.Name, stmt.Name.Name) + }, + func(c *model.DatabaseConnection) bool { return c.Excluded }, + ); ok { + if !stmt.CreateOrModify { + modName := h.GetModuleName(h.FindModuleID(ex.ContainerID)) + return mdlerrors.NewAlreadyExistsMsg("database connection", modName+"."+ex.Name, fmt.Sprintf("database connection already exists: %s.%s (use create or modify to update)", modName, ex.Name)) } + existingConnID = ex.ID + existingExcluded = ex.Excluded } // A literal where Mendix stores a ConstantIdentifier writes a project that @@ -76,6 +81,7 @@ func createDatabaseConnection(ctx *ExecContext, stmt *ast.CreateDatabaseConnecti UserName: userName, Password: password, ExportLevel: "Hidden", + Excluded: existingExcluded, } // Build queries diff --git a/mdl/executor/cmd_enumerations.go b/mdl/executor/cmd_enumerations.go index 43801fa90..a72d1edd8 100644 --- a/mdl/executor/cmd_enumerations.go +++ b/mdl/executor/cmd_enumerations.go @@ -78,6 +78,9 @@ func execCreateEnumeration(ctx *ExecContext, s *ast.CreateEnumerationStmt) error // In-place update: preserve the existing UUID so BSON git-diff sees a // modification rather than a delete+insert pair. enum.ID = existingEnum.ID + // Excluded is model state, not script state: MDL cannot express it for + // an enumeration, so the stored value is the one that survives (#914). + enum.Excluded = existingEnum.Excluded if err := ctx.Backend.UpdateEnumeration(enum); err != nil { return mdlerrors.NewBackend("update enumeration", err) } @@ -110,12 +113,15 @@ func findEnumeration(ctx *ExecContext, moduleName, enumName string) *model.Enume return nil } - for _, enum := range enums { - modID := h.FindModuleID(enum.ContainerID) - modName := h.GetModuleName(modID) - if enum.Name == enumName && modName == moduleName { - return enum - } + // Prefer the live enumeration: a module may hold an excluded twin of this + // name (#914), and the app only has the active one. + if enum, ok := pickLive(enums, + func(e *model.Enumeration) bool { + return e.Name == enumName && h.GetModuleName(h.FindModuleID(e.ContainerID)) == moduleName + }, + func(e *model.Enumeration) bool { return e.Excluded }, + ); ok { + return enum } return nil } diff --git a/mdl/executor/cmd_export_mappings.go b/mdl/executor/cmd_export_mappings.go index 477660464..ae778c545 100644 --- a/mdl/executor/cmd_export_mappings.go +++ b/mdl/executor/cmd_export_mappings.go @@ -199,6 +199,10 @@ func execCreateExportMapping(ctx *ExecContext, s *ast.CreateExportMappingStmt) e ExportLevel: "Hidden", NullValueOption: s.NullValueOption, } + if existing != nil { + // Excluded is model state, not script state (#914). + em.Excluded = existing.Excluded + } if em.NullValueOption == "" { em.NullValueOption = "LeaveOutElement" } diff --git a/mdl/executor/cmd_imagecollections.go b/mdl/executor/cmd_imagecollections.go index 8076ef07f..74524daab 100644 --- a/mdl/executor/cmd_imagecollections.go +++ b/mdl/executor/cmd_imagecollections.go @@ -46,6 +46,8 @@ func execCreateImageCollection(ctx *ExecContext, s *ast.CreateImageCollectionStm } if existing != nil { ic.ID = existing.ID + // Excluded is model state, not script state (#914). + ic.Excluded = existing.Excluded } // Load image files diff --git a/mdl/executor/cmd_import_mappings.go b/mdl/executor/cmd_import_mappings.go index 468a97930..fd96629f5 100644 --- a/mdl/executor/cmd_import_mappings.go +++ b/mdl/executor/cmd_import_mappings.go @@ -211,6 +211,10 @@ func execCreateImportMapping(ctx *ExecContext, s *ast.CreateImportMappingStmt) e Name: s.Name.Name, ExportLevel: "Hidden", } + if existing != nil { + // Excluded is model state, not script state (#914). + im.Excluded = existing.Excluded + } // Set schema source reference switch s.SchemaKind { diff --git a/mdl/executor/cmd_javaactions.go b/mdl/executor/cmd_javaactions.go index 8a5564370..4867a4b60 100644 --- a/mdl/executor/cmd_javaactions.go +++ b/mdl/executor/cmd_javaactions.go @@ -319,16 +319,19 @@ func execCreateJavaAction(ctx *ExecContext, s *ast.CreateJavaActionStmt) error { return mdlerrors.NewBackend("list java actions", err) } var existingJAID model.ID - for _, existing := range jas { - existingModID := h.FindModuleID(existing.ContainerID) - existingModName := h.GetModuleName(existingModID) - if existingModName == s.Name.Module && existing.Name == s.Name.Name { - if !s.CreateOrModify { - return mdlerrors.NewAlreadyExists("java action", s.Name.Module+"."+s.Name.Name) - } - existingJAID = existing.ID - break + // Target the live action and carry its exclusion forward (#914). + existingExcluded := false + if existing, ok := pickLive(jas, + func(ja *types.JavaAction) bool { + return h.GetModuleName(h.FindModuleID(ja.ContainerID)) == s.Name.Module && ja.Name == s.Name.Name + }, + func(ja *types.JavaAction) bool { return ja.Excluded }, + ); ok { + if !s.CreateOrModify { + return mdlerrors.NewAlreadyExists("java action", s.Name.Module+"."+s.Name.Name) } + existingJAID = existing.ID + existingExcluded = existing.Excluded } newID := model.ID(types.GenerateID()) @@ -342,6 +345,7 @@ func execCreateJavaAction(ctx *ExecContext, s *ast.CreateJavaActionStmt) error { ID: newID, TypeName: "JavaActions$JavaAction", }, + Excluded: existingExcluded, ContainerID: containerID, Name: s.Name.Name, Documentation: s.Documentation, diff --git a/mdl/executor/cmd_javascript_actions_write.go b/mdl/executor/cmd_javascript_actions_write.go index 714393f2c..387bddcf6 100644 --- a/mdl/executor/cmd_javascript_actions_write.go +++ b/mdl/executor/cmd_javascript_actions_write.go @@ -48,15 +48,19 @@ func execCreateJavaScriptAction(ctx *ExecContext, s *ast.CreateJavaScriptActionS return mdlerrors.NewBackend("list javascript actions", err) } var existingID model.ID - for _, ex := range existing { - exModName := h.GetModuleName(h.FindModuleID(ex.ContainerID)) - if exModName == s.Name.Module && ex.Name == s.Name.Name { - if !s.CreateOrModify { - return mdlerrors.NewAlreadyExists("javascript action", s.Name.Module+"."+s.Name.Name) - } - existingID = ex.ID - break + // Target the live action and carry its exclusion forward (#914). + existingExcluded := false + if ex, ok := pickLive(existing, + func(a *types.JavaScriptAction) bool { + return h.GetModuleName(h.FindModuleID(a.ContainerID)) == s.Name.Module && a.Name == s.Name.Name + }, + func(a *types.JavaScriptAction) bool { return a.Excluded }, + ); ok { + if !s.CreateOrModify { + return mdlerrors.NewAlreadyExists("javascript action", s.Name.Module+"."+s.Name.Name) } + existingID = ex.ID + existingExcluded = ex.Excluded } newID := model.ID(types.GenerateID()) @@ -69,6 +73,7 @@ func execCreateJavaScriptAction(ctx *ExecContext, s *ast.CreateJavaScriptActionS ContainerID: containerID, Name: s.Name.Name, Documentation: s.Documentation, + Excluded: existingExcluded, ExportLevel: "Public", ActionDefaultReturnName: "ReturnValueName", Platform: platformOrDefault(s.Platform), diff --git a/mdl/executor/cmd_jsonstructures.go b/mdl/executor/cmd_jsonstructures.go index ed663b17b..6cca8e908 100644 --- a/mdl/executor/cmd_jsonstructures.go +++ b/mdl/executor/cmd_jsonstructures.go @@ -217,6 +217,10 @@ func execCreateJsonStructure(ctx *ExecContext, s *ast.CreateJsonStructureStmt) e JsonSnippet: types.PrettyPrintJSON(s.JsonSnippet), Elements: elements, } + if existing != nil { + // Excluded is model state, not script state (#914). + js.Excluded = existing.Excluded + } if existing != nil { js.ID = existing.ID diff --git a/mdl/executor/cmd_microflows_create.go b/mdl/executor/cmd_microflows_create.go index 18783e561..813f5a6f6 100644 --- a/mdl/executor/cmd_microflows_create.go +++ b/mdl/executor/cmd_microflows_create.go @@ -72,21 +72,30 @@ func execCreateMicroflow(ctx *ExecContext, s *ast.CreateMicroflowStmt) error { var existingContainerID model.ID var existingAllowedRoles []model.ID preserveAllowedRoles := false + // Excluded is model state, not script state: an absent @excluded must not + // clear a stored exclusion (#914). + existingExcluded := false existingMicroflows, err := ctx.Backend.ListMicroflows() if err != nil { return mdlerrors.NewBackend("check existing microflows", err) } - for _, existing := range existingMicroflows { - if existing.Name == s.Name.Name && getModuleID(ctx, existing.ContainerID) == module.ID { - if !s.CreateOrModify { - return mdlerrors.NewAlreadyExistsMsg("microflow", s.Name.Module+"."+s.Name.Name, "microflow '"+s.Name.Module+"."+s.Name.Name+"' already exists (use create or modify to overwrite)") - } - existingID = existing.ID - existingContainerID = existing.ContainerID - existingAllowedRoles = cloneRoleIDs(existing.AllowedModuleRoles) - preserveAllowedRoles = true - break + // A module may hold several microflows with this name as long as all but one + // are excluded, so target the live one rather than whichever comes first + // (#914). + if existing, ok := pickLive(existingMicroflows, + func(m *microflows.Microflow) bool { + return m.Name == s.Name.Name && getModuleID(ctx, m.ContainerID) == module.ID + }, + func(m *microflows.Microflow) bool { return m.Excluded }, + ); ok { + if !s.CreateOrModify { + return mdlerrors.NewAlreadyExistsMsg("microflow", s.Name.Module+"."+s.Name.Name, "microflow '"+s.Name.Module+"."+s.Name.Name+"' already exists (use create or modify to overwrite)") } + existingID = existing.ID + existingContainerID = existing.ContainerID + existingAllowedRoles = cloneRoleIDs(existing.AllowedModuleRoles) + preserveAllowedRoles = true + existingExcluded = existing.Excluded } // For CREATE OR REPLACE/MODIFY, reuse the existing ID to preserve references @@ -132,7 +141,7 @@ func execCreateMicroflow(ctx *ExecContext, s *ast.CreateMicroflowStmt) error { Documentation: s.Documentation, AllowConcurrentExecution: true, // Default: allow concurrent execution MarkAsUsed: false, - Excluded: s.Excluded, + Excluded: s.Excluded || existingExcluded, } if preserveAllowedRoles { mf.AllowedModuleRoles = existingAllowedRoles diff --git a/mdl/executor/cmd_microflows_show.go b/mdl/executor/cmd_microflows_show.go index 04a222467..3f4f32de5 100644 --- a/mdl/executor/cmd_microflows_show.go +++ b/mdl/executor/cmd_microflows_show.go @@ -215,15 +215,14 @@ func describeMicroflow(ctx *ExecContext, name ast.QualifiedName) error { } } - var targetMf *microflows.Microflow - for _, mf := range allMicroflows { - modID := h.FindModuleID(mf.ContainerID) - modName := h.GetModuleName(modID) - if modName == name.Module && mf.Name == name.Name { - targetMf = mf - break - } - } + // Describe the live microflow: a module may hold an excluded twin of this + // name, and describing that one shows a body the app does not run (#914). + targetMf, _ := pickLive(allMicroflows, + func(mf *microflows.Microflow) bool { + return h.GetModuleName(h.FindModuleID(mf.ContainerID)) == name.Module && mf.Name == name.Name + }, + func(mf *microflows.Microflow) bool { return mf.Excluded }, + ) if targetMf == nil { return mdlerrors.NewNotFound("microflow", name.String()) @@ -365,15 +364,13 @@ func describeNanoflow(ctx *ExecContext, name ast.QualifiedName) error { microflowNames[nf.ID] = h.GetQualifiedName(nf.ContainerID, nf.Name) } - var targetNf *microflows.Nanoflow - for _, nf := range allNanoflows { - modID := h.FindModuleID(nf.ContainerID) - modName := h.GetModuleName(modID) - if modName == name.Module && nf.Name == name.Name { - targetNf = nf - break - } - } + // Describe the live nanoflow, not an excluded twin of the same name (#914). + targetNf, _ := pickLive(allNanoflows, + func(nf *microflows.Nanoflow) bool { + return h.GetModuleName(h.FindModuleID(nf.ContainerID)) == name.Module && nf.Name == name.Name + }, + func(nf *microflows.Nanoflow) bool { return nf.Excluded }, + ) if targetNf == nil { return mdlerrors.NewNotFound("nanoflow", name.String()) @@ -484,15 +481,14 @@ func describeMicroflowToString(ctx *ExecContext, name ast.QualifiedName) (string microflowNames[mf.ID] = h.GetQualifiedName(mf.ContainerID, mf.Name) } - var targetMf *microflows.Microflow - for _, mf := range allMicroflows { - modID := h.FindModuleID(mf.ContainerID) - modName := h.GetModuleName(modID) - if modName == name.Module && mf.Name == name.Name { - targetMf = mf - break - } - } + // Describe the live microflow: a module may hold an excluded twin of this + // name, and describing that one shows a body the app does not run (#914). + targetMf, _ := pickLive(allMicroflows, + func(mf *microflows.Microflow) bool { + return h.GetModuleName(h.FindModuleID(mf.ContainerID)) == name.Module && mf.Name == name.Name + }, + func(mf *microflows.Microflow) bool { return mf.Excluded }, + ) if targetMf == nil { return "", nil, mdlerrors.NewNotFound("microflow", name.String()) @@ -540,15 +536,13 @@ func describeNanoflowToString(ctx *ExecContext, name ast.QualifiedName) (string, microflowNames[nf.ID] = h.GetQualifiedName(nf.ContainerID, nf.Name) } - var targetNf *microflows.Nanoflow - for _, nf := range allNanoflows { - modID := h.FindModuleID(nf.ContainerID) - modName := h.GetModuleName(modID) - if modName == name.Module && nf.Name == name.Name { - targetNf = nf - break - } - } + // Describe the live nanoflow, not an excluded twin of the same name (#914). + targetNf, _ := pickLive(allNanoflows, + func(nf *microflows.Nanoflow) bool { + return h.GetModuleName(h.FindModuleID(nf.ContainerID)) == name.Module && nf.Name == name.Name + }, + func(nf *microflows.Nanoflow) bool { return nf.Excluded }, + ) if targetNf == nil { return "", nil, mdlerrors.NewNotFound("nanoflow", name.String()) diff --git a/mdl/executor/cmd_nanoflows_create.go b/mdl/executor/cmd_nanoflows_create.go index 9ecc4c8d0..ad7b4e5d1 100644 --- a/mdl/executor/cmd_nanoflows_create.go +++ b/mdl/executor/cmd_nanoflows_create.go @@ -48,21 +48,29 @@ func execCreateNanoflow(ctx *ExecContext, s *ast.CreateNanoflowStmt) error { var existingContainerID model.ID var existingAllowedRoles []model.ID preserveAllowedRoles := false + // Excluded is model state, not script state: an absent @excluded must not + // clear a stored exclusion (#914). + existingExcluded := false existingNanoflows, err := ctx.Backend.ListNanoflows() if err != nil { return mdlerrors.NewBackend("check existing nanoflows", err) } - for _, existing := range existingNanoflows { - if existing.Name == s.Name.Name && getModuleID(ctx, existing.ContainerID) == module.ID { - if !s.CreateOrModify { - return mdlerrors.NewAlreadyExistsMsg("nanoflow", s.Name.Module+"."+s.Name.Name, "nanoflow '"+s.Name.Module+"."+s.Name.Name+"' already exists (use create or modify to overwrite)") - } - existingID = existing.ID - existingContainerID = existing.ContainerID - existingAllowedRoles = cloneRoleIDs(existing.AllowedModuleRoles) - preserveAllowedRoles = true - break + // A module may hold several nanoflows with this name as long as all but one + // are excluded, so target the live one rather than whichever comes first. + if existing, ok := pickLive(existingNanoflows, + func(n *microflows.Nanoflow) bool { + return n.Name == s.Name.Name && getModuleID(ctx, n.ContainerID) == module.ID + }, + func(n *microflows.Nanoflow) bool { return n.Excluded }, + ); ok { + if !s.CreateOrModify { + return mdlerrors.NewAlreadyExistsMsg("nanoflow", s.Name.Module+"."+s.Name.Name, "nanoflow '"+s.Name.Module+"."+s.Name.Name+"' already exists (use create or modify to overwrite)") } + existingID = existing.ID + existingContainerID = existing.ContainerID + existingAllowedRoles = cloneRoleIDs(existing.AllowedModuleRoles) + preserveAllowedRoles = true + existingExcluded = existing.Excluded } // For CREATE OR REPLACE/MODIFY, reuse the existing ID to preserve references @@ -93,7 +101,7 @@ func execCreateNanoflow(ctx *ExecContext, s *ast.CreateNanoflowStmt) error { Name: s.Name.Name, Documentation: s.Documentation, MarkAsUsed: false, - Excluded: s.Excluded, + Excluded: s.Excluded || existingExcluded, } if preserveAllowedRoles { nf.AllowedModuleRoles = existingAllowedRoles diff --git a/mdl/executor/cmd_pages_create_v3.go b/mdl/executor/cmd_pages_create_v3.go index e4af80621..7427a0ffd 100644 --- a/mdl/executor/cmd_pages_create_v3.go +++ b/mdl/executor/cmd_pages_create_v3.go @@ -8,6 +8,7 @@ import ( "github.com/mendixlabs/mxcli/mdl/ast" mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/pages" ) // ============================================================================ @@ -41,19 +42,43 @@ func execCreatePageV3(ctx *ExecContext, s *ast.CreatePageStmtV3) error { var pagesToDelete []model.ID var existingAllowedRoles []model.ID preserveAllowedRoles := false + // Mendix allows several pages in one module to share a name as long as all + // but one are excluded, so the replace set is the LIVE pages only. An + // excluded twin is a different document that the app does not render: it is + // neither rewritten nor deleted here, and its exclusion is carried forward + // when it is the only match (#914). + existingExcluded := false + var excludedMatches []*pages.Page + matches := 0 for _, p := range existingPages { modID := getModuleID(ctx, p.ContainerID) modName := getModuleName(ctx, modID) - if modName == s.Name.Module && p.Name == s.Name.Name { - if !s.IsReplace && !s.IsModify && len(pagesToDelete) == 0 { - return mdlerrors.NewAlreadyExists("page", s.Name.String()) - } - if len(pagesToDelete) == 0 { - existingAllowedRoles = cloneRoleIDs(p.AllowedRoles) - preserveAllowedRoles = true - } - pagesToDelete = append(pagesToDelete, p.ID) + if modName != s.Name.Module || p.Name != s.Name.Name { + continue + } + matches++ + if !s.IsReplace && !s.IsModify && matches == 1 { + return mdlerrors.NewAlreadyExists("page", s.Name.String()) + } + if p.Excluded { + excludedMatches = append(excludedMatches, p) + continue + } + if len(pagesToDelete) == 0 { + existingAllowedRoles = cloneRoleIDs(p.AllowedRoles) + preserveAllowedRoles = true } + pagesToDelete = append(pagesToDelete, p.ID) + } + if len(pagesToDelete) == 0 && len(excludedMatches) > 0 { + // Every page with this name is excluded: rewrite the first of them in + // place and keep it excluded, rather than adding an active page the + // script never asked to un-exclude. + p := excludedMatches[0] + existingAllowedRoles = cloneRoleIDs(p.AllowedRoles) + preserveAllowedRoles = true + existingExcluded = true + pagesToDelete = append(pagesToDelete, p.ID) } // Build the page BEFORE deleting the old one (atomic: if build fails, old page is preserved) @@ -75,6 +100,7 @@ func execCreatePageV3(ctx *ExecContext, s *ast.CreatePageStmtV3) error { if err != nil { return mdlerrors.NewBackend("build page", err) } + page.Excluded = page.Excluded || existingExcluded if preserveAllowedRoles { page.AllowedRoles = existingAllowedRoles } else if len(page.AllowedRoles) == 0 { @@ -126,15 +152,31 @@ func execCreateSnippetV3(ctx *ExecContext, s *ast.CreateSnippetStmtV3) error { // Check if snippet already exists - collect ALL duplicates existingSnippets, _ := ctx.Backend.ListSnippets() var snippetsToDelete []model.ID + // As for pages: an excluded twin of this name is a separate document that + // Mendix allows, so it is neither replaced nor deleted, and an exclusion is + // carried forward when every match is excluded (#914). + existingExcluded := false + var excludedSnippets []*pages.Snippet + matches := 0 for _, snip := range existingSnippets { modID := getModuleID(ctx, snip.ContainerID) modName := getModuleName(ctx, modID) - if modName == s.Name.Module && snip.Name == s.Name.Name { - if !s.IsReplace && !s.IsModify && len(snippetsToDelete) == 0 { - return mdlerrors.NewAlreadyExists("snippet", s.Name.String()) - } - snippetsToDelete = append(snippetsToDelete, snip.ID) + if modName != s.Name.Module || snip.Name != s.Name.Name { + continue + } + matches++ + if !s.IsReplace && !s.IsModify && matches == 1 { + return mdlerrors.NewAlreadyExists("snippet", s.Name.String()) } + if snip.Excluded { + excludedSnippets = append(excludedSnippets, snip) + continue + } + snippetsToDelete = append(snippetsToDelete, snip.ID) + } + if len(snippetsToDelete) == 0 && len(excludedSnippets) > 0 { + existingExcluded = true + snippetsToDelete = append(snippetsToDelete, excludedSnippets[0].ID) } // Build the snippet BEFORE deleting the old one (atomic: if build fails, old snippet is preserved) @@ -156,6 +198,7 @@ func execCreateSnippetV3(ctx *ExecContext, s *ast.CreateSnippetStmtV3) error { if err != nil { return mdlerrors.NewBackend("build snippet", err) } + snippet.Excluded = snippet.Excluded || existingExcluded // Delete old snippets only after successful build for _, id := range snippetsToDelete { diff --git a/mdl/executor/cmd_published_rest.go b/mdl/executor/cmd_published_rest.go index 1036554fc..efb552fd7 100644 --- a/mdl/executor/cmd_published_rest.go +++ b/mdl/executor/cmd_published_rest.go @@ -168,12 +168,14 @@ func findPublishedRestService(ctx *ExecContext, moduleName, name string) (*model if err != nil { return nil, err } - for _, svc := range services { - modID := h.FindModuleID(svc.ContainerID) - modName := h.GetModuleName(modID) - if modName == moduleName && svc.Name == name { - return svc, nil - } + // Prefer the live service over an excluded twin of the same name (#914). + if svc, ok := pickLive(services, + func(svc *model.PublishedRestService) bool { + return h.GetModuleName(h.FindModuleID(svc.ContainerID)) == moduleName && svc.Name == name + }, + func(svc *model.PublishedRestService) bool { return svc.Excluded }, + ); ok { + return svc, nil } return nil, mdlerrors.NewNotFound("published rest service", moduleName+"."+name) } @@ -225,6 +227,8 @@ func execCreatePublishedRestService(ctx *ExecContext, s *ast.CreatePublishedRest if existing != nil { svc.ID = existing.ID svc.AllowedRoles = existing.AllowedRoles + // Excluded is model state, not script state (#914). + svc.Excluded = existing.Excluded } for _, resDef := range s.Resources { diff --git a/mdl/executor/cmd_queues.go b/mdl/executor/cmd_queues.go index 76600c65b..d69901413 100644 --- a/mdl/executor/cmd_queues.go +++ b/mdl/executor/cmd_queues.go @@ -27,14 +27,16 @@ func findQueue(ctx *ExecContext, moduleName, name string) *types.Queue { if err != nil { return nil } - for _, q := range queues { - if !strings.EqualFold(q.Name, name) { - continue - } - mod := h.GetModuleName(h.FindModuleID(q.ContainerID)) - if strings.EqualFold(mod, moduleName) { - return q - } + // Prefer the live queue: a module may hold an excluded twin of this name + // (#914), and the app only has the active one. + if q, ok := pickLive(queues, + func(q *types.Queue) bool { + return strings.EqualFold(q.Name, name) && + strings.EqualFold(h.GetModuleName(h.FindModuleID(q.ContainerID)), moduleName) + }, + func(q *types.Queue) bool { return q.Excluded }, + ); ok { + return q } return nil } @@ -74,6 +76,10 @@ func execCreateQueue(ctx *ExecContext, s *ast.CreateQueueStmt) error { if existing != nil { q.ID = existing.ID + // Excluded is model state, not script state: a rewrite must not clear a + // stored exclusion (#914). MDL cannot express it for a queue, so the + // stored value is always the one that survives. + q.Excluded = existing.Excluded if err := ctx.Backend.UpdateQueue(q); err != nil { return mdlerrors.NewBackend("update queue", err) } diff --git a/mdl/executor/cmd_workflows_write.go b/mdl/executor/cmd_workflows_write.go index 95222f38e..6f6d24959 100644 --- a/mdl/executor/cmd_workflows_write.go +++ b/mdl/executor/cmd_workflows_write.go @@ -53,19 +53,25 @@ func execCreateWorkflow(ctx *ExecContext, s *ast.CreateWorkflowStmt) error { } var existingID model.ID - for _, existing := range existingWorkflows { - modID := h.FindModuleID(existing.ContainerID) - modName := h.GetModuleName(modID) - if modName == s.Name.Module && existing.Name == s.Name.Name { - if !s.CreateOrModify { - return mdlerrors.NewAlreadyExistsMsg("workflow", s.Name.Module+"."+s.Name.Name, "workflow '"+s.Name.Module+"."+s.Name.Name+"' already exists (use create or modify to overwrite)") - } - existingID = existing.ID - break + // Excluded is model state, not script state, and a module may hold an + // excluded twin of this name — target the live workflow and carry its + // exclusion forward (#914). + existingExcluded := false + if existing, ok := pickLive(existingWorkflows, + func(w *workflows.Workflow) bool { + return h.GetModuleName(h.FindModuleID(w.ContainerID)) == s.Name.Module && w.Name == s.Name.Name + }, + func(w *workflows.Workflow) bool { return w.Excluded }, + ); ok { + if !s.CreateOrModify { + return mdlerrors.NewAlreadyExistsMsg("workflow", s.Name.Module+"."+s.Name.Name, "workflow '"+s.Name.Module+"."+s.Name.Name+"' already exists (use create or modify to overwrite)") } + existingID = existing.ID + existingExcluded = existing.Excluded } wf := &workflows.Workflow{} + wf.Excluded = existingExcluded wf.ContainerID = module.ID wf.Name = s.Name.Name wf.Documentation = s.Documentation diff --git a/mdl/executor/excluded_docs.go b/mdl/executor/excluded_docs.go new file mode 100644 index 000000000..1c362d56a --- /dev/null +++ b/mdl/executor/excluded_docs.go @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +// Excluded documents ("Exclude from project" in Studio Pro) make a document +// name non-unique. Mendix allows two documents in one module to share a name as +// long as at most one of them is active: mxbuild reports CE0122 "Duplicate +// document name" only for the active ones. Measured on 11.13.0 — two microflows +// named MyFirstModule.Calc with one Excluded build at 0 errors; the same pair +// with both active is 1 error, CE0122. +// +// Two consequences for every by-name lookup and every rewrite: +// +// 1. A name is not a unique key, so picking the first match is order-dependent. +// It can resolve to a document the app does not contain — DESCRIBE then +// shows the wrong body, and CREATE OR REPLACE edits the wrong document +// while the live one silently keeps its old behaviour. +// 2. Excluded belongs to the model, not to the MDL script. A rebuild that does +// not carry it forward turns an excluded document active, which makes a +// valid project fail CE0122 (guard-don't-drop, ADR-0005) — a build broken by +// a statement that never mentioned exclusion. +// +// pickLive addresses (1). preserveExcluded (each call site, one line next to the +// ID/roles it already preserves) addresses (2). +// +// Issue #914. + +// pickLive returns the document the running app uses: the active (non-excluded) +// match when there is one, otherwise the first match so that a lookup against an +// all-excluded set still resolves rather than reporting "not found". +// +// matches selects the candidates (typically name + module); excluded reports a +// candidate's Excluded flag. The bool result is false only when nothing matched. +func pickLive[T any](items []T, matches func(T) bool, excluded func(T) bool) (T, bool) { + var first T + found := false + for _, it := range items { + if !matches(it) { + continue + } + if !excluded(it) { + return it, true + } + if !found { + first, found = it, true + } + } + return first, found +} + +// pickLiveIndex is pickLive for call sites that need the position rather than +// the element (slice mutation in place, or a parallel slice). +func pickLiveIndex[T any](items []T, matches func(T) bool, excluded func(T) bool) int { + first := -1 + for i, it := range items { + if !matches(it) { + continue + } + if !excluded(it) { + return i + } + if first < 0 { + first = i + } + } + return first +} diff --git a/mdl/executor/excluded_docs_test.go b/mdl/executor/excluded_docs_test.go new file mode 100644 index 000000000..fec5ba953 --- /dev/null +++ b/mdl/executor/excluded_docs_test.go @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// TestPickLive covers the selector every by-name lookup now goes through. +// The order cases are the point: before #914 the lookups took the first match, +// so which document they resolved to depended on enumeration order rather than +// on which one the app contains. +func TestPickLive(t *testing.T) { + type doc struct { + name string + excl bool + } + match := func(want string) func(doc) bool { + return func(d doc) bool { return d.name == want } + } + excluded := func(d doc) bool { return d.excl } + + t.Run("excluded first", func(t *testing.T) { + got, ok := pickLive([]doc{{"A", true}, {"A", false}}, match("A"), excluded) + if !ok || got.excl { + t.Fatalf("want the live document, got %+v (ok=%v)", got, ok) + } + }) + + t.Run("live first", func(t *testing.T) { + got, ok := pickLive([]doc{{"A", false}, {"A", true}}, match("A"), excluded) + if !ok || got.excl { + t.Fatalf("want the live document, got %+v (ok=%v)", got, ok) + } + }) + + t.Run("all excluded falls back to the first match", func(t *testing.T) { + got, ok := pickLive([]doc{{"B", true}, {"A", true}, {"A", true}}, match("A"), excluded) + if !ok || !got.excl { + t.Fatalf("an all-excluded set must still resolve, got %+v (ok=%v)", got, ok) + } + }) + + t.Run("no match", func(t *testing.T) { + if _, ok := pickLive([]doc{{"B", false}}, match("A"), excluded); ok { + t.Fatal("no match must report false") + } + }) +} + +// microflowWriteProbe wires a backend whose ListMicroflows returns stored and +// captures whatever CreateMicroflow is handed. +func microflowWriteProbe(t *testing.T, stored []*microflows.Microflow, moduleID model.ID) (*ExecContext, **microflows.Microflow) { + t.Helper() + var written *microflows.Microflow + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListModulesFunc: func() ([]*model.Module, error) { + return []*model.Module{{ + BaseElement: model.BaseElement{ID: moduleID}, + Name: "MyModule", + }}, nil + }, + GetModuleByNameFunc: func(name string) (*model.Module, error) { + if name != "MyModule" { + return nil, nil + } + return &model.Module{BaseElement: model.BaseElement{ID: moduleID}, Name: "MyModule"}, nil + }, + ListMicroflowsFunc: func() ([]*microflows.Microflow, error) { return stored, nil }, + CreateMicroflowFunc: func(mf *microflows.Microflow) error { + written = mf + return nil + }, + // An existing document is rewritten through UpdateMicroflow, which is + // the path both #914 halves run down. + UpdateMicroflowFunc: func(mf *microflows.Microflow) error { + written = mf + return nil + }, + } + ctx, _ := newMockCtx(t, withBackend(mb)) + return ctx, &written +} + +// TestCreateOrModifyMicroflow_PreservesStoredExclusion is the #914 write half: +// a statement that never mentions exclusion must not clear one. Before the fix +// the rebuild wrote Excluded=false from the AST default, which turned an +// excluded document active — and with a same-named live twin present that is +// CE0122 "Duplicate document name", i.e. a valid project made unbuildable by a +// statement that only meant to edit a body. +func TestCreateOrModifyMicroflow_PreservesStoredExclusion(t *testing.T) { + const moduleID = model.ID("module-1") + stored := []*microflows.Microflow{{ + BaseElement: model.BaseElement{ID: "mf-excluded"}, + ContainerID: moduleID, + Name: "Calc", + Excluded: true, + }} + ctx, written := microflowWriteProbe(t, stored, moduleID) + + stmt := &ast.CreateMicroflowStmt{ + Name: ast.QualifiedName{Module: "MyModule", Name: "Calc"}, + CreateOrModify: true, + } + if err := execCreateMicroflow(ctx, stmt); err != nil { + t.Fatalf("CREATE OR MODIFY MICROFLOW failed: %v", err) + } + if *written == nil { + t.Fatal("no microflow was written") + } + if !(*written).Excluded { + t.Error("a rewrite cleared the stored Excluded flag; the document would become active and collide with its twin (CE0122)") + } +} + +// TestCreateOrModifyMicroflow_TargetsLiveTwin is the #914 read half: with two +// documents of the same name, the write must land on the one the app contains. +// The excluded twin is deliberately first so that a first-match lookup picks +// the wrong document — which is exactly what happened before the fix, leaving +// the live microflow silently unchanged. +func TestCreateOrModifyMicroflow_TargetsLiveTwin(t *testing.T) { + const moduleID = model.ID("module-1") + stored := []*microflows.Microflow{ + { + BaseElement: model.BaseElement{ID: "mf-excluded"}, + ContainerID: moduleID, + Name: "Calc", + Excluded: true, + }, + { + BaseElement: model.BaseElement{ID: "mf-live"}, + ContainerID: moduleID, + Name: "Calc", + }, + } + ctx, written := microflowWriteProbe(t, stored, moduleID) + + stmt := &ast.CreateMicroflowStmt{ + Name: ast.QualifiedName{Module: "MyModule", Name: "Calc"}, + CreateOrModify: true, + } + if err := execCreateMicroflow(ctx, stmt); err != nil { + t.Fatalf("CREATE OR MODIFY MICROFLOW failed: %v", err) + } + if *written == nil { + t.Fatal("no microflow was written") + } + if got := (*written).ID; got != "mf-live" { + t.Errorf("write targeted %q, want the live document %q", got, "mf-live") + } + if (*written).Excluded { + t.Error("the live document must not inherit the excluded twin's flag") + } +} diff --git a/mdl/types/java.go b/mdl/types/java.go index ae98ed410..bce6e88a4 100644 --- a/mdl/types/java.go +++ b/mdl/types/java.go @@ -12,6 +12,9 @@ type JavaAction struct { ContainerID model.ID `json:"containerId"` Name string `json:"name"` Documentation string `json:"documentation,omitempty"` + // Excluded mirrors Studio Pro's "Exclude from project". Reads must supply + // it so a rewrite can carry it forward instead of clearing it (#914). + Excluded bool `json:"excluded,omitempty"` } // GetName returns the Java action's name. diff --git a/mdl/types/mapping.go b/mdl/types/mapping.go index 5600175a7..a71c279f6 100644 --- a/mdl/types/mapping.go +++ b/mdl/types/mapping.go @@ -47,7 +47,10 @@ type ImageCollection struct { Name string `json:"name"` ExportLevel string `json:"exportLevel,omitempty"` Documentation string `json:"documentation,omitempty"` - Images []Image `json:"images,omitempty"` + // Excluded mirrors Studio Pro's "Exclude from project". Reads must supply + // it so a rewrite can carry it forward instead of clearing it (#914). + Excluded bool `json:"excluded,omitempty"` + Images []Image `json:"images,omitempty"` } // GetName returns the image collection's name. diff --git a/model/types.go b/model/types.go index 3ed465321..36f174118 100644 --- a/model/types.go +++ b/model/types.go @@ -196,10 +196,14 @@ func (c *Constant) GetContainerID() ID { // Enumeration represents an enumeration type. type Enumeration struct { BaseElement - ContainerID ID `json:"containerId"` - Name string `json:"name"` - Documentation string `json:"documentation,omitempty"` - Values []EnumerationValue `json:"values,omitempty"` + ContainerID ID `json:"containerId"` + Name string `json:"name"` + Documentation string `json:"documentation,omitempty"` + // Excluded mirrors Studio Pro's "Exclude from project". It is model state + // that MDL cannot express for an enumeration, so every write path must + // carry the stored value rather than default it to false (#914). + Excluded bool `json:"excluded,omitempty"` + Values []EnumerationValue `json:"values,omitempty"` } // GetName returns the enumeration's name. diff --git a/sdk/mpr/parser_enumeration.go b/sdk/mpr/parser_enumeration.go index 964ae548e..75a74dffb 100644 --- a/sdk/mpr/parser_enumeration.go +++ b/sdk/mpr/parser_enumeration.go @@ -33,6 +33,11 @@ func (r *Reader) parseEnumeration(unitID, containerID string, contents []byte) ( if doc, ok := raw["Documentation"].(string); ok { enum.Documentation = doc } + // Excluded must survive a read→rebuild→write cycle; defaulting it to false + // un-excludes the document on the next CREATE OR MODIFY (#914). + if excl, ok := raw["Excluded"].(bool); ok { + enum.Excluded = excl + } // Parse values - array may start with a version number, skip non-map elements if values, ok := raw["Values"].(bson.A); ok { diff --git a/sdk/mpr/parser_misc.go b/sdk/mpr/parser_misc.go index 07f5258f7..bf687a621 100644 --- a/sdk/mpr/parser_misc.go +++ b/sdk/mpr/parser_misc.go @@ -72,6 +72,10 @@ func (r *Reader) parseSnippet(unitID, containerID string, contents []byte) (*pag if doc, ok := raw["Documentation"].(string); ok { snippet.Documentation = doc } + // Excluded must survive read→rebuild→write (#914). + if excl, ok := raw["Excluded"].(bool); ok { + snippet.Excluded = excl + } if entityID := extractID(raw["Entity"]); entityID != "" { snippet.EntityID = model.ID(entityID) } @@ -151,6 +155,11 @@ func (r *Reader) parseJavaAction(unitID, containerID string, contents []byte) (* if doc, ok := raw["Documentation"].(string); ok { ja.Documentation = doc } + // Excluded must survive read→rebuild→write; defaulting it to false + // un-excludes the document on the next CREATE OR MODIFY (#914). + if excl, ok := raw["Excluded"].(bool); ok { + ja.Excluded = excl + } return ja, nil } diff --git a/sdk/mpr/writer_enumeration.go b/sdk/mpr/writer_enumeration.go index 6789db1e8..c03f861f2 100644 --- a/sdk/mpr/writer_enumeration.go +++ b/sdk/mpr/writer_enumeration.go @@ -130,7 +130,7 @@ func (w *Writer) serializeEnumeration(enum *model.Enumeration) ([]byte, error) { {Key: "$Type", Value: "Enumerations$Enumeration"}, {Key: "Name", Value: enum.Name}, {Key: "Documentation", Value: enum.Documentation}, - {Key: "Excluded", Value: false}, + {Key: "Excluded", Value: enum.Excluded}, {Key: "ExportLevel", Value: "Hidden"}, {Key: "RemoteSource", Value: nil}, {Key: "Values", Value: values}, diff --git a/sdk/pages/pages.go b/sdk/pages/pages.go index 3077e986b..71076ad5c 100644 --- a/sdk/pages/pages.go +++ b/sdk/pages/pages.go @@ -77,13 +77,16 @@ const ( // Snippet represents a reusable page snippet. type Snippet struct { model.BaseElement - ContainerID model.ID `json:"containerId"` - Name string `json:"name"` - Documentation string `json:"documentation,omitempty"` - EntityID model.ID `json:"entityId,omitempty"` - Parameters []*SnippetParameter `json:"parameters,omitempty"` - Variables []*LocalVariable `json:"variables,omitempty"` - Widgets []Widget `json:"widgets,omitempty"` + ContainerID model.ID `json:"containerId"` + Name string `json:"name"` + Documentation string `json:"documentation,omitempty"` + // Excluded mirrors Studio Pro's "Exclude from project". Reads must supply + // it so a rewrite can carry it forward instead of clearing it (#914). + Excluded bool `json:"excluded,omitempty"` + EntityID model.ID `json:"entityId,omitempty"` + Parameters []*SnippetParameter `json:"parameters,omitempty"` + Variables []*LocalVariable `json:"variables,omitempty"` + Widgets []Widget `json:"widgets,omitempty"` } // GetName returns the snippet's name. From cefa3b829ad8cb1fc531bc1c75f47f49bcda35b7 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 18 Aug 2026 07:14:04 +0000 Subject: [PATCH 35/35] fix(microflows): keep a replaced microflow's StartEvent position A describe -> exec round-trip moved the start. A Studio-Pro-authored flow whose StartEvent sat at 145;200 came back at 100;200, while every other coordinate in it -- five activities, a split and two end events -- survived exactly. It was the one piece of a hand-laid-out microflow mxcli could not preserve. The start has no MDL statement to annotate and DESCRIBE cannot emit its position, so the builder always derives one: (first annotated activity X) minus one spacing unit, 260 - 160 = 100. Studio Pro's 145 is not derivable from anything in the description. Carried over from the microflow being replaced instead, the way the folder and the allowed module roles already are on CREATE OR MODIFY. Nil on a fresh CREATE, which keeps the existing derivation, so only a rebuild of an existing flow changes. Reading the stored flow is best-effort: a backend that cannot answer yields the derived placement rather than failing the statement. Verified on the full coordinate set rather than the start alone -- every RelativeMiddlePoint in Administration.SaveNewAccount dumped before and after, diff empty. Control with the carry-over stubbed reproduces the 100;200 drift. Re-running the round-trip script is a no-op, with the MXCLI_ALWAYS_WRITE=1 control confirming the comparison detects writes. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013uQvFDd5R4eNqqita59jM8 --- .claude/skills/fix-issue.md | 1 + CHANGELOG.md | 4 ++ mdl/executor/cmd_microflows_builder.go | 9 ++- mdl/executor/cmd_microflows_builder_graph.go | 9 ++- mdl/executor/cmd_microflows_create.go | 30 ++++++++- mdl/executor/start_event_position_test.go | 69 ++++++++++++++++++++ 6 files changed, 119 insertions(+), 3 deletions(-) create mode 100644 mdl/executor/start_event_position_test.go diff --git a/.claude/skills/fix-issue.md b/.claude/skills/fix-issue.md index ac6da4774..1270edb8a 100644 --- a/.claude/skills/fix-issue.md +++ b/.claude/skills/fix-issue.md @@ -540,3 +540,4 @@ extracting `OffsetExpression`/`LimitExpression`. | `mxcli check` reports **MDL003** *"microflow returns X but not all code paths have a return statement"* on a microflow that is correct and builds cleanly. Since `exec` began refusing scripts whose checks report an error, the script also will not run without `--no-check` — so a false positive became a hard block. Seven shipped `mdl-examples` scripts were in this state | The microflow uses the documented `RETURNS T AS $Var` form and assigns `$Var` instead of writing an explicit `RETURN`. `buildFlowGraph` sets the final EndEvent's `ReturnValue` to `"$"+Var` **whenever the AS clause is present** (`cmd_microflows_builder_graph.go`), so the return is synthesized either way — that is what the clause is for. The check demanded a `RETURN` statement in the source on top of it | `mdl/executor/validate_microflow.go` (Check 5 in `microflowValidator.validate`) | Skip MDL003 when `returnType.Variable != ""`, mirroring the builder's own condition exactly rather than inventing a second rule. **Verify against mxbuild, not intuition**: the flagged microflow builds with **0 errors** on 11.13.0, `describe microflow` renders the synthesized `return $Found;`, and the stored EndEvent carries a `ReturnValue` — three independent confirmations the check was wrong, not the script. Keep a companion test for the no-AS-clause case, or a fix that simply disables the rule passes. **Sweeping examples as a control**: exclude `*.test.mdl` (those are `mxcli test` files with `@test`/`@expect` blocks, unparseable by `check` by design) and `*.fail.mdl`, and diff failure SETS rather than counts | | A guard that walks a stored BSON document works on the **modelsdk** engine and is a **silent no-op on legacy** (or vice versa) — same project, same document, opposite behaviour. Here `checkNoQueuedCalls` refused a rewrite that would drop a task-queue binding under modelsdk, and let it through under `--engine legacy`, destroying the binding exactly as the guard existed to prevent | The two readers return **different Go shapes for the same BSON**: `modelsdk` yields `[]interface{}` for arrays, the legacy `mpr` reader yields **`bson.A`**. `bson.A` is a NAMED slice type (`type A []interface{}`), so a type switch `case []any:` does **not** match it and the walk never descends into `ObjectCollection.Objects` | `mdl/executor/validate_queued_calls.go` (`queuedCallTargets`, `storedQueueRetries`); the same blind spot exists in `mdl/backend/mcp/page.go`, `mdl/backend/mcp/page_mutator.go`, `mdl/executor/widget_sync_apply.go` | Add `case bson.A:` alongside `case []any:` in every raw-BSON walk. The house pattern already does this (`mdl/backend/bsonnav`, `mdl/catalog/builder_pages.go`, `mdl/xpathrefs/scan.go` and ~7 more) — the ones missing it are the outliers: `for f in $(grep -rln "case \[\]any:" --include=*.go mdl/); do grep -q "bson.A" $f \|\| echo $f; done`. **Test both shapes of the same document in one test**, not just the one your engine returns. **A unit test alone is not enough to find this**: it only surfaced by running the same command under `MXCLI_ENGINE=legacy` and diffing against modelsdk — engine parity has to be exercised, not assumed. Reported as FINDINGS #25 | | A generated loop/while box does not fit its contents: activities sit outside the container, or the box is far wider than what is in it. An explicit `@position` on a loop child moves the child but not the box | The container's `Size` came from `measureStatementsSpan`, a pre-pass over the AST run BEFORE the body was built, so it was a function of statement COUNT alone. Varying only the children's positions changed nothing: 2 activities at x=150/310, at x=1500/2000 and at x=160/170 all produced `480;160`, and in the second case both children sat entirely outside their own container. Nothing catches it — the model carries no geometry rules, so `mx check` is silent | `mdl/executor/layout.go` (`containerBounds`, `fitContainerSize`), called from `addLoopStatement` / `addWhileStatement` in `cmd_microflows_builder_control.go` | Size the box AFTER the body is built, from the real child bounding box (`Position ± Size/2`). Nesting needs no extra code — an inner loop is sized when its own `addLoopStatement` returns, so the outer measures a correct inner box. **Do NOT translate the children to fit**, however tempting: their positions round-trip through DESCRIBE as `@position`, so moving them makes a describe→exec cycle store different coordinates than it read, which under ADR-0008 turns a quiet re-run into a write. Grow the box instead. **Two measurement traps hit while verifying this.** A helper that sets an annotation via `ast.StatementAnnotations` is a silent no-op when the field is nil (it returns nil rather than allocating) — the explicit-`@position` test passed while testing nothing until the field was assigned directly. And an idempotence check over `mprcontents` proved blind because the re-run script began with `create module`/`create entity` and aborted at "already exists" before reaching the microflow; the `MXCLI_ALWAYS_WRITE=1` control is what exposed it, so never assert "nothing changed" without it. Repro `mdl-examples/bug-tests/container-autosize-884.mdl`. Issue #884 problem 1 | +| A `describe microflow` → `exec` round-trip moves the **StartEvent**: a Studio-Pro-authored flow whose start sat at `145;200` comes back at `100;200`, while every other coordinate in the flow survives exactly | The start has no MDL statement to annotate and DESCRIBE cannot emit its position, so the builder always derives one — `(first annotated activity X) − spacing`, i.e. `260 − 160 = 100`. Studio Pro's `145` is not derivable from anything in the description | `mdl/executor/cmd_microflows_create.go` (`storedStartPosition`), `mdl/executor/cmd_microflows_builder.go` (`flowBuilder.startPosition`), `mdl/executor/cmd_microflows_builder_graph.go` (the StartEvent construction) | Carry the position over from the microflow being replaced, the way the folder and the allowed module roles already are on CREATE OR MODIFY — read it off the stored document and let it win over the derived value. Nil on a fresh CREATE keeps the existing derivation, so only a rebuild of an existing flow is affected. Best-effort: a backend that cannot read the stored flow yields the derived placement rather than failing the statement. **Verify with the whole coordinate set, not just the start** — dump every `RelativeMiddlePoint` before and after and diff, or a fix that pins the start while shifting something else reads as success. Control: stub the `fb.startPosition` branch and the `100;200` drift returns. Issue #884 follow-up | diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a7ad0816..29d8ffd81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +### Fixed + +- **A microflow's StartEvent no longer moves on a describe→exec round-trip** — the start has no MDL statement to annotate and `DESCRIBE` cannot emit its position, so the builder always derived one (first annotated activity minus one spacing unit). A Studio-Pro-authored flow whose start sat at `145;200` came back at `100;200` — the only coordinate in it that did not survive. The position is now carried over from the microflow being replaced, the way the folder and allowed module roles already are; a fresh `CREATE` still derives it. + ### Added - **`MPR011` — loop child containment** — a new lint rule flagging a microflow activity positioned outside the loop container that holds it. This is the only automated check for the condition: the Mendix model carries no geometry rules, so such a flow passes `mx check` with zero errors and builds and runs normally, it is just drawn wrong when opened. It catches the condition however it arrives — a project written by an older mxcli, hand-edited in Studio Pro, or a future layout regression. Nested loops are checked in their own coordinate space, and an unpositioned child at the origin is skipped rather than flagged. diff --git a/mdl/executor/cmd_microflows_builder.go b/mdl/executor/cmd_microflows_builder.go index 9d29f0be9..c35a486ff 100644 --- a/mdl/executor/cmd_microflows_builder.go +++ b/mdl/executor/cmd_microflows_builder.go @@ -50,7 +50,14 @@ type flowBuilder struct { // curveByOrigin holds each statement's @curve keyed by the activity it was // written on, applied to that activity's outgoing flows by applyFlowCurves // once the graph is complete. (#884) - curveByOrigin map[model.ID]*ast.FlowCurve + curveByOrigin map[model.ID]*ast.FlowCurve + // startPosition is the StartEvent position read off the microflow being + // REPLACED, carried over so a hand-laid-out start survives the rebuild. The + // start has no MDL statement to annotate and DESCRIBE cannot emit it, so + // without this a describe→exec round-trip silently moved it (a Studio Pro + // flow's 145;200 became 100;200). Nil on a fresh CREATE, where the position + // is derived from the first annotated activity as before. + startPosition *model.Point backend backend.FullBackend // For looking up page/microflow references hierarchy *ContainerHierarchy // For resolving container IDs to module names pendingAnnotations *ast.ActivityAnnotations // Pending annotations to attach to next activity diff --git a/mdl/executor/cmd_microflows_builder_graph.go b/mdl/executor/cmd_microflows_builder_graph.go index ebcd0149e..b36b378eb 100644 --- a/mdl/executor/cmd_microflows_builder_graph.go +++ b/mdl/executor/cmd_microflows_builder_graph.go @@ -49,10 +49,17 @@ func (fb *flowBuilder) buildFlowGraph(stmts []ast.MicroflowStatement, returns *a } // Create StartEvent - Position is the CENTER point (RelativeMiddlePoint in Mendix) + // A position carried over from the microflow being replaced wins: the start + // has no statement to annotate, so a rebuild would otherwise move a + // hand-laid-out one to the derived spot. + startX, startY := fb.posX, fb.posY + if fb.startPosition != nil { + startX, startY = fb.startPosition.X, fb.startPosition.Y + } startEvent := µflows.StartEvent{ BaseMicroflowObject: microflows.BaseMicroflowObject{ BaseElement: model.BaseElement{ID: model.ID(types.GenerateID())}, - Position: model.Point{X: fb.posX, Y: fb.posY}, + Position: model.Point{X: startX, Y: startY}, Size: model.Size{Width: EventSize, Height: EventSize}, }, } diff --git a/mdl/executor/cmd_microflows_create.go b/mdl/executor/cmd_microflows_create.go index 18783e561..7c93e2aa1 100644 --- a/mdl/executor/cmd_microflows_create.go +++ b/mdl/executor/cmd_microflows_create.go @@ -258,7 +258,14 @@ func execCreateMicroflow(ctx *ExecContext, s *ast.CreateMicroflowStmt) error { restServices, _ := loadRestServices(ctx) builder := &flowBuilder{ - posX: 200, + // Carry over the StartEvent position of the microflow being replaced. + // The start has no MDL statement to annotate and DESCRIBE cannot emit + // it, so a rebuild would otherwise move a hand-laid-out one to the + // derived spot — a Studio Pro flow's 145;200 became 100;200 on a + // describe→exec round-trip, the only coordinate in it that did not + // survive. Preserved the way the folder and allowed roles already are. + startPosition: storedStartPosition(ctx, existingID), + posX: 200, posY: 200, baseY: 200, // Base Y for happy path spacing: HorizontalSpacing, @@ -304,3 +311,24 @@ func execCreateMicroflow(ctx *ExecContext, s *ast.CreateMicroflowStmt) error { invalidateHierarchy(ctx) return nil } + +// storedStartPosition reads the StartEvent position off the microflow being +// replaced, or nil for a fresh CREATE (where the position is derived from the +// first annotated activity). Best-effort: a backend that cannot read the flow +// yields the derived placement rather than failing the statement. +func storedStartPosition(ctx *ExecContext, existingID model.ID) *model.Point { + if existingID == "" || ctx.Backend == nil { + return nil + } + mf, err := ctx.Backend.GetMicroflow(existingID) + if err != nil || mf == nil || mf.ObjectCollection == nil { + return nil + } + for _, o := range mf.ObjectCollection.Objects { + if se, ok := o.(*microflows.StartEvent); ok { + p := se.GetPosition() + return &p + } + } + return nil +} diff --git a/mdl/executor/start_event_position_test.go b/mdl/executor/start_event_position_test.go new file mode 100644 index 000000000..bfa7b95f6 --- /dev/null +++ b/mdl/executor/start_event_position_test.go @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: Apache-2.0 + +// A microflow's StartEvent position is not expressible in MDL and is not +// captured by DESCRIBE, so a describe→exec round-trip used to move it: a +// Studio-Pro-authored flow whose start sat at 145;200 came back at 100;200, +// because the builder derives the start as (first annotated activity X − +// spacing) and 145 is not derivable from 260. +// +// Every other coordinate in that flow round-trips exactly, so this was the one +// piece of a hand-laid-out microflow mxcli could not preserve. It is preserved +// the way the folder and the allowed module roles already are on CREATE OR +// MODIFY: read off the stored document and carried over. +package executor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +func startEventOf(objects []microflows.MicroflowObject) *microflows.StartEvent { + for _, o := range objects { + if se, ok := o.(*microflows.StartEvent); ok { + return se + } + } + return nil +} + +func buildFlowWithStart(t *testing.T, start *model.Point) *microflows.StartEvent { + t.Helper() + fb := &flowBuilder{ + posX: 100, posY: 200, baseY: 200, spacing: HorizontalSpacing, + varTypes: map[string]string{}, + declaredVars: map[string]string{}, + measurer: &layoutMeasurer{}, + startPosition: start, + } + stmt := &ast.LogStmt{ + Message: &ast.LiteralExpr{Kind: ast.LiteralString, Value: "x"}, + Annotations: &ast.ActivityAnnotations{Position: &ast.Position{X: 260, Y: 200}}, + } + fb.buildFlowGraph([]ast.MicroflowStatement{stmt}, nil) + se := startEventOf(fb.objects) + if se == nil { + t.Fatal("no StartEvent in the built flow") + } + return se +} + +// The stored position wins over the derived one. +func TestStartEvent_PreservesStoredPosition(t *testing.T) { + se := buildFlowWithStart(t, &model.Point{X: 145, Y: 200}) + if se.Position.X != 145 || se.Position.Y != 200 { + t.Errorf("StartEvent at %d;%d, want 145;200 — a hand-laid-out start must survive a rebuild", + se.Position.X, se.Position.Y) + } +} + +// With nothing stored (a fresh CREATE) the derived placement is unchanged: +// one spacing unit left of the first annotated activity. +func TestStartEvent_DerivesWhenNothingStored(t *testing.T) { + se := buildFlowWithStart(t, nil) + if want := 260 - HorizontalSpacing; se.Position.X != want { + t.Errorf("StartEvent at %d, want %d (derived)", se.Position.X, want) + } +}