From 3c84e3558cded76318cdb6d408785ca9e9110871 Mon Sep 17 00:00:00 2001 From: Jacek Tomaszewski Date: Mon, 12 Oct 2026 12:01:42 +0200 Subject: [PATCH 1/2] feat(mssql): a SQL Server engine family (P4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit treeman gives every Conductor worktree its own Postgres database and a clone per test worker. A project whose source of truth is SQL Server got nothing: one shared container, hardcoded ids, and a test tier that two worktrees — or two jest workers — cannot run at once. SQL Server has no `CREATE DATABASE … TEMPLATE`, and the two things that look like one are not: `DBCC CLONEDATABASE` copies schema and statistics without rows, and a database snapshot is read-only. The only copy a client can drive is BACKUP + RESTORE, so a template here is a pair — the template database, which is what treeman's cache-hit probe asks about for every engine, plus one `.bak` beside it that each clone restores from. Captured once, hydrated N times. Everything runs over TDS, file paths included: `TO DISK`, `FROM DISK` and `MOVE … TO` all take variables, so only database names are interpolated, through the same strict `[A-Za-z0-9_]` validator the other engines use. Three engine differences worth knowing: * Restored files carry a random token rather than the database's name. The spare claim is `ALTER DATABASE … MODIFY NAME`, which moves the database but not its files, so a claimed spare keeps files named for the spare — and naming by database would collide on the next restore of that freed name. * Drops take the database single-user first. There is no pg_terminate_backend, and DROP fails outright while anyone is connected, which a jest worker's idle pool routinely is. * `dump:` is rejected, with the reason in the message: RESTORE reads a path on the server, which for a containerised server is a different filesystem. Seed from `seed:` instead, which is a command and can get a file there. `db_dump` refuses for the same reason. `WriteWatermark` declines rather than guessing. The caller SKIPS a capture when the token is unchanged, and SQL Server's per-database write counters are per index — writes to a heap move nothing — so a token that can miss a write is worse than none. Verified against SQL Server 2022 and 2025 (Developer): 12 e2e tests covering the prepare cycle end to end and the driver's own behaviour — eight concurrent restores of one template, the collation reaching the databases treeman creates, LIKE-escaping against the sibling-wipe, the rename race having exactly one winner, a drop with a live connection open, and a restore after the backup file was removed behind the template's back. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/fsh-vet.yml | 15 + FORK.md | 25 +- README.md | 23 +- cmd/treeman/cmd/doctor_engines.go | 6 + cmd/treeman/cmd/misc.go | 10 +- docs/cli.md | 6 +- docs/config-reference.md | 13 +- docs/configuration.md | 14 +- docs/internals.md | 35 +- docs/mcp-tools.md | 2 +- e2e/README.md | 5 +- e2e/mssql/docker-compose.yml | 31 ++ e2e/mssql/driver_test.go | 444 +++++++++++++++++++ e2e/mssql/fixtures/migrate.sh | 49 ++ e2e/mssql/fixtures/migrations/001_init.sql | 8 + e2e/mssql/mssql_test.go | 278 ++++++++++++ go.mod | 12 +- go.sum | 56 ++- internal/config/config.go | 118 ++++- internal/config/connections.go | 18 +- internal/config/connections_test.go | 3 + internal/config/dsn_test.go | 33 ++ internal/config/validate.go | 31 +- internal/db/engineconn/capabilities_test.go | 4 +- internal/db/engineconn/engineconn.go | 51 +++ internal/db/engineconn/engineconn_test.go | 5 +- internal/db/engineconn/registry_drivers.go | 21 + internal/db/ident/mssql.go | 15 + internal/db/mssql/mssql.go | 436 ++++++++++++++++++ internal/db/mssql/mssql_test.go | 180 ++++++++ internal/db/mssql/rowcount.go | 98 ++++ internal/db/mssql/snapshot.go | 306 +++++++++++++ internal/db/reachability/reachability.go | 2 + internal/engine/engine.go | 4 + internal/engine/engine_test.go | 3 + internal/initgen/initgen.go | 13 +- internal/mcp/server.go | 4 +- internal/mcp/tools_engine.go | 191 +++++++- internal/mcp/tools_write.go | 4 +- internal/prepare/branchreap.go | 3 + internal/prepare/branchstate.go | 49 ++ internal/prepare/coldbuild_invariant_test.go | 3 +- internal/prepare/prepare.go | 249 +++++++++++ internal/resolve/resolve.go | 103 +++++ internal/schema/schema.go | 1 + internal/snapshot/orphans.go | 46 +- justfile | 16 +- schemas/treeman.global.schema.json | 75 ++++ schemas/treeman.schema.json | 75 ++++ 49 files changed, 3091 insertions(+), 101 deletions(-) create mode 100644 e2e/mssql/docker-compose.yml create mode 100644 e2e/mssql/driver_test.go create mode 100755 e2e/mssql/fixtures/migrate.sh create mode 100644 e2e/mssql/fixtures/migrations/001_init.sql create mode 100644 e2e/mssql/mssql_test.go create mode 100644 internal/db/ident/mssql.go create mode 100644 internal/db/mssql/mssql.go create mode 100644 internal/db/mssql/mssql_test.go create mode 100644 internal/db/mssql/rowcount.go create mode 100644 internal/db/mssql/snapshot.go diff --git a/.github/workflows/fsh-vet.yml b/.github/workflows/fsh-vet.yml index 163ce726..4b1cf03e 100644 --- a/.github/workflows/fsh-vet.yml +++ b/.github/workflows/fsh-vet.yml @@ -56,7 +56,22 @@ jobs: # Files we are allowed to add or change on top of upstream. Anything # else means we have started editing his tree — which is a decision, # not an accident, so it has to be made here, in this list. + # + # P4, the SQL Server driver, is the first entry that edits upstream's + # own files rather than only adding ours: an engine family touches the + # engine registry, the config schema, prepare, resolve and the docs. + # The FORK.md row carries the why and the drop-when; this list carries + # the blast radius. Keep the two in step. allowed='^(FORK\.md|PATCHES\.md|\.fsh/|\.github/workflows/fsh-)' + allowed="$allowed"'|^(internal/db/mssql/|e2e/mssql/|internal/db/ident/mssql\.go)' + allowed="$allowed"'|^(go\.mod|go\.sum|justfile|README\.md|docs/|schemas/|e2e/README\.md)' + allowed="$allowed"'|^internal/(engine/engine(_test)?\.go|schema/schema\.go)' + allowed="$allowed"'|^internal/config/(config|connections|validate|connections_test|dsn_test)\.go' + allowed="$allowed"'|^internal/db/(engineconn/|reachability/reachability\.go)' + allowed="$allowed"'|^internal/prepare/(prepare|branchstate|branchreap|coldbuild_invariant_test)\.go' + allowed="$allowed"'|^internal/(resolve/resolve\.go|snapshot/orphans\.go|initgen/initgen\.go)' + allowed="$allowed"'|^internal/mcp/(tools_engine|tools_write|server)\.go' + allowed="$allowed"'|^cmd/treeman/cmd/(misc|doctor_engines)\.go' base="$(git merge-base HEAD upstream/master)" unexpected="$(git diff --name-only "$base"..HEAD | grep -Ev "$allowed" || true)" if [ -n "$unexpected" ]; then diff --git a/FORK.md b/FORK.md index 16fe59c7..aa2f100c 100644 --- a/FORK.md +++ b/FORK.md @@ -11,6 +11,11 @@ Our diff against upstream is **new files**, plus the smallest possible edits to We do **not** delete the parts we don't use (the MCP server, the TUI, the MySQL/Mongo drivers, `treeman git`). Dead code costs disk; a 20k-line deletion costs us every future merge. +A new engine family is the one shape that cannot honour "new files only" — the engine registry, +the config schema, prepare, resolve and the generated docs all have to learn it. The patch index +below names each such patch and `.github/workflows/fsh-vet.yml` carries the matching allowlist, so +the blast radius is written down in both places rather than discovered during a merge. + - `master` — untouched mirror of upstream. Never commit here; it only fast-forwards. - `fsh` — our branch, and the one we release from. Upstream arrives by **pull request** (`fsh-sync.yml`), never by an unattended rebase, so somebody reads the delta before we ship it. @@ -23,14 +28,14 @@ We do **not** delete the parts we don't use (the MCP server, the TUI, the MySQL/ been abused — it is the same mutable-tag pattern most repos have, ours included — but it means a published binary is not cryptographically tied to the source. Building it ourselves makes that question moot. -2. **We need a Meilisearch driver**, which upstream has no support for and 7 of our 8 local - stacks require. +2. **We need drivers upstream does not have** — Meilisearch, which 7 of our 8 local stacks + require, and SQL Server, which one client's whole ERP is. 3. **Two safety fixes we would rather not wait for** (see the patch index). Everything in (1) is fixable upstream with ~5 lines (`actions/attest-build-provenance`, pinning `xilo@v1`, a `dist/` gitignore entry). We intend to offer those. **If upstream takes them, this -fork's reason to exist shrinks to the Meilisearch driver, and we should say so here and shrink -it.** A fork that outlives its justification is just technical debt with a remote. +fork's reason to exist shrinks to the drivers, and we should say so here and shrink it.** A fork +that outlives its justification is just technical debt with a remote. ## What we actually run @@ -45,7 +50,7 @@ Every patch is one commit, and one row here. A row without a **Drop when** condi | # | Patch | Why | Upstream | Drop when | |---|---|---|---|---| -| — | _none yet_ | the branch currently carries only the additive files below | — | — | +| P4 | SQL Server (`mssql`) engine family | Groomershop's ERP is Subiekt GT on SQL Server. Its test tier runs against one shared container with hardcoded ids, so two worktrees — or two jest workers — cannot run it at once; Postgres gets per-worktree databases and per-worker clones from treeman, SQL Server got nothing. Upstream supports Postgres/MySQL/Mongo/Redis/ES-OpenSearch/S3 and has a clean driver interface to extend | offer once it has run a few weeks for us | upstream merges it | Planned, not yet written: @@ -68,6 +73,9 @@ Planned, not yet written: | `.fsh/upstream-osv.json` | baseline of upstream's known-vulnerable dependencies | | `.fsh/osv.py` | writes that baseline and compares against it | | `.fsh/socket/go.mod` | a manifest nothing builds, so Socket scores upstream's own code — see [Vetting](#vetting) | +| `internal/db/mssql/` | the SQL Server driver (P4) | +| `internal/db/ident/mssql.go` | bracket-quoting for SQL Server identifiers (P4) | +| `e2e/mssql/` | its docker-tier suite: the prepare cycle, plus the driver's clone path, spare claim and rename race (P4) | ## Vetting @@ -97,6 +105,13 @@ source imports no `x/crypto` package, so reachability is doubtful, and a `go get clear all 37). A check that is red on day one teaches everyone to ignore it. These go red only when something *moves*. +The one dependency we have added is `github.com/microsoft/go-mssqldb` (P4). It carries no OSV +advisory of its own, and it drags `golang.org/x/crypto` from 0.33.0 up to 0.55.0 — which clears 34 +of the 37 advisories that single stale indirect contributes to the baseline. The three that remain +on 0.55.0 are all baselined already, so the deps check stays green without a re-baseline. Its +Azure, Key Vault and Kerberos modules are in its `go.mod` but not in ours: they belong to packages +we do not import, and module-graph pruning keeps them out of `go.sum`. + Two honest limits, so nobody mistakes a green run for an audit: - **Scorecard measures practices, not intent.** 3.3/10 means an attack on this project would not diff --git a/README.md b/README.md index c0c0e1ff..e0e4a919 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,10 @@ databases, search indices, and parallel test clones for each git worktree — torn down on delete, kept in sync as your migrations and fixtures change. Language- and framework-agnostic. -Pure wire-protocol DB access (Go `database/sql` for MySQL + -PostgreSQL, the official Mongo / Redis / Elasticsearch SDKs) — no -shelling out to `mysql` / `psql` / `mongosh` / `redis-cli`. A single +Pure wire-protocol DB access (Go `database/sql` for MySQL, +PostgreSQL + SQL Server, the official Mongo / Redis / Elasticsearch +SDKs) — no shelling out to `mysql` / `psql` / `sqlcmd` / `mongosh` / +`redis-cli`. A single user-mode daemon, a thin CLI client, and a SQLite-backed event log. --- @@ -28,14 +29,24 @@ commands you type. |---|---|---| | MySQL | MariaDB, TiDB | Separate database | | PostgreSQL | — | Separate database | +| SQL Server | — | Separate database | | MongoDB | — | Separate database | | Redis | Valkey, DragonflyDB | Key-prefix in DB 0 (cluster-safe, no 16-DB cap) | | Elasticsearch | OpenSearch | Index-name prefix | Variants are first-class aliases — declare `engine: mariadb`, `tidb`, -`valkey`, `dragonfly`, or `opensearch` and treeman routes it to the -parent engine's driver (Valkey/DragonflyDB ride the Redis wire -protocol). +`sqlserver`, `valkey`, `dragonfly`, or `opensearch` and treeman routes +it to the parent engine's driver (Valkey/DragonflyDB ride the Redis +wire protocol). + +SQL Server clones through BACKUP + RESTORE, the only database copy a +client can drive there — so a template is the template database plus +one `.bak` file beside it, captured once and restored per clone. Two +consequences worth knowing before you declare one: the connection +needs BACKUP/RESTORE rights (`sysadmin` in practice, since restoring a +database that does not yet exist is sysadmin-only), and `dump:` is +rejected, because RESTORE reads a path on the *server* — load a dump +from `seed:` instead. ## Features diff --git a/cmd/treeman/cmd/doctor_engines.go b/cmd/treeman/cmd/doctor_engines.go index 9a65d91a..5378ea47 100644 --- a/cmd/treeman/cmd/doctor_engines.go +++ b/cmd/treeman/cmd/doctor_engines.go @@ -81,6 +81,12 @@ func engineProbes(cfg *config.Config) []engineProbe { engineProbe{name: "postgres", ref: p.ContainerRef, host: orDefault(p.Host, "127.0.0.1"), port: orDefaultPort(p.Port, 5432)}, ) } + if ms := c.Mssql; ms != nil { + out = append( + out, + engineProbe{name: "mssql", ref: ms.ContainerRef, host: orDefault(ms.Host, "127.0.0.1"), port: orDefaultPort(ms.Port, 1433)}, + ) + } if mg := c.Mongodb; mg != nil { out = append(out, engineProbe{name: "mongodb", ref: mg.ContainerRef, url: mg.URI}) } diff --git a/cmd/treeman/cmd/misc.go b/cmd/treeman/cmd/misc.go index 8646c040..5e54aa59 100644 --- a/cmd/treeman/cmd/misc.go +++ b/cmd/treeman/cmd/misc.go @@ -409,7 +409,7 @@ func DbCmd() *cli.Command { &cli.StringFlag{Name: "repo", Aliases: []string{"r"}}, &cli.StringFlag{ Name: "engine", - Usage: "restrict the reset to one engine family (mysql, postgres, mongodb, redis, elasticsearch; aliases like mariadb/postgresql/valkey/dragonfly accepted)", + Usage: "restrict the reset to one engine family (mysql, postgres, mssql, mongodb, redis, elasticsearch; aliases like mariadb/postgresql/sqlserver/valkey/dragonfly accepted)", }, &cli.BoolFlag{Name: "json"}, &cli.BoolFlag{ @@ -452,7 +452,7 @@ func DbCmd() *cli.Command { &cli.StringFlag{Name: "repo", Aliases: []string{"r"}}, &cli.StringFlag{ Name: "engine", - Usage: "restrict the save to one engine family (mysql, postgres, mongodb, redis, elasticsearch; aliases like mariadb/postgresql/valkey/dragonfly accepted)", + Usage: "restrict the save to one engine family (mysql, postgres, mssql, mongodb, redis, elasticsearch; aliases like mariadb/postgresql/sqlserver/valkey/dragonfly accepted)", }, &cli.BoolFlag{Name: "json"}, &cli.BoolFlag{ @@ -710,6 +710,7 @@ func ConfigCmd() *cli.Command { _, _ = fmt.Fprintln(ui.Out, "# resolved connections") printResolved("mysql", r.Mysql) printResolved("postgres", r.Postgres) + printResolved("mssql", r.Mssql) printResolved("mongodb", r.Mongodb) printResolved("redis", r.Redis) printResolved("elasticsearch", r.Elasticsearch) @@ -1478,7 +1479,7 @@ func InitCmd() *cli.Command { }, &cli.StringFlag{ Name: "engine", - Usage: "comma-separated engines to activate in databases: (e.g. mysql,postgres,redis) — emits minimal blocks when detection found no framework", + Usage: "comma-separated engines to activate in databases: (e.g. mysql,postgres,mssql,redis) — emits minimal blocks when detection found no framework", }, &cli.BoolFlag{ Name: "interactive", @@ -1637,11 +1638,12 @@ func pickEngines() ([]string, error) { items := []string{ "mysql — MySQL / MariaDB / TiDB", "postgres — PostgreSQL", + "mssql — Microsoft SQL Server", "mongodb — MongoDB", "redis — Redis / Valkey / Dragonfly", "elasticsearch — Elasticsearch / OpenSearch", } - values := []string{"mysql", "postgres", "mongodb", "redis", "elasticsearch"} + values := []string{"mysql", "postgres", "mssql", "mongodb", "redis", "elasticsearch"} res, err := tui.Select(items, tui.Options{ Prompt: "engines to activate in databases: (space marks, enter confirms, esc = none)", Values: values, diff --git a/docs/cli.md b/docs/cli.md index 796e7033..c0133d6f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -523,7 +523,7 @@ re-sync branch_scoped databases from the live base branch (defaults to the cwd's | Flag | Usage | |---|---| | `-r`, `--repo` | | -| `--engine` | restrict the reset to one engine family (mysql, postgres, mongodb, redis, elasticsearch; aliases like mariadb/postgresql/valkey/dragonfly accepted) | +| `--engine` | restrict the reset to one engine family (mysql, postgres, mssql, mongodb, redis, elasticsearch; aliases like mariadb/postgresql/sqlserver/valkey/dragonfly accepted) | | `--json` | | | `--dry-run` | list the branch_scoped namespaces that would be dropped + re-seeded; change nothing, connect nowhere | | `-f`, `--wait`, `--foreground` | stream the daemon's live progress and block until done (default: dispatch and return) | @@ -535,7 +535,7 @@ capture branch_scoped databases into the current branch's durable copies without | Flag | Usage | |---|---| | `-r`, `--repo` | | -| `--engine` | restrict the save to one engine family (mysql, postgres, mongodb, redis, elasticsearch; aliases like mariadb/postgresql/valkey/dragonfly accepted) | +| `--engine` | restrict the save to one engine family (mysql, postgres, mssql, mongodb, redis, elasticsearch; aliases like mariadb/postgresql/sqlserver/valkey/dragonfly accepted) | | `--json` | | | `-f`, `--wait`, `--foreground` | stream the daemon's live progress and block until done (default: dispatch and return) | @@ -866,7 +866,7 @@ print the slug treeman derives for a worktree | `--force` | | | `--json` | | | `--global` | scaffold the user-global ~/.config/treeman/config.yaml (machine-wide defaults) instead of a per-repo .treeman.yaml | -| `--engine` | comma-separated engines to activate in databases: (e.g. mysql,postgres,redis) — emits minimal blocks when detection found no framework | +| `--engine` | comma-separated engines to activate in databases: (e.g. mysql,postgres,mssql,redis) — emits minimal blocks when detection found no framework | | `--interactive` | pick engines from a terminal picker instead of --engine (declines gracefully when stdin is not a TTY) | ### `treeman doctor` diff --git a/docs/config-reference.md b/docs/config-reference.md index 0be6cb06..932ab398 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -57,6 +57,7 @@ daemon: connections: mysql: '...' postgres: '...' + mssql: '...' mongodb: '...' redis: '...' elasticsearch: '...' @@ -154,6 +155,7 @@ include: connections: mysql: '...' postgres: '...' + mssql: '...' mongodb: '...' redis: '...' elasticsearch: '...' @@ -530,6 +532,11 @@ create clones, dump templates. PostgreSQL connection. Required when any `databases:` entry uses `engine: postgres` (or the `postgresql` alias). +#### `mssql` *(one of: [MssqlConn](#mssqlconn), map of name → [MssqlConn](#mssqlconn))* + +Microsoft SQL Server connection. Required when any `databases:` +entry uses `engine: mssql` (or the `sqlserver` alias). + #### `mongodb` *(one of: [MongoConn](#mongoconn), map of name → [MongoConn](#mongoconn))* MongoDB connection. URI form (`mongodb://...`); host/port get @@ -662,7 +669,7 @@ for `redis` (same wire protocol, same key-prefix scoping). The rendered schema's enum is injected from engine.Known by schema.Reflect, so the alias list lives in exactly one place. -_Allowed: `mysql`, `mariadb`, `tidb`, `postgres`, `postgresql`, `mongodb`, `redis`, `valkey`, `dragonfly`, `elasticsearch`, `opensearch`, `s3`, `sqlite`, `duckdb`_ +_Allowed: `mysql`, `mariadb`, `tidb`, `postgres`, `postgresql`, `mssql`, `sqlserver`, `mongodb`, `redis`, `valkey`, `dragonfly`, `elasticsearch`, `opensearch`, `s3`, `sqlite`, `duckdb`_ #### `connection` *(string)* @@ -1091,6 +1098,10 @@ declared count are a config error caught at load time. mongodb connection — bare URL string OR structured object. +### MssqlConn + +Microsoft SQL Server connection — bare DSN string OR structured object. + ### MysqlConn MySQL connection — bare DSN string OR structured object. diff --git a/docs/configuration.md b/docs/configuration.md index edd5df1c..aef461eb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -56,17 +56,24 @@ connections: port: 5432 user: postgres password: $PGPASSWORD + mssql: + host: 127.0.0.1 + port: 1433 + user: sa + password: $MSSQL_SA_PASSWORD + collation: Polish_CI_AS # CREATE DATABASE collation; default is the server's own + backup_dir: /var/opt/mssql/data # ON THE SERVER: where template .bak files go mongodb: { uri: "mongodb://127.0.0.1:27017" } redis: { url: "redis://127.0.0.1:6379" } elasticsearch: { url: "http://127.0.0.1:9200" } databases: - - engine: mysql # mysql|mariadb|tidb|postgres|postgresql|mongodb|redis|elasticsearch|opensearch + - engine: mysql # mysql|mariadb|tidb|postgres|postgresql|mssql|sqlserver|mongodb|redis|elasticsearch|opensearch name_template: "myapp_testing_{slug}" dump: { path: storage/dumps/seed.sql.gz } - fanout: 0 # 0 = safe per-engine default (mysql 4, pg GOMAXPROCS, mongo 6, es 8). + fanout: 0 # 0 = safe per-engine default (mysql 4, pg GOMAXPROCS, mssql 4, mongo 6, es 8). # raise only if the server is provisioned (max_connections bumped, etc.). - prewarm: 0 # Postgres only: keep N spare clones pre-restored from the cached + prewarm: 0 # Postgres, MySQL, SQL Server: keep N spare clones pre-restored from the cached # template; a cache-hit create claims one via ALTER DATABASE … RENAME # (milliseconds, size-independent) instead of a full template copy. # The pool replenishes in the background and is dropped with its @@ -493,6 +500,7 @@ Supported flavours per engine: |---|---| | MySQL | `DB_TEST_*` → `DB_*` (Laravel); `MYSQL_URL`; `SPRING_DATASOURCE_URL` (jdbc:mysql://…) + `SPRING_DATASOURCE_{USERNAME,PASSWORD}`; `MYSQL_PASSWORD` / `MYSQL_PWD` | | Postgres | `DB_TEST_*` → `DB_*` (Laravel); `DATABASE_URL`; `PG*` (`PGHOST`, `PGPORT`, `PGUSER`, `PGPASSWORD`); `SPRING_DATASOURCE_URL` (jdbc:postgresql://…) | +| SQL Server | `DB_TEST_*` → `DB_*` (Laravel); `MSSQL_URL`; `DATABASE_URL` (`sqlserver://…`); `SPRING_DATASOURCE_URL` (jdbc:sqlserver://…); `MSSQL_SA_PASSWORD` / `SA_PASSWORD` | | MongoDB | `MONGODB_URI` / `MONGO_URL` / `DATABASE_URL`; `MONGO_DB_HOST` + `MONGO_DB_PORT` composed; `SPRING_DATA_MONGODB_*` | | Redis | `REDIS_URL`; `REDIS_HOST` + `REDIS_PORT` + `REDIS_PASSWORD` composed; literal `null` / `(null)` passwords skipped | | Elasticsearch | `ELASTICSEARCH_URL` / `ELASTICSEARCH_HOSTS` (auto-prefixes `http://` if missing) | diff --git a/docs/internals.md b/docs/internals.md index 9537374e..038ab4e9 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -106,18 +106,41 @@ in `registry_drivers.go`). Everything else — `engineconn.Connect`, `engineconn.Configured`, prepare, teardown, GC, and stale-worktree recovery — is registry-driven and picks a new engine up for free. -Checklist for a new engine family (≤4 touchpoints, #47): +Checklist for a new engine family (#47): 1. **Driver package** `internal/db/`: dial + the `Conn` primitives (Drop/Exists/ListMatching/DropMatching/…). 2. **Registry entry** in `engineconn/registry_drivers.go`: a - `Configured` field-check and a `Connect` dialer. -3. **Config**: a `ConnectionsConfig` field, a `databases[].engine` - alias pair in `engine.Known` + `engine.Canonical`, and a - `validate.go` rule if the family can't support some modes. + `Configured` field-check and a `Connect` dialer, plus the + `Conn` wrapper in `engineconn.go` and — if the driver can + claim a pre-warmed spare — the `SnapshotCreator`/`SpareClaimer` + methods and a `config.PrewarmCapableEngines` entry. +3. **Config**: a `ConnectionsConfig` field + its `*Named` map, a + `databases[].engine` alias pair in `engine.Known` + + `engine.Canonical`, the family arms in `connections.go` + (`UnmarshalYAML`, `Resolve`, `familyShape`, `hasNamedBlock`, + `namedNames`), `schema.connFamilyDefs`, and a `validate.go` rule if + the family can't support some modes. 4. **Extras** (only what applies): a patcher driver if config files need rewriting, an `initgen` preset if a framework pairs with it, - docker-tier e2e under `e2e/`. + docker-tier e2e under `e2e/` (registered in the justfile's + `BATCHES[engines]`). + +The registry covers prepare, teardown, GC and recovery. Four places +switch on the CONCRETE adapter type instead and need an arm of their +own for a name-scoped engine — a missing one fails silently, not +loudly: + +- `prepare.go`'s `prepareFamily` plus a `prepare` and + `ColdBuildSteps` (and the `fanOutLimits` / + `innerConnsPerRestore` entries that size its fan-out); +- `branchstate.go`'s `NS` adapter and `connectBranchEngine` + arm, or `branch_scoped` quietly no-ops; +- `branchreap.go`'s `untrackedMarker` entry and + `listDurableCandidates` arm, or untracked durables leak forever; +- `resolve.go`'s `Resolved` field, `ApplyEnvCredentials` copy-back and + container-env password filler, or a `password: $NAME` ref resolves + to empty. The jsonschema `engine` enum is GENERATED from `engine.Known` (`just sync-schema`) — adding an alias cannot half-land. History: diff --git a/docs/mcp-tools.md b/docs/mcp-tools.md index eb9d24da..bdd4527f 100644 --- a/docs/mcp-tools.md +++ b/docs/mcp-tools.md @@ -57,7 +57,7 @@ through the `tools` gateway (`action=list` / `action=enable`). | `db_query` | | Read OR write a configured engine directly — no shelling out to mysql/psql/mongosh/redis-cli. | | `db_reset` | | Reset a worktree's branch_scoped DBs to the base branch's data — drops the active namespace + current branch's durable copy, then re-seeds from the parent. | | `db_save` | | Capture a worktree's branch_scoped active namespaces into the CURRENT branch's durable copies without switching branches — a manual checkpoint of the swap lifecycle's capture half. | -| `db_schema_dump` | | Dump the live schema for ONE database — mysql/postgres: CREATE TABLEs, mongo: collection list + samples, ES: index mapping, redis: SCAN-driven key summary. | +| `db_schema_dump` | | Dump the live schema for ONE database — mysql/postgres: CREATE TABLEs, mssql: tables + columns, mongo: collection list + samples, ES: index mapping, redis: SCAN-driven key summary. | ## doctor diff --git a/e2e/README.md b/e2e/README.md index 35f11493..944aa495 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -17,8 +17,8 @@ First run pulls ~6 GB of images and compiles ~2 GB of Rust deps (sqlx-cli + diesel_cli). Subsequent runs are container-cached. Tests run sequentially against the host's docker daemon — each suite -binds a unique host port (13306–15562, 16380, 19200–19201, 27117–27118) -so you don't need `-p 1`. +binds a unique host port (11433, 13306–15562, 16380, 19200–19201, +27117–27118) so you don't need `-p 1`. ## Coverage matrix @@ -28,6 +28,7 @@ so you don't need `-p 1`. |---|---|---| | `mysql/` | MySQL 8.4 | Cold build → cache hit → cold rebuild on input edit | | `postgres/` | Postgres 16 | Same cycle | +| `mssql/` | SQL Server 2022 | Same cycle, plus the driver's BACKUP/RESTORE clone path, the rename-based spare claim and its race | | `mongo/` | MongoDB 7 | Seed → snapshot → cache hit cycle | | `redis/` | Redis 7 | Prefix-keyed template + fingerprint change | | `elasticsearch/` | ES 8.15 | NDJSON `_bulk` dump → snapshot → fanout | diff --git a/e2e/mssql/docker-compose.yml b/e2e/mssql/docker-compose.yml new file mode 100644 index 00000000..ffb267e2 --- /dev/null +++ b/e2e/mssql/docker-compose.yml @@ -0,0 +1,31 @@ +# SQL Server e2e fixture. Port 11433 is outside common conflicts so a +# developer's local instance on 1433 doesn't block the test run. +# +# MSSQL_MEMORY_LIMIT_MB matters: SQL Server's default is "all of it", +# which on a 2-CPU/7 GB runner shared with a Go build is how the kernel +# ends up choosing between them. Developer edition because Express caps +# a database at 10 GB and refuses backup compression. +services: + mssql: + image: mcr.microsoft.com/mssql/server:2022-latest + container_name: treeman-e2e-mssql + environment: + ACCEPT_EULA: "Y" + MSSQL_SA_PASSWORD: "treemanE2E!pw1" + MSSQL_PID: Developer + MSSQL_MEMORY_LIMIT_MB: "2048" + ports: + - "11433:1433" + healthcheck: + # -C trusts the self-signed certificate the server generates for + # itself; without it sqlcmd 18 refuses to connect and the + # healthcheck never passes. + test: + [ + "CMD-SHELL", + "/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P \"$$MSSQL_SA_PASSWORD\" -C -Q 'SELECT 1' || exit 1", + ] + interval: 3s + timeout: 5s + retries: 40 + start_period: 20s diff --git a/e2e/mssql/driver_test.go b/e2e/mssql/driver_test.go new file mode 100644 index 00000000..399efcfa --- /dev/null +++ b/e2e/mssql/driver_test.go @@ -0,0 +1,444 @@ +//go:build e2e + +// Driver-level SQL Server checks: the clone lifecycle against a real +// server, which is the only place the engine's own behaviour can be +// observed. A mocked driver here would re-assert what we already +// believe — and the BACKUP/RESTORE clone path, the rename-based spare +// claim and the LIKE-escaping guard are exactly where that belief has +// somewhere to be wrong. +package mssql_e2e + +import ( + "context" + "fmt" + "sync" + "testing" + + "github.com/stubbedev/treeman/e2e/harness" + "github.com/stubbedev/treeman/internal/config" + dbmssql "github.com/stubbedev/treeman/internal/db/mssql" +) + +// driverConn is the connection the driver tests dial. Collation is set +// so the EnsureDB path is covered. +func driverConn() config.MssqlConn { + return config.MssqlConn{ + Host: "127.0.0.1", Port: 11433, User: "sa", Password: password, + PoolMax: 8, Collation: "Polish_CI_AS", + } +} + +const ( + drvPrefix = "tmdrv_" + drvSource = drvPrefix + "src" + drvTemplate = "_tm_" + "deadbeefdeadbeef" +) + +func connectDriver(t *testing.T) (*dbmssql.Driver, context.Context) { + t.Helper() + harness.SkipIfNoDocker(t) + t.Cleanup(harness.ComposeUp(t, harness.MustAbs("."))) + waitForServer(t) + ctx := context.Background() + d, err := dbmssql.Connect(ctx, driverConn()) + if err != nil { + t.Fatalf("connect: %v", err) + } + t.Cleanup(func() { _ = d.Close() }) + cleanup := func() { + if _, err := d.DropMatching(ctx, drvPrefix); err != nil { + t.Logf("cleanup DropMatching: %v", err) + } + if err := d.DropSnapshot(ctx, drvTemplate); err != nil { + t.Logf("cleanup DropSnapshot: %v", err) + } + } + cleanup() + t.Cleanup(cleanup) + return d, ctx +} + +// seedSource builds a source database with the things a clone is +// easiest to get wrong about: rows in a non-dbo schema, a heap (no +// index at all), a scalar function, and non-ASCII text. +func seedSource(t *testing.T, d *dbmssql.Driver, ctx context.Context) { + t.Helper() + if err := d.EnsureDB(ctx, drvSource); err != nil { + t.Fatalf("EnsureDB: %v", err) + } + // Twice: EnsureDB is called on every cold build and must be a no-op + // on the second. + if err := d.EnsureDB(ctx, drvSource); err != nil { + t.Fatalf("EnsureDB (second call): %v", err) + } + scoped, err := d.OpenScoped(ctx, drvSource) + if err != nil { + t.Fatalf("OpenScoped: %v", err) + } + defer func() { _ = scoped.Close() }() + for _, q := range []string{ + "CREATE SCHEMA app", + "CREATE TABLE app.orders (id int IDENTITY PRIMARY KEY, note nvarchar(100))", + "CREATE TABLE dbo.heap (v nvarchar(50))", + "CREATE FUNCTION dbo.answer() RETURNS int AS BEGIN RETURN 42 END", + "INSERT INTO app.orders(note) VALUES (N'zażółć'), (N'gęślą'), (N'jaźń'), (N'x'), (N'y')", + "INSERT INTO dbo.heap(v) VALUES (N'a'), (N'b')", + } { + if _, err := scoped.ExecContext(ctx, q); err != nil { + t.Fatalf("seed %q: %v", q, err) + } + } +} + +// assertClone proves a clone is a copy of the source rather than a +// same-shaped empty: the rows, the scalar function body, and the +// non-ASCII text round-tripped through BACKUP/RESTORE. +func assertClone(t *testing.T, d *dbmssql.Driver, ctx context.Context, name string) { + t.Helper() + scoped, err := d.OpenScoped(ctx, name) + if err != nil { + t.Errorf("OpenScoped %s: %v", name, err) + return + } + defer func() { _ = scoped.Close() }() + var rows, answer int + var note string + if err := scoped.QueryRowContext(ctx, "SELECT COUNT(*) FROM app.orders").Scan(&rows); err != nil { + t.Errorf("%s count: %v", name, err) + return + } + if err := scoped.QueryRowContext(ctx, "SELECT dbo.answer()").Scan(&answer); err != nil { + t.Errorf("%s function: %v", name, err) + return + } + if err := scoped.QueryRowContext(ctx, "SELECT note FROM app.orders WHERE id = 1").Scan(¬e); err != nil { + t.Errorf("%s note: %v", name, err) + return + } + if rows != 5 || answer != 42 || note != "zażółć" { + t.Errorf("%s: rows=%d answer=%d note=%q — want 5, 42, %q", name, rows, answer, note, "zażółć") + } +} + +// TestSnapshotCloneLifecycle is the core contract: capture once, +// hydrate many, concurrently, from one template. +func TestSnapshotCloneLifecycle(t *testing.T) { + d, ctx := connectDriver(t) + seedSource(t, d, ctx) + + if err := d.SnapshotCreate(ctx, drvSource, drvTemplate); err != nil { + t.Fatalf("SnapshotCreate: %v", err) + } + // The template is a real database, because DatabaseExists is what + // the cache-hit probe asks — the same question for every engine. + ok, err := d.DatabaseExists(ctx, drvTemplate) + if err != nil || !ok { + t.Fatalf("template database missing after SnapshotCreate (ok=%v err=%v)", ok, err) + } + assertClone(t, d, ctx, drvTemplate) + + // The source must be untouched: callers treat it as read-only and + // restore the template over the clones, not over it. + assertClone(t, d, ctx, drvSource) + + // A fan-out is concurrent restores of ONE template — which for this + // engine means several readers of one backup file at once. + const clones = 8 + var wg sync.WaitGroup + errs := make([]error, clones) + for i := range clones { + wg.Add(1) + go func() { + defer wg.Done() + errs[i] = d.SnapshotRestore(ctx, drvTemplate, fmt.Sprintf("%sw%d", drvPrefix, i+1)) + }() + } + wg.Wait() + for i, err := range errs { + if err != nil { + t.Errorf("concurrent SnapshotRestore %d: %v", i+1, err) + } + } + for i := range clones { + assertClone(t, d, ctx, fmt.Sprintf("%sw%d", drvPrefix, i+1)) + } + + // Restoring over an existing clone is how a stale one is replaced. + if err := d.SnapshotRestore(ctx, drvTemplate, drvPrefix+"w1"); err != nil { + t.Errorf("SnapshotRestore over an existing target: %v", err) + } + assertClone(t, d, ctx, drvPrefix+"w1") +} + +// TestEnsureDBAppliesCollation pins the connection's collation reaching +// the databases treeman creates, and SIMPLE recovery reaching both the +// created and the restored ones — a FULL-recovery clone grows a log +// nobody will ever back up. +func TestEnsureDBAppliesCollation(t *testing.T) { + d, ctx := connectDriver(t) + seedSource(t, d, ctx) + if err := d.SnapshotCreate(ctx, drvSource, drvTemplate); err != nil { + t.Fatalf("SnapshotCreate: %v", err) + } + for _, name := range []string{drvSource, drvTemplate} { + scoped, err := d.OpenScoped(ctx, name) + if err != nil { + t.Fatalf("OpenScoped %s: %v", name, err) + } + var collation, recovery string + err = scoped.QueryRowContext(ctx, ` + SELECT CONVERT(nvarchar(128), DATABASEPROPERTYEX(DB_NAME(), 'Collation')), + CONVERT(nvarchar(32), DATABASEPROPERTYEX(DB_NAME(), 'Recovery'))`, + ).Scan(&collation, &recovery) + _ = scoped.Close() + if err != nil { + t.Fatalf("properties of %s: %v", name, err) + } + if collation != "Polish_CI_AS" { + t.Errorf("%s collation = %q, want Polish_CI_AS", name, collation) + } + if recovery != "SIMPLE" { + t.Errorf("%s recovery = %q, want SIMPLE", name, recovery) + } + } +} + +// TestListMatchingEscapesUnderscore is the sibling-wipe guard. Every +// templated name carries a literal `_`; unescaped it is a +// single-character wildcard, and a prefix reap would then match another +// worktree's databases. +func TestListMatchingEscapesUnderscore(t *testing.T) { + d, ctx := connectDriver(t) + if err := d.EnsureDB(ctx, drvSource); err != nil { + t.Fatalf("EnsureDB: %v", err) + } + // Same length as drvSource, differing only where the `_` is — a + // wildcard match, never a prefix match. + decoy := "tmdrvXsrc" + if err := d.EnsureDB(ctx, decoy); err != nil { + t.Fatalf("EnsureDB decoy: %v", err) + } + t.Cleanup(func() { _ = d.DropDatabase(ctx, decoy) }) + + names, err := d.ListMatching(ctx, drvPrefix+"s") + if err != nil { + t.Fatalf("ListMatching: %v", err) + } + for _, n := range names { + if n == decoy { + t.Fatalf("ListMatching(%q) matched %q — LIKE metacharacters are not escaped", drvPrefix+"s", decoy) + } + } + if len(names) != 1 || names[0] != drvSource { + t.Errorf("ListMatching(%q) = %v, want [%s]", drvPrefix+"s", names, drvSource) + } +} + +// TestClaimSpareRename covers the prewarm claim and the file-name trap +// behind it: MODIFY NAME moves the database but not its files, so a +// restore that named files after the database would collide with files +// the renamed one still holds open. +func TestClaimSpareRename(t *testing.T) { + d, ctx := connectDriver(t) + seedSource(t, d, ctx) + if err := d.SnapshotCreate(ctx, drvSource, drvTemplate); err != nil { + t.Fatalf("SnapshotCreate: %v", err) + } + spare, claimed := drvPrefix+"spare1", drvPrefix+"claimed" + if err := d.SnapshotRestore(ctx, drvTemplate, spare); err != nil { + t.Fatalf("restore spare: %v", err) + } + if err := d.RenameDatabase(ctx, spare, claimed); err != nil { + t.Fatalf("RenameDatabase: %v", err) + } + if ok, _ := d.DatabaseExists(ctx, spare); ok { + t.Errorf("%s still present after rename", spare) + } + assertClone(t, d, ctx, claimed) + + // Replenishment: the freed spare name must be restorable again + // while the claimed database still holds the original files. + if err := d.SnapshotRestore(ctx, drvTemplate, spare); err != nil { + t.Fatalf("re-restore of the freed spare name: %v", err) + } + assertClone(t, d, ctx, spare) +} + +// TestRenameRaceHasOneWinner is the atomicity the spare-claim path +// leans on across processes: of two claimers of one target, exactly one +// wins and the other fails rather than overwriting. +func TestRenameRaceHasOneWinner(t *testing.T) { + d, ctx := connectDriver(t) + seedSource(t, d, ctx) + if err := d.SnapshotCreate(ctx, drvSource, drvTemplate); err != nil { + t.Fatalf("SnapshotCreate: %v", err) + } + a, b, target := drvPrefix+"racea", drvPrefix+"raceb", drvPrefix+"won" + for _, n := range []string{a, b} { + if err := d.SnapshotRestore(ctx, drvTemplate, n); err != nil { + t.Fatalf("restore %s: %v", n, err) + } + } + var wg sync.WaitGroup + errs := make([]error, 2) + for i, from := range []string{a, b} { + wg.Add(1) + go func() { + defer wg.Done() + errs[i] = d.RenameDatabase(ctx, from, target) + }() + } + wg.Wait() + won := 0 + for _, err := range errs { + if err == nil { + won++ + } + } + if won != 1 { + t.Fatalf("%d of 2 racing claimers won, want exactly 1 (errs: %v)", won, errs) + } + assertClone(t, d, ctx, target) + // The loser must still be usable — the claim path moves on to the + // next slot rather than giving up, and a database left in + // single-user mode would wedge it. + loser := a + if errs[0] == nil { + loser = b + } + assertClone(t, d, ctx, loser) +} + +// TestDropSnapshotRemovesBackup proves the template's backup file goes +// with the template database. The file is on the server, so the +// observable is that a restore can no longer be served from it. +func TestDropSnapshotRemovesBackup(t *testing.T) { + d, ctx := connectDriver(t) + seedSource(t, d, ctx) + if err := d.SnapshotCreate(ctx, drvSource, drvTemplate); err != nil { + t.Fatalf("SnapshotCreate: %v", err) + } + if err := d.DropSnapshot(ctx, drvTemplate); err != nil { + t.Fatalf("DropSnapshot: %v", err) + } + if ok, _ := d.DatabaseExists(ctx, drvTemplate); ok { + t.Fatalf("template %s survived DropSnapshot", drvTemplate) + } + // Idempotent — GC and the orphan sweep both call it speculatively. + if err := d.DropSnapshot(ctx, drvTemplate); err != nil { + t.Errorf("DropSnapshot (second call): %v", err) + } + if err := d.SnapshotRestore(ctx, drvTemplate, drvPrefix+"w1"); err == nil { + t.Error("SnapshotRestore from a dropped snapshot must fail — the backup file was left behind") + } +} + +// TestSnapshotRestoreRederivesBackup covers the half-state: the +// template database is still good but its backup file is gone (someone +// cleaned the directory, or the template predates this driver). A cold +// rebuild of a demonstrably-good template is the wrong answer. +func TestSnapshotRestoreRederivesBackup(t *testing.T) { + d, ctx := connectDriver(t) + seedSource(t, d, ctx) + if err := d.SnapshotCreate(ctx, drvSource, drvTemplate); err != nil { + t.Fatalf("SnapshotCreate: %v", err) + } + // Remove the file behind the template's back, the way a cleaned + // backup directory would. + bak := d.SnapshotBackupPath(drvTemplate) + if _, err := d.DB.ExecContext(ctx, "EXEC master.sys.xp_delete_files @p1", bak); err != nil { + t.Fatalf("remove %s: %v", bak, err) + } + if err := d.SnapshotRestore(ctx, drvTemplate, drvPrefix+"w1"); err != nil { + t.Fatalf("SnapshotRestore after the backup file was removed: %v", err) + } + assertClone(t, d, ctx, drvPrefix+"w1") +} + +// TestDropMatchingEvictsConnections pins the difference from Postgres: +// there is no pg_terminate_backend here, so a DROP with an idle pooled +// connection open on the database fails unless the driver takes it +// single-user first. A jest worker leaving a pool behind is the normal +// case, not the exotic one. +func TestDropMatchingEvictsConnections(t *testing.T) { + d, ctx := connectDriver(t) + seedSource(t, d, ctx) + if err := d.SnapshotCreate(ctx, drvSource, drvTemplate); err != nil { + t.Fatalf("SnapshotCreate: %v", err) + } + if err := d.SnapshotRestore(ctx, drvTemplate, drvPrefix+"w1"); err != nil { + t.Fatalf("restore: %v", err) + } + live, err := d.OpenScoped(ctx, drvPrefix+"w1") + if err != nil { + t.Fatalf("OpenScoped: %v", err) + } + if err := live.PingContext(ctx); err != nil { + t.Fatalf("ping: %v", err) + } + dropped, err := d.DropMatching(ctx, drvPrefix) + _ = live.Close() + if err != nil { + t.Fatalf("DropMatching with a live connection: %v", err) + } + if len(dropped) == 0 { + t.Fatal("DropMatching dropped nothing") + } + left, err := d.ListMatching(ctx, drvPrefix) + if err != nil { + t.Fatalf("ListMatching: %v", err) + } + if len(left) != 0 { + t.Errorf("databases left after DropMatching: %v", left) + } +} + +// TestRowCountSampleMatchesAcrossClone is what capture verification +// rests on: the sample the source yields must count the same in the +// copy. The sample names carry their schema, because a table name +// alone is not unique in a SQL Server database. +func TestRowCountSampleMatchesAcrossClone(t *testing.T) { + d, ctx := connectDriver(t) + seedSource(t, d, ctx) + sample, err := d.RowCountSample(ctx, drvSource) + if err != nil { + t.Fatalf("RowCountSample: %v", err) + } + if len(sample) == 0 { + t.Fatal("RowCountSample is empty — capture verification would record every capture unverified") + } + if _, ok := sample["app.orders"]; !ok { + t.Errorf("sample %v does not name app.orders schema-qualified", sample) + } + if err := d.SnapshotCreate(ctx, drvSource, drvTemplate); err != nil { + t.Fatalf("SnapshotCreate: %v", err) + } + names := make([]string, 0, len(sample)) + for k := range sample { + names = append(names, k) + } + got, err := d.RowCountsFor(ctx, drvTemplate, names) + if err != nil { + t.Fatalf("RowCountsFor: %v", err) + } + for k, want := range sample { + if got[k] != want { + t.Errorf("clone %s = %d, source = %d", k, got[k], want) + } + } +} + +// TestWriteWatermarkDeclines pins the deliberate refusal. A token that +// can miss a write is worse than none: the caller SKIPS a capture when +// the token is unchanged. +func TestWriteWatermarkDeclines(t *testing.T) { + d, ctx := connectDriver(t) + seedSource(t, d, ctx) + wm, err := d.WriteWatermark(ctx, drvSource) + if err != nil { + t.Fatalf("WriteWatermark: %v", err) + } + if wm != "" { + t.Errorf("WriteWatermark = %q, want \"\" (no sound per-database write counter on this engine)", wm) + } +} diff --git a/e2e/mssql/fixtures/migrate.sh b/e2e/mssql/fixtures/migrate.sh new file mode 100755 index 00000000..51095004 --- /dev/null +++ b/e2e/mssql/fixtures/migrate.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env sh +# Stand-in for a framework's migrate CLI. Applies pending .sql files +# from fixtures/migrations/ in lexical order AND records each +# application in a `_treeman_e2e_migrations` ledger so re-runs skip +# the already-applied files. That ledger is what makes the +# incremental-from-ancestor path correct: cloning a cached template +# carries the partial ledger with it, so this script applies only the +# new files on top. +set -eu +: "${DB_DATABASE:?DB_DATABASE not set}" +container="${MSSQL_CONTAINER:-treeman-e2e-mssql}" +password="${MSSQL_PASSWORD:-treemanE2E!pw1}" +dir="$(dirname "$0")/migrations" + +# -b makes sqlcmd exit non-zero on a T-SQL error; without it a failed +# migration is a silent success and the suite proves nothing. -C +# trusts the certificate the server generated for itself. +sqlcmd_in() { + docker exec -i "$container" /opt/mssql-tools18/bin/sqlcmd \ + -S localhost -U sa -P "$password" -C -b -d "$DB_DATABASE" "$@" +} + +# Ledger table. The IF NOT EXISTS guard is idempotent across cold + +# incremental runs. +sqlcmd_in <&2 + exit 1 + ;; + esac + sqlcmd_in <"$f" + sqlcmd_in -Q "INSERT INTO dbo._treeman_e2e_migrations(name) VALUES (N'$name')" +done diff --git a/e2e/mssql/fixtures/migrations/001_init.sql b/e2e/mssql/fixtures/migrations/001_init.sql new file mode 100644 index 00000000..74c5f251 --- /dev/null +++ b/e2e/mssql/fixtures/migrations/001_init.sql @@ -0,0 +1,8 @@ +CREATE TABLE dbo.products (id int IDENTITY PRIMARY KEY, name nvarchar(100) NOT NULL); +GO +CREATE TABLE dbo.orders (id int IDENTITY PRIMARY KEY, product_id int NOT NULL REFERENCES dbo.products(id)); +GO +INSERT INTO dbo.products(name) VALUES (N'Alpha'), (N'Beta'), (N'zażółć gęślą jaźń'); +GO +INSERT INTO dbo.orders(product_id) VALUES (1), (2); +GO diff --git a/e2e/mssql/mssql_test.go b/e2e/mssql/mssql_test.go new file mode 100644 index 00000000..22f97ac7 --- /dev/null +++ b/e2e/mssql/mssql_test.go @@ -0,0 +1,278 @@ +//go:build e2e + +// SQL Server end-to-end: cold build → cache hit → input edit → +// rebuild, with the fan-out clones and `wt delete` teardown. Same +// shape as the postgres suite; what differs is that there is no +// `dump:` here (config rejects it for this family — RESTORE reads the +// server's filesystem, not treeman's), so the schema arrives through +// `migrate:` alone. +package mssql_e2e + +import ( + "context" + "database/sql" + "fmt" + "net" + "os" + "path/filepath" + "strings" + "testing" + "time" + + _ "github.com/microsoft/go-mssqldb" + + "github.com/stubbedev/treeman/e2e/harness" + "github.com/stubbedev/treeman/internal/config" + "github.com/stubbedev/treeman/internal/prepare" +) + +const ( + addr = "127.0.0.1:11433" + password = "treemanE2E!pw1" +) + +func TestMSSQLEndToEnd(t *testing.T) { + harness.SkipIfNoDocker(t) + t.Cleanup(harness.ComposeUp(t, harness.MustAbs("."))) + waitForServer(t) + + wt := t.TempDir() + copyTree(t, "fixtures", filepath.Join(wt, "fixtures")) + cfg := buildConfig() + env := harness.NewEnv(t, wt) + + outs := env.RunPrepare(t, cfg) + o1 := harness.AssertOutcome(t, outs, "mssql", false) + t.Logf("pass1: template=%s sourceDB=%s", o1.TemplateName, o1.SourceDB) + assertTables(t, o1.SourceDB, []string{"products", "orders"}) + + // The collation on the connection must reach the database it + // creates — a clone that orders and compares text differently from + // the real server is the subtle failure this knob exists to stop. + if got := collationOf(t, o1.SourceDB); got != "Polish_CI_AS" { + t.Errorf("source DB collation = %q, want Polish_CI_AS", got) + } + + outs = env.RunPrepare(t, cfg) + o2 := harness.AssertOutcome(t, outs, "mssql", true) + if o2.Fingerprint != o1.Fingerprint { + t.Errorf("fingerprint drift: %s vs %s", o1.Fingerprint, o2.Fingerprint) + } + // A cache hit restores from the template, so the clone must still + // carry the data — a RESTORE that only brought the schema would + // pass every "does the table exist" check. + assertRowCount(t, o2.SourceDB, "dbo.products", 3) + + addMigration(t, wt) + outs = env.RunPrepare(t, cfg) + o3 := harness.AssertOutcome(t, outs, "mssql", false) + if o3.Fingerprint == o1.Fingerprint { + t.Errorf("fingerprint unchanged after edit") + } + assertTables(t, o3.SourceDB, []string{"products", "orders", "shipments"}) + + if len(o3.Clones) != 2 { + t.Fatalf("fanout: got %d clones, want 2 (%v)", len(o3.Clones), o3.Clones) + } + for _, c := range o3.Clones { + if !dbExists(t, c) { + t.Errorf("clone DB %s missing after fanout", c) + continue + } + assertTables(t, c, []string{"products", "orders", "shipments"}) + assertRowCount(t, c, "dbo.products", 3) + } + + // ── teardown: `wt delete` drops the per-worktree DBs, keeps the cache ── + if err := prepare.TeardownDatabases(env.Ctx, cfg, env.Slug.Value, env.RepoID, env.WTID, env.Store); err != nil { + t.Fatalf("TeardownDatabases: %v", err) + } + if dbExists(t, o3.SourceDB) { + t.Errorf("source DB %s still exists after teardown", o3.SourceDB) + } + for _, c := range o3.Clones { + if dbExists(t, c) { + t.Errorf("clone DB %s still exists after teardown", c) + } + } + if !dbExists(t, o3.TemplateName) { + t.Errorf("template DB %s was dropped by teardown (cache must survive)", o3.TemplateName) + } +} + +// TestRejectsDump pins the one place this family deliberately refuses +// what the other SQL engines accept. The message has to name the +// reason, because "dump: is not supported" alone reads as an oversight. +func TestRejectsDump(t *testing.T) { + cfg := buildConfig() + cfg.Databases[0].Dump = config.DumpList{{Path: "fixtures/seed.sql"}} + err := cfg.Validate() + if err == nil { + t.Fatal("a `dump:` on an mssql entry must be rejected") + } + for _, want := range []string{"dump:", "RESTORE reads the path on the SQL Server host"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err, want) + } + } +} + +func buildConfig() *config.Config { + return &config.Config{ + Connections: config.ConnectionsConfig{ + Mssql: &config.MssqlConn{ + Host: "127.0.0.1", + Port: 11433, + User: "sa", + Password: password, + Collation: "Polish_CI_AS", + }, + }, + Databases: []config.DatabaseConfig{ + { + Engine: "mssql", + NameTemplate: "treeman_e2e_{slug}", + Migrate: &config.Step{ + Run: "./fixtures/migrate.sh", + Env: map[string]string{ + "DB_DATABASE": "{target_db}", + "MSSQL_CONTAINER": "treeman-e2e-mssql", + "MSSQL_PASSWORD": password, + }, + }, + Inputs: []config.Input{ + {Glob: "fixtures/migrations/*.sql", Label: "migrations"}, + }, + // Fanout: 2 clones, so teardown's per-worktree cleanup + // (source + clones) is exercised. + TestClones: &config.TestClonesSpec{ + Clones: config.ClonesSetting{Fixed: 2}, + NameTemplate: "treeman_e2e_{slug}_w{n}", + }, + }, + }, + } +} + +func waitForServer(t *testing.T) { + t.Helper() + // The compose healthcheck already gates on sqlcmd, but the TCP + // probe is what the driver will do, and on an emulated arm64 host + // the two can be seconds apart. + harness.WaitForReady(t, "mssql:11433", 180*time.Second, func() error { + c, err := net.DialTimeout("tcp", addr, 2*time.Second) + if err != nil { + return err + } + _ = c.Close() + return nil + }) +} + +func open(t *testing.T, dbName string) *sql.DB { + t.Helper() + dsn := fmt.Sprintf("sqlserver://sa:%s@%s?database=%s&encrypt=disable", + "treemanE2E%21pw1", addr, dbName) + db, err := sql.Open("sqlserver", dsn) + if err != nil { + t.Fatalf("open %s: %v", dbName, err) + } + t.Cleanup(func() { _ = db.Close() }) + return db +} + +func assertTables(t *testing.T, dbName string, want []string) { + t.Helper() + db := open(t, dbName) + got := map[string]bool{} + rows, err := db.QueryContext(context.Background(), + "SELECT name FROM sys.tables WHERE schema_id = SCHEMA_ID('dbo')") + if err != nil { + t.Fatalf("list tables in %s: %v", dbName, err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var n string + if err := rows.Scan(&n); err != nil { + t.Fatal(err) + } + got[n] = true + } + for _, w := range want { + if !got[w] { + t.Errorf("%s: table %q missing (have %v)", dbName, w, got) + } + } +} + +func assertRowCount(t *testing.T, dbName, table string, want int) { + t.Helper() + db := open(t, dbName) + var n int + if err := db.QueryRowContext(context.Background(), "SELECT COUNT(*) FROM "+table).Scan(&n); err != nil { + t.Fatalf("count %s.%s: %v", dbName, table, err) + } + if n != want { + t.Errorf("%s.%s has %d rows, want %d", dbName, table, n, want) + } +} + +func collationOf(t *testing.T, dbName string) string { + t.Helper() + db := open(t, "master") + var c string + if err := db.QueryRowContext(context.Background(), + "SELECT CONVERT(nvarchar(128), DATABASEPROPERTYEX(@p1, 'Collation'))", dbName).Scan(&c); err != nil { + t.Fatalf("collation of %s: %v", dbName, err) + } + return c +} + +func dbExists(t *testing.T, dbName string) bool { + t.Helper() + db := open(t, "master") + var n int + err := db.QueryRowContext(context.Background(), + "SELECT COUNT(*) FROM sys.databases WHERE name = @p1", dbName).Scan(&n) + if err != nil { + t.Fatalf("exists %s: %v", dbName, err) + } + return n > 0 +} + +func addMigration(t *testing.T, wt string) { + t.Helper() + body := "CREATE TABLE dbo.shipments (id int IDENTITY PRIMARY KEY, order_id int NOT NULL REFERENCES dbo.orders(id));\nGO\n" + if err := os.WriteFile(filepath.Join(wt, "fixtures/migrations/002_shipments.sql"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } +} + +func copyTree(t *testing.T, src, dst string) { + t.Helper() + if err := os.MkdirAll(dst, 0o755); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(src) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + s, d := filepath.Join(src, e.Name()), filepath.Join(dst, e.Name()) + if e.IsDir() { + copyTree(t, s, d) + continue + } + info, err := e.Info() + if err != nil { + t.Fatal(err) + } + body, err := os.ReadFile(s) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(d, body, info.Mode()); err != nil { + t.Fatal(err) + } + } +} diff --git a/go.mod b/go.mod index 7f670579..1d1167e6 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/jackc/pgx/v5 v5.10.0 github.com/klauspost/compress v1.18.6 github.com/mattn/go-isatty v0.0.22 + github.com/microsoft/go-mssqldb v1.11.2 github.com/modelcontextprotocol/go-sdk v1.7.0 github.com/pb33f/ordered-map/v2 v2.3.1 github.com/pelletier/go-toml/v2 v2.3.1 @@ -27,8 +28,8 @@ require ( github.com/ulikunitz/xz v0.5.15 github.com/urfave/cli/v3 v3.9.1 go.mongodb.org/mongo-driver/v2 v2.6.0 - golang.org/x/sync v0.21.0 - golang.org/x/sys v0.46.0 + golang.org/x/sync v0.22.0 + golang.org/x/sys v0.47.0 gopkg.in/ini.v1 v1.67.2 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/blake3 v1.4.1 @@ -63,6 +64,8 @@ require ( github.com/clipperhouse/uax29/v2 v2.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect + github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect @@ -81,6 +84,7 @@ require ( github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/segmentio/asm v1.1.3 // indirect github.com/segmentio/encoding v0.5.4 // indirect + github.com/shopspring/decimal v1.4.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.2.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect @@ -89,9 +93,9 @@ require ( github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect go.uber.org/atomic v1.11.0 // indirect go.yaml.in/yaml/v4 v4.0.0-rc.2 // indirect - golang.org/x/crypto v0.33.0 // indirect + golang.org/x/crypto v0.55.0 // indirect golang.org/x/oauth2 v0.35.0 // indirect - golang.org/x/text v0.38.0 // indirect + golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.15.0 // indirect modernc.org/libc v1.72.3 // indirect modernc.org/mathutil v1.7.1 // indirect diff --git a/go.sum b/go.sum index 53471419..1e85d1ed 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,17 @@ filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 h1:zvXfGJCWvywnCA814d8ZiVyt+fm9nnTE8xSb99zRyfo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 h1:u93s+zU2JD62im61Bm5CZIc1ZrOJaIAWEg0WOrMVkEo= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1/go.mod h1:oXtinPO4OLj9d1DOTrqrL1oRwGhcqadvAmrl6wTeGlk= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0 h1:MaKvxE6D0KkjOg6Wd9M00iqP5PR0kUxCfiezes4JweM= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0/go.mod h1:i2h9fsTFKZorh8RdV2IcSUf/Qj98GlTkrTvUbX/s8as= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA= +github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= @@ -86,6 +98,10 @@ github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxo github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= +github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= +github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0= @@ -114,6 +130,8 @@ github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= @@ -122,6 +140,8 @@ github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2J github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/microsoft/go-mssqldb v1.11.2 h1:FCgeBIK8um2+X4tbun6Q71N1KsfyCDPKY41e1yGVjSE= +github.com/microsoft/go-mssqldb v1.11.2/go.mod h1:CYgwG5AMXFojbjTg+GNP5G/y6uz1BhTyZaPqQWzkGnQ= github.com/modelcontextprotocol/go-sdk v1.7.0 h1:yqjY2dsbKAC0LSuWZVBMrHgiG8ukXv6NRo0JiALay44= github.com/modelcontextprotocol/go-sdk v1.7.0/go.mod h1:dL7u98E/zjJTGzEq+j30jQ8K2k1mb6LeAH4inEcSGts= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= @@ -136,7 +156,8 @@ github.com/pb33f/ordered-map/v2 v2.3.1 h1:5319HDO0aw4DA4gzi+zv4FXU9UlSs3xGZ40wcP github.com/pb33f/ordered-map/v2 v2.3.1/go.mod h1:qxFQgd0PkVUtOMCkTapqotNgzRhMPL7VvaHKbd1HnmQ= github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc= github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.20.1 h1:sfCU6A8P3dXbKyWes02uxA2baehGux9dZHfEKtsTB1w= github.com/redis/go-redis/v9 v9.20.1/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA= @@ -150,6 +171,8 @@ github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -159,8 +182,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli/v3 v3.9.1 h1:OLU13atWZ0M+a4xmyBuBNOLZsSRYXyPeMeNjOvgYP54= @@ -184,49 +208,53 @@ go.mongodb.org/mongo-driver/v2 v2.6.0 h1:b9sJOYrkmt4l8bY43ZenFBcPlhYIjaOfYHLtbB/ go.mongodb.org/mongo-driver/v2 v2.6.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= go.yaml.in/yaml/v4 v4.0.0-rc.2 h1:/FrI8D64VSr4HtGIlUtlFMGsm7H7pWTbj6vOLVZcA6s= go.yaml.in/yaml/v4 v4.0.0-rc.2/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= -golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= -golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= -golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= -golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/internal/config/config.go b/internal/config/config.go index 10553411..74d07513 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -404,6 +404,10 @@ type ConnectionsConfig struct { // uses `engine: postgres` (or the `postgresql` alias). Postgres *PostgresConn `yaml:"postgres,omitempty"` + // Microsoft SQL Server connection. Required when any `databases:` + // entry uses `engine: mssql` (or the `sqlserver` alias). + Mssql *MssqlConn `yaml:"mssql,omitempty"` + // MongoDB connection. URI form (`mongodb://...`); host/port get // rewritten at dial time when a `container` ref is set. Mongodb *MongoConn `yaml:"mongodb,omitempty"` @@ -448,6 +452,7 @@ type ConnectionsConfig struct { // - { engine: postgres, name_template: "bi_{slug}", connection: analytics } MysqlNamed map[string]*MysqlConn `yaml:"-"` PostgresNamed map[string]*PostgresConn `yaml:"-"` + MssqlNamed map[string]*MssqlConn `yaml:"-"` MongodbNamed map[string]*MongoConn `yaml:"-"` RedisNamed map[string]*RedisConn `yaml:"-"` ElasticsearchNamed map[string]*EsConn `yaml:"-"` @@ -604,9 +609,10 @@ func (c *MysqlConn) UnmarshalYAML(node *yaml.Node) error { // JSONSchema for MysqlConn: scalar DSN OR full structured object. // sqlConnStructSchema reflects the structured-object form shared by the -// MySQL and Postgres connection types — they carry the identical field -// set, so the schema is generated once here. MySQL extras (charset, -// collation) are layered on by mysqlStructSchema. +// MySQL, Postgres and SQL Server connection types — they carry the +// identical field set, so the schema is generated once here. The extras +// (MySQL's charset/collation, SQL Server's collation and server-side +// directories) are layered on by the per-engine struct schemas. func sqlConnStructSchema() *jsonschema.Schema { r := &jsonschema.Reflector{Anonymous: true, ExpandedStruct: true, FieldNameTag: "yaml"} return r.Reflect(&struct { @@ -791,6 +797,112 @@ func parsePostgresDSN(dsn string, cfg *PostgresConn) error { return nil } +// MssqlConn — Microsoft SQL Server. Same shape as PostgresConn plus a +// collation and the two server-side directories the backup/restore +// clone path writes into. +type MssqlConn struct { + // Hostname or IP. Defaults to `127.0.0.1`. ContainerRef overrides + // it at dial time when set. There is no named-pipe or shared-memory + // mode: treeman speaks TDS over TCP only. + Host string `yaml:"host,omitempty"` + + // TCP port. Defaults to 1433. + Port uint16 `yaml:"port,omitempty"` + + // Login. Required. Needs `dbcreator` (CREATE/DROP DATABASE) plus + // BACKUP DATABASE and RESTORE DATABASE — in practice `sysadmin`, + // because RESTORE of a database that does not yet exist is a + // sysadmin-only operation. + User string `yaml:"user"` + + // Password is either a literal value or a `$NAME` / `${NAME}` + // env-var reference. See MysqlConn.Password for the resolution + // order and the security warning about literals. + Password string `yaml:"password,omitempty"` + + // Maximum open connections in the daemon's pool. + PoolMax uint32 `yaml:"pool_max,omitempty"` + + // Collation for CREATE DATABASE (`Polish_CI_AS`, + // `Latin1_General_100_CI_AS`, …). Empty means the server's own + // default, which on a stock install is SQL_Latin1_General_CP1_CI_AS + // — rarely what a non-English application actually uses. Set this + // to whatever the real server uses: a clone that compares and + // orders text differently from production is an expensive + // difference to debug. Clones restored from a template inherit the + // template's collation and ignore this field. + Collation string `yaml:"collation,omitempty"` + + // Directory, ON THE SERVER, that template `.bak` files are written + // to. Defaults to `SERVERPROPERTY('InstanceDefaultBackupPath')`. + // Set it when the service account cannot write there, or to put the + // backups on a different volume from the data files. + // + // Server-side, not treeman-side: with SQL Server in a container + // this is a path inside the container, and treeman never opens it. + BackupDir string `yaml:"backup_dir,omitempty"` + + // Directory, ON THE SERVER, that restored `.mdf`/`.ldf` files are + // written to. Defaults to + // `SERVERPROPERTY('InstanceDefaultDataPath')`. Same server-side + // caveat as `backup_dir`. + DataDir string `yaml:"data_dir,omitempty"` + ContainerRef ` yaml:",inline"` +} + +// UnmarshalYAML accepts either a bare DSN string +// (`sqlserver://user:pass@host:port`) or the structured object. +// Mirrors MysqlConn — see its doc-comment for the trade-offs. +func (c *MssqlConn) UnmarshalYAML(node *yaml.Node) error { + if node.Kind == yaml.ScalarNode { + return parseMssqlDSN(node.Value, c) + } + if node.Kind != yaml.MappingNode { + return fmt.Errorf("mssql connection (line %d): want a DSN string or mapping", node.Line) + } + type alias MssqlConn + return node.Decode((*alias)(c)) +} + +// mssqlStructSchema is sqlConnStructSchema plus the SQL-Server-only +// knobs: the CREATE DATABASE collation and the two server-side +// directories the clone path writes into. +func mssqlStructSchema() *jsonschema.Schema { + r := &jsonschema.Reflector{Anonymous: true, ExpandedStruct: true, FieldNameTag: "yaml"} + return r.Reflect(&struct { + Host string `yaml:"host,omitempty"` + Port uint16 `yaml:"port,omitempty"` + User string `yaml:"user"` + Password string `yaml:"password,omitempty"` + PoolMax uint32 `yaml:"pool_max,omitempty"` + Collation string `yaml:"collation,omitempty"` + BackupDir string `yaml:"backup_dir,omitempty"` + DataDir string `yaml:"data_dir,omitempty"` + ContainerRef ContainerRef `yaml:",inline"` + }{}) +} + +// JSONSchema for MssqlConn: scalar DSN OR full structured object. +func (MssqlConn) JSONSchema() *jsonschema.Schema { + return dsnOrStructSchema( + "DSN: `sqlserver://user:pass@host:port`. Equivalent to the structured form below.", + "Microsoft SQL Server connection — bare DSN string OR structured object.", + mssqlStructSchema(), + ) +} + +// parseMssqlDSN fills cfg from a URL-form DSN. `sqlserver://` is the +// scheme go-mssqldb itself documents; `mssql://` is accepted because it +// is what the engine alias is called. +func parseMssqlDSN(dsn string, cfg *MssqlConn) error { + p, err := parseConnDSN(dsn, "mssql", "sqlserver://", "sqlserver", "mssql") + if err != nil { + return err + } + applyDSNParts(p, &cfg.Host, &cfg.Port, &cfg.User, &cfg.Password) + return nil +} + // MongoConn — `mongodb://…` URI. When a ContainerRef is set, the // URI's host/port are rewritten at dial time. type MongoConn struct { diff --git a/internal/config/connections.go b/internal/config/connections.go index 9c3dc50f..c0d5464c 100644 --- a/internal/config/connections.go +++ b/internal/config/connections.go @@ -72,7 +72,7 @@ func isSingleConnNode[T any](node *yaml.Node) bool { // UnmarshalYAML decodes the `connections:` block, routing each family // key through the single-or-named union decode. ConnectionsConfig is -// exactly the seven family keys, so the mapping is walked directly — +// exactly the eight family keys, so the mapping is walked directly — // decoding through an alias-with-shadowed-fields breaks on scalar DSN // values — and any other key fails loud, matching the decoder's // KnownFields contract. @@ -89,6 +89,8 @@ func (c *ConnectionsConfig) UnmarshalYAML(node *yaml.Node) error { err = decodeConnUnion(val, &c.Mysql, &c.MysqlNamed) case "postgres": err = decodeConnUnion(val, &c.Postgres, &c.PostgresNamed) + case "mssql": + err = decodeConnUnion(val, &c.Mssql, &c.MssqlNamed) case "mongodb": err = decodeConnUnion(val, &c.Mongodb, &c.MongodbNamed) case "redis": @@ -100,7 +102,7 @@ func (c *ConnectionsConfig) UnmarshalYAML(node *yaml.Node) error { case "sqlite": err = decodeConnUnion(val, &c.Sqlite, &c.SqliteNamed) default: - err = fmt.Errorf("unknown key %q (expected one of: mysql, postgres, mongodb, redis, elasticsearch, s3, sqlite)", key) + err = fmt.Errorf("unknown key %q (expected one of: mysql, postgres, mssql, mongodb, redis, elasticsearch, s3, sqlite)", key) } if err != nil { return fmt.Errorf("connections.%s: %w", key, err) @@ -135,6 +137,11 @@ func (c *ConnectionsConfig) ResolvePostgres(name string) (*PostgresConn, error) return resolveNamed(c.Postgres, c.PostgresNamed, "postgres", name) } +// ResolveMssql is ResolveMysql for SQL Server. +func (c *ConnectionsConfig) ResolveMssql(name string) (*MssqlConn, error) { + return resolveNamed(c.Mssql, c.MssqlNamed, "mssql", name) +} + // ResolveMongodb is ResolveMysql for mongodb. func (c *ConnectionsConfig) ResolveMongodb(name string) (*MongoConn, error) { return resolveNamed(c.Mongodb, c.MongodbNamed, "mongodb", name) @@ -170,6 +177,8 @@ func (c *ConnectionsConfig) familyShape(fam engine.Family) (hasSingle bool, name return c.Mysql != nil, len(c.MysqlNamed) case engine.FamilyPostgres: return c.Postgres != nil, len(c.PostgresNamed) + case engine.FamilyMSSQL: + return c.Mssql != nil, len(c.MssqlNamed) case engine.FamilyMongo: return c.Mongodb != nil, len(c.MongodbNamed) case engine.FamilyRedis: @@ -195,6 +204,9 @@ func (c *ConnectionsConfig) hasNamedBlock(fam engine.Family, name string) bool { case engine.FamilyPostgres: _, ok := c.PostgresNamed[name] return ok + case engine.FamilyMSSQL: + _, ok := c.MssqlNamed[name] + return ok case engine.FamilyMongo: _, ok := c.MongodbNamed[name] return ok @@ -259,6 +271,8 @@ func (c *ConnectionsConfig) namedNames(fam engine.Family) map[string]struct{} { addNames(c.MysqlNamed, out) case engine.FamilyPostgres: addNames(c.PostgresNamed, out) + case engine.FamilyMSSQL: + addNames(c.MssqlNamed, out) case engine.FamilyMongo: addNames(c.MongodbNamed, out) case engine.FamilyRedis: diff --git a/internal/config/connections_test.go b/internal/config/connections_test.go index 03062d02..7917c86d 100644 --- a/internal/config/connections_test.go +++ b/internal/config/connections_test.go @@ -191,6 +191,9 @@ func TestSingleConnBlockWithContainerRef(t *testing.T) { {"postgres", `{host: 127.0.0.1, user: app, ` + ref + `}`, func(c *ConnectionsConfig) (bool, int) { return c.Postgres != nil && c.Postgres.Container == "app-db", len(c.PostgresNamed) }}, + {"mssql", `{host: 127.0.0.1, port: 1433, user: sa, ` + ref + `}`, func(c *ConnectionsConfig) (bool, int) { + return c.Mssql != nil && c.Mssql.Container == "app-db", len(c.MssqlNamed) + }}, {"mongodb", `{uri: "mongodb://localhost:27017", ` + ref + `}`, func(c *ConnectionsConfig) (bool, int) { return c.Mongodb != nil && c.Mongodb.Container == "app-db", len(c.MongodbNamed) }}, diff --git a/internal/config/dsn_test.go b/internal/config/dsn_test.go index e9f27657..ac7aa6a7 100644 --- a/internal/config/dsn_test.go +++ b/internal/config/dsn_test.go @@ -12,6 +12,7 @@ func TestConnectionsAcceptDSNStrings(t *testing.T) { connections: mysql: "mysql://root:secret@10.0.0.5:3307" postgres: "postgres://postgres:pgpw@127.0.0.1:5432" + mssql: "sqlserver://sa:Str0ng!pw@127.0.0.1:14333" mongodb: "mongodb://127.0.0.1:27017" redis: "redis://127.0.0.1:6379" elasticsearch: "http://127.0.0.1:9200" @@ -44,6 +45,20 @@ connections: if cfg.Connections.Postgres.Port != 5432 { t.Errorf("postgres port = %d", cfg.Connections.Postgres.Port) } + if cfg.Connections.Mssql == nil { + t.Fatal("mssql nil") + } + if cfg.Connections.Mssql.User != "sa" { + t.Errorf("mssql user = %q", cfg.Connections.Mssql.User) + } + if cfg.Connections.Mssql.Port != 14333 { + t.Errorf("mssql port = %d", cfg.Connections.Mssql.Port) + } + // The `!` is legal in userinfo and must survive the parse — SQL + // Server's password policy all but guarantees one. + if cfg.Connections.Mssql.Password != "Str0ng!pw" { + t.Errorf("mssql password = %q", cfg.Connections.Mssql.Password) + } } func TestStructuredFormStillWorks(t *testing.T) { @@ -79,6 +94,24 @@ connections: } } +// TestMssqlDSNRejectsWrongScheme: `mssql://` and `sqlserver://` are +// the two accepted schemes, and anything else must say so rather than +// dial a SQL Server with a Postgres URL's defaults. +func TestMssqlDSNRejectsWrongScheme(t *testing.T) { + y := ` +connections: + mssql: "postgres://x@y" +` + var cfg Config + err := yaml.Unmarshal([]byte(y), &cfg) + if err == nil { + t.Fatal("want scheme-mismatch error") + } + if !strings.Contains(err.Error(), "scheme must be sqlserver://") { + t.Errorf("unexpected error: %v", err) + } +} + func TestDSNRejectsWrongScheme(t *testing.T) { y := ` connections: diff --git a/internal/config/validate.go b/internal/config/validate.go index 6ded7231..83d92f0b 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -390,11 +390,34 @@ func (d DatabaseConfig) validateFamily(fam engine.Family, path string) error { return d.validateS3(path) case engine.FamilyFile: return d.validateFile(path) + case engine.FamilyMSSQL: + return d.validateMssql(path) default: return nil } } +// validateMssql rejects `dump:`. Every other engine's dump loader reads +// the file treeman is pointed at; SQL Server's RESTORE reads a path on +// the SERVER, so a host path means nothing to it — and with the server +// in a container the two filesystems are unrelated. Rejecting is the +// honest answer: the alternative silently restores whatever happens to +// sit at that path inside the container, or nothing. +// +// Seed through `migrate:`/`seed:` instead — those are commands, so they +// can be `sqlcmd -i`, `docker cp` + RESTORE, or anything else that knows +// how to get a file to the server. +func (d DatabaseConfig) validateMssql(path string) error { + if len(d.Dump) == 0 { + return nil + } + return fmt.Errorf( + "%s: engine %q does not support `dump:` — RESTORE reads the path on the SQL Server host, not on this machine, so a local file is not reachable. Load the dump from `seed:` instead (e.g. a command that copies the .bak to the server and RESTOREs it)", + path, + d.Engine, + ) +} + func (d DatabaseConfig) validate(path string) error { var errs []error if d.Engine == "" { @@ -472,14 +495,16 @@ func (d DatabaseConfig) validate(path string) error { // PrewarmCapableEngines lists the engine families whose driver // implements the spare capabilities (engineconn.SnapshotCreator + -// SpareClaimer): Postgres claims spares via a constant-time catalog -// rename, MySQL via a staged physical clone. It lives here because +// SpareClaimer): Postgres and SQL Server claim spares via a +// constant-time catalog rename, MySQL via a staged physical clone. It +// lives here because // engineconn imports config — a cycle would result the other way — // and engineconn's tests cross-check that every listed family's conn // really implements both interfaces, so the set can't drift. var PrewarmCapableEngines = map[string]bool{ "mysql": true, "postgres": true, + "mssql": true, } // validatePrewarm guards `databases[].prewarm`. Accepts any family @@ -494,7 +519,7 @@ func (d DatabaseConfig) validatePrewarm(path string) []error { fam, ok := engine.Canonical(d.Engine) if !ok || !PrewarmCapableEngines[string(fam)] { errs = append(errs, fmt.Errorf( - "%s: prewarm is not supported for engine %q (capable: mysql, postgres) — it has no cheap spare-claim path", + "%s: prewarm is not supported for engine %q (capable: mysql, postgres, mssql) — it has no cheap spare-claim path", path, d.Engine)) } if d.BranchScoped { diff --git a/internal/db/engineconn/capabilities_test.go b/internal/db/engineconn/capabilities_test.go index c3f4fed1..4ddd0c77 100644 --- a/internal/db/engineconn/capabilities_test.go +++ b/internal/db/engineconn/capabilities_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/stubbedev/treeman/internal/config" + dbmssql "github.com/stubbedev/treeman/internal/db/mssql" dbmysql "github.com/stubbedev/treeman/internal/db/mysql" dbpostgres "github.com/stubbedev/treeman/internal/db/postgres" "github.com/stubbedev/treeman/internal/engine" @@ -19,6 +20,7 @@ func TestPrewarmCapableEnginesMatchCapabilities(t *testing.T) { conns := map[string]Conn{ "mysql": NewMySQLConn(&dbmysql.Driver{}), "postgres": NewPostgresConn(&dbpostgres.Driver{}), + "mssql": NewMSSQLConn(&dbmssql.Driver{}), } for fam := range config.PrewarmCapableEngines { conn, ok := conns[fam] @@ -42,7 +44,7 @@ func TestPrewarmCapableEnginesMatchCapabilities(t *testing.T) { if !ok { continue } - if fam == engine.FamilyMySQL || fam == engine.FamilyPostgres { + if fam == engine.FamilyMySQL || fam == engine.FamilyPostgres || fam == engine.FamilyMSSQL { continue } conn, ok := conns[string(fam)] diff --git a/internal/db/engineconn/engineconn.go b/internal/db/engineconn/engineconn.go index b0094c32..83a8919c 100644 --- a/internal/db/engineconn/engineconn.go +++ b/internal/db/engineconn/engineconn.go @@ -19,6 +19,7 @@ import ( "github.com/stubbedev/treeman/internal/config" dbes "github.com/stubbedev/treeman/internal/db/es" dbmongo "github.com/stubbedev/treeman/internal/db/mongo" + dbmssql "github.com/stubbedev/treeman/internal/db/mssql" dbmysql "github.com/stubbedev/treeman/internal/db/mysql" dbpostgres "github.com/stubbedev/treeman/internal/db/postgres" dbredis "github.com/stubbedev/treeman/internal/db/redis" @@ -80,6 +81,9 @@ func NewPostgresConn(d *dbpostgres.Driver) Conn { return postgresConn{d} } // NewMySQLConn is NewPostgresConn for MySQL. func NewMySQLConn(d *dbmysql.Driver) Conn { return mysqlConn{d} } +// NewMSSQLConn is NewPostgresConn for SQL Server. +func NewMSSQLConn(d *dbmssql.Driver) Conn { return mssqlConn{d} } + func (c postgresConn) CreateSnapshot(ctx context.Context, template, target string) error { return c.d.SnapshotRestore(ctx, template, target) } @@ -96,6 +100,53 @@ func (c mysqlConn) ClaimSpare(ctx context.Context, spare, target string) error { return c.d.SnapshotRestore(ctx, spare, target) } +func (c mssqlConn) CreateSnapshot(ctx context.Context, template, target string) error { + return c.d.SnapshotRestore(ctx, template, target) +} + +// ClaimSpare renames the spare onto the target, like Postgres: a +// catalog-only ALTER DATABASE … MODIFY NAME rather than another +// restore. Cheap enough to be worth the spare pool at all — a restore +// copies every page. +func (c mssqlConn) ClaimSpare(ctx context.Context, spare, target string) error { + return c.d.RenameDatabase(ctx, spare, target) +} + +type mssqlConn struct{ d *dbmssql.Driver } + +func (c mssqlConn) Close() error { return c.d.Close() } +func (c mssqlConn) EngineVersion(ctx context.Context) (string, error) { + return c.d.EngineVersion(ctx) +} + +func (c mssqlConn) Exists(ctx context.Context, n string) (bool, error) { + return c.d.DatabaseExists(ctx, n) +} + +func (c mssqlConn) DropMatching(ctx context.Context, n string) (int, error) { + dropped, err := c.d.DropMatching(ctx, n) + return len(dropped), err +} + +func (c mssqlConn) DropSnapshot(ctx context.Context, n string) error { + return c.d.DropSnapshot(ctx, n) +} + +func (c mssqlConn) ListMatching(ctx context.Context, p string) ([]string, error) { + return c.d.ListMatching(ctx, p) +} + +// SizeKB sums the database's data and log files. sys.master_files is +// server-wide, so unlike DB_NAME-scoped alternatives it answers for a +// database this connection is not inside; `size` is in 8 KiB pages. +func (c mssqlConn) SizeKB(ctx context.Context, n string) int64 { + var size int64 + _ = c.d.DB.QueryRowContext(ctx, + `SELECT ISNULL(SUM(CONVERT(bigint, size)), 0) * 8 + FROM sys.master_files WHERE database_id = DB_ID(@p1)`, n).Scan(&size) + return size +} + type mysqlConn struct{ d *dbmysql.Driver } func (c mysqlConn) Close() error { return c.d.Close() } diff --git a/internal/db/engineconn/engineconn_test.go b/internal/db/engineconn/engineconn_test.go index e8f99ee0..0be49871 100644 --- a/internal/db/engineconn/engineconn_test.go +++ b/internal/db/engineconn/engineconn_test.go @@ -14,8 +14,8 @@ import ( // block. Connect's not-configured path must return (nil, false, nil). func TestConfiguredMatchesConnect(t *testing.T) { families := []engine.Family{ - engine.FamilyMySQL, engine.FamilyPostgres, engine.FamilyMongo, - engine.FamilyRedis, engine.FamilyES, + engine.FamilyMySQL, engine.FamilyPostgres, engine.FamilyMSSQL, + engine.FamilyMongo, engine.FamilyRedis, engine.FamilyES, } empty := &config.Config{} for _, fam := range families { @@ -31,6 +31,7 @@ func TestConfiguredMatchesConnect(t *testing.T) { full := &config.Config{Connections: config.ConnectionsConfig{ Mysql: &config.MysqlConn{Host: "127.0.0.1", Port: 1, User: "u"}, Postgres: &config.PostgresConn{Host: "127.0.0.1", Port: 1, User: "u"}, + Mssql: &config.MssqlConn{Host: "127.0.0.1", Port: 1, User: "u"}, Mongodb: &config.MongoConn{URI: "mongodb://127.0.0.1:1"}, Redis: &config.RedisConn{URL: "redis://127.0.0.1:1"}, Elasticsearch: &config.EsConn{URL: "http://127.0.0.1:1"}, diff --git a/internal/db/engineconn/registry_drivers.go b/internal/db/engineconn/registry_drivers.go index cf5ae66a..46a9fdd6 100644 --- a/internal/db/engineconn/registry_drivers.go +++ b/internal/db/engineconn/registry_drivers.go @@ -14,6 +14,7 @@ import ( "github.com/stubbedev/treeman/internal/config" dbes "github.com/stubbedev/treeman/internal/db/es" dbmongo "github.com/stubbedev/treeman/internal/db/mongo" + dbmssql "github.com/stubbedev/treeman/internal/db/mssql" dbmysql "github.com/stubbedev/treeman/internal/db/mysql" dbpostgres "github.com/stubbedev/treeman/internal/db/postgres" dbredis "github.com/stubbedev/treeman/internal/db/redis" @@ -73,6 +74,26 @@ func init() { return postgresConn{d}, true, nil }, }) + Register(engine.FamilyMSSQL, DriverFactory{ + Configured: func(cfg *config.Config, name string) bool { + conn, err := cfg.Connections.ResolveMssql(name) + return err == nil && conn != nil + }, + Connect: func(ctx context.Context, cfg *config.Config, name string) (Conn, bool, error) { + conn, err := cfg.Connections.ResolveMssql(name) + if err != nil { + return nil, true, err + } + if conn == nil { + return nil, false, nil + } + d, err := dbmssql.Connect(ctx, *conn) + if err != nil { + return nil, true, err + } + return mssqlConn{d}, true, nil + }, + }) Register(engine.FamilyMongo, DriverFactory{ Configured: func(cfg *config.Config, name string) bool { conn, err := cfg.Connections.ResolveMongodb(name) diff --git a/internal/db/ident/mssql.go b/internal/db/ident/mssql.go new file mode 100644 index 00000000..d145b3a2 --- /dev/null +++ b/internal/db/ident/mssql.go @@ -0,0 +1,15 @@ +package ident + +// ValidateMSSQL reports whether s is safe to interpolate into a +// bracket-quoted SQL Server identifier. +func ValidateMSSQL(s string) error { return validate(s) } + +// QuoteMSSQL returns s wrapped in brackets, or an error if s is not a +// valid identifier. Brackets rather than double quotes because +// QUOTED_IDENTIFIER is a session setting and brackets work either way. +func QuoteMSSQL(s string) (string, error) { + if err := validate(s); err != nil { + return "", err + } + return "[" + s + "]", nil +} diff --git a/internal/db/mssql/mssql.go b/internal/db/mssql/mssql.go new file mode 100644 index 00000000..5e2849c4 --- /dev/null +++ b/internal/db/mssql/mssql.go @@ -0,0 +1,436 @@ +// Package mssql is the treeman Microsoft SQL Server driver. Uses +// microsoft/go-mssqldb (the `sqlserver` database/sql driver) over TDS. +// +// WHAT MAKES THIS DRIVER DIFFERENT FROM postgres/mysql: SQL Server has +// no `CREATE DATABASE … TEMPLATE` and no cross-database file clone a +// client can drive. `DBCC CLONEDATABASE` copies schema and statistics +// but no rows; a database snapshot (`CREATE DATABASE … AS SNAPSHOT OF`) +// is read-only. The only client-drivable copy is BACKUP + RESTORE, so +// a template here is a pair: the template DATABASE (which is what +// treeman's `DatabaseExists` cache-hit probe asks about, same as every +// other engine) and a `.bak` file beside it that every clone restores +// from. One backup per template, one restore per clone. +// +// All of it runs over TDS, including the file paths: `TO DISK`, `FROM +// DISK` and `MOVE … TO` all accept variables, so paths are bound as +// parameters and only database names are interpolated — through +// ident.QuoteMSSQL, whose charset is the same strict [A-Za-z0-9_] the +// other engines use. +package mssql + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net" + "net/url" + "strconv" + "strings" + "time" + + // Register the go-mssqldb database/sql driver under "sqlserver". + _ "github.com/microsoft/go-mssqldb" + + "golang.org/x/sync/errgroup" + + "github.com/stubbedev/treeman/internal/config" + "github.com/stubbedev/treeman/internal/db/adapter" + "github.com/stubbedev/treeman/internal/db/containerip" + "github.com/stubbedev/treeman/internal/db/ident" +) + +// DefaultPort is the SQL Server default TDS port. +const DefaultPort uint16 = 1433 + +// Driver wraps a pool against `master` so database-level DDL works. +type Driver struct { + DB *sql.DB + cfg config.MssqlConn + + // Server-side directories, resolved once on Connect from config or + // SERVERPROPERTY. They are paths on the SERVER, not on the machine + // treeman runs on: with SQL Server in a container the two have no + // relation, which is why neither is ever touched with os.*. + dataDir string + logDir string + backupDir string + sep string // path separator the server uses: "/" or "\" +} + +// Connect opens a pool against `master`, from which CREATE / DROP / +// ALTER DATABASE and BACKUP / RESTORE are all issued. Container and +// compose refs are resolved exactly as in the postgres driver; there is +// no unix-socket mode (SQL Server speaks TDS over TCP only). +func Connect(ctx context.Context, cfg config.MssqlConn) (*Driver, error) { + if cfg.Port == 0 { + cfg.Port = DefaultPort + } + if cfg.Host == "" { + cfg.Host = "127.0.0.1" + } + opts := containerip.Opts{ + Container: cfg.Container, + ComposeService: cfg.ComposeService, + ComposeProject: cfg.ComposeProject, + Engine: cfg.ContainerEngine, + Network: cfg.Network, + InternalPort: cfg.Port, + } + if err := adapter.ResolveAndProbe(ctx, "mssql", opts, &cfg.Host, &cfg.Port); err != nil { + return nil, err + } + db, err := sql.Open("sqlserver", dsn(cfg, "master")) + if err != nil { + return nil, fmt.Errorf("open mssql: %w", err) + } + adapter.ConfigurePool(db, int(cfg.PoolMax)) + if err := db.PingContext(ctx); err != nil { + _ = db.Close() + return nil, fmt.Errorf("ping mssql: %w", err) + } + d := &Driver{DB: db, cfg: cfg} + if err := d.resolveServerDirs(ctx); err != nil { + _ = db.Close() + return nil, err + } + return d, nil +} + +// dsn renders a `sqlserver://` URL for one database. encrypt=disable +// because every supported target is a local or CI server reached over a +// private network — the same posture the other drivers take with +// sslmode=disable — and SQL Server 2022+ otherwise refuses a +// self-signed certificate it generated itself. +func dsn(cfg config.MssqlConn, database string) string { + u := url.URL{ + Scheme: "sqlserver", + User: url.UserPassword(cfg.User, cfg.Password), + Host: net.JoinHostPort(cfg.Host, strconv.Itoa(int(cfg.Port))), + } + q := url.Values{} + q.Set("database", database) + q.Set("encrypt", "disable") + // A connect that hangs past the caller's patience is worse than one + // that fails: every call site wraps this in a context, and the + // driver's own default is 30s. + q.Set("dial timeout", "10") + u.RawQuery = q.Encode() + return u.String() +} + +// resolveServerDirs fixes the three server-side directories this driver +// writes into, from SERVERPROPERTY. +func (d *Driver) resolveServerDirs(ctx context.Context) error { + var data, logp, backup sql.NullString + err := d.DB.QueryRowContext(ctx, ` + SELECT CONVERT(nvarchar(4000), SERVERPROPERTY('InstanceDefaultDataPath')), + CONVERT(nvarchar(4000), SERVERPROPERTY('InstanceDefaultLogPath')), + CONVERT(nvarchar(4000), SERVERPROPERTY('InstanceDefaultBackupPath'))`, + ).Scan(&data, &logp, &backup) + if err != nil { + return fmt.Errorf("read mssql default paths: %w", err) + } + return d.applyServerDirs(data.String, logp.String, backup.String) +} + +// applyServerDirs resolves the directories against the config. Config +// wins; otherwise the server's own answers do, which are right for a +// stock container (`/var/opt/mssql/data/`) and for a Windows install +// alike. `data_dir` covers the log directory too when set, because a +// caller relocating the data files almost never means to leave the +// logs behind — and the separator is read off whichever path came back +// rather than assumed, because the server's OS is not ours. +func (d *Driver) applyServerDirs(data, logp, backup string) error { + d.sep = "/" + for _, p := range []string{data, logp, backup} { + if strings.Contains(p, `\`) { + d.sep = `\` + break + } + } + d.dataDir = trimSep(firstNonEmpty(d.cfg.DataDir, data), d.sep) + d.logDir = trimSep(firstNonEmpty(d.cfg.DataDir, logp, data), d.sep) + d.backupDir = trimSep(firstNonEmpty(d.cfg.BackupDir, backup, data), d.sep) + if d.dataDir == "" || d.backupDir == "" { + return errors.New("mssql: server reported no default data/backup path — set `data_dir` and `backup_dir` on the connection") + } + return nil +} + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if v != "" { + return v + } + } + return "" +} + +func trimSep(p, sep string) string { return strings.TrimRight(p, sep) } + +func (d *Driver) join(dir, file string) string { return dir + d.sep + file } + +// backupPath is where a template's `.bak` lives. One file per template +// name, overwritten WITH INIT, so the set of files on disk is bounded +// by the set of live templates. +func (d *Driver) backupPath(template string) string { + return d.join(d.backupDir, template+".bak") +} + +func (d *Driver) Close() error { return d.DB.Close() } + +// OpenScoped returns a fresh pool scoped to `dbName`. SQL Server does +// have `USE`, but a pooled connection's current database is not a +// property a caller can rely on, so each target database gets its own +// pool — same contract as the postgres driver. The caller MUST Close it. +func (d *Driver) OpenScoped(ctx context.Context, dbName string) (*sql.DB, error) { + if err := ident.ValidateMSSQL(dbName); err != nil { + return nil, err + } + db, err := sql.Open("sqlserver", dsn(d.cfg, dbName)) + if err != nil { + return nil, fmt.Errorf("open mssql %s: %w", dbName, err) + } + adapter.ConfigurePool(db, int(d.cfg.PoolMax)) + if err := db.PingContext(ctx); err != nil { + _ = db.Close() + return nil, fmt.Errorf("ping mssql %s: %w", dbName, err) + } + return db, nil +} + +func (d *Driver) EngineVersion(ctx context.Context) (string, error) { + var v string + if err := d.DB.QueryRowContext(ctx, "SELECT @@VERSION").Scan(&v); err != nil { + return "", err + } + return v, nil +} + +// MaxConnections returns the server's user-connection cap. The +// configured value is 0 when it means "unlimited", in which case the +// engine's own ceiling (32767) is the honest answer for the fanout +// auto-tuner — it sizes a goroutine pool against a connection budget, +// and 0 would starve it. +func (d *Driver) MaxConnections(ctx context.Context) (int, error) { + var n int + if err := d.DB.QueryRowContext(ctx, "SELECT @@MAX_CONNECTIONS").Scan(&n); err != nil { + return 0, err + } + if n <= 0 { + n = 32767 + } + return n, nil +} + +// WriteWatermark declines to answer. The caller skips work when the +// token is unchanged between two calls, so a token that can miss a +// write is worse than none: SQL Server's per-database write counters +// (`sys.dm_db_index_usage_stats.user_updates`) are per INDEX, so writes +// to a heap with no index move nothing, and the rows are dropped when a +// database goes offline or the instance restarts. "" means "no sound +// token", which makes the caller do the work rather than skip it. +func (d *Driver) WriteWatermark(context.Context, string) (string, error) { return "", nil } + +// EnsureDB creates `name` if it is absent. The collation comes from the +// connection when set: a clone that sorts and compares differently from +// the app's real server is a subtle, expensive difference to debug, and +// SQL Server's install default (SQL_Latin1_General_CP1_CI_AS) is rarely +// what a non-English application uses. +func (d *Driver) EnsureDB(ctx context.Context, name string) error { + qname, err := ident.QuoteMSSQL(name) + if err != nil { + return err + } + exists, err := d.DatabaseExists(ctx, name) + if err != nil { + return err + } + if exists { + return nil + } + collate, err := d.createCollateClause() + if err != nil { + return err + } + //nolint:gosec // qname via ident.QuoteMSSQL, collation via the same validator + stmt := "CREATE DATABASE " + qname + collate + if _, err := d.DB.ExecContext(ctx, stmt); err != nil { + return fmt.Errorf("CREATE DATABASE %s: %w", qname, err) + } + // SIMPLE recovery on every database treeman owns: these are + // disposable, nobody restores them to a point in time, and FULL + // recovery grows a log file that is never backed up and so never + // truncates. + return d.setSimpleRecovery(ctx, name) +} + +// createCollateClause is the ` COLLATE ` suffix for CREATE +// DATABASE, or "" to take the server's default. The collation runs +// through the identifier validator even though it is not an +// identifier: SQL Server collation names are exactly [A-Za-z0-9_], +// and the value is interpolated into DDL. +func (d *Driver) createCollateClause() (string, error) { + c := d.cfg.Collation + if c == "" { + return "", nil + } + if err := ident.ValidateMSSQL(c); err != nil { + return "", fmt.Errorf("mssql collation %q: %w", c, err) + } + return " COLLATE " + c, nil +} + +func (d *Driver) setSimpleRecovery(ctx context.Context, name string) error { + qname, err := ident.QuoteMSSQL(name) + if err != nil { + return err + } + if _, err := d.DB.ExecContext(ctx, "ALTER DATABASE "+qname+" SET RECOVERY SIMPLE"); err != nil { + return fmt.Errorf("ALTER DATABASE %s SET RECOVERY SIMPLE: %w", qname, err) + } + return nil +} + +// DropMatching drops every database whose name starts with `prefix` — +// the worktree's own database plus its per-test clones. Four at a time: +// each DROP takes an exclusive lock on the database being dropped, not +// a server-wide one, so they genuinely overlap, and the limit keeps the +// connection count modest. +func (d *Driver) DropMatching(ctx context.Context, prefix string) ([]string, error) { + matched, err := d.ListMatching(ctx, prefix) + if err != nil { + return nil, err + } + g, gctx := errgroup.WithContext(ctx) + limit := min(4, max(len(matched), 1)) + g.SetLimit(limit) + for _, n := range matched { + g.Go(func() error { return d.DropDatabase(gctx, n) }) + } + if err := g.Wait(); err != nil { + return nil, err + } + return matched, nil +} + +// ListMatching returns every database named `prefix…`, ordered. LIKE +// metacharacters in the prefix are escaped, because every templated +// name carries a literal `_` and an unescaped one is a single-character +// wildcard — which would make a sibling worktree's database a match for +// this one's prefix. Same guard as the mysql and postgres drivers. +func (d *Driver) ListMatching(ctx context.Context, prefix string) ([]string, error) { + like := strings.NewReplacer(`\`, `\\`, `_`, `\_`, `%`, `\%`, `[`, `\[`).Replace(prefix) + "%" + rows, err := d.DB.QueryContext(ctx, + `SELECT name FROM sys.databases + WHERE name LIKE @p1 ESCAPE '\' AND database_id > 4 + ORDER BY name`, like) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + var out []string + for rows.Next() { + var s string + if err := rows.Scan(&s); err != nil { + return nil, err + } + out = append(out, s) + } + return out, rows.Err() +} + +// DatabaseExists reports whether the named database is present. +func (d *Driver) DatabaseExists(ctx context.Context, name string) (bool, error) { + var n int + err := d.DB.QueryRowContext(ctx, "SELECT 1 FROM sys.databases WHERE name = @p1", name).Scan(&n) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } else if err != nil { + return false, err + } + return true, nil +} + +// DropDatabase drops EXACTLY the named database — never a prefix match. +// branch_scoped teardown must use this rather than DropMatching: the +// namespace there is often a bare application database name that is a +// prefix of every linked worktree's `name_`, so a prefix drop +// would take the siblings with it. Idempotent. +// +// SINGLE_USER WITH ROLLBACK IMMEDIATE first, because unlike Postgres +// there is no pg_terminate_backend to call and DROP DATABASE fails +// outright while anybody is connected — including a pool that a jest +// worker left idle. +func (d *Driver) DropDatabase(ctx context.Context, name string) error { + qname, err := ident.QuoteMSSQL(name) + if err != nil { + return err + } + exists, err := d.DatabaseExists(ctx, name) + if err != nil { + return err + } + if !exists { + return nil + } + // Best-effort: a database already in single-user mode, or one with + // no connections at all, makes this a no-op either way, and the + // DROP below is the statement whose failure matters. + _, _ = d.DB.ExecContext(ctx, "ALTER DATABASE "+qname+" SET SINGLE_USER WITH ROLLBACK IMMEDIATE") + if _, err := d.DB.ExecContext(ctx, "DROP DATABASE IF EXISTS "+qname); err != nil { + return fmt.Errorf("DROP DATABASE %s: %w", qname, err) + } + return nil +} + +// RenameDatabase renames `from` to `to` — a catalog-only operation, +// constant-time whatever the database holds. It fails, never +// overwrites, when `to` is already taken, which is the atomicity the +// spare-claim path leans on: of two racing claimers exactly one wins. +// +// It does NOT rename the files on disk, which is why every restore in +// this driver names its files after a fresh random token rather than +// after the database: a spare renamed to `…_test_w1` keeps the spare's +// `.mdf`, and a later restore of the spare's own name would otherwise +// collide with a file that is still in use. +func (d *Driver) RenameDatabase(ctx context.Context, from, to string) error { + qfrom, err := ident.QuoteMSSQL(from) + if err != nil { + return err + } + if err := ident.ValidateMSSQL(to); err != nil { + return err + } + // MODIFY NAME is refused while anyone else is connected. The spare + // being claimed is idle by construction, but a doctor probe or a + // leftover pool is enough to lose the race, so the lock is taken + // explicitly and released whatever happens. + if _, err := d.DB.ExecContext(ctx, "ALTER DATABASE "+qfrom+" SET SINGLE_USER WITH ROLLBACK IMMEDIATE"); err != nil { + return fmt.Errorf("ALTER DATABASE %s SET SINGLE_USER: %w", qfrom, err) + } + _, err = d.DB.ExecContext(ctx, "ALTER DATABASE "+qfrom+" MODIFY NAME = ["+to+"]") + if err != nil { + // Hand the database back before returning, or a lost race + // leaves a single-user spare nothing can ever claim. + d.restoreMultiUser(ctx, from) + return fmt.Errorf("ALTER DATABASE %s MODIFY NAME: %w", qfrom, err) + } + d.restoreMultiUser(ctx, to) + return nil +} + +// restoreMultiUser puts a database back into MULTI_USER on a fresh, +// short-lived context, so it runs even when the caller's ctx is already +// cancelled — otherwise a cancellation mid-rename strands a database +// that admits exactly one session forever. +func (d *Driver) restoreMultiUser(ctx context.Context, name string) { + qname, err := ident.QuoteMSSQL(name) + if err != nil { + return + } + bg, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + _, _ = d.DB.ExecContext(bg, "ALTER DATABASE "+qname+" SET MULTI_USER") +} diff --git a/internal/db/mssql/mssql_test.go b/internal/db/mssql/mssql_test.go new file mode 100644 index 00000000..ba29c185 --- /dev/null +++ b/internal/db/mssql/mssql_test.go @@ -0,0 +1,180 @@ +package mssql + +import ( + "strings" + "testing" + + "github.com/stubbedev/treeman/internal/config" +) + +// TestDSNHidesNothingAndEscapes pins the two things a hand-built DSN +// gets wrong: a password with URL-significant characters, and an +// omitted database. SQL Server's own sa password policy pushes people +// toward `!`, `@` and `#`, so this is the common case, not the edge. +func TestDSNHidesNothingAndEscapes(t *testing.T) { + got := dsn(config.MssqlConn{ + Host: "127.0.0.1", Port: 14333, User: "sa", Password: "p@ss/wo rd!#", + }, "master") + for _, want := range []string{ + "sqlserver://", + "sa:p%40ss%2Fwo%20rd%21%23@127.0.0.1:14333", + "database=master", + "encrypt=disable", + } { + if !strings.Contains(got, want) { + t.Errorf("dsn = %q, missing %q", got, want) + } + } + // The password must never survive into the DSN unescaped, or the + // `/` would re-anchor the URL and the `@` would split the host. + if strings.Contains(got, "p@ss") { + t.Errorf("dsn = %q: password is not escaped", got) + } +} + +// TestResolveServerDirsPrefersConfig covers the precedence the +// server-side directories follow, without a server: config over +// SERVERPROPERTY, data_dir standing in for a missing log path, backup +// falling back to data. Exercised through the same helpers Connect +// uses, since that is where a wrong fallback would land. +func TestResolveServerDirsPrefersConfig(t *testing.T) { + cases := []struct { + name string + cfg config.MssqlConn + data, logp, backup string + wantData, wantLog, wantBak string + wantSep string + }{ + { + name: "server defaults, posix", + data: "/var/opt/mssql/data/", + logp: "/var/opt/mssql/data/", + backup: "/var/opt/mssql/backup", + wantData: "/var/opt/mssql/data", + wantLog: "/var/opt/mssql/data", + wantBak: "/var/opt/mssql/backup", + wantSep: "/", + }, + { + name: "windows separator detected from the server's answer", + data: `C:\SQL\DATA\`, + logp: `C:\SQL\LOG\`, + backup: `C:\SQL\BACKUP`, + wantData: `C:\SQL\DATA`, + wantLog: `C:\SQL\LOG`, + wantBak: `C:\SQL\BACKUP`, + wantSep: `\`, + }, + { + name: "config wins, and data_dir stands in for the log dir", + cfg: config.MssqlConn{DataDir: "/mnt/fast", BackupDir: "/mnt/bak"}, + data: "/var/opt/mssql/data/", + logp: "/var/opt/mssql/log/", + backup: "/var/opt/mssql/backup", + wantData: "/mnt/fast", + wantLog: "/mnt/fast", + wantBak: "/mnt/bak", + wantSep: "/", + }, + { + name: "no backup path reported falls back to the data dir", + data: "/var/opt/mssql/data/", + logp: "/var/opt/mssql/data/", + wantData: "/var/opt/mssql/data", + wantLog: "/var/opt/mssql/data", + wantBak: "/var/opt/mssql/data", + wantSep: "/", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + d := &Driver{cfg: c.cfg} + if err := d.applyServerDirs(c.data, c.logp, c.backup); err != nil { + t.Fatalf("applyServerDirs: %v", err) + } + if d.dataDir != c.wantData || d.logDir != c.wantLog || d.backupDir != c.wantBak { + t.Errorf("dirs = (%q, %q, %q), want (%q, %q, %q)", + d.dataDir, d.logDir, d.backupDir, c.wantData, c.wantLog, c.wantBak) + } + if d.sep != c.wantSep { + t.Errorf("sep = %q, want %q", d.sep, c.wantSep) + } + }) + } +} + +// TestResolveServerDirsRefusesEmpty: a server that reports no default +// paths at all (an unusual install, or a login without VIEW SERVER +// STATE) must fail at Connect with the remedy named, not later with a +// RESTORE writing to the filesystem root. +func TestResolveServerDirsRefusesEmpty(t *testing.T) { + d := &Driver{} + err := d.applyServerDirs("", "", "") + if err == nil { + t.Fatal("empty server paths must be refused") + } + for _, want := range []string{"data_dir", "backup_dir"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not name %q", err, want) + } + } +} + +// TestBackupPathIsOneFilePerTemplate pins the naming the GC leans on: +// one `.bak` per template name, so the files on disk are bounded by the +// live templates and a re-capture overwrites rather than accumulates. +func TestBackupPathIsOneFilePerTemplate(t *testing.T) { + d := &Driver{} + if err := d.applyServerDirs("/var/opt/mssql/data/", "", "/bak"); err != nil { + t.Fatal(err) + } + if got, want := d.backupPath("_tm_0123456789abcdef"), "/bak/_tm_0123456789abcdef.bak"; got != want { + t.Errorf("backupPath = %q, want %q", got, want) + } + if d.backupPath("_tm_a") == d.backupPath("_tm_b") { + t.Error("two templates share one backup file") + } +} + +// TestFileTokenIsRandom: restored files are named after a token rather +// than after the database, because a claimed spare keeps files named +// for the spare. Two tokens colliding would reintroduce exactly the +// collision the token exists to avoid. +func TestFileTokenIsRandom(t *testing.T) { + seen := map[string]bool{} + for range 100 { + tok, err := fileToken() + if err != nil { + t.Fatalf("fileToken: %v", err) + } + if len(tok) != 8 { + t.Fatalf("fileToken = %q, want 8 hex chars", tok) + } + if seen[tok] { + t.Fatalf("fileToken repeated %q within 100 draws", tok) + } + seen[tok] = true + } +} + +// TestCellString covers the one conversion RESTORE FILELISTONLY forces: +// its columns arrive as `any`, and a driver that hands back []byte +// instead of string would silently yield an empty logical file name — +// which RESTORE would then reject with a message about a file that is +// not in the backup set. +func TestCellString(t *testing.T) { + cases := []struct { + in any + want string + }{ + {"logical", "logical"}, + {[]byte("logical"), "logical"}, + {nil, ""}, + {42, ""}, + } + for _, c := range cases { + if got := cellString(c.in); got != c.want { + t.Errorf("cellString(%#v) = %q, want %q", c.in, got, c.want) + } + } +} diff --git a/internal/db/mssql/rowcount.go b/internal/db/mssql/rowcount.go new file mode 100644 index 00000000..5e9f2956 --- /dev/null +++ b/internal/db/mssql/rowcount.go @@ -0,0 +1,98 @@ +package mssql + +import ( + "context" + "fmt" + "strings" + + "github.com/stubbedev/treeman/internal/db/ident" + "github.com/stubbedev/treeman/internal/db/rowcount" +) + +// nonEmptyTables lists up to rowcount.SampleLimit of `db`'s smallest +// non-empty tables, as `schema.table`. Unlike MySQL and Postgres a +// table name alone is not unique within a database here — SQL Server +// has real schemas, and a vendor database puts the same table name in +// several — so the schema travels with it and countOf splits it back +// apart. Estimates only PICK the sample; exact counts decide. +func (d *Driver) nonEmptyTables(ctx context.Context, db string) ([]string, error) { + qdb, err := ident.QuoteMSSQL(db) + if err != nil { + return nil, err + } + // Three-part names reach another database's catalog views, so this + // runs on the master pool without a scoped connection. index_id + // 0/1 is the heap or the clustered index — the one partition set + // that holds the rows rather than a copy of them. + rows, err := d.DB.QueryContext(ctx, fmt.Sprintf(` + SELECT TOP (@p1) s.name, t.name + FROM %[1]s.sys.tables t + JOIN %[1]s.sys.schemas s ON s.schema_id = t.schema_id + JOIN %[1]s.sys.partitions p ON p.object_id = t.object_id AND p.index_id IN (0, 1) + GROUP BY s.name, t.name + HAVING SUM(p.rows) > 0 + ORDER BY SUM(p.rows) ASC`, qdb), rowcount.SampleLimit) + if err != nil { + return nil, fmt.Errorf("row-count sample list %s: %w", db, err) + } + defer func() { _ = rows.Close() }() + var tables []string + for rows.Next() { + var schema, name string + if err := rows.Scan(&schema, &name); err != nil { + return nil, err + } + tables = append(tables, schema+"."+name) + } + return tables, rows.Err() +} + +// countOf counts one `schema.table` with identifier validation and +// quoting on each of the three parts. +func (d *Driver) countOf(ctx context.Context, db, table string) (int64, error) { + qdb, err := ident.QuoteMSSQL(db) + if err != nil { + return 0, err + } + schema, name, ok := strings.Cut(table, ".") + if !ok { + // A bare name can only have come from a caller that did not go + // through nonEmptyTables; `dbo` is the only defensible guess. + schema, name = "dbo", table + } + qschema, err := ident.QuoteMSSQL(schema) + if err != nil { + return 0, err + } + qname, err := ident.QuoteMSSQL(name) + if err != nil { + return 0, err + } + var n int64 + if err := d.DB.QueryRowContext(ctx, + fmt.Sprintf("SELECT COUNT_BIG(*) FROM %s.%s.%s", qdb, qschema, qname)).Scan(&n); err != nil { + return 0, err + } + return n, nil +} + +// RowCountSample returns exact row counts for a small sample of `db`'s +// tables (see nonEmptyTables). Capture/restore verification compares +// the source's sample against the copy's; an empty map means "no table +// the catalog believes non-empty" and is NOT a verification failure. +func (d *Driver) RowCountSample(ctx context.Context, db string) (map[string]int64, error) { + if err := ident.ValidateMSSQL(db); err != nil { + return nil, err + } + return rowcount.Sample(ctx, d.nonEmptyTables, d.countOf, db) +} + +// RowCountsFor returns exact row counts for the NAMED tables in `db`. +// Capture verification uses it to count the SOURCE's sample tables in +// the COPY — by name, so the copy's own catalog estimates never decide. +func (d *Driver) RowCountsFor(ctx context.Context, db string, tables []string) (map[string]int64, error) { + if err := ident.ValidateMSSQL(db); err != nil { + return nil, err + } + return rowcount.CountsFor(ctx, d.countOf, db, tables) +} diff --git a/internal/db/mssql/snapshot.go b/internal/db/mssql/snapshot.go new file mode 100644 index 00000000..0aeb08d4 --- /dev/null +++ b/internal/db/mssql/snapshot.go @@ -0,0 +1,306 @@ +package mssql + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/hex" + "fmt" + "strings" + + "github.com/stubbedev/treeman/internal/db/ident" +) + +// backupFile describes one file inside a backup set, as +// `RESTORE FILELISTONLY` reports it. Every RESTORE must name each of +// them in a MOVE clause, or the restore lands on the paths the source +// database occupies — which are taken. +type backupFile struct { + logical string + isLog bool +} + +// SnapshotCreate captures `source` as the template `template`. +// +// Two statements, because SQL Server has no `CREATE DATABASE … TEMPLATE`: +// BACKUP source to the template's `.bak`, then RESTORE that file as the +// template database. The second half is what keeps the template a real +// database — treeman's cache-hit probe is `DatabaseExists(template)`, +// the same for every engine — and it is not wasted work either: the +// `.bak` it leaves behind is what every clone restores from, so a +// template is captured once and hydrated N times. +// +// `source` is read-only here, and unlike the postgres driver nothing is +// fenced: BACKUP is an online, transactionally consistent read, so +// there is no window to close and no unfence that can leak if the +// context is cancelled mid-capture. +func (d *Driver) SnapshotCreate(ctx context.Context, source, template string) error { + qsource, err := ident.QuoteMSSQL(source) + if err != nil { + return err + } + if err := ident.ValidateMSSQL(template); err != nil { + return err + } + if err := d.backupTo(ctx, qsource, d.backupPath(template)); err != nil { + return err + } + return d.restoreFromBackup(ctx, d.backupPath(template), template) +} + +// SnapshotRestore hydrates `target` from the template `template` — one +// RESTORE from the `.bak` SnapshotCreate left behind. The template +// survives: a prepare restores it concurrently into the worktree's own +// database and every per-test clone. +// +// When the `.bak` is gone but the template database is still there +// (someone cleaned the backup directory, or the template was captured +// by an older treeman), it is re-derived from the template rather than +// failing — the alternative is a cold rebuild of a template that is +// demonstrably still good. +func (d *Driver) SnapshotRestore(ctx context.Context, template, target string) error { + qtemplate, err := ident.QuoteMSSQL(template) + if err != nil { + return err + } + if err := ident.ValidateMSSQL(target); err != nil { + return err + } + bak := d.backupPath(template) + present, err := d.serverFileExists(ctx, bak) + if err != nil { + return err + } + if !present { + exists, err := d.DatabaseExists(ctx, template) + if err != nil { + return err + } + if !exists { + return fmt.Errorf("mssql snapshot %q: neither the template database nor %s exists", template, bak) + } + if err := d.backupTo(ctx, qtemplate, bak); err != nil { + return err + } + } + return d.restoreFromBackup(ctx, bak, target) +} + +// DropSnapshot drops exactly the named database and removes the `.bak` +// that belongs to it. Idempotent, and callable on a name that was never +// a template — the spare-claim path uses it to clear a target before a +// claim, where there is no backup file to find. +func (d *Driver) DropSnapshot(ctx context.Context, template string) error { + if err := d.DropDatabase(ctx, template); err != nil { + return err + } + return d.deleteServerFile(ctx, d.backupPath(template)) +} + +// SnapshotBackupPath is where the template's backup set lives ON THE +// SERVER. Exported because when a RESTORE fails there is nothing on +// this machine to look at, and the path is the first thing a human +// needs — `docker exec ls -l` on it answers most of the +// questions a restore error raises. +func (d *Driver) SnapshotBackupPath(template string) string { + return d.backupPath(template) +} + +// backupTo writes a fresh, self-contained backup set over whatever is +// at `path`. +// +// - FORMAT + INIT: one backup set per file, never appended to, so the +// file's size is the database's size and `WITH FILE = 1` is always +// the right set. An appending backup device grows without bound and +// makes every restore guess at a set number. +// - COPY_ONLY: leaves the source's differential base alone. Nothing +// treeman clones is on a backup schedule, but a database that is +// (a `dump:` source pointed at something real) must not have its +// chain disturbed by a provisioning run. +// - No COMPRESSION: unsupported on Express, and it buys little at the +// sizes a test template runs to. +func (d *Driver) backupTo(ctx context.Context, quotedDB, path string) error { + const q = "BACKUP DATABASE %s TO DISK = @p1 WITH FORMAT, INIT, COPY_ONLY, SKIP" + if err := drain(ctx, d.DB, fmt.Sprintf(q, quotedDB), path); err != nil { + return fmt.Errorf("BACKUP DATABASE %s TO %s: %w", quotedDB, path, err) + } + return nil +} + +// restoreFromBackup restores the backup set at `path` as the database +// `name`, which is dropped first so the restore is an overwrite by +// exact name (never a prefix match — a target name is routinely a +// prefix of a sibling worktree's databases). +// +// Every file in the set is moved to a path carrying a fresh random +// token. Naming them after `name` would be the obvious choice and is +// wrong: `RenameDatabase` (the spare claim) renames the database but +// not its files, so a claimed spare keeps files named for the spare, +// and the next restore of that spare's own name would collide with +// files still in use. +func (d *Driver) restoreFromBackup(ctx context.Context, path, name string) error { + qname, err := ident.QuoteMSSQL(name) + if err != nil { + return err + } + files, err := d.backupFileList(ctx, path) + if err != nil { + return err + } + if len(files) == 0 { + return fmt.Errorf("mssql restore %s: backup set %s lists no files", qname, path) + } + if err := d.DropDatabase(ctx, name); err != nil { + return err + } + token, err := fileToken() + if err != nil { + return err + } + var q strings.Builder + q.WriteString("RESTORE DATABASE " + qname + " FROM DISK = @p1 WITH RECOVERY, REPLACE") + args := []any{path} + for i, f := range files { + dir, ext := d.dataDir, ".mdf" + if f.isLog { + dir, ext = d.logDir, ".ldf" + } + if i > 0 { + // Secondary data files cannot all be `.mdf`; the suffix + // only has to be unique and stable within one restore. + ext = fmt.Sprintf("_%d%s", i, ext) + } + args = append(args, f.logical, d.join(dir, name+"_"+token+ext)) + fmt.Fprintf(&q, ", MOVE @p%d TO @p%d", len(args)-1, len(args)) + } + if err := drain(ctx, d.DB, q.String(), args...); err != nil { + return fmt.Errorf("RESTORE DATABASE %s FROM %s: %w", qname, path, err) + } + // The restored database inherits the source's recovery model, which + // is SIMPLE for anything EnsureDB made — but a `dump:` source or a + // hand-built database need not be, and a FULL-recovery clone grows + // a log nobody will ever back up. + return d.setSimpleRecovery(ctx, name) +} + +// backupFileList reads the logical file names out of a backup set. +// Columns are located by name rather than by position: FILELISTONLY's +// result set has grown with nearly every release (22 columns on 2025), +// and only three of them have ever mattered here. +func (d *Driver) backupFileList(ctx context.Context, path string) ([]backupFile, error) { + rows, err := d.DB.QueryContext(ctx, "RESTORE FILELISTONLY FROM DISK = @p1", path) + if err != nil { + return nil, fmt.Errorf("RESTORE FILELISTONLY FROM %s: %w", path, err) + } + defer func() { _ = rows.Close() }() + cols, err := rows.Columns() + if err != nil { + return nil, err + } + iLogical, iType := -1, -1 + for i, c := range cols { + switch c { + case "LogicalName": + iLogical = i + case "Type": + iType = i + } + } + if iLogical < 0 || iType < 0 { + return nil, fmt.Errorf("RESTORE FILELISTONLY FROM %s: no LogicalName/Type column in %v", path, cols) + } + cells := make([]any, len(cols)) + ptrs := make([]any, len(cols)) + for i := range cells { + ptrs[i] = &cells[i] + } + var out []backupFile + for rows.Next() { + if err := rows.Scan(ptrs...); err != nil { + return nil, err + } + // 'D' data, 'L' log, 'F'/'S' FILESTREAM and full-text — all of + // which restore as data files. + out = append(out, backupFile{ + logical: cellString(cells[iLogical]), + isLog: cellString(cells[iType]) == "L", + }) + } + return out, rows.Err() +} + +// serverFileExists asks the SERVER whether a path is a readable file. +// os.Stat would answer about treeman's own filesystem, which for a +// containerised server is a different machine. +func (d *Driver) serverFileExists(ctx context.Context, path string) (bool, error) { + var isFile, isDir, parentExists sql.NullInt64 + err := d.DB.QueryRowContext(ctx, + `DECLARE @r TABLE (f int, d int, p int); + INSERT INTO @r EXEC master.sys.xp_fileexist @p1; + SELECT f, d, p FROM @r`, path).Scan(&isFile, &isDir, &parentExists) + if err != nil { + return false, fmt.Errorf("xp_fileexist %s: %w", path, err) + } + return isFile.Int64 == 1, nil +} + +// deleteServerFile removes a file from the server's filesystem. +// Best-effort by design: sys.xp_delete_files arrived in SQL Server 2019 +// and needs sysadmin, so on an older or less-privileged server a stale +// `.bak` is left behind rather than failing a teardown over it. The +// leak is bounded — one file per template name, overwritten in place by +// the next capture of that template. +func (d *Driver) deleteServerFile(ctx context.Context, path string) error { + present, err := d.serverFileExists(ctx, path) + if err != nil || !present { + return nil //nolint:nilerr // a probe failure is not a teardown failure + } + _, _ = d.DB.ExecContext(ctx, "EXEC master.sys.xp_delete_files @p1", path) + return nil +} + +// fileToken is the random component of a restored database's file +// names. Eight hex characters: these live in one directory alongside +// at most a few hundred siblings, so collision resistance matters far +// less than the names staying readable in a `ls`. +func fileToken() (string, error) { + var b [4]byte + if _, err := rand.Read(b[:]); err != nil { + return "", fmt.Errorf("mssql file token: %w", err) + } + return hex.EncodeToString(b[:]), nil +} + +func cellString(v any) string { + switch t := v.(type) { + case string: + return t + case []byte: + return string(t) + } + return "" +} + +// drain runs a statement that reports progress as result sets rather +// than as a row count — BACKUP and RESTORE both do ("Processed N pages +// …"). ExecContext discards those sets, and with them any error the +// server raises after the first one, so the statement is issued as a +// query and every set is walked to completion. +func drain(ctx context.Context, db *sql.DB, q string, args ...any) error { + rows, err := db.QueryContext(ctx, q, args...) + if err != nil { + return err + } + defer func() { _ = rows.Close() }() + for { + for rows.Next() { + } + if err := rows.Err(); err != nil { + return err + } + if !rows.NextResultSet() { + break + } + } + return rows.Err() +} diff --git a/internal/db/reachability/reachability.go b/internal/db/reachability/reachability.go index 73dc8994..4f06c9f6 100644 --- a/internal/db/reachability/reachability.go +++ b/internal/db/reachability/reachability.go @@ -123,6 +123,8 @@ func defaultPort(scheme string) (uint16, bool) { return 3306, true case "postgres", "postgresql": return 5432, true + case "sqlserver", "mssql": + return 1433, true } return 0, false } diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 7a91f528..ba562775 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -18,6 +18,7 @@ type Family string const ( FamilyMySQL Family = "mysql" // mysql / mariadb / tidb FamilyPostgres Family = "postgres" // postgres / postgresql + FamilyMSSQL Family = "mssql" // mssql / sqlserver FamilyMongo Family = "mongodb" // mongodb FamilyRedis Family = "redis" // redis / valkey / dragonfly FamilyES Family = "elasticsearch" // elasticsearch / opensearch @@ -33,6 +34,7 @@ const ( var Known = []string{ "mysql", "mariadb", "tidb", "postgres", "postgresql", + "mssql", "sqlserver", "mongodb", "redis", "valkey", "dragonfly", "elasticsearch", "opensearch", @@ -55,6 +57,8 @@ func Canonical(eng string) (Family, bool) { return FamilyMySQL, true case "postgres", "postgresql": return FamilyPostgres, true + case "mssql", "sqlserver": + return FamilyMSSQL, true case "mongodb": return FamilyMongo, true case "redis", "valkey", "dragonfly": diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go index 23412ef2..6d76bc6c 100644 --- a/internal/engine/engine_test.go +++ b/internal/engine/engine_test.go @@ -17,6 +17,8 @@ func TestCanonical(t *testing.T) { "tidb": {FamilyMySQL, true}, "postgres": {FamilyPostgres, true}, "postgresql": {FamilyPostgres, true}, + "mssql": {FamilyMSSQL, true}, + "sqlserver": {FamilyMSSQL, true}, "mongodb": {FamilyMongo, true}, "redis": {FamilyRedis, true}, "valkey": {FamilyRedis, true}, @@ -43,6 +45,7 @@ func TestFamilyScope(t *testing.T) { cases := map[Family]Scope{ FamilyMySQL: ScopeName, FamilyPostgres: ScopeName, + FamilyMSSQL: ScopeName, FamilyMongo: ScopeName, FamilyRedis: ScopePrefix, FamilyES: ScopePrefix, diff --git a/internal/initgen/initgen.go b/internal/initgen/initgen.go index d70c69cf..1b36ea54 100644 --- a/internal/initgen/initgen.go +++ b/internal/initgen/initgen.go @@ -125,9 +125,9 @@ func engineEntry(name, engine string, fam enginepkg.Family) *yaml.Node { "clones", scalar("auto"), "name_template", scalar(name+"_testing_{slug}_test_{n}"), )) - if fam == enginepkg.FamilyPostgres || fam == enginepkg.FamilyMySQL { + if fam == enginepkg.FamilyPostgres || fam == enginepkg.FamilyMySQL || fam == enginepkg.FamilyMSSQL { mapSet(db, "prewarm", scalar("2")) - mapKeyNode(db, "prewarm").LineComment = "spare clones pre-restored from the template; cache-hit creates claim one cheaply (rename on postgres, physical clone on mysql)" + mapKeyNode(db, "prewarm").LineComment = "spare clones pre-restored from the template; cache-hit creates claim one cheaply (rename on postgres and mssql, physical clone on mysql)" } } return db @@ -190,14 +190,15 @@ func frameworkDbEntry(name string, spec framework.Spec) *yaml.Node { )) // prewarm only applies to the spare-capable engines — scaffold it // where it's valid so users discover the knob. - if fam, ok := enginepkg.Canonical(spec.EngineHint); ok && (fam == enginepkg.FamilyPostgres || fam == enginepkg.FamilyMySQL) { + if fam, ok := enginepkg.Canonical(spec.EngineHint); ok && + (fam == enginepkg.FamilyPostgres || fam == enginepkg.FamilyMySQL || fam == enginepkg.FamilyMSSQL) { mapSet(db, "prewarm", scalar("2")) - mapKeyNode(db, "prewarm").LineComment = "spare clones pre-restored from the template; cache-hit creates claim one cheaply (rename on postgres, physical clone on mysql)" + mapKeyNode(db, "prewarm").LineComment = "spare clones pre-restored from the template; cache-hit creates claim one cheaply (rename on postgres and mssql, physical clone on mysql)" } return db } -// inferEngineFromEnv picks mysql or postgres from DATABASE_URL — the +// inferEngineFromEnv picks mysql, postgres or mssql from DATABASE_URL — the // process env first, then a repo-root .env — so a hintless framework // (e.g. rails) on a known engine still scaffolds the right one. "" // when neither source yields a usable scheme. @@ -225,6 +226,8 @@ func inferEngineFromEnv(cwd string) string { return "postgres" case "mysql", "mariadb": return "mysql" + case "sqlserver", "mssql": + return "mssql" default: return "" } diff --git a/internal/mcp/server.go b/internal/mcp/server.go index e5429169..e4cfed2c 100644 --- a/internal/mcp/server.go +++ b/internal/mcp/server.go @@ -80,13 +80,13 @@ func buildServer() (*mcpsdk.Server, []*toolEntry) { // into the conversation and the highest-leverage entry points, so an // agent doesn't fall back to raw `git`/`psql` when treeman owns the // flow. Keep it short — clients prepend it to the model's context. -const serverInstructions = `Treeman gives every git worktree its own ephemeral MySQL/Postgres/Mongo/Redis/ES instance, cold-built from a dump + migrations and cached as a template for fast reuse. +const serverInstructions = `Treeman gives every git worktree its own ephemeral MySQL/Postgres/SQL Server/Mongo/Redis/ES instance, cold-built from a dump + migrations and cached as a template for fast reuse. REACH FOR TREEMAN when the user mentions: worktree, branch-scoped DB, snapshot/template, prepare, dump+migrate+seed, ` + "`.treeman.yaml`" + `, treemand, finalize, fingerprint, slug. Only a few core tools load up front (doctor, status_overview, worktree_create, worktree_list, prepare_run, logs_query). For everything else — config, engine access, snapshots, registry, daemon control, init, the rest of the worktree/logs surface, and the guided multi-step prompts — call the ` + "`tools`" + ` tool: action=list catalogs every capability by category (with a routing guide), action=enable loads the ones the task needs. -Prefer treeman over raw ` + "`git worktree`, `psql`, `mysql`, `mongosh`, `redis-cli`, `curl`" + `: it reaches every engine through its own configured, authenticated drivers and keeps SQLite + engine state in sync — bypassing it desyncs them. Destructive tools take dry_run=true to preview.` +Prefer treeman over raw ` + "`git worktree`, `psql`, `mysql`, `sqlcmd`, `mongosh`, `redis-cli`, `curl`" + `: it reaches every engine through its own configured, authenticated drivers and keeps SQLite + engine state in sync — bypassing it desyncs them. Destructive tools take dry_run=true to preview.` // Serve boots the MCP server on stdio and blocks until the client // disconnects or ctx is cancelled. Returns nil on clean shutdown. diff --git a/internal/mcp/tools_engine.go b/internal/mcp/tools_engine.go index d177a474..635b4aff 100644 --- a/internal/mcp/tools_engine.go +++ b/internal/mcp/tools_engine.go @@ -28,6 +28,7 @@ import ( dbes "github.com/stubbedev/treeman/internal/db/es" "github.com/stubbedev/treeman/internal/db/ident" dbmongo "github.com/stubbedev/treeman/internal/db/mongo" + dbmssql "github.com/stubbedev/treeman/internal/db/mssql" dbmysql "github.com/stubbedev/treeman/internal/db/mysql" dbpostgres "github.com/stubbedev/treeman/internal/db/postgres" dbredis "github.com/stubbedev/treeman/internal/db/redis" @@ -49,7 +50,7 @@ func registerEngineReadTools(srv *mcpsdk.Server) { addTool(srv, &mcpsdk.Tool{ Name: "db_schema_dump", - Description: "Dump the live schema for ONE database — mysql/postgres: CREATE TABLEs, mongo: collection list + samples, ES: index mapping, redis: SCAN-driven key summary. Use to reason about live shape vs. what migrations expect. db = the rendered per-worktree name (find via worktree_show or snapshots_inspect). Capped at max_tables (default 200).", + Description: "Dump the live schema for ONE database — mysql/postgres: CREATE TABLEs, mssql: tables + columns, mongo: collection list + samples, ES: index mapping, redis: SCAN-driven key summary. Use to reason about live shape vs. what migrations expect. db = the rendered per-worktree name (find via worktree_show or snapshots_inspect). Capped at max_tables (default 200).", Annotations: readOnlyAnno("Dump live schema", true), }, dbSchemaDumpTool) @@ -79,7 +80,7 @@ func registerEngineReadTools(srv *mcpsdk.Server) { addTool(srv, &mcpsdk.Tool{ Name: "connection_probe", - Description: "Dry-test a connection string against an engine without writing it to .treeman.yaml. Tightens the setup loop — iterate on credentials, observe reachable/version/latency, then commit via config_set. dsn forms: mysql://user:pw@host:port, postgres://…, mongodb://…, redis://…, http(s)://…(ES). Omit dsn to probe the repo's currently-configured connection.", + Description: "Dry-test a connection string against an engine without writing it to .treeman.yaml. Tightens the setup loop — iterate on credentials, observe reachable/version/latency, then commit via config_set. dsn forms: mysql://user:pw@host:port, postgres://…, sqlserver://…, mongodb://…, redis://…, http(s)://…(ES). Omit dsn to probe the repo's currently-configured connection.", Annotations: readOnlyAnno("Probe connection", true), }, connectionProbeTool) @@ -99,7 +100,7 @@ func registerEngineWriteTools(srv *mcpsdk.Server) { addTool(srv, &mcpsdk.Tool{ Name: "db_dump", - Description: "Refresh the seed dump used for cold builds — runs mysqldump / pg_dump / mongodump / ES scroll-bulk. Commit the new file then trigger prepare_run. Redis intentionally not implemented (uses a seed step, not a dump). output_dir defaults to /storage/dumps.", + Description: "Refresh the seed dump used for cold builds — runs mysqldump / pg_dump / mongodump / ES scroll-bulk. Commit the new file then trigger prepare_run. Redis and mssql intentionally not implemented (redis cold-builds from a seed step, and SQL Server has no client-side dump format at all). output_dir defaults to /storage/dumps.", Annotations: writeAnno("Dump database", false, false, true), }, dbDumpTool) } @@ -132,7 +133,7 @@ func engineStatusTool(ctx context.Context, _ *mcpsdk.CallToolRequest, in engineS // written to distinct slice indices (no shared mutation) and the // fixed order is preserved. probes := []func(context.Context, *config.Config) engineProbeResult{ - probeMySQL, probePostgres, probeMongo, probeRedis, probeES, probeS3, + probeMySQL, probePostgres, probeMssql, probeMongo, probeRedis, probeES, probeS3, } results := make([]engineProbeResult, len(probes)) var wg sync.WaitGroup @@ -183,6 +184,28 @@ func probePostgres(ctx context.Context, cfg *config.Config) engineProbeResult { return r } +func probeMssql(ctx context.Context, cfg *config.Config) engineProbeResult { + r := engineProbeResult{Engine: "mssql"} + if cfg.Connections.Mssql == nil { + return r + } + r.Configured = true + drv, err := dbmssql.Connect(ctx, *cfg.Connections.Mssql) + if err != nil { + r.Error = err.Error() + return r + } + defer func() { _ = drv.Close() }() + r.Reachable = true + r.Version, _ = drv.EngineVersion(ctx) + // ListMatching("") already excludes the four system databases + // (database_id > 4), so there is no listMssqlDatabases twin of the + // mysql/postgres helpers. + dbs, _ := drv.ListMatching(ctx, "") + r.Detail = databasesDetail(dbs) + return r +} + func probeMongo(ctx context.Context, cfg *config.Config) engineProbeResult { r := engineProbeResult{Engine: "mongodb"} if cfg.Connections.Mongodb == nil { @@ -323,7 +346,7 @@ func s3LiteralPrefixes(cfg *config.Config) []string { type dbSchemaIn struct { Repo string `json:"repo,omitempty"` - Engine string `json:"engine" jsonschema:"mysql|mariadb|tidb|postgres|postgresql|mongodb|redis|elasticsearch"` + Engine string `json:"engine" jsonschema:"mysql|mariadb|tidb|postgres|postgresql|mssql|sqlserver|mongodb|redis|elasticsearch"` DB string `json:"db"` MaxTables int `json:"max_tables,omitempty" jsonschema:"sql/mongo: cap on the number of tables/collections returned (default 200). truncated=true is set in the output when the cap kicks in."` } @@ -350,6 +373,8 @@ func dbSchemaDumpTool(ctx context.Context, _ *mcpsdk.CallToolRequest, in dbSchem schema, err = dbSchemaMySQL(ctx, cfg, in.DB) case engine.FamilyPostgres: schema, err = dbSchemaPostgres(ctx, cfg, in.DB) + case engine.FamilyMSSQL: + schema, err = dbSchemaMssql(ctx, cfg, in.DB) case engine.FamilyMongo: schema, err = dbSchemaMongo(ctx, cfg, in.DB) case engine.FamilyES: @@ -437,6 +462,18 @@ func dbSchemaPostgres(ctx context.Context, cfg *config.Config, db string) (map[s return postgresSchema(ctx, drv, db) } +func dbSchemaMssql(ctx context.Context, cfg *config.Config, db string) (map[string]any, error) { + if cfg.Connections.Mssql == nil { + return nil, errors.New("connections.mssql not configured") + } + drv, err := dbmssql.Connect(ctx, *cfg.Connections.Mssql) + if err != nil { + return nil, err + } + defer func() { _ = drv.Close() }() + return mssqlSchema(ctx, drv, db) +} + func dbSchemaMongo(ctx context.Context, cfg *config.Config, db string) (map[string]any, error) { if cfg.Connections.Mongodb == nil { return nil, errors.New("connections.mongodb not configured") @@ -478,7 +515,7 @@ type dbQueryIn struct { Repo string `json:"repo,omitempty"` Engine string `json:"engine"` DB string `json:"db"` - Query string `json:"query,omitempty" jsonschema:"SQL for mysql/postgres; raw command for redis"` + Query string `json:"query,omitempty" jsonschema:"SQL for mysql/postgres/mssql; raw command for redis"` Collection string `json:"collection,omitempty" jsonschema:"mongodb collection name (read/find path)"` FilterJSON string `json:"filter_json,omitempty" jsonschema:"mongodb filter document (read/find path)"` CommandJSON string `json:"command_json,omitempty" jsonschema:"mongodb: a raw command document run via RunCommand, e.g. {\"insert\":\"users\",\"documents\":[...]}, {\"aggregate\":\"c\",\"pipeline\":[...],\"cursor\":{}}, {\"count\":\"c\"}. Read commands run with write=false; write commands need write=true+ack."` @@ -524,6 +561,8 @@ func dbQueryTool(ctx context.Context, _ *mcpsdk.CallToolRequest, in dbQueryIn) ( rows, cols, affected, err = dbQueryMySQL(ctx, cfg, in) case engine.FamilyPostgres: rows, cols, affected, err = dbQueryPostgres(ctx, cfg, in) + case engine.FamilyMSSQL: + rows, cols, affected, err = dbQueryMssql(ctx, cfg, in) case engine.FamilyMongo: rows, err = dbQueryMongo(ctx, cfg, in) case engine.FamilyES: @@ -601,6 +640,33 @@ func dbQueryPostgres(ctx context.Context, cfg *config.Config, in dbQueryIn) ([]a return rows, cols, nil, err } +func dbQueryMssql(ctx context.Context, cfg *config.Config, in dbQueryIn) ([]any, []string, *int64, error) { + if !in.Write { + if err := assertReadOnlySQL(in.Query); err != nil { + return nil, nil, nil, err + } + } + if cfg.Connections.Mssql == nil { + return nil, nil, nil, errors.New("connections.mssql not configured") + } + drv, err := dbmssql.Connect(ctx, *cfg.Connections.Mssql) + if err != nil { + return nil, nil, nil, err + } + defer func() { _ = drv.Close() }() + scoped, err := drv.OpenScoped(ctx, in.DB) + if err != nil { + return nil, nil, nil, err + } + defer func() { _ = scoped.Close() }() + if in.Write { + rows, aff, err := execSQL(ctx, scoped, in.Query) + return rows, nil, aff, err + } + rows, cols, err := runSQLQuery(ctx, scoped, in.Query, in.Limit) + return rows, cols, nil, err +} + // execSQL runs a mutating statement and reports rows-affected (plus the // driver's last-insert-id when it exposes a useful one). The write path // for db_query's SQL engines. @@ -1136,6 +1202,10 @@ func dbDumpTool(ctx context.Context, _ *mcpsdk.CallToolRequest, in dbDumpIn) (*m p += ".gz" } return runESDump(ctx, cfg.Connections.Elasticsearch, in.DB, p, in.Gzip) + case engine.FamilyMSSQL: + return nil, dbDumpOut{}, errors.New( + "db_dump: mssql dump is not implemented — sqlcmd is not a pg_dump equivalent (it cannot emit schema+data as SQL), and BACKUP DATABASE writes its .bak on the SQL Server host's filesystem rather than here. That is also why `dump:` is rejected in config for mssql entries: seed them from `seed:`/`migrate:` commands, which can get a file to the server and RESTORE it", + ) case engine.FamilyRedis: return nil, dbDumpOut{}, errors.New( "db_dump: redis dump is not implemented — redis cold-build uses a `seed:` step rather than a dump file, so there is no restore counterpart. If you need a redis snapshot, use redis-cli BGSAVE / SAVE manually and reference the resulting RDB out-of-band", @@ -1368,6 +1438,77 @@ func postgresTableColumns(ctx context.Context, scoped *sql.DB, schema, name stri return cols, true } +func mssqlSchema(ctx context.Context, drv *dbmssql.Driver, db string) (map[string]any, error) { + scoped, err := drv.OpenScoped(ctx, db) + if err != nil { + return nil, err + } + defer func() { _ = scoped.Close() }() + // sys.tables rather than INFORMATION_SCHEMA.TABLES: the catalog view + // returns only base tables and already excludes system objects, + // whereas the information_schema view also lists views. + rows, err := scoped.QueryContext(ctx, ` + SELECT s.name, t.name + FROM sys.tables AS t + JOIN sys.schemas AS s ON s.schema_id = t.schema_id + ORDER BY s.name, t.name + `) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + type tablePair struct{ schema, name string } + var tables []tablePair + for rows.Next() { + var p tablePair + _ = rows.Scan(&p.schema, &p.name) + tables = append(tables, p) + } + if err := rows.Err(); err != nil { + return nil, err + } + schema := map[string]any{} + for _, t := range tables { + cols, ok := mssqlTableColumns(ctx, scoped, t.schema, t.name) + if !ok { + continue + } + schema[t.schema+"."+t.name] = cols + } + return map[string]any{"tables": schema}, nil +} + +// mssqlTableColumns is postgresTableColumns for SQL Server — same +// partial-schema contract (ok=false drops the one unreadable table +// rather than the whole dump), different placeholder syntax. +func mssqlTableColumns(ctx context.Context, scoped *sql.DB, schema, name string) ([]map[string]any, bool) { + colRows, err := scoped.QueryContext(ctx, ` + SELECT COLUMN_NAME, DATA_TYPE, IS_NULLABLE, COLUMN_DEFAULT + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = @p1 AND TABLE_NAME = @p2 + ORDER BY ORDINAL_POSITION + `, schema, name) + if err != nil { + return nil, false + } + defer func() { _ = colRows.Close() }() + var cols []map[string]any + for colRows.Next() { + var colName, dtype, nullable string + var def sql.NullString + _ = colRows.Scan(&colName, &dtype, &nullable, &def) + cols = append(cols, map[string]any{ + "name": colName, "type": dtype, + "nullable": nullable == "YES", + "default": def.String, + }) + } + if err := colRows.Err(); err != nil { + return nil, false + } + return cols, true +} + func mongoSchema(ctx context.Context, drv *dbmongo.Driver, db string) (map[string]any, error) { cols, err := drv.Client.Database(db).ListCollectionNames(ctx, struct{}{}) if err != nil { @@ -1506,8 +1647,8 @@ func decodeSQLValue(v any) any { // ─── connection_probe ─────────────────────────────────────────────── type connectionProbeIn struct { - Engine string `json:"engine" jsonschema:"mysql|mariadb|tidb|postgres|postgresql|mongodb|redis|elasticsearch|opensearch"` - DSN string `json:"dsn,omitempty" jsonschema:"connection string. mysql://user:pw@host:port, postgres://…, mongodb://…, redis://…, http(s)://…(ES). Omit to probe the repo's configured connection for this engine."` + Engine string `json:"engine" jsonschema:"mysql|mariadb|tidb|postgres|postgresql|mssql|sqlserver|mongodb|redis|elasticsearch|opensearch"` + DSN string `json:"dsn,omitempty" jsonschema:"connection string. mysql://user:pw@host:port, postgres://…, sqlserver://…, mongodb://…, redis://…, http(s)://…(ES). Omit to probe the repo's configured connection for this engine."` Repo string `json:"repo,omitempty" jsonschema:"only used when dsn is empty — selects the .treeman.yaml whose connection block to probe"` } @@ -1564,6 +1705,8 @@ func probeConnection(ctx context.Context, fam engine.Family, dsn, repo string) ( return probeMySQLConn(ctx, dsn, repo) case engine.FamilyPostgres: return probePostgresConn(ctx, dsn, repo) + case engine.FamilyMSSQL: + return probeMssqlConn(ctx, dsn, repo) case engine.FamilyMongo: return probeMongoConn(ctx, dsn, repo) case engine.FamilyRedis: @@ -1634,6 +1777,31 @@ func probePostgresConn(ctx context.Context, dsn, repo string) (bool, string, err return true, v, nil } +func probeMssqlConn(ctx context.Context, dsn, repo string) (bool, string, error) { + var cfg config.MssqlConn + if dsn != "" { + if err := yamlScalar(&cfg, dsn); err != nil { + return false, "", err + } + } else { + c, err := loadCfgForRepo(ctx, repo) + if err != nil { + return false, "", err + } + if c.Connections.Mssql == nil { + return false, "", errors.New("connections.mssql not configured") + } + cfg = *c.Connections.Mssql + } + drv, err := dbmssql.Connect(ctx, cfg) + if err != nil { + return false, "", err + } + defer func() { _ = drv.Close() }() + v, _ := drv.EngineVersion(ctx) + return true, v, nil +} + func probeMongoConn(ctx context.Context, dsn, repo string) (bool, string, error) { var cfg config.MongoConn if dsn != "" { @@ -1706,7 +1874,7 @@ func probeESConn(ctx context.Context, dsn, repo string) (bool, string, error) { type engineLogsIn struct { Repo string `json:"repo,omitempty"` - Engine string `json:"engine" jsonschema:"mysql|mariadb|tidb|postgres|postgresql|mongodb|redis|elasticsearch|opensearch"` + Engine string `json:"engine" jsonschema:"mysql|mariadb|tidb|postgres|postgresql|mssql|sqlserver|mongodb|redis|elasticsearch|opensearch"` Lines int `json:"lines,omitempty" jsonschema:"max lines returned (default 200, max 5000)"` Since string `json:"since,omitempty" jsonschema:"only logs newer than this — duration like 10m|2h (default empty = no since filter)"` } @@ -1805,6 +1973,11 @@ func containerRefForEngine(cfg *config.Config, eng string) (*config.ContainerRef return nil, errors.New("connections.postgres not configured") } ref = &cfg.Connections.Postgres.ContainerRef + case engine.FamilyMSSQL: + if cfg.Connections.Mssql == nil { + return nil, errors.New("connections.mssql not configured") + } + ref = &cfg.Connections.Mssql.ContainerRef case engine.FamilyMongo: if cfg.Connections.Mongodb == nil { return nil, errors.New("connections.mongodb not configured") diff --git a/internal/mcp/tools_write.go b/internal/mcp/tools_write.go index d6b2dcf7..950efd61 100644 --- a/internal/mcp/tools_write.go +++ b/internal/mcp/tools_write.go @@ -204,7 +204,7 @@ func prepareTool(ctx context.Context, _ *mcpsdk.CallToolRequest, in prepareIn) ( type dbResetIn struct { Worktree string `json:"worktree,omitempty" jsonschema:"defaults to cwd's worktree"` Repo string `json:"repo,omitempty"` - Engine string `json:"engine,omitempty" jsonschema:"restrict to one engine family: mysql|postgres|mongodb|redis|elasticsearch (aliases like mariadb/postgresql/valkey/dragonfly accepted); omit to reset all branch_scoped databases"` + Engine string `json:"engine,omitempty" jsonschema:"restrict to one engine family: mysql|postgres|mssql|mongodb|redis|elasticsearch (aliases like mariadb/postgresql/sqlserver/valkey/dragonfly accepted); omit to reset all branch_scoped databases"` DryRun bool `json:"dry_run,omitempty" jsonschema:"plan only — return the active-namespace + current-branch durable names that WOULD be dropped without performing the reset"` Ack bool `json:"ack,omitempty" jsonschema:"skip the elicitation confirmation prompt"` } @@ -249,7 +249,7 @@ func dbResetTool(ctx context.Context, req *mcpsdk.CallToolRequest, in dbResetIn) type dbSaveIn struct { Worktree string `json:"worktree,omitempty" jsonschema:"defaults to cwd's worktree"` Repo string `json:"repo,omitempty"` - Engine string `json:"engine,omitempty" jsonschema:"restrict to one engine family: mysql|postgres|mongodb|redis|elasticsearch (aliases like mariadb/postgresql/valkey/dragonfly accepted); omit to save all branch_scoped databases"` + Engine string `json:"engine,omitempty" jsonschema:"restrict to one engine family: mysql|postgres|mssql|mongodb|redis|elasticsearch (aliases like mariadb/postgresql/sqlserver/valkey/dragonfly accepted); omit to save all branch_scoped databases"` } type dbSaveOut struct { Saved []prepare.BranchScopedSave `json:"saved" jsonschema:"per-database result; a non-empty skipped field explains why no capture ran (no marker, missing active, or unchanged watermark)"` diff --git a/internal/prepare/branchreap.go b/internal/prepare/branchreap.go index 8260cf59..85f0da87 100644 --- a/internal/prepare/branchreap.go +++ b/internal/prepare/branchreap.go @@ -272,6 +272,7 @@ func isBSHash(s string) bool { var untrackedMarker = map[string]string{ "mysql": "_tmbs_", "postgres": "_tmbs_", + "mssql": "_tmbs_", "mongodb": "_tmbs_", "s3": "tmbs-", "elasticsearch": esDurableMarker, @@ -308,6 +309,8 @@ func listDurableCandidates(ctx context.Context, drv nsDriver, marker string) ([] names, err = a.d.ListMatching(ctx, marker) case postgresNS: names, err = a.d.ListMatching(ctx, marker) + case mssqlNS: + names, err = a.d.ListMatching(ctx, marker) case mongoNS: names, err = a.d.ListMatching(ctx, marker) case s3NS: diff --git a/internal/prepare/branchstate.go b/internal/prepare/branchstate.go index d1313fe9..c9ede200 100644 --- a/internal/prepare/branchstate.go +++ b/internal/prepare/branchstate.go @@ -15,6 +15,7 @@ import ( "github.com/stubbedev/treeman/internal/config" dbes "github.com/stubbedev/treeman/internal/db/es" dbmongo "github.com/stubbedev/treeman/internal/db/mongo" + dbmssql "github.com/stubbedev/treeman/internal/db/mssql" dbmysql "github.com/stubbedev/treeman/internal/db/mysql" dbpostgres "github.com/stubbedev/treeman/internal/db/postgres" dbredis "github.com/stubbedev/treeman/internal/db/redis" @@ -315,6 +316,44 @@ func (a postgresNS) Watermark(ctx context.Context, ns string) (string, error) { return a.d.WriteWatermark(ctx, ns) } +type mssqlNS struct{ d *dbmssql.Driver } + +func (a mssqlNS) Exists(ctx context.Context, ns string) (bool, error) { + return a.d.DatabaseExists(ctx, ns) +} + +func (a mssqlNS) Capture(ctx context.Context, active, durable string) error { + return a.d.SnapshotCreate(ctx, active, durable) +} + +func (a mssqlNS) Restore(ctx context.Context, durable, active string) error { + return a.d.SnapshotRestore(ctx, durable, active) +} + +func (a mssqlNS) RestoreParent(ctx context.Context, parent, active string, _ func(string) bool) error { + return a.d.SnapshotRestore(ctx, parent, active) +} + +func (a mssqlNS) Empty(ctx context.Context, active string) error { + // EXACT drop, not DropMatching — see mysqlNS.Empty. + if err := a.d.DropDatabase(ctx, active); err != nil { + return err + } + return a.d.EnsureDB(ctx, active) +} + +func (a mssqlNS) Drop(ctx context.Context, ns string) error { + return a.d.DropDatabase(ctx, ns) +} + +func (a mssqlNS) DropDurable(ctx context.Context, durable string) error { + return a.d.DropSnapshot(ctx, durable) +} + +func (a mssqlNS) Watermark(ctx context.Context, ns string) (string, error) { + return a.d.WriteWatermark(ctx, ns) +} + type mongoNS struct{ d *dbmongo.Driver } func (a mongoNS) Exists(ctx context.Context, ns string) (bool, error) { @@ -588,6 +627,16 @@ func connectBranchEngine( return nil, func() {}, err } return &branchEngine{drv: postgresNS{drv}, scope: scope, engine: label}, func() { _ = drv.Close() }, nil + case "mssql": + ms, rerr := cfg.Connections.ResolveMssql(connection) + if rerr != nil || ms == nil { + return nil, func() {}, errors.New("connections.mssql not configured") + } + drv, err := dbmssql.Connect(ctx, *ms) + if err != nil { + return nil, func() {}, err + } + return &branchEngine{drv: mssqlNS{drv}, scope: scope, engine: label}, func() { _ = drv.Close() }, nil case "mongodb": mc, rerr := cfg.Connections.ResolveMongodb(connection) if rerr != nil || mc == nil { diff --git a/internal/prepare/coldbuild_invariant_test.go b/internal/prepare/coldbuild_invariant_test.go index 5448f860..2de11bd9 100644 --- a/internal/prepare/coldbuild_invariant_test.go +++ b/internal/prepare/coldbuild_invariant_test.go @@ -29,6 +29,7 @@ func TestColdBuildEmitsDropEventPerEngine(t *testing.T) { labels := []string{ "mysqlColdBuild:", "pgColdBuild:", + "mssqlColdBuild:", "mongoColdBuild:", "redisColdBuild:", "esColdBuild:", @@ -93,7 +94,7 @@ func TestColdBuildUsesSlugAnchoredDrop(t *testing.T) { // `DropPrefixFiltered` (which take a third predicate arg) don't // false-positive on the bare 2-arg form. banned := regexp.MustCompile(`drv\.(DropMatching|DropPrefix)\(ctx, [^,]+\)`) - labels := []string{"mysqlColdBuild:", "pgColdBuild:", "mongoColdBuild:", "redisColdBuild:", "esColdBuild:"} + labels := []string{"mysqlColdBuild:", "pgColdBuild:", "mssqlColdBuild:", "mongoColdBuild:", "redisColdBuild:", "esColdBuild:"} for i, lbl := range labels { t.Run(strings.TrimSuffix(lbl, ":"), func(t *testing.T) { block := coldBuildBlock(t, body, lbl, labels, i) diff --git a/internal/prepare/prepare.go b/internal/prepare/prepare.go index 6fa76ea4..d9715950 100644 --- a/internal/prepare/prepare.go +++ b/internal/prepare/prepare.go @@ -37,6 +37,7 @@ import ( dbes "github.com/stubbedev/treeman/internal/db/es" dbfile "github.com/stubbedev/treeman/internal/db/filedb" dbmongo "github.com/stubbedev/treeman/internal/db/mongo" + dbmssql "github.com/stubbedev/treeman/internal/db/mssql" dbmysql "github.com/stubbedev/treeman/internal/db/mysql" dbpostgres "github.com/stubbedev/treeman/internal/db/postgres" dbredis "github.com/stubbedev/treeman/internal/db/redis" @@ -293,9 +294,14 @@ type cloneRestorer func(ctx context.Context, template, target string) error // `CREATE DATABASE … TEMPLATE` statement; the bottleneck is disk // throughput (block-copy of template files), so an outer wider than // 8 just thrashes seeks without improving wall-clock — cap there. +// SQL Server's RESTORE is a page-level copy like Postgres's TEMPLATE, +// but it reads one shared backup file per template rather than the +// source's own files, so the outer fan-out is held to 4 — four parallel +// readers of one file is where sequential read-ahead stops helping. var fanOutLimits = map[engine.Family]int{ engine.FamilyMySQL: 4, engine.FamilyPostgres: 8, + engine.FamilyMSSQL: 4, // File copies: no connection budget to respect — the cap is disk // throughput, and reflink clones are metadata-only where supported. engine.FamilyFile: 8, @@ -311,6 +317,7 @@ var fanOutLimits = map[engine.Family]int{ var innerConnsPerRestore = map[engine.Family]int{ engine.FamilyMySQL: mysqlInnerPerRestore, engine.FamilyPostgres: 1, + engine.FamilyMSSQL: 1, } // mysqlInnerPerRestore mirrors mysqlCloneFanout in the mysql driver. @@ -778,6 +785,8 @@ func prepareFamily( return prepareMySQL(ctx, cfg, d, dbIdx, tplCtx, worktreePath, st, repoID, worktreeID, inheritedEnv) case engine.FamilyPostgres: return preparePostgres(ctx, cfg, d, dbIdx, tplCtx, worktreePath, st, repoID, worktreeID, inheritedEnv) + case engine.FamilyMSSQL: + return prepareMSSQL(ctx, cfg, d, dbIdx, tplCtx, worktreePath, st, repoID, worktreeID, inheritedEnv) case engine.FamilyMongo: return prepareMongo(ctx, cfg, d, dbIdx, tplCtx, worktreePath, st, repoID, worktreeID, inheritedEnv) case engine.FamilyRedis: @@ -2405,6 +2414,246 @@ func postgresColdBuildSteps( return nil } +// prepareMSSQL readies the per-worktree SQL Server database. Same +// linear flow as preparePostgres with two differences, both from the +// engine: a template is a database plus a `.bak` beside it (SQL Server +// has no `CREATE DATABASE … TEMPLATE`), and `dump:` is rejected by +// config validation because RESTORE reads the server's filesystem, not +// treeman's — so the dump-only template path has nothing to build from +// and is not wired up here. +// +//nolint:funlen // mirrors the linear cache-hit / incremental / cold-build / fanout flow used by every engine; extracting helpers just spreads the same conditions across functions +func prepareMSSQL( + ctx context.Context, + cfg *config.Config, + d config.DatabaseConfig, + dbIdx int, + tplCtx template.Context, + worktreePath string, + st *store.Store, + repoID, worktreeID int64, + inheritedEnv map[string]string, +) (out Outcome, err error) { + started := time.Now() + sourceDB, err := template.Render(d.NameTemplate, tplCtx) + if err != nil { + return Outcome{}, fmt.Errorf("render name_template: %w", err) + } + msConn, err := cfg.Connections.ResolveMssql(d.Connection) + if err != nil { + return Outcome{}, err + } + if msConn == nil { + return emitPrepareSkipped(ctx, st, repoID, worktreeID, d.Engine, sourceDB, + "connections.mssql not configured"), nil + } + + drv, err := dbmssql.Connect(ctx, *msConn) + if err != nil { + return Outcome{}, err + } + defer func() { _ = drv.Close() }() + + // Probe keys ride the connection selector — different servers must + // not share probe results (see prepareMySQL, #44). + version, _ := cachedProbe(ctx, "mssql-version:"+d.Connection, func() (string, error) { return drv.EngineVersion(ctx) }) + maxConns, _ := cachedProbe(ctx, "mssql-maxconns:"+d.Connection, func() (int, error) { return drv.MaxConnections(ctx) }) + + if d.BranchScoped { + return runBranchScoped(ctx, branchScopedArgs{ + cfg: cfg, d: d, dbIdx: dbIdx, tplCtx: tplCtx, worktreePath: worktreePath, + st: st, repoID: repoID, worktreeID: worktreeID, inheritedEnv: inheritedEnv, + migrateFP: computeSnapshotKey(ctx, st, d, worktreePath, version).Fingerprint(), + eng: &branchEngine{drv: mssqlNS{drv}, scope: scopeName, engine: "mssql"}, + // Unreachable: config.validateMssql rejects `dump:` for this + // family, so a branch-scoped mssql entry never carries one. + // Stated rather than left as a nil panic. + loadDump: func(context.Context, string, dumpFile) error { + return errors.New("mssql: `dump:` is not supported — see config validation") + }, + }) + } + + key, inputs := computeKeyAndVectors(ctx, st, d, worktreePath, version) + templateName := key.TemplateName() + + // See prepareMySQL's pin comment — same race, same fix. + unpinTemplate := snapshot.Pin(key.Fingerprint()) + defer unpinTemplate() + + _ = st.WriteEvent(ctx, store.LevelInfo, store.EvtPrepareStart, + fmt.Sprintf("engine=mssql source=%s template=%s", sourceDB, templateName), + repoID, worktreeID, "", 0, map[string]string{ + "engine": "mssql", + "source_db": sourceDB, + "template": templateName, + "fingerprint": key.Fingerprint(), + }) + + // Cache hit? Fingerprint covers every declared input. With a spare + // pool configured, restores first try to claim a pre-warmed spare by + // rename before paying a full RESTORE. + restore := restoreFor(d.Engine, engineconn.NewMSSQLConn(drv), drv.SnapshotRestore, + st, repoID, worktreeID, d) + out, done, flight, err := cacheHitGeneric( + ctx, + drv.DatabaseExists, + restore, + d, + tplCtx, + worktreePath, + st, + repoID, + worktreeID, + sourceDB, + key, + maxConns, + started, + ) + defer flight.finish() + if done || err != nil { + return out, err + } + + ops := incrementalOps{ + exists: drv.DatabaseExists, + snapshotRestore: drv.SnapshotRestore, + snapshotCreate: drv.SnapshotCreate, + } + + out, done, err = tryIncrementalBuild(ctx, cfg, d, tplCtx, worktreePath, st, + repoID, worktreeID, sourceDB, templateName, version, maxConns, key, inputs, + inheritedEnv, started, ops) + if done || err != nil { + return out, err + } + + out, done, err = tryRollbackIncrementalBuild(ctx, cfg, d, tplCtx, worktreePath, st, + repoID, worktreeID, sourceDB, templateName, version, maxConns, key, inputs, + inheritedEnv, started, ops) + if done || err != nil { + return out, err + } + + if err := mssqlColdBuildSteps( + ctx, + drv, + d, + tplCtx, + worktreePath, + st, + repoID, + worktreeID, + sourceDB, + inheritedEnv, + ); err != nil { + return Outcome{}, err + } + snapStart := time.Now() + if err := drv.SnapshotCreate(ctx, sourceDB, templateName); err != nil { + return Outcome{}, fmt.Errorf("snapshot create %s → %s: %w", sourceDB, templateName, err) + } + emitPhaseDone(ctx, st, repoID, worktreeID, d.Engine, sourceDB, "snapshot-create", snapStart) + recordSnapshot(ctx, st, key, d.Engine, version, sourceDB, templateName, inputs, repoID) + spawnEvict(cfg, st, repoID) + + clones, err := resolveCloneNames(d.TestClones, tplCtx, worktreePath) + if err != nil { + return Outcome{}, err + } + if err := fanOutClones( + ctx, + st, + repoID, + worktreeID, + drv.SnapshotRestore, + templateName, + clones, + d.Engine, + d.Fanout, + maxConns, + ); err != nil { + return Outcome{}, err + } + ms := time.Since(started).Milliseconds() + _ = st.WriteEvent(ctx, store.LevelInfo, store.EvtPrepareEnd, + fmt.Sprintf("cold_build clones=%d duration=%dms", len(clones), ms), + repoID, worktreeID, "", 0, map[string]string{ + "engine": "mssql", + "source_db": sourceDB, + "template": templateName, + "clones": strconv.Itoa(len(clones)), + "cache_hit": "false", + "duration_ms": strconv.FormatInt(ms, 10), + }) + return Outcome{ + Engine: d.Engine, SourceDB: sourceDB, TemplateName: templateName, + Fingerprint: key.Fingerprint(), CacheHit: false, Clones: clones, + }, nil +} + +// mssqlColdBuildSteps runs the SQL Server cold-build populate phase: +// drop + recreate the source DB, then migrate and seed. No dump step — +// `dump:` is rejected for this family (config.validateMssql), so there +// is no intermediate dump-only template to seed either. +func mssqlColdBuildSteps( + ctx context.Context, + drv *dbmssql.Driver, + d config.DatabaseConfig, + tplCtx template.Context, + worktreePath string, + st *store.Store, + repoID, worktreeID int64, + sourceDB string, + inheritedEnv map[string]string, +) error { + // mssqlColdBuild: cold-build pre-drop site (see coldbuild_invariant_test). + // DropDatabase, NOT DropMatching — see the comment on the matching + // mysql cold-build site. + if err := drv.DropDatabase(ctx, sourceDB); err != nil { + return err + } + emitColdBuildDrop(ctx, st, repoID, worktreeID, "mssql", sourceDB) + if err := drv.EnsureDB(ctx, sourceDB); err != nil { + return err + } + if d.Migrate != nil { + if err := runPhase( + ctx, + st, + repoID, + worktreeID, + d, + tplCtx, + worktreePath, + sourceDB, + "migrate", + runner.FromMigrate(*d.Migrate), + inheritedEnv, + ); err != nil { + return err + } + } + if d.Seed != nil { + if err := runPhase( + ctx, + st, + repoID, + worktreeID, + d, + tplCtx, + worktreePath, + sourceDB, + "seed", + runner.FromSeed(*d.Seed), + inheritedEnv, + ); err != nil { + return err + } + } + return nil +} + // prepareMongo readies the per-worktree MongoDB database. Cold-build // order matches the SQL engines: drop, optionally `mongorestore` a // dump archive, run the seed step, snapshot the per-collection diff --git a/internal/resolve/resolve.go b/internal/resolve/resolve.go index 5c7e64a4..f1b81f7f 100644 --- a/internal/resolve/resolve.go +++ b/internal/resolve/resolve.go @@ -62,6 +62,7 @@ func (s Source) String() string { type Resolved struct { Mysql *resolvedConn[config.MysqlConn] Postgres *resolvedConn[config.PostgresConn] + Mssql *resolvedConn[config.MssqlConn] Mongodb *resolvedConn[config.MongoConn] Redis *resolvedConn[config.RedisConn] Elasticsearch *resolvedConn[config.EsConn] @@ -93,6 +94,7 @@ func resolveRoots(cfg *config.Config, roots ...string) Resolved { return Resolved{ Mysql: resolveMysql(cfg, env), Postgres: resolvePostgres(cfg, env), + Mssql: resolveMssql(cfg, env), Mongodb: resolveMongodb(cfg, env), Redis: resolveRedis(cfg, env), Elasticsearch: resolveElasticsearch(cfg, env), @@ -120,6 +122,10 @@ func ApplyEnvCredentials(cfg *config.Config, roots ...string) { v := r.Postgres.Conn cfg.Connections.Postgres = &v } + if r.Mssql != nil { + v := r.Mssql.Conn + cfg.Connections.Mssql = &v + } if r.Mongodb != nil { v := r.Mongodb.Conn cfg.Connections.Mongodb = &v @@ -152,6 +158,10 @@ func fillFromContainerEnv(cfg *config.Config) { for _, p := range cfg.Connections.PostgresNamed { fillPostgresPasswordFromContainerEnv(p) } + fillMssqlPasswordFromContainerEnv(cfg.Connections.Mssql) + for _, m := range cfg.Connections.MssqlNamed { + fillMssqlPasswordFromContainerEnv(m) + } fillS3FromContainerEnv(cfg.Connections.S3) for _, s := range cfg.Connections.S3Named { fillS3FromContainerEnv(s) @@ -194,6 +204,27 @@ func fillPostgresPasswordFromContainerEnv(p *config.PostgresConn) { } } +// fillMssqlPasswordFromContainerEnv is the postgres filler for SQL +// Server. MSSQL_SA_PASSWORD is what the official image documents; +// SA_PASSWORD is the older name it still honours. +func fillMssqlPasswordFromContainerEnv(m *config.MssqlConn) { + if m == nil || m.Password != "" || (m.Container == "" && m.ComposeService == "") { + return + } + env, _ := containerip.EnvLookup(context.Background(), containerip.Opts{ + Container: m.Container, + ComposeService: m.ComposeService, + ComposeProject: m.ComposeProject, + Engine: m.ContainerEngine, + }) + for _, k := range []string{"MSSQL_SA_PASSWORD", "SA_PASSWORD"} { + if v, ok := env[k]; ok && nonEmpty(v) { + m.Password = v + return + } + } +} + // fillS3FromContainerEnv back-fills S3 access/secret keys from the // container's own environment when they weren't supplied in config — // MinIO/Garage publish their root creds as well-known env vars @@ -642,6 +673,8 @@ func normalizeEngineScheme(s string) string { return "postgres" case "cockroach", "cockroachdb": return "cockroach" + case "sqlserver", "mssql": + return "mssql" case "redis", "rediss": return "redis" case "mongodb", "mongodb+srv": @@ -686,6 +719,76 @@ func mysqlFromURL(u *url.URL) config.MysqlConn { return config.MysqlConn{Host: host, Port: port, User: user, PoolMax: 8} } +func resolveMssql(cfg *config.Config, env envfile.EnvFile) *resolvedConn[config.MssqlConn] { + if cfg.Connections.Mssql != nil { + m := *cfg.Connections.Mssql + m.Password = resolveMssqlPassword(env, m.Password) + return &resolvedConn[config.MssqlConn]{Conn: m, Source: Source{Kind: SourceYaml}} + } + if m, ok := springMssql(env); ok { + return &resolvedConn[config.MssqlConn]{Conn: m, Source: repoSrc(env)} + } + if u, src, ok := pickURL(env, "MSSQL_URL", []string{"sqlserver", "mssql"}); ok { + m := mssqlFromURL(u) + if pw, ok := urlPassword(u); ok { + m.Password = pw + } + return &resolvedConn[config.MssqlConn]{Conn: m, Source: src} + } + return nil +} + +func springMssql(env envfile.EnvFile) (config.MssqlConn, bool) { + v, ok := env.Get("SPRING_DATASOURCE_URL") + if !ok { + return config.MssqlConn{}, false + } + u, eng := parseJDBC(v) + if eng != "mssql" { + return config.MssqlConn{}, false + } + m := mssqlFromURL(u) + if pw, ok := springPassword(env, u); ok { + m.Password = pw + } + if user, ok := env.Get("SPRING_DATASOURCE_USERNAME"); ok { + m.User = user + } + return m, true +} + +func resolveMssqlPassword(env envfile.EnvFile, configured string) string { + if v := resolvePasswordValue(env, configured); v != "" { + return v + } + for _, k := range []string{"DB_TEST_PASSWORD", "DB_PASSWORD", "MSSQL_SA_PASSWORD", "SA_PASSWORD"} { + if v, ok := env.Get(k); ok && nonEmpty(v) { + return v + } + } + return "" +} + +func mssqlFromURL(u *url.URL) config.MssqlConn { + host := u.Hostname() + if host == "" { + host = "127.0.0.1" + } + port := uint16(1433) + if p := u.Port(); p != "" { + if n, err := strconv.ParseUint(p, 10, 16); err == nil { + port = uint16(n) + } + } + user := "sa" + if u.User != nil { + if n := u.User.Username(); n != "" { + user = n + } + } + return config.MssqlConn{Host: host, Port: port, User: user, PoolMax: 8} +} + func postgresFromURL(u *url.URL) config.PostgresConn { host := u.Hostname() if host == "" { diff --git a/internal/schema/schema.go b/internal/schema/schema.go index 57c0eee3..8ed43d63 100644 --- a/internal/schema/schema.go +++ b/internal/schema/schema.go @@ -87,6 +87,7 @@ func injectEngineEnum(s *jsonschema.Schema) { var connFamilyDefs = map[string]string{ "mysql": "MysqlConn", "postgres": "PostgresConn", + "mssql": "MssqlConn", "mongodb": "MongoConn", "redis": "RedisConn", "elasticsearch": "EsConn", diff --git a/internal/snapshot/orphans.go b/internal/snapshot/orphans.go index 8d16e558..ed5b9598 100644 --- a/internal/snapshot/orphans.go +++ b/internal/snapshot/orphans.go @@ -46,8 +46,8 @@ var redisTemplatePrefix = regexp.MustCompile(`^(_tm:[0-9a-f]{16}:)`) // `_tm:` prefixes are findable (and DropSnapshot-able) without a // manual FLUSHDB. var orphanFamilies = []engine.Family{ - engine.FamilyMySQL, engine.FamilyPostgres, engine.FamilyMongo, engine.FamilyES, - engine.FamilyRedis, + engine.FamilyMySQL, engine.FamilyPostgres, engine.FamilyMSSQL, engine.FamilyMongo, + engine.FamilyES, engine.FamilyRedis, } // templateNameSource is the store subset FindOrphans needs. @@ -116,7 +116,7 @@ func listTemplates(ctx context.Context, conn engineconn.Conn, fam engine.Family) // cleanup for the file family goes through GC + PurgeRepo, which // act on recorded snapshot rows. return nil, nil - case engine.FamilyMySQL, engine.FamilyPostgres, engine.FamilyMongo, engine.FamilyS3: + case engine.FamilyMySQL, engine.FamilyPostgres, engine.FamilyMSSQL, engine.FamilyMongo, engine.FamilyS3: // name-scoped default already set } names, err := conn.ListMatching(ctx, prefix) @@ -140,32 +140,48 @@ func listTemplates(ctx context.Context, conn engineconn.Conn, fam engine.Family) // capture group 1 is the owning template. var spareNamePattern = regexp.MustCompile(`^(_tm_(?:tmpl_[a-z]+_)?[0-9a-f]{16})_spare[0-9]+$`) +// spareFamilies is every family whose driver can hold a pre-warmed +// spare pool — the same set config.PrewarmCapableEngines admits, as a +// fixed-order slice so SpareCounts walks them deterministically. +var spareFamilies = []engine.Family{ + engine.FamilyPostgres, engine.FamilyMySQL, engine.FamilyMSSQL, +} + // SpareCounts returns the number of pre-warmed spare databases per -// template name, as they exist engine-side right now. Postgres-only -// today (the only engine prewarm supports); returns an empty map -// without error when postgres isn't configured. Feeds the doctor's -// snapshots probe and the `snapshots list` SPARES column — the only -// surfaces that make the otherwise-anonymous pool observable. +// template name, as they exist engine-side right now, across every +// prewarm-capable family that is configured. Template names carry the +// fingerprint, so two families cannot collide on a key. Feeds the +// doctor's snapshots probe and the `snapshots list` SPARES column — the +// only surfaces that make the otherwise-anonymous pool observable. func SpareCounts(ctx context.Context, cfg *config.Config) (map[string]int, error) { - if !engineconn.Configured(cfg, engine.FamilyPostgres, "") { - return map[string]int{}, nil + out := map[string]int{} + for _, fam := range spareFamilies { + if !engineconn.Configured(cfg, fam, "") { + continue + } + if err := addSpareCounts(ctx, cfg, fam, out); err != nil { + return nil, err + } } - conn, _, err := engineconn.Connect(ctx, cfg, engine.FamilyPostgres, "") + return out, nil +} + +func addSpareCounts(ctx context.Context, cfg *config.Config, fam engine.Family, out map[string]int) error { + conn, _, err := engineconn.Connect(ctx, cfg, fam, "") if err != nil { - return nil, err + return err } defer func() { _ = conn.Close() }() names, err := conn.ListMatching(ctx, "_tm_") if err != nil { - return nil, err + return err } - out := map[string]int{} for _, name := range names { if m := spareNamePattern.FindStringSubmatch(name); m != nil { out[m[1]]++ } } - return out, nil + return nil } // DropOrphans reclaims the given orphans: each template is dropped diff --git a/justfile b/justfile index f1277f47..ea5884fa 100644 --- a/justfile +++ b/justfile @@ -71,10 +71,12 @@ test: go test ./... # Run the e2e suite against real docker-backed engines. Pulls -# ~3GB of images on first run (mysql, postgres, mongo, redis, es) -# and binds local ports 13306-13356 + 15432 + 27117 + 16379 + 19200 -# — make sure those are free. Each subtest brings up + tears down -# its own docker-compose stack. +# ~5GB of images on first run (mysql, postgres, sql server, mongo, +# redis, es — SQL Server is ~1.5GB of that and ships no arm64 image, +# so on an arm64 host its suite runs under emulation) and binds local +# ports 13306-13356 + 15432 + 11433 + 27117 + 16379 + 19200 — make +# sure those are free. Each subtest brings up + tears down its own +# docker-compose stack. # # Suites are grouped into batches so docker doesn't drown under the # full ~60-suite fanout. Within a batch up to `parallel` compose @@ -91,7 +93,7 @@ test-e2e batch="all" parallel="4": set -euo pipefail declare -A BATCHES - BATCHES[engines]="mysql postgres mongo redis elasticsearch s3" + BATCHES[engines]="mysql postgres mssql mongo redis elasticsearch s3" BATCHES[fw1]="fw_alembic fw_diesel fw_django fw_flyway fw_laravel fw_rails fw_sqlx" BATCHES[fw2]="fw_drizzle fw_golang_migrate fw_knex fw_mikro fw_prisma fw_sequelize fw_typeorm" BATCHES[matrix]="matrix_es matrix_mongo matrix_postgres matrix_redis containerref containerref-all conn_forms engine_aliases named_conns" @@ -163,7 +165,7 @@ e2e-prepull batch: set -euo pipefail declare -A BATCHES - BATCHES[engines]="mysql postgres mongo redis elasticsearch s3" + BATCHES[engines]="mysql postgres mssql mongo redis elasticsearch s3" BATCHES[fw1]="fw_alembic fw_diesel fw_django fw_flyway fw_laravel fw_rails fw_sqlx" BATCHES[fw2]="fw_drizzle fw_golang_migrate fw_knex fw_mikro fw_prisma fw_sequelize fw_typeorm" BATCHES[matrix]="matrix_es matrix_mongo matrix_postgres matrix_redis containerref containerref-all conn_forms engine_aliases named_conns" @@ -200,7 +202,7 @@ e2e-prepull batch: test-e2e-list: #!/usr/bin/env bash set -euo pipefail - echo "engines : mysql postgres mongo redis elasticsearch s3" + echo "engines : mysql postgres mssql mongo redis elasticsearch s3" echo "fw1 : fw_alembic fw_diesel fw_django fw_flyway fw_laravel fw_rails fw_sqlx" echo "fw2 : fw_drizzle fw_golang_migrate fw_knex fw_mikro fw_prisma fw_sequelize fw_typeorm" echo "matrix : matrix_es matrix_mongo matrix_postgres matrix_redis containerref containerref-all conn_forms engine_aliases named_conns" diff --git a/schemas/treeman.global.schema.json b/schemas/treeman.global.schema.json index e284746b..e38d570e 100644 --- a/schemas/treeman.global.schema.json +++ b/schemas/treeman.global.schema.json @@ -124,6 +124,21 @@ ], "description": "PostgreSQL connection. Required when any `databases:` entry\nuses `engine: postgres` (or the `postgresql` alias)." }, + "mssql": { + "anyOf": [ + { + "$ref": "#/$defs/MssqlConn" + }, + { + "additionalProperties": { + "$ref": "#/$defs/MssqlConn" + }, + "type": "object", + "description": "Named connection blocks — a `databases[].connection` selector picks one" + } + ], + "description": "Microsoft SQL Server connection. Required when any `databases:`\nentry uses `engine: mssql` (or the `sqlserver` alias)." + }, "mongodb": { "anyOf": [ { @@ -288,6 +303,8 @@ "tidb", "postgres", "postgresql", + "mssql", + "sqlserver", "mongodb", "redis", "valkey", @@ -707,6 +724,64 @@ ], "description": "mongodb connection — bare URL string OR structured object." }, + "MssqlConn": { + "oneOf": [ + { + "type": "string", + "description": "DSN: `sqlserver://user:pass@host:port`. Equivalent to the structured form below." + }, + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "host": { + "type": "string" + }, + "port": { + "type": "integer" + }, + "user": { + "type": "string" + }, + "password": { + "type": "string" + }, + "pool_max": { + "type": "integer" + }, + "collation": { + "type": "string" + }, + "backup_dir": { + "type": "string" + }, + "data_dir": { + "type": "string" + }, + "container": { + "type": "string" + }, + "compose_service": { + "type": "string" + }, + "compose_project": { + "type": "string" + }, + "container_engine": { + "type": "string" + }, + "container_network": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "user" + ] + } + ], + "description": "Microsoft SQL Server connection — bare DSN string OR structured object." + }, "MysqlConn": { "oneOf": [ { diff --git a/schemas/treeman.schema.json b/schemas/treeman.schema.json index 24a6f8b7..1b573984 100644 --- a/schemas/treeman.schema.json +++ b/schemas/treeman.schema.json @@ -124,6 +124,21 @@ ], "description": "PostgreSQL connection. Required when any `databases:` entry\nuses `engine: postgres` (or the `postgresql` alias)." }, + "mssql": { + "anyOf": [ + { + "$ref": "#/$defs/MssqlConn" + }, + { + "additionalProperties": { + "$ref": "#/$defs/MssqlConn" + }, + "type": "object", + "description": "Named connection blocks — a `databases[].connection` selector picks one" + } + ], + "description": "Microsoft SQL Server connection. Required when any `databases:`\nentry uses `engine: mssql` (or the `sqlserver` alias)." + }, "mongodb": { "anyOf": [ { @@ -288,6 +303,8 @@ "tidb", "postgres", "postgresql", + "mssql", + "sqlserver", "mongodb", "redis", "valkey", @@ -707,6 +724,64 @@ ], "description": "mongodb connection — bare URL string OR structured object." }, + "MssqlConn": { + "oneOf": [ + { + "type": "string", + "description": "DSN: `sqlserver://user:pass@host:port`. Equivalent to the structured form below." + }, + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "host": { + "type": "string" + }, + "port": { + "type": "integer" + }, + "user": { + "type": "string" + }, + "password": { + "type": "string" + }, + "pool_max": { + "type": "integer" + }, + "collation": { + "type": "string" + }, + "backup_dir": { + "type": "string" + }, + "data_dir": { + "type": "string" + }, + "container": { + "type": "string" + }, + "compose_service": { + "type": "string" + }, + "compose_project": { + "type": "string" + }, + "container_engine": { + "type": "string" + }, + "container_network": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "user" + ] + } + ], + "description": "Microsoft SQL Server connection — bare DSN string OR structured object." + }, "MysqlConn": { "oneOf": [ { From 09c5ef2bc1901645fb064d695f47ae9128f46fe3 Mon Sep 17 00:00:00 2001 From: Jacek Tomaszewski Date: Mon, 12 Oct 2026 12:04:03 +0200 Subject: [PATCH 2/2] ci(fsh-vet): build, vet and test a PR into `fsh` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream's `ci.yml` triggers on PRs to main/master only, so a PR into `fsh` — which is every patch this fork carries — reaches merge with no build, vet or test gate at all. The three vet jobs answer questions about upstream and about the shape of our diff; none of them compiles anything. Not a copy of upstream's job: no lint (the flake-pinned golangci-lint is a nix download this job's egress policy would have to admit, and a finding in upstream's tree is not ours to gate on), no macOS matrix, no e2e tier — that one needs docker engines and is run by hand. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/fsh-vet.yml | 46 +++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/.github/workflows/fsh-vet.yml b/.github/workflows/fsh-vet.yml index 4b1cf03e..9cffb9f4 100644 --- a/.github/workflows/fsh-vet.yml +++ b/.github/workflows/fsh-vet.yml @@ -4,6 +4,11 @@ # 1. Is upstream's security hygiene getting worse? (scorecard vs a baseline) # 2. Do our dependencies have known vulnerabilities? (osv-scanner) # 3. Is our fork still additive-only? (diff vs upstream) +# 4. Does the code we are about to build still build and pass? (go test) +# +# (4) is here because upstream's own ci.yml only triggers on PRs to main/master +# — a PR into `fsh` gets none of it, so without this job our own patches would +# merge with no build, vet or test gate at all. # # None of this proves upstream code is benign — Scorecard measures practices, # not intent. It is a prior and a drift detector, not a verdict. See FORK.md. @@ -120,6 +125,47 @@ jobs: python3 .fsh/osv.py compare .fsh/upstream-osv.json /tmp/osv.json \ >> "$GITHUB_STEP_SUMMARY" + # Build, vet and test. Deliberately NOT a copy of upstream's full ci.yml: + # no lint (the flake-pinned golangci-lint is a nix download this job's + # egress policy would have to admit, and a lint finding in upstream's tree + # is not ours to gate on), no macOS matrix, no e2e tier — e2e needs docker + # engines and is run by hand. What it does cover is the gate a patch into + # `fsh` otherwise has none of. + build-test: + runs-on: ubuntu-latest + steps: + - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + with: + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + objects.githubusercontent.com:443 + proxy.golang.org:443 + release-assets.githubusercontent.com:443 + storage.googleapis.com:443 + sum.golang.org:443 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version-file: go.mod + cache: true + - run: go build ./... + - run: go vet ./... + # The e2e suites are behind `//go:build e2e`, so the untagged vet above + # never type-checks them — a signature change reaches them only here. + - run: go vet -tags e2e ./e2e/... + - run: go test ./... + - name: gofmt + run: | + set -euo pipefail + unformatted="$(gofmt -l .)" + if [ -n "$unformatted" ]; then + echo "::error::gofmt would rewrite:" + echo "$unformatted" + exit 1 + fi + # Has upstream's hygiene regressed since we last looked? A drop in # Token-Permissions, Dangerous-Workflow or Pinned-Dependencies is exactly the # shape a hostile or careless change to his release path would take.