From a0074f474f625bf45ac56e7754cf1b360135d1f7 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Wed, 21 Oct 2026 21:26:24 +0500 Subject: [PATCH] Issue (#663): Add in-core attach_node/detach_node alongside SQL add_node/remove_node Growing a cluster needed more than Spock itself: samples/Z0DAN/zodan.sql had to be loaded onto the joining node and dblink installed before spock.add_node() existed to be called. spock.attach_node() and spock.detach_node() now do that work inside the extension, over the cluster's own connections. They create the node, wire replication both ways, synchronize the initial data, line up slots and origins, and enable replication, with every wait bounded so a join that cannot progress fails with a clear error rather than hanging. The SQL-script path is untouched: spock.add_node() and spock.remove_node() keep working for anyone who prefers orchestration they can read and adjust, under names that do not collide. The Zodan documentation now covers both, and the installcheck containers attach with the in-core procedure, so dblink and the script load are no longer part of that setup. get_database_name() is reached through utils/lsyscache.h as well as commands/dbcommands.h, since PostgreSQL 19 moved its declaration. --- .codacy.yaml | 2 + README.md | 2 +- docs/modify/zodan/index.md | 72 +- docs/modify/zodan/zodan_readme.md | 189 +- docs/modify/zodan/zodan_tutorial.md | 69 +- mkdocs.yml | 2 +- sql/spock--5.0.12--6.0.0.sql | 46 + sql/spock--6.0.0.sql | 46 + src/spock_apply.c | 6 +- src/spock_zodan.c | 2581 +++++++++++++++++ tests/docker/entrypoint.sh | 6 +- tests/tap/schedule | 8 + tests/tap/schedule-nightly | 3 + tests/tap/t/034_attach_detach_node_incore.pl | 350 +++ .../t/035_attach_node_sync_third_incore.pl | 450 +++ tests/tap/t/036_attach_node_basics_incore.pl | 119 + .../t/037_attach_node_3n_timeout_incore.pl | 197 ++ .../t/040_zodan_lolor_attach_node_incore.pl | 182 ++ utils/pgindent/typedefs.list | 3 + 19 files changed, 4195 insertions(+), 138 deletions(-) create mode 100644 src/spock_zodan.c create mode 100755 tests/tap/t/034_attach_detach_node_incore.pl create mode 100755 tests/tap/t/035_attach_node_sync_third_incore.pl create mode 100755 tests/tap/t/036_attach_node_basics_incore.pl create mode 100755 tests/tap/t/037_attach_node_3n_timeout_incore.pl create mode 100644 tests/tap/t/040_zodan_lolor_attach_node_incore.pl diff --git a/.codacy.yaml b/.codacy.yaml index 63559d9a9..c174e254d 100644 --- a/.codacy.yaml +++ b/.codacy.yaml @@ -31,6 +31,7 @@ engines: - "src/spock_sequences.c" - "src/spock_sync.c" - "src/spock_worker.c" + - "src/spock_zodan.c" lizard: exclude_paths: - "src/spock_apply.c" @@ -58,3 +59,4 @@ engines: - "src/spock_sequences.c" - "src/spock_sync.c" - "src/spock_worker.c" + - "src/spock_zodan.c" diff --git a/README.md b/README.md index ddc81aade..9dee200c5 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ - [Modifying a Cluster](docs/modify/index.md) - Using Zodan - [Modifying your Cluster with Zodan](docs/modify/zodan/index.md) - - [Using Zodan Scripts and Workflows](docs/modify/zodan/zodan_readme.md) + - [Using Zodan (attach_node/detach_node and the SQL scripts)](docs/modify/zodan/zodan_readme.md) - [Adding a Node with Zero Downtime](docs/modify/zodan/zodan_tutorial.md) - [Adding a Node with Minimal Downtime with pgBackRest](docs/modify/add_node_pgbackrest.md) - Monitoring a Cluster diff --git a/docs/modify/zodan/index.md b/docs/modify/zodan/index.md index 3c3e07eb7..45a603b0f 100644 --- a/docs/modify/zodan/index.md +++ b/docs/modify/zodan/index.md @@ -1,37 +1,49 @@ # Modifying a Cluster with Zodan -Zodan provides tools to add or remove a node with zero downtime. The -scripts are located in the -[samples/Z0DAN](https://github.com/pgEdge/spock/tree/main/samples/Z0DAN) -directory of the [Spock GitHub](https://github.com/pgEdge/spock) repository. +Zodan (Zero Downtime Add/Remove Node) adds or removes a node with zero +downtime for the existing nodes. During node addition it manages creation of +the new node, subscription management (both to and from the node), replication +slot creation, data synchronization, replication slot advancement, and final +activation of subscriptions. During node removal it drops the node's +subscriptions, replication sets, slots, and origins in the correct order, +without deleting any Postgres artifacts (the database, data directory, log +files, and so on). + +Spock offers two ways to run this workflow. Both perform the same steps and +you can pick whichever fits your environment: + +- **In-core procedures (recommended).** `spock.attach_node` and + `spock.detach_node` are built into the Spock extension. A single + `CREATE EXTENSION spock` makes them available: there is no script to load + and no `dblink` dependency. All orchestration runs inside Spock and reaches + the other nodes over libpq. + +- **SQL scripts.** `spock.add_node` and `spock.remove_node` are loaded from + the SQL scripts in the + [samples/Z0DAN](https://github.com/pgEdge/spock/tree/main/samples/Z0DAN) + directory and reach the other nodes through the `dblink` extension. This + method is useful where you prefer to keep the orchestration in a script you + can read and modify. -During node addition, Zodan seamlessly manages creation of the new node, -subscription management (both to and from the node), replication slot -creation, data synchronization, replication slot advancement, and final -activation of subscriptions. +!!! note -During node removal, Zodan simplifies removing fully-functional or failed -nodes from a cluster. When you remove a node from a cluster, the removal -does not delete Postgres artifacts (the database, data directory, log -files, etc.). + Whichever method you use, the add procedure (`attach_node` or `add_node`) + must be run on the new node being added, and the remove procedure + (`detach_node` or `remove_node`) must be run on the node being removed. !!! hint - Zodan simplifies removing partially added nodes created during failed - node add operations. Additional cleanup steps may be required before - attempting another node deployment on the target host. - -!!! note - - Each script must be run from the target node being added or removed. + Zodan simplifies removing partially added nodes created during failed node + add operations. Additional cleanup steps may be required before attempting + another node deployment on the target host. ## Key Differences Between using Zodan and the Manual Process -The following differences highlight how Zodan automates and simplifies -node addition: +The following differences highlight how Zodan automates and simplifies node +addition: -- Zodan stores sync LSNs and uses them later to ensure subscriptions - start from the correct point even if hours pass between steps. +- Zodan stores sync LSNs and uses them later to ensure subscriptions start + from the correct point even if time passes between steps. - Zodan automatically detects existing schemas on the new node and populates the `skip_schema` parameter, preventing conflicts during structure sync. @@ -43,15 +55,16 @@ node addition: mixed cluster from any source node; a new node older than an existing node is rejected. -- Zodan includes the `verify_subscription_replicating()` function after - enabling subscriptions to ensure they reach replicating status. +- Zodan waits for each new subscription to reach the replicating state before + proceeding. - When adding to a single-node cluster, Zodan handles the process - differently — no disabled subscriptions are needed. - -- Zodan shows final status of all nodes and subscriptions across the - entire cluster, not just the new node. + differently, since no disabled subscriptions are needed. +- With the in-core procedures, every internal wait is bounded by the + `timeout_sec` argument (default 180 seconds), so a join that cannot make + progress fails quickly instead of blocking. Pass a larger `timeout_sec` if + your environment needs more headroom. For more information, review the following resources: @@ -59,4 +72,3 @@ For more information, review the following resources: - [Zodan Tutorial](zodan_tutorial.md) - [Zodan Scripts and Workflows](https://github.com/pgEdge/spock/tree/main/samples/Z0DAN) - [Spock Documentation](https://docs.pgedge.com/spock-v5/) - diff --git a/docs/modify/zodan/zodan_readme.md b/docs/modify/zodan/zodan_readme.md index cf0423c74..b4a213c01 100644 --- a/docs/modify/zodan/zodan_readme.md +++ b/docs/modify/zodan/zodan_readme.md @@ -1,110 +1,149 @@ -# Zodan: Zero-Downtime Node Addition for Spock +# Using Zodan: Zero-Downtime Node Addition and Removal -Zodan provides tools to add or remove a node with zero downtime. The -scripts are located in the -[samples/Z0DAN](https://github.com/pgEdge/spock/tree/main/samples/Z0DAN) -directory of the [Spock GitHub](https://github.com/pgEdge/spock) -repository. +Zodan adds or removes a node from a Spock cluster with zero downtime for the +existing nodes. Spock ships two implementations of the same workflow: the +in-core procedures `spock.attach_node` / `spock.detach_node`, and the SQL +scripts that provide `spock.add_node` / `spock.remove_node`. Both are covered +below. -Zodan's workflows and scripts streamline the process of adding a node to -or removing a node from a Spock cluster. Zodan features the following -scripts and workflows: +## In-core procedures: attach_node and detach_node -- The [zodan.sql](#using-the-zodansql-sql-workflow) workflow is a complete - SQL-based workflow that uses `dblink` to perform the same add node - operations from within Postgres. -- The [zodremove.sql](#the-zodremovesql-workflow) workflow is a complete - SQL-based workflow that uses `dblink` to perform the same removal - operations from within Postgres. +The in-core implementation is built into the Spock extension. A single +`CREATE EXTENSION spock` makes both procedures available. There is nothing to +install beyond the extension itself: the `dblink` extension is not required, +and there are no SQL scripts to load. All local work runs over SPI and all +cross-node work runs over libpq. -## Components +- `spock.attach_node` must be run on the new node being added. +- `spock.detach_node` must be run on the node being removed. -The following scripts and workflows are available via Zodan. +### Adding a node -### Using the zodan.sql SQL Workflow +`spock.attach_node` orchestrates the full join: it validates prerequisites, +creates the node, sets up replication slots and subscriptions in both +directions, coordinates synchronization events, advances replication slots and +origins to a consistent point, and enables replication. It supports both a two +node cluster and the general multi node case. -The SQL-based implementation utilizes the Postgres `dblink` extension to -handle node addition directly from within the database. This method is -ideal for environments where you may not have access to a shell or Python. +```sql +CALL spock.attach_node( + src_node_name => 'source_node_name', + src_dsn => 'src_dsn', + new_node_name => 'new_node_name', + new_node_dsn => 'new_node_dsn', + verb => false, -- verbose progress output, optional + new_node_location => 'NY', -- optional + new_node_country => 'USA', -- optional + new_node_info => '{}'::jsonb, -- optional metadata + timeout_sec => 180 -- bound on each wait, optional +); +``` -Within the workflow, SQL commands orchestrate the following operations: +In the following example, the command adds node `n4` to the cluster. Run it +while connected to `n4`: -- `add_node` - The main procedure to orchestrate the full workflow. -- `create_node` - Register the new node via `spock.node_create`. -- `get_spock_nodes` - Fetch current node metadata from a remote node. -- `create_sub` and `enable_sub` - Manage subscription creation and - activation. -- `create_replication_slot` - Create and configure logical replication - slots. -- `sync_event` and `wait_for_sync_event` - Coordinate data synchronization - events. -- `get_commit_timestamp` and `advance_replication_slot` - Align - replication states. +```sql +CALL spock.attach_node( + 'n1', + 'host=127.0.0.1 dbname=pgedge port=5431 user=pgedge password=', + 'n4', + 'host=127.0.0.1 dbname=pgedge port=5434 user=pgedge password=' +); +``` + +The `timeout_sec` argument bounds every internal wait loop, so a join that +cannot make progress fails quickly with a clear error rather than blocking for +a long fixed period. It defaults to 180 seconds; pass a larger value if your +environment needs more headroom. -To use the workflow, execute the following command in your Postgres -session. In the following example, the `spock.add_node` procedure adds a -new node to the cluster: +### Removing a node + +`spock.detach_node` removes a node in the correct order: it drops the inbound +subscriptions on each surviving node, drops the subscriptions local to the +node being removed, drops the node's replication sets, and finally drops the +node from the catalog. Replication slot and origin cleanup is handled as part +of dropping the subscriptions. A surviving node that cannot be reached during +teardown is skipped with a warning rather than aborting the removal. ```sql -CALL spock.add_node( - 'source_node_name', - 'src_dsn', - 'new_node_name', - 'new_node_dsn', - true|false, -- verbose? optional - 'new_node_location', -- optional - 'new_node_country', -- optional - '{}'::jsonb -- optional info +CALL spock.detach_node( + target_node_name => 'target_node_name', + target_node_dsn => 'target_dsn', -- DSN of the node being removed + verbose_mode => true -- verbose progress output, optional ); ``` -In the following example, the command adds node `n4` to the cluster: +`detach_node` runs on the node being removed and does all of its work locally +and against the surviving nodes it already knows about. `target_node_dsn` must +be the DSN of that same node: `detach_node` connects to it and compares the +system identifier and database name against the local ones, refusing to run if +they differ. This is the mirror of the `new_node_dsn` check in `attach_node`, +and it is what stops `detach_node` from tearing down the replication of +whichever node it was accidentally run on. + +In the following example, the command removes node `n4` from the cluster. Run +it while connected to `n4`: ```sql -CALL spock.add_node( - 'n1', - 'host=127.0.0.1 dbname=pgedge port=5431 user=pgedge password=', +CALL spock.detach_node( 'n4', 'host=127.0.0.1 dbname=pgedge port=5434 user=pgedge password=' ); ``` -### The zodremove.sql Workflow +## SQL scripts: add_node and remove_node -The SQL-based implementation utilizes the Postgres `dblink` extension to -handle node removal directly from within the database. This method is -ideal for environments where you may not have access to a shell or Python. +The SQL-based implementation uses the Postgres `dblink` extension to run the +same node add and remove operations from within the database. The scripts are +located in the +[samples/Z0DAN](https://github.com/pgEdge/spock/tree/main/samples/Z0DAN) +directory of the [Spock GitHub](https://github.com/pgEdge/spock) repository. +This method is useful where you prefer to keep the orchestration in a script +you can read and modify. Load `zodan.sql` and `zodremove.sql` on the node you +run the procedures from, and make sure `dblink` is installed there. -Within the workflow, SQL commands orchestrate the following operations: +### Adding a node with zodan.sql -- `spock.remove_node` - Main procedure to orchestrate the full workflow. -- `spock.remove_node_subscriptions` - Manages removing subscriptions. Also - removes the replication slot if there are no remaining subscriptions. -- `spock.remove_node_replication_sets` - Removes published repsets on the - node being removed. -- `spock.remove_node_from_cluster_registry` - Removes the node from the - cluster. +The `zodan.sql` workflow orchestrates the following operations: -The workflow is located in the -[samples/Z0DAN](https://github.com/pgEdge/spock/tree/main/samples/Z0DAN) -directory of the [Spock GitHub](https://github.com/pgEdge/spock) -repository. +- `add_node` - the main procedure to orchestrate the full workflow. +- `create_node` - register the new node via `spock.node_create`. +- `get_spock_nodes` - fetch current node metadata from a remote node. +- `create_sub` and `enable_sub` - manage subscription creation and activation. +- `create_replication_slot` - create and configure logical replication slots. +- `sync_event` and `wait_for_sync_event` - coordinate data synchronization + events. +- `get_commit_timestamp` and `advance_replication_slot` - align replication + states. -To use the workflow, call a command from your Postgres session. In the -following example, the Z0DAN `spock.remove_node` procedure removes a node -from the cluster. Note that `spock.remove_node` is a Z0DAN utility -procedure provided by `zodremove.sql`; it is not a built-in function of -the Spock extension. +To use the workflow, load `zodan.sql` and call `spock.add_node`. In the +following example, the command adds node `n4` to the cluster: ```sql -CALL spock.remove_node( - 'target_node_name', - 'target_dsn', - true -- verbose_mode, optional boolean +CALL spock.add_node( + 'n1', + 'host=127.0.0.1 dbname=pgedge port=5431 user=pgedge password=', + 'n4', + 'host=127.0.0.1 dbname=pgedge port=5434 user=pgedge password=' ); ``` -In the following example, the command removes node `n4` from the cluster: +### Removing a node with zodremove.sql + +The `zodremove.sql` workflow orchestrates the following operations: + +- `spock.remove_node` - the main procedure to orchestrate the full workflow. +- `spock.remove_node_subscriptions` - removes subscriptions, and the + replication slot once no subscriptions remain. +- `spock.remove_node_replication_sets` - removes published repsets on the node + being removed. +- `spock.remove_node_from_cluster_registry` - removes the node from the + cluster. + +To use the workflow, load `zodremove.sql` and call `spock.remove_node`. Note +that `spock.remove_node` is a Zodan utility procedure provided by +`zodremove.sql`; it is not a built-in function of the Spock extension. In the +following example, the command removes node `n4` from the cluster: ```sql CALL spock.remove_node( diff --git a/docs/modify/zodan/zodan_tutorial.md b/docs/modify/zodan/zodan_tutorial.md index c1a4a8038..d7b9065e0 100644 --- a/docs/modify/zodan/zodan_tutorial.md +++ b/docs/modify/zodan/zodan_tutorial.md @@ -23,8 +23,10 @@ node), `n2`, `n3`, and the new node is `n4`. - All nodes in your cluster must be available to Spock for the duration of the node addition. - The procedure should be performed on the new node being added. - - The `dblink` extension must be installed on the node from which - commands like `SELECT spock.add_node()` are being run. + - `spock.attach_node` and `spock.detach_node` are built into the Spock + extension. They reach the other nodes over libpq, so the `dblink` + extension is not required. (`dblink` is only needed if you choose to + follow the optional manual walkthrough later in this tutorial.) - Prepare the new node to meet all of the prerequisites described here. - If the process fails, do not immediately retry a command until you ensure that all artifacts created by the workflow have been removed. @@ -85,36 +87,27 @@ psql -c "CREATE DATABASE inventory;" psql -c "CREATE USER pgedge WITH PASSWORD '1safepassword';" psql -c "GRANT ALL ON DATABASE inventory TO pgedge;" -# Install Spock extension +# Install Spock extension (provides attach_node and detach_node) psql -d inventory -c "CREATE EXTENSION spock;" psql -d inventory -c "CREATE EXTENSION dblink;" ``` ## Using the Zodan Procedure to Add a Node -After creating the node, you can use [Zodan scripts](zodan_readme.md) to -simplify adding a node to a cluster. - -To use the SQL script, connect to the new node that you wish to add to the -pgEdge cluster. In the following example, the `psql` command connects to the -new node: +`spock.attach_node` is built into the extension, so there is nothing to load. +Connect to the new node that you wish to add to the pgEdge cluster. In the +following example, the `psql` command connects to the new node: ```bash psql -h 127.0.0.1 -p 5435 -d inventory -U pgedge ``` -Load the Zodan procedures with the following command: - -```sql -\i /path/to/zodan.sql -``` - -Then, use `spock.add_node()` from the new node to create the node -definition. In the following example, the `spock.add_node` procedure adds node `n4` to -the cluster: +Then, use `spock.attach_node()` from the new node to add it to the cluster. In +the following example, the `spock.attach_node` procedure adds node `n4` to the +cluster: ```sql -CALL spock.add_node( +CALL spock.attach_node( src_node_name := 'n1', src_dsn := 'host=127.0.0.1 dbname=inventory port=5432 user=pgedge password=1safepassword', new_node_name := 'n4', @@ -126,13 +119,41 @@ CALL spock.add_node( ); ``` -The `spock.add_node` function executes the steps required to add a node to +The `spock.attach_node` function executes the steps required to add a node to the cluster; a detailed explanation of the steps performed follows below. -Should a problem occur during this process, you can source the -`zodremove.sql` script and call the `spock.remove_node` procedure to -remove the node or reverse partially completed steps. The -`spock.remove_node` procedure should be called on the node being removed. +Should a problem occur during this process, you can call the +`spock.detach_node` procedure to remove the node or reverse partially +completed steps. The `spock.detach_node` procedure should be called on the +node being removed. + +### Setting the Synchronisation Timeout + +`spock.add_node()` waits at several points for the nodes to synchronise - +for a sync event to arrive on a peer, for the new node to catch up, and +for a subscription to start replicating. Each of those waits is allowed +180 seconds (3 minutes) by default, which is sized for a small cluster +where a node joins in seconds. On a large database, catchup can +legitimately take minutes or hours, and the procedure fails with a timeout +long before the work is done. + +To raise the budget, set +[`spock.sync_timeout`](../../configuring.md#spock-sync_timeout) in the +same session, before calling the add_node procedure: + +```sql +SET spock.sync_timeout = '2h'; +``` + +The new value applies to every synchronisation wait the procedure +performs, and it reverts when the session ends, so a long-running node +addition does not need a cluster-wide configuration change. To raise the +limit for every session, set the parameter in `postgresql.conf` instead. + +The setting bounds a whole wait, not an individual probe of a remote node; +steps that also bound each remote probe - so that one unresponsive peer +cannot consume the entire budget - keep their own, much smaller limit for +that purpose. ### Setting the Synchronisation Timeout diff --git a/mkdocs.yml b/mkdocs.yml index a47f881dd..88692d015 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -72,7 +72,7 @@ nav: - Modifying a Cluster: modify/index.md - Using Z0DAN: - Modifying your Cluster with Zodan: modify/zodan/index.md - - Using Zodan Scripts and Workflows: modify/zodan/zodan_readme.md + - Using Zodan (attach_node/detach_node and SQL scripts): modify/zodan/zodan_readme.md - Adding a Node with Zero Downtime: modify/zodan/zodan_tutorial.md - Adding a Node with Minimal Downtime with pgBackRest: modify/add_node_pgbackrest.md - Recovering from Catastrophic Node Failure: recovery/catastrophic_node_failure.md diff --git a/sql/spock--5.0.12--6.0.0.sql b/sql/spock--5.0.12--6.0.0.sql index df73da851..e67191026 100644 --- a/sql/spock--5.0.12--6.0.0.sql +++ b/sql/spock--5.0.12--6.0.0.sql @@ -325,6 +325,52 @@ RETURNS boolean AS 'MODULE_PATHNAME', 'spock_alter_subscription_options' LANGUAGE C STRICT VOLATILE; +-- ---------------------------------------------------------------------------- +-- Zero Downtime Add/Remove Node (ZODAN), in-core C orchestration +-- +-- attach_node/detach_node orchestrate joining or removing a node entirely in +-- C, reaching the other cluster nodes over libpq (no dblink dependency). +-- +-- These are the in-core equivalents of the SQL/dblink procedures shipped in +-- samples/Z0DAN/zodan.sql (spock.add_node) and zodremove.sql +-- (spock.remove_node); users may choose either orchestration. +-- +-- attach_node must be run on the new node being added. +-- detach_node must be run on the node being removed. +-- +-- Neither procedure is atomic across the cluster. Both commit local state at +-- phase boundaries, and the subscriptions they create or drop on the other nodes +-- are committed one node at a time by the remote server. A failure part-way +-- through therefore leaves the cluster in an intermediate state that is not +-- rolled back: recovery from a failed attach_node is spock.detach_node() on the +-- new node, then a fresh attach_node(). See the header of src/spock_zodan.c. +-- +-- Both take an arbitrary connection string and reach out to it, and detach_node +-- drops every subscription, replication set and the node itself, so EXECUTE is +-- revoked from PUBLIC below. +-- ---------------------------------------------------------------------------- +CREATE PROCEDURE spock.attach_node( + src_node_name text, + src_dsn text, + new_node_name text, + new_node_dsn text, + verb boolean DEFAULT false, + new_node_location text DEFAULT 'NY', + new_node_country text DEFAULT 'USA', + new_node_info jsonb DEFAULT '{}'::jsonb, + timeout_sec int DEFAULT 180 +) LANGUAGE c AS 'MODULE_PATHNAME', 'spock_attach_node'; + +CREATE PROCEDURE spock.detach_node( + target_node_name text, + target_node_dsn text, + verbose_mode boolean DEFAULT true +) LANGUAGE c AS 'MODULE_PATHNAME', 'spock_detach_node'; + +REVOKE ALL ON PROCEDURE spock.attach_node(text, text, text, text, boolean, + text, text, jsonb, int) FROM PUBLIC; +REVOKE ALL ON PROCEDURE spock.detach_node(text, text, boolean) FROM PUBLIC; + -- ---- -- Enable the "failover" flag on spock's existing logical replication slots. -- diff --git a/sql/spock--6.0.0.sql b/sql/spock--6.0.0.sql index 578ddd74e..f1f3e1ecf 100644 --- a/sql/spock--6.0.0.sql +++ b/sql/spock--6.0.0.sql @@ -1126,3 +1126,49 @@ CREATE FUNCTION spock.reset_subscription_stats(subid oid DEFAULT NULL) RETURNS void AS 'MODULE_PATHNAME', 'spock_reset_subscription_stats' LANGUAGE C CALLED ON NULL INPUT VOLATILE; + +-- ---------------------------------------------------------------------------- +-- Zero Downtime Add/Remove Node (ZODAN), in-core C orchestration +-- +-- attach_node/detach_node orchestrate joining or removing a node entirely in +-- C, reaching the other cluster nodes over libpq (no dblink dependency). +-- +-- These are the in-core equivalents of the SQL/dblink procedures shipped in +-- samples/Z0DAN/zodan.sql (spock.add_node) and zodremove.sql +-- (spock.remove_node); users may choose either orchestration. +-- +-- attach_node must be run on the new node being added. +-- detach_node must be run on the node being removed. +-- +-- Neither procedure is atomic across the cluster. Both commit local state at +-- phase boundaries, and the subscriptions they create or drop on the other nodes +-- are committed one node at a time by the remote server. A failure part-way +-- through therefore leaves the cluster in an intermediate state that is not +-- rolled back: recovery from a failed attach_node is spock.detach_node() on the +-- new node, then a fresh attach_node(). See the header of src/spock_zodan.c. +-- +-- Both take an arbitrary connection string and reach out to it, and detach_node +-- drops every subscription, replication set and the node itself, so EXECUTE is +-- revoked from PUBLIC below. +-- ---------------------------------------------------------------------------- +CREATE PROCEDURE spock.attach_node( + src_node_name text, + src_dsn text, + new_node_name text, + new_node_dsn text, + verb boolean DEFAULT false, + new_node_location text DEFAULT 'NY', + new_node_country text DEFAULT 'USA', + new_node_info jsonb DEFAULT '{}'::jsonb, + timeout_sec int DEFAULT 180 +) LANGUAGE c AS 'MODULE_PATHNAME', 'spock_attach_node'; + +CREATE PROCEDURE spock.detach_node( + target_node_name text, + target_node_dsn text, + verbose_mode boolean DEFAULT true +) LANGUAGE c AS 'MODULE_PATHNAME', 'spock_detach_node'; + +REVOKE ALL ON PROCEDURE spock.attach_node(text, text, text, text, boolean, + text, text, jsonb, int) FROM PUBLIC; +REVOKE ALL ON PROCEDURE spock.detach_node(text, text, boolean) FROM PUBLIC; diff --git a/src/spock_apply.c b/src/spock_apply.c index 10a5dfc27..ba066fd87 100644 --- a/src/spock_apply.c +++ b/src/spock_apply.c @@ -630,7 +630,7 @@ action_error_callback(void *arg) */ /* - * If the pause flag is set (slot creation in progress for add_node), + * If the pause flag is set (slot creation in progress for attach_node), * sleep on the ConditionVariable until resumed or timed out. */ static void @@ -674,8 +674,8 @@ begin_replication_step(void) if (!IsTransactionState()) { /* - * Check if slot creation (add_node) needs us to pause. This only - * fires during add_node (a rare operation). The fast path is a + * Check if slot creation (attach_node) needs us to pause. This only + * fires during attach_node (a rare operation). The fast path is a * single atomic read that almost always sees 0. * * Runs before StartTransactionCommand so the worker has no xid while diff --git a/src/spock_zodan.c b/src/spock_zodan.c new file mode 100644 index 000000000..5f86ecfa4 --- /dev/null +++ b/src/spock_zodan.c @@ -0,0 +1,2581 @@ +/*------------------------------------------------------------------------- + * + * spock_zodan.c + * Zero Downtime Add/Remove Node (ZODAN) orchestration. + * + * Implements spock.attach_node() and spock.detach_node() entirely in C. These + * are the in-core counterparts of the PL/pgSQL procedures still shipped in + * samples/Z0DAN/zodan.sql (spock.add_node) and zodremove.sql + * (spock.remove_node), which reach every other node in the cluster through the + * dblink extension. Users may choose either orchestration. The in-core + * orchestration is driven from C: local work is done over SPI (calling the + * same spock.* SQL functions), and all cross-node work is done over libpq via + * spock_connect(), so the dblink dependency is gone. + * + * Both procedures are LANGUAGE c PROCEDUREs: they run non-atomically (invoked + * via top-level CALL) and use SPI_commit() at the phase boundaries where the + * original scripts relied on COMMIT so that apply/sync workers pick up newly + * created or enabled subscriptions. + * + * attach_node must be run on the new node being added. + * detach_node must be run on the node being removed. + * + * NEITHER PROCEDURE IS ATOMIC ACROSS THE CLUSTER. Local state is committed at + * the phase boundaries listed above, and the subscriptions created on the other + * nodes are committed one node at a time by the remote server (libpq + * autocommit). A failure part-way through therefore leaves the cluster in an + * intermediate state -- for example, attach_node failing on the third of five + * nodes leaves the first two subscribed to the new node -- and nothing is rolled + * back or compensated. Recovery is spock.detach_node() on the new node, + * followed by a fresh attach_node() once the cause has been addressed. This + * matches the behavior of the SQL/dblink spock.add_node(). + * + * Copyright (c) 2022-2026, pgEdge, Inc. + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, The Regents of the University of California + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "libpq-fe.h" + +#include "access/xact.h" +#include "executor/spi.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "commands/dbcommands.h" +#include "pgstat.h" +#include "storage/latch.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" +#include "utils/pg_lsn.h" +#if PG_VERSION_NUM >= 190000 +#include "nodes/miscnodes.h" +#endif +#include "utils/resowner.h" +#include "utils/timestamp.h" + +#include "spock.h" +#include "spock_node.h" + +PG_FUNCTION_INFO_V1(spock_attach_node); +PG_FUNCTION_INFO_V1(spock_detach_node); + +/* + * Replication sets used for a ZODAN-managed subscription when its provider + * reports none at all, which should not happen: spock.node_create() always + * creates these three. The real list is read from the provider, so a cluster + * with user-created sets does not silently lose them on the new node. + */ +#define ZODAN_DEFAULT_REPSETS "ARRAY['default', 'default_insert_only', 'ddl_sql']" + +/* Minimum Spock version required on every node participating in attach_node. */ +#define ZODAN_MIN_VERSION "5.0.9" + +/* + * An order-independent digest of a node's whole replication set configuration: + * the set definitions, the table memberships with their column lists and row + * filters, and the sequence memberships. Run on the source and on the new node + * to confirm the mirror took, and that the source did not move underneath it. + */ +#define ZODAN_REPSET_DIGEST_SQL \ + "SELECT md5(coalesce(string_agg(entry, E'\\n' ORDER BY entry), '')) FROM (" \ + " SELECT rs.set_name || ' ' || rs.replicate_insert || rs.replicate_update ||" \ + " rs.replicate_delete || rs.replicate_truncate AS entry" \ + " FROM spock.replication_set rs" \ + " JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id" \ + " UNION ALL" \ + " SELECT rs.set_name || ' ' ||" \ + " quote_ident(n.nspname) || '.' || quote_ident(c.relname) || ' ' ||" \ + " coalesce(rst.set_att_list::text, '') || ' ' ||" \ + " coalesce(pg_get_expr(rst.set_row_filter, rst.set_reloid), '')" \ + " FROM spock.replication_set_table rst" \ + " JOIN spock.replication_set rs ON rs.set_id = rst.set_id" \ + " JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id" \ + " JOIN pg_catalog.pg_class c ON c.oid = rst.set_reloid" \ + " JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace" \ + " UNION ALL" \ + " SELECT rs.set_name || ' seq ' ||" \ + " quote_ident(n.nspname) || '.' || quote_ident(c.relname)" \ + " FROM spock.replication_set_seq rss" \ + " JOIN spock.replication_set rs ON rs.set_id = rss.set_id" \ + " JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id" \ + " JOIN pg_catalog.pg_class c ON c.oid = rss.set_seqoid" \ + " JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace" \ + ") s" + +/* + * A remote node and the DSN we reach it on. Built by connecting to the source + * node and reading spock.node JOIN spock.node_interface. + */ +typedef struct ZNode +{ + char *name; + char *dsn; + char *location; + char *country; + char *info; +} ZNode; + +/* + * Everything the attach_node phases need, so we don't thread a dozen arguments + * through each helper. All strings live in the dedicated memory context + * (see spock_attach_node) so they survive the SPI_commit() calls between phases. + */ +typedef struct ZodanAddCtx +{ + char *src_node_name; + char *src_dsn; + char *new_node_name; + char *new_node_dsn; + bool verb; + char *new_node_location; + char *new_node_country; + char *new_node_info; /* jsonb rendered as text */ + int timeout_sec; + + /* Cluster snapshot fetched from the source node (all existing nodes). */ + ZNode *nodes; + int nnodes; + + /* Source node's Spock version, from zodan_check_versions(). */ + int src_major; + int src_minor; + int src_patch; + + MemoryContext mcxt; /* survives SPI_commit() */ +} ZodanAddCtx; + +/* A two-column row snapshotted out of SPI_tuptable; see zodan_local_pairs(). */ +typedef struct ZPair +{ + char *a; + char *b; +} ZPair; + +/* + * Progress reporting. All progress output is gated on verbose mode so a + * normal run is quiet; only the final confirmation is emitted unconditionally + * by the entry points. zodan_verbose is set once per top-level call, from the + * procedure's verbose argument, so the macro needs no per-call flag. + */ +static bool zodan_verbose = false; + +#define ZNOTICE(...) \ + do { if (zodan_verbose) ereport(NOTICE, (errmsg(__VA_ARGS__))); } while (0) + +/* + * A boolean column of a PGresult as a SQL literal. libpq renders booleans as + * "t"/"f", which parses as a column reference rather than a value when it is + * interpolated into a query. + */ +#define ZBOOL(res, row, col) \ + (PQgetvalue((res), (row), (col))[0] == 't' ? "true" : "false") + +/* ------------------------------------------------------------------------ + * Small utilities + * ------------------------------------------------------------------------ */ + +/* + * Sleep for the given number of milliseconds, honoring query cancel and + * postmaster death. Used by the various catch-up/sync wait loops. + */ +static void +zodan_sleep_ms(long ms) +{ + int rc; + + CHECK_FOR_INTERRUPTS(); + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + ms, PG_WAIT_EXTENSION); + ResetLatch(MyLatch); + if (rc & WL_LATCH_SET) + CHECK_FOR_INTERRUPTS(); +} + +/* + * Extract the database name from a libpq connection string. Errors out if the + * DSN does not name a database, matching the old extract_dbname_from_dsn(). + */ +static char * +zodan_dbname_from_dsn(const char *dsn) +{ + char *parse_err = NULL; + PQconninfoOption *opts; + PQconninfoOption *o; + char *dbname = NULL; + + opts = PQconninfoParse(dsn, &parse_err); + if (opts == NULL) + { + char buf[1024]; + + snprintf(buf, sizeof(buf), "%s", parse_err ? parse_err : "unknown error"); + if (parse_err) + PQfreemem(parse_err); + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("could not parse DSN \"%s\": %s", dsn, buf))); + } + + for (o = opts; o->keyword != NULL; o++) + { + if (strcmp(o->keyword, "dbname") == 0 && o->val != NULL && o->val[0] != '\0') + { + dbname = pstrdup(o->val); + break; + } + } + PQconninfoFree(opts); + + if (dbname == NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("database name must be included in the DSN string: %s", + dsn))); + return dbname; +} + +/* Build the subscription name sub__. */ +static char * +zodan_gen_sub_name(const char *provider, const char *subscriber) +{ + return psprintf("sub_%s_%s", provider, subscriber); +} + +/* + * Build the replication slot name for a subscription on this node the way + * sub_create() does: from the local database name (the subscriber's, not the + * provider's), the provider node name and the subscription name as the name + * type stores it, so a name longer than NAMEDATALEN - 1 hashes the same. + */ +static char * +zodan_gen_slot_name(const char *provider_node, const char *sub_name) +{ + NameData slot; + NameData sub; + + namestrcpy(&sub, sub_name); + gen_slot_name(&slot, get_database_name(MyDatabaseId), provider_node, + NameStr(sub)); + return pstrdup(NameStr(slot)); +} + +/* + * Parse "5.0.10-devel" into {major, minor, patch}; ignores any suffix. label + * names the node the version came from, for the error message: silently + * treating an unparseable version as 0.0.0 would surface as a bogus "has + * version 0.0.0" mismatch instead of the real problem. + */ +static void +zodan_parse_version(const char *v, const char *label, + int *major, int *minor, int *patch) +{ + *major = *minor = *patch = 0; + if (v == NULL || sscanf(v, "%d.%d.%d", major, minor, patch) != 3) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("could not parse Spock version \"%s\" on %s", + v ? v : "(null)", label))); +} + +/* Convert a pg_lsn text value to XLogRecPtr, ERROR if malformed. */ +static XLogRecPtr +zodan_parse_lsn(const char *lsn) +{ + bool have_error = false; + XLogRecPtr ret; + + if (lsn == NULL) + ereport(ERROR, (errmsg("expected an LSN value, got NULL"))); + +#if PG_VERSION_NUM >= 190000 + { + /* PG19 replaced the have_error flag with soft error reporting. */ + ErrorSaveContext escontext = {T_ErrorSaveContext}; + + ret = pg_lsn_in_safe(lsn, (Node *) &escontext); + have_error = escontext.error_occurred; + } +#else + ret = pg_lsn_in_internal(lsn, &have_error); +#endif + if (have_error) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid LSN value \"%s\"", lsn))); + return ret; +} + +/* Return <0, 0, >0 comparing the left major.minor.patch to the right one. */ +static int +zodan_version_cmp(int left_major, int left_minor, int left_patch, + int right_major, int right_minor, int right_patch) +{ + if (left_major != right_major) + return left_major - right_major; + if (left_minor != right_minor) + return left_minor - right_minor; + return left_patch - right_patch; +} + +/* + * Name of the spock.progress column holding the LSN of the last commit applied + * from a peer, on a node running the given Spock version. 6.0.0 renamed it + * from remote_lsn to remote_commit_lsn, so a query sent to a remote node has + * to use the name that node understands. + */ +static const char * +zodan_progress_lsn_column(int major, int minor, int patch) +{ + /* An unset version would silently pick the pre-6.0.0 name. */ + Assert(major > 0); + + if (zodan_version_cmp(major, minor, patch, 6, 0, 0) < 0) + return "remote_lsn"; + return "remote_commit_lsn"; +} + +/* ------------------------------------------------------------------------ + * libpq (remote node) helpers -- these replace dblink() + * ------------------------------------------------------------------------ */ + +/* + * Open libpq connections and PGresults are tracked here so that a phase helper + * which ereport(ERROR)s mid-orchestration does not strand its socket, or leak + * the result's malloc'd memory, for the rest of the backend's life. + * spock_connect() does not register the PGconn with any resource owner, PGresult + * has no resource-owner integration at all, and the phase helpers keep both in + * local variables -- so on error the entry point drains these registries (see + * zodan_release_all). A handful of slots is far more than the two or three + * connections and single result ever live at once. + */ +#define ZODAN_MAX_CONNS 16 +static PGconn *zodan_conns[ZODAN_MAX_CONNS]; +static int zodan_nconns = 0; + +#define ZODAN_MAX_RESULTS 16 +static PGresult *zodan_results[ZODAN_MAX_RESULTS]; +static int zodan_nresults = 0; + +/* Connect to a remote node, ERROR on failure. Tracks the connection. */ +static PGconn * +zodan_connect(const char *dsn, const char *purpose) +{ + PGconn *conn; + + /* + * Refuse rather than hand back an untracked connection: overflow would + * mean a leak on the error path, which is exactly what the registry + * exists to prevent. Reserve the slot before connecting so nothing can + * be dropped. + */ + if (zodan_nconns >= ZODAN_MAX_CONNS) + elog(ERROR, "too many concurrent zodan connections (max %d)", + ZODAN_MAX_CONNS); + + conn = spock_connect(dsn, "spock_zodan", purpose); + if (conn != NULL) + zodan_conns[zodan_nconns++] = conn; + return conn; +} + +/* Close a tracked connection (no-op on NULL) and forget it. */ +static void +zodan_disconnect(PGconn *conn) +{ + int i; + + if (conn == NULL) + return; + for (i = 0; i < zodan_nconns; i++) + { + if (zodan_conns[i] == conn) + { + zodan_conns[i] = zodan_conns[--zodan_nconns]; + break; + } + } + PQfinish(conn); +} + +/* Clear a tracked PGresult (no-op on NULL) and forget it. */ +static void +zodan_clear(PGresult *res) +{ + int i; + + if (res == NULL) + return; + for (i = 0; i < zodan_nresults; i++) + { + if (zodan_results[i] == res) + { + zodan_results[i] = zodan_results[--zodan_nresults]; + break; + } + } + PQclear(res); +} + +/* + * Release every still-live tracked result and connection. Used for cleanup on + * error, and to assert a clean slate on entry. + */ +static void +zodan_release_all(void) +{ + while (zodan_nresults > 0) + PQclear(zodan_results[--zodan_nresults]); + while (zodan_nconns > 0) + PQfinish(zodan_conns[--zodan_nconns]); +} + +/* + * Run a query on a remote node that is expected to return tuples. The result is + * tracked; the caller owns it and must release it with zodan_clear(). ERROR on + * failure. + */ +static PGresult * +zodan_remote_query(PGconn *conn, const char *sql) +{ + PGresult *res; + + if (zodan_nresults >= ZODAN_MAX_RESULTS) + elog(ERROR, "too many concurrent zodan results (max %d)", + ZODAN_MAX_RESULTS); + + res = PQexec(conn, sql); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + char msg[1024]; + + snprintf(msg, sizeof(msg), "%s", PQerrorMessage(conn)); + PQclear(res); + ereport(ERROR, + (errmsg("remote query failed on node"), + errdetail("query: %s", sql), + errdetail("error: %s", msg))); + } + zodan_results[zodan_nresults++] = res; + return res; +} + +/* + * Run a command on a remote node (COMMAND_OK or TUPLES_OK both accepted). + * ERROR on failure. + */ +static void +zodan_remote_command(PGconn *conn, const char *sql) +{ + PGresult *res = PQexec(conn, sql); + ExecStatusType st = PQresultStatus(res); + + if (st != PGRES_COMMAND_OK && st != PGRES_TUPLES_OK) + { + char msg[1024]; + + snprintf(msg, sizeof(msg), "%s", PQerrorMessage(conn)); + PQclear(res); + ereport(ERROR, + (errmsg("remote command failed on node"), + errdetail("command: %s", sql), + errdetail("error: %s", msg))); + } + PQclear(res); +} + +/* + * Run a query on a remote node and return the first column of the first row as + * a palloc'd string (NULL if no rows or a SQL NULL). + */ +static char * +zodan_remote_scalar(PGconn *conn, const char *sql) +{ + PGresult *res = zodan_remote_query(conn, sql); + char *ret = NULL; + + if (PQntuples(res) > 0 && !PQgetisnull(res, 0, 0)) + ret = pstrdup(PQgetvalue(res, 0, 0)); + zodan_clear(res); + return ret; +} + +/* ------------------------------------------------------------------------ + * SPI (local node) helpers + * ------------------------------------------------------------------------ */ + +/* Execute a local command over SPI, ERROR on failure. */ +static void +zodan_local_command(const char *sql) +{ + int rc = SPI_execute(sql, false, 0); + + if (rc < 0) + elog(ERROR, "SPI_execute failed for: %s", sql); +} + +/* + * Execute a local query over SPI and return the first column of the first row + * as a palloc'd string in the caller's context (NULL if no rows / SQL NULL). + * + * read_only is false so every call takes a fresh snapshot; the wait/poll loops + * depend on observing rows committed by the apply and sync workers while the + * loop is running. + */ +static char * +zodan_local_scalar(const char *sql) +{ + int rc; + char *ret = NULL; + + rc = SPI_execute(sql, false, 0); + if (rc != SPI_OK_SELECT) + elog(ERROR, "SPI_execute (select) failed for: %s", sql); + + /* SPI_getvalue() returns NULL for a SQL NULL, which is what we want. */ + if (SPI_processed > 0) + ret = SPI_getvalue(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1); + + return ret; +} + +/* + * Snapshot a one- or two-column local query into palloc'd strings in mcxt. + * + * Callers act on the rows by issuing further SPI_execute() calls, and each of + * those replaces SPI_tuptable, so the rows have to be copied out first. A SQL + * NULL, and a missing second column, come back as NULL. + */ +static ZPair * +zodan_local_pairs(const char *sql, MemoryContext mcxt, int *nrows) +{ + ZPair *rows; + int n; + int i; + int natts; + MemoryContext old; + + if (SPI_execute(sql, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute (select) failed for: %s", sql); + + n = SPI_processed; + natts = SPI_tuptable->tupdesc->natts; + + /* SPI_getvalue() allocates in the current context, so switch first. */ + old = MemoryContextSwitchTo(mcxt); + rows = (ZPair *) palloc0(sizeof(ZPair) * Max(n, 1)); + for (i = 0; i < n; i++) + { + rows[i].a = SPI_getvalue(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 1); + if (natts >= 2) + rows[i].b = SPI_getvalue(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 2); + } + MemoryContextSwitchTo(old); + + *nrows = n; + return rows; +} + +/* Convenience: local scalar as int64 (0 if NULL). */ +static int64 +zodan_local_count(const char *sql) +{ + char *s = zodan_local_scalar(sql); + int64 v = 0; + + if (s != NULL) + { + v = pg_strtoint64(s); + pfree(s); + } + return v; +} + +/* ------------------------------------------------------------------------ + * Cluster inspection + * ------------------------------------------------------------------------ */ + +/* + * Fetch the list of all nodes known to the source cluster (from the source + * node), storing them in ctx->nodes/ctx->nnodes in ctx->mcxt. + */ +static void +zodan_fetch_cluster_nodes(ZodanAddCtx *ctx) +{ + PGconn *conn; + PGresult *res; + int n; + int i; + MemoryContext old; + + conn = zodan_connect(ctx->src_dsn, "nodes"); + res = zodan_remote_query(conn, + "SELECT n.node_name, i.if_dsn, " + "COALESCE(n.location,''), COALESCE(n.country,''), " + "COALESCE(n.info::text,'') " + "FROM spock.node n " + "JOIN spock.node_interface i ON n.node_id = i.if_nodeid " + "ORDER BY n.node_name"); + n = PQntuples(res); + + old = MemoryContextSwitchTo(ctx->mcxt); + ctx->nodes = (ZNode *) palloc0(sizeof(ZNode) * Max(n, 1)); + for (i = 0; i < n; i++) + { + ctx->nodes[i].name = pstrdup(PQgetvalue(res, i, 0)); + ctx->nodes[i].dsn = pstrdup(PQgetvalue(res, i, 1)); + ctx->nodes[i].location = pstrdup(PQgetvalue(res, i, 2)); + ctx->nodes[i].country = pstrdup(PQgetvalue(res, i, 3)); + ctx->nodes[i].info = pstrdup(PQgetvalue(res, i, 4)); + } + ctx->nnodes = n; + MemoryContextSwitchTo(old); + + zodan_clear(res); + zodan_disconnect(conn); + + /* + * src_node_name is only ever used to compare against these names -- to + * pick the source out of the cluster, to count the "other" nodes (which + * selects the 2-node vs multi-node path in several phases), and to build + * subscription names. A name that does not match any node therefore does + * not fail here; it silently reclassifies the real source as an "other" + * node and fails much later, in a confusing place. Reject it up front. + */ + { + StringInfoData known; + + for (i = 0; i < n; i++) + { + if (strcmp(ctx->nodes[i].name, ctx->src_node_name) == 0) + return; + } + + initStringInfo(&known); + for (i = 0; i < n; i++) + appendStringInfo(&known, "%s%s", i > 0 ? ", " : "", + ctx->nodes[i].name); + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("source node \"%s\" is not a node in the cluster reached by src_dsn", + ctx->src_node_name), + errdetail("Nodes known to that cluster: %s", + known.len > 0 ? known.data : "(none)"))); + } +} + +/* + * Render a PGresult's first column as the SQL array literal ARRAY['a', 'b'], + * or ZODAN_DEFAULT_REPSETS when it has no rows. + */ +static char * +zodan_repset_array(PGresult *res) +{ + StringInfoData buf; + int n = PQntuples(res); + int i; + + if (n == 0) + { + /* + * A Spock node always has the built-ins, so no rows means the DSN + * does not point at one. Fall back rather than build an empty array + * that would replicate nothing. + */ + return pstrdup(ZODAN_DEFAULT_REPSETS); + } + + initStringInfo(&buf); + appendStringInfoString(&buf, "ARRAY["); + for (i = 0; i < n; i++) + appendStringInfo(&buf, "%s%s", i > 0 ? ", " : "", + quote_literal_cstr(PQgetvalue(res, i, 0))); + appendStringInfoChar(&buf, ']'); + return buf.data; +} + +/* + * A node's replication sets, as the SQL array literal for the replication_sets + * argument of sub_create(). + * + * Subscriptions used to be pinned to the three built-in sets, which silently + * dropped every table that only lives in a user-created set: the new node + * neither received those tables nor sent them, with no error anywhere. + * + * This must always be called with the subscription's PROVIDER dsn. A + * subscription's set names are resolved on the provider, by + * get_replication_sets() in spock_repset.c, and a name the provider does not + * have is an ERROR there ("replication set %s not found"), not something it + * ignores. The walsender then refuses START_REPLICATION and the apply worker + * retries forever. Replication sets are per-node local state, so two existing + * nodes can legitimately have different ones. + */ +static char * +zodan_node_repset_list(const char *node_dsn) +{ + PGconn *conn; + PGresult *res; + char *arr; + + conn = zodan_connect(node_dsn, "repsets"); + res = zodan_remote_query(conn, + "SELECT rs.set_name " + "FROM spock.replication_set rs " + "JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id " + "ORDER BY rs.set_name"); + arr = zodan_repset_array(res); + zodan_clear(res); + zodan_disconnect(conn); + + return arr; +} + +/* Number of nodes that are neither the source nor the new node. */ +static int +zodan_num_other_nodes(ZodanAddCtx *ctx) +{ + int i; + int cnt = 0; + + for (i = 0; i < ctx->nnodes; i++) + { + if (strcmp(ctx->nodes[i].name, ctx->src_node_name) != 0 && + strcmp(ctx->nodes[i].name, ctx->new_node_name) != 0) + cnt++; + } + return cnt; +} + +/* + * Check the Spock extension version on a node reachable via conn, returning + * major/minor/patch. ERROR if the extension is missing. + */ +static void +zodan_remote_spock_version(PGconn *conn, const char *label, + int *major, int *minor, int *patch) +{ + char *v = zodan_remote_scalar(conn, + "SELECT extversion FROM pg_extension WHERE extname = 'spock'"); + + if (v == NULL) + ereport(ERROR, + (errmsg("Spock extension not found on %s", label))); + zodan_parse_version(v, label, major, minor, patch); + pfree(v); +} + +/* ------------------------------------------------------------------------ + * attach_node phases + * ------------------------------------------------------------------------ */ + +/* + * Phase 0: verify the Spock version on the source node, the new node and every + * existing cluster node. + * + * Every node must be >= ZODAN_MIN_VERSION, and the new node must run the same + * or a newer major.minor than every existing node, the source included: only + * the newer sync worker knows how to talk to an older provider. The existing + * nodes may differ from each other, since a cluster is mixed from the first + * newer node on and further nodes still have to be able to join it. + */ +static void +zodan_check_versions(ZodanAddCtx *ctx) +{ + int min_major, + min_minor, + min_patch; + int src_major, + src_minor, + src_patch; + int new_major, + new_minor, + new_patch; + StringInfoData existing; + bool mixed = false; + PGconn *conn; + int i; + + zodan_parse_version(ZODAN_MIN_VERSION, "ZODAN_MIN_VERSION", + &min_major, &min_minor, &min_patch); + + ZNOTICE("Checking Spock version on source node"); + conn = zodan_connect(ctx->src_dsn, "ver"); + zodan_remote_spock_version(conn, "source node", + &src_major, &src_minor, &src_patch); + zodan_disconnect(conn); + + if (zodan_version_cmp(src_major, src_minor, src_patch, + min_major, min_minor, min_patch) < 0) + ereport(ERROR, + (errmsg("Spock version mismatch: source node has version %d.%d.%d, " + "but minimum required version is %s", + src_major, src_minor, src_patch, ZODAN_MIN_VERSION))); + + ZNOTICE("Checking Spock version on new node"); + conn = zodan_connect(ctx->new_node_dsn, "ver"); + zodan_remote_spock_version(conn, "new node", + &new_major, &new_minor, &new_patch); + zodan_disconnect(conn); + + if (zodan_version_cmp(new_major, new_minor, new_patch, + min_major, min_minor, min_patch) < 0) + ereport(ERROR, + (errmsg("Spock version mismatch: new node has version %d.%d.%d, " + "but minimum required version is %s", + new_major, new_minor, new_patch, ZODAN_MIN_VERSION))); + + if (zodan_version_cmp(new_major, new_minor, 0, + src_major, src_minor, 0) < 0) + ereport(ERROR, + (errmsg("Spock version mismatch: new node has version %d.%d.%d, " + "but source node has version %d.%d.%d; the new node must " + "run the same or a newer major.minor version than every " + "existing node", + new_major, new_minor, new_patch, + src_major, src_minor, src_patch))); + + /* + * The existing nodes may differ from each other: a cluster is mixed from + * the first newer node on, and further nodes have to be able to join it. + * What none of them may be is newer than the new node. + */ + initStringInfo(&existing); + appendStringInfo(&existing, "%s %d.%d.%d", ctx->src_node_name, + src_major, src_minor, src_patch); + + for (i = 0; i < ctx->nnodes; i++) + { + int node_major, + node_minor, + node_patch; + + conn = zodan_connect(ctx->nodes[i].dsn, "ver"); + zodan_remote_spock_version(conn, ctx->nodes[i].name, + &node_major, &node_minor, &node_patch); + zodan_disconnect(conn); + + if (zodan_version_cmp(node_major, node_minor, node_patch, + min_major, min_minor, min_patch) < 0) + ereport(ERROR, + (errmsg("Spock version mismatch: node %s has version %d.%d.%d, " + "but required version is at least %s", + ctx->nodes[i].name, + node_major, node_minor, node_patch, + ZODAN_MIN_VERSION))); + + if (zodan_version_cmp(new_major, new_minor, 0, + node_major, node_minor, 0) < 0) + ereport(ERROR, + (errmsg("Spock version mismatch: new node has version %d.%d.%d, " + "but node %s has version %d.%d.%d; the new node must " + "run the same or a newer major.minor version than " + "every existing node", + new_major, new_minor, new_patch, + ctx->nodes[i].name, + node_major, node_minor, node_patch))); + + if (node_major != new_major || node_minor != new_minor) + mixed = true; + + appendStringInfo(&existing, ", %s %d.%d.%d", ctx->nodes[i].name, + node_major, node_minor, node_patch); + } + + /* zodan_progress_lsn_column() needs this once the copy is under way. */ + ctx->src_major = src_major; + ctx->src_minor = src_minor; + ctx->src_patch = src_patch; + + /* Worth saying without verb: this is not the usual shape of a cluster. */ + if (mixed || new_major != src_major || new_minor != src_minor) + ereport(NOTICE, + (errmsg("mixed-version attach: new node runs Spock %d.%d.%d, " + "existing nodes run %s", + new_major, new_minor, new_patch, existing.data))); + + ZNOTICE("Version check passed: source %d.%d.%d, new node %d.%d.%d", + src_major, src_minor, src_patch, new_major, new_minor, new_patch); +} + +/* + * Phase 1: verify prerequisites. Most importantly this enforces that attach_node + * is being run on the new node (by comparing the local system identifier and + * database name against what new_node_dsn points at). + */ +static void +zodan_verify_prerequisites(ZodanAddCtx *ctx) +{ + char *local_sysid; + char *local_dbname; + char *remote_sysid; + char *remote_dbname; + char *new_dbname; + PGconn *conn; + int64 cnt; + + ZNOTICE("Phase 1: Validating source and new node prerequisites"); + + /* attach_node must be run on the new node. */ + local_sysid = zodan_local_scalar("SELECT system_identifier::text FROM pg_control_system()"); + local_dbname = zodan_local_scalar("SELECT current_database()"); + + conn = zodan_connect(ctx->new_node_dsn, "prereq"); + remote_sysid = zodan_remote_scalar(conn, + "SELECT system_identifier::text FROM pg_control_system()"); + remote_dbname = zodan_remote_scalar(conn, "SELECT current_database()"); + zodan_disconnect(conn); + + if (local_sysid == NULL || remote_sysid == NULL || + strcmp(local_sysid, remote_sysid) != 0 || + local_dbname == NULL || remote_dbname == NULL || + strcmp(local_dbname, remote_dbname) != 0) + ereport(ERROR, + (errmsg("attach_node must be run on the new node being added"), + errdetail("new_node_dsn (%s) does not match the current database connection", + ctx->new_node_dsn), + errhint("Connect to the new node and re-run attach_node."))); + + ZNOTICE(" OK: attach_node is running on the new node"); + + /* Sanity-check the database named by the new node DSN. */ + new_dbname = zodan_dbname_from_dsn(ctx->new_node_dsn); + + /* + * lolor state on the new node. Having the lolor extension installed is + * fine (and required when the source replicates lolor tables), but any + * pre-existing large object data would conflict with the data + * synchronized from the source, so the lolor tables must be empty. The + * lolor structure itself is excluded from the structure dump (see + * skip_schema[]) and comes from CREATE EXTENSION lolor, so the extension + * must already exist on the new node when the source replicates lolor + * tables. + */ + { + bool new_lolor_installed; + int64 src_lolor_repset_tables; + + new_lolor_installed = zodan_local_count( + "SELECT count(*) FROM pg_catalog.pg_extension WHERE extname = 'lolor'") > 0; + + if (new_lolor_installed) + { + int64 lolor_rows = zodan_local_count( + "SELECT (SELECT count(*) FROM lolor.pg_largeobject) + " + "(SELECT count(*) FROM lolor.pg_largeobject_metadata)"); + + if (lolor_rows > 0) + ereport(ERROR, + (errmsg("database %s on the new node has pre-existing large object data in the lolor tables", + new_dbname), + errhint("The lolor tables must be empty so the large object data from the source node can be synchronized."))); + ZNOTICE(" OK: database %s lolor tables are empty", new_dbname); + } + + /* + * If the source node replicates lolor tables, the new node must have + * lolor installed or the initial data synchronization will fail. + */ + { + PGconn *src_conn = zodan_connect(ctx->src_dsn, "prereq"); + char *s; + + s = zodan_remote_scalar(src_conn, + "SELECT count(*) FROM spock.tables " + "WHERE nspname = 'lolor' AND set_name IS NOT NULL"); + zodan_disconnect(src_conn); + + src_lolor_repset_tables = (s != NULL) ? pg_strtoint64(s) : 0; + if (s != NULL) + pfree(s); + } + + if (src_lolor_repset_tables > 0 && !new_lolor_installed) + ereport(ERROR, + (errmsg("source node replicates lolor tables but database %s on the new node does not have the lolor extension installed", + new_dbname), + errhint("Run CREATE EXTENSION lolor on the new node first."))); + ZNOTICE(" OK: lolor requirements satisfied for database %s", new_dbname); + } + + /* + * No user tables on the new node. 'lolor' is excluded: its tables are + * created by CREATE EXTENSION lolor, which is permitted (and required) + * when the source replicates lolor data, and were already validated as + * empty above. + */ + cnt = zodan_local_count( + "SELECT count(*) FROM pg_tables " + "WHERE schemaname NOT IN ('information_schema','pg_catalog','pg_toast','spock','lolor') " + "AND schemaname NOT LIKE 'pg_temp_%' " + "AND schemaname NOT LIKE 'pg_toast_temp_%'"); + if (cnt > 0) + ereport(ERROR, + (errmsg("database %s on the new node has " INT64_FORMAT " user-created tables", + new_dbname, cnt), + errhint("The new node must be a freshly created database with no user tables."))); + ZNOTICE(" OK: database %s has no user tables", new_dbname); + + /* Every source login role must exist on the new node. */ + { + PGconn *src_conn = zodan_connect(ctx->src_dsn, "prereq"); + PGresult *res; + int i; + StringInfoData missing; + + initStringInfo(&missing); + res = zodan_remote_query(src_conn, + "SELECT rolname FROM pg_roles " + "WHERE rolcanlogin = true " + "AND rolname NOT IN ('postgres','rdsadmin','rdsrepladmin','rds_superuser') " + "ORDER BY rolname"); + for (i = 0; i < PQntuples(res); i++) + { + char *rolname = PQgetvalue(res, i, 0); + char *sql = psprintf( + "SELECT count(*) FROM pg_roles WHERE rolname = %s AND rolcanlogin = true", + quote_literal_cstr(rolname)); + int64 exists = zodan_local_count(sql); + + pfree(sql); + if (exists == 0) + { + if (missing.len > 0) + appendStringInfoString(&missing, ", "); + appendStringInfoString(&missing, rolname); + } + } + zodan_clear(res); + zodan_disconnect(src_conn); + + if (missing.len > 0) + ereport(ERROR, + (errmsg("new node is missing roles that exist on the source node: %s", + missing.data), + errhint("Create these roles on the new node before adding it to the cluster."))); + pfree(missing.data); + ZNOTICE(" OK: new node has all source-node login roles"); + } + + /* Every existing cluster node must have only enabled subscriptions. */ + { + int i; + + for (i = 0; i < ctx->nnodes; i++) + { + PGconn *nconn; + PGresult *res; + int j; + + if (strcmp(ctx->nodes[i].name, ctx->new_node_name) == 0) + continue; + + nconn = zodan_connect(ctx->nodes[i].dsn, "prereq"); + res = zodan_remote_query(nconn, + "SELECT sub_name, sub_enabled FROM spock.subscription"); + for (j = 0; j < PQntuples(res); j++) + { + char *en = PQgetvalue(res, j, 1); + + if (en[0] != 't') + { + /* + * Copy the name out of the PGresult before releasing it: + * PQgetvalue() points into the result's own storage. + */ + char *sub = pstrdup(PQgetvalue(res, j, 0)); + + zodan_clear(res); + zodan_disconnect(nconn); + ereport(ERROR, + (errmsg("node %s has disabled subscription %s", + ctx->nodes[i].name, sub), + errhint("All subscriptions must be enabled before adding a node."))); + } + } + zodan_clear(res); + zodan_disconnect(nconn); + } + ZNOTICE(" OK: every cluster node has only enabled subscriptions"); + } + + /* The new node must not already exist locally with subs/repsets. */ + cnt = zodan_local_count(psprintf( + "SELECT count(*) FROM spock.node WHERE node_name = %s", + quote_literal_cstr(ctx->new_node_name))); + if (cnt > 0) + ereport(ERROR, (errmsg("new node %s already exists", ctx->new_node_name))); + + /* + * Nor in the cluster the source knows about, which a failed earlier + * attach can leave behind: sub_create on the source would fail in Phase + * 11 after the data sync. + */ + { + int i; + + for (i = 0; i < ctx->nnodes; i++) + if (strcmp(ctx->nodes[i].name, ctx->new_node_name) == 0) + ereport(ERROR, + (errmsg("new node %s already exists in the cluster with dsn %s", + ctx->new_node_name, ctx->nodes[i].dsn))); + } + + cnt = zodan_local_count( + "SELECT count(*) FROM spock.subscription"); + if (cnt > 0) + ereport(ERROR, + (errmsg("new node already has subscriptions; it must be a clean node"))); + + ZNOTICE(" OK: new node %s is clean", ctx->new_node_name); + + pfree(new_dbname); +} + +/* + * Phase 2: create the new node in the local (new node's) database. The source + * node already exists in its own database, and the local representation of the + * source node is created automatically by spock.sub_create() when the + * source->new subscription is created, so we only create the new node here. + */ +static void +zodan_create_nodes(ZodanAddCtx *ctx) +{ + StringInfoData sql; + + ZNOTICE("Phase 2: Creating nodes"); + + /* Create the new (local) node. */ + initStringInfo(&sql); + appendStringInfo(&sql, + "SELECT spock.node_create(node_name := %s, dsn := %s, " + "location := %s, country := %s, info := %s::jsonb)", + quote_literal_cstr(ctx->new_node_name), + quote_literal_cstr(ctx->new_node_dsn), + quote_literal_cstr(ctx->new_node_location), + quote_literal_cstr(ctx->new_node_country), + quote_literal_cstr(ctx->new_node_info)); + zodan_local_command(sql.data); + pfree(sql.data); + ZNOTICE(" OK: created new node %s", ctx->new_node_name); + + /* Commit so the node rows are visible cluster-wide before we continue. */ + SPI_commit(); +} + +/* + * Create a logical replication slot on a remote node if it does not already + * exist. On PG17+ the slot is created with failover = true, matching Spock's + * own slot creation path. Returns the slot's LSN as a palloc'd string, or NULL + * if the slot already existed. + */ +static char * +zodan_remote_create_slot(PGconn *conn, const char *slot_name) +{ + char *exists; + char *server_ver; + int vernum; + StringInfoData sql; + char *lsn; + + exists = zodan_remote_scalar(conn, psprintf( + "SELECT count(*) FROM pg_replication_slots WHERE slot_name = %s", + quote_literal_cstr(slot_name))); + if (exists != NULL) + { + bool already = strcmp(exists, "0") != 0; + + pfree(exists); + if (already) + return NULL; + } + + server_ver = zodan_remote_scalar(conn, "SHOW server_version_num"); + vernum = server_ver ? atoi(server_ver) : 0; + if (server_ver != NULL) + pfree(server_ver); + + initStringInfo(&sql); + if (vernum >= 170000) + appendStringInfo(&sql, + "SELECT slot_name, lsn FROM pg_create_logical_replication_slot(%s, 'spock_output', false, false, true)", + quote_literal_cstr(slot_name)); + else + appendStringInfo(&sql, + "SELECT slot_name, lsn FROM pg_create_logical_replication_slot(%s, 'spock_output')", + quote_literal_cstr(slot_name)); + + lsn = NULL; + { + PGresult *res = zodan_remote_query(conn, sql.data); + + if (PQntuples(res) > 0 && !PQgetisnull(res, 0, 1)) + lsn = pstrdup(PQgetvalue(res, 0, 1)); + zodan_clear(res); + } + pfree(sql.data); + return lsn; +} + +/* + * Wait until the source node has applied changes from origin_node up to + * target_lsn (observed through spock.progress.remote_commit_lsn on the source). + * Bounded by ctx->timeout_sec. + */ +static void +zodan_wait_source_caughtup(ZodanAddCtx *ctx, const char *origin_node, + const char *target_lsn) +{ + PGconn *conn = zodan_connect(ctx->src_dsn, "catchup"); + char *progress_sql; + XLogRecPtr target; + TimestampTz start = GetCurrentTimestamp(); + + /* The column name is the source node's, not ours: it may predate 6.0.0. */ + progress_sql = psprintf( + "SELECT p.%s " + "FROM spock.progress p " + "JOIN spock.node n ON n.node_id = p.remote_node_id " + "WHERE p.node_id = (SELECT node_id FROM spock.node_info()) " + "AND n.node_name = %s", + zodan_progress_lsn_column(ctx->src_major, ctx->src_minor, + ctx->src_patch), + quote_literal_cstr(origin_node)); + + ZNOTICE(" - Waiting for source node %s to apply %s changes up to %s", + ctx->src_node_name, origin_node, target_lsn); + + target = zodan_parse_lsn(target_lsn); + + for (;;) + { + char *cur = zodan_remote_scalar(conn, progress_sql); + + if (cur != NULL) + { + bool reached = zodan_parse_lsn(cur) >= target; + + pfree(cur); + if (reached) + break; + } + + if (TimestampDifferenceExceeds(start, GetCurrentTimestamp(), + ctx->timeout_sec * 1000)) + { + zodan_disconnect(conn); + ereport(ERROR, + (errmsg("timed out waiting for source node %s to apply %s changes through %s", + ctx->src_node_name, origin_node, target_lsn))); + } + zodan_sleep_ms(500); + } + + zodan_disconnect(conn); + pfree(progress_sql); +} + +/* + * Trigger a sync event on a remote node (conn) and return its LSN as a palloc'd + * string. transactional selects spock.sync_event(true) vs spock.sync_event(). + */ +static char * +zodan_remote_sync_event(PGconn *conn, bool transactional) +{ + return zodan_remote_scalar(conn, + transactional + ? "SELECT spock.sync_event(true)" + : "SELECT spock.sync_event()"); +} + +/* + * Wait for a sync event (origin_node / lsn) to be applied on the node reached + * by conn, by calling spock.wait_for_sync_event() there. ERROR on timeout. + */ +static void +zodan_remote_wait_for_sync_event(ZodanAddCtx *ctx, PGconn *conn, + const char *origin_node, const char *lsn) +{ + StringInfoData sql; + char *ok; + + initStringInfo(&sql); + appendStringInfo(&sql, + "CALL spock.wait_for_sync_event(true, %s, %s::pg_lsn, %d, true)", + quote_literal_cstr(origin_node), + quote_literal_cstr(lsn), + ctx->timeout_sec); + ok = zodan_remote_scalar(conn, sql.data); + pfree(sql.data); + + if (ok == NULL || ok[0] != 't') + ereport(ERROR, + (errmsg("wait_for_sync_event timed out for %s (lsn %s)", + origin_node, lsn))); + pfree(ok); +} + +/* + * Wait for a sync event locally (we are the new node) by polling + * spock.progress until we have applied origin_node's changes up to lsn. + */ +static void +zodan_local_wait_for_sync_event(ZodanAddCtx *ctx, const char *origin_node, + const char *lsn) +{ + char *progress_sql; + XLogRecPtr target = zodan_parse_lsn(lsn); + TimestampTz start = GetCurrentTimestamp(); + + progress_sql = psprintf( + "SELECT p.remote_commit_lsn " + "FROM spock.progress p " + "JOIN spock.node n ON n.node_id = p.remote_node_id " + "WHERE p.node_id = (SELECT node_id FROM spock.node_info()) " + "AND n.node_name = %s", + quote_literal_cstr(origin_node)); + + for (;;) + { + char *cur = zodan_local_scalar(progress_sql); + + if (cur != NULL) + { + bool reached = zodan_parse_lsn(cur) >= target; + + pfree(cur); + if (reached) + break; + } + + if (TimestampDifferenceExceeds(start, GetCurrentTimestamp(), + ctx->timeout_sec * 1000)) + ereport(ERROR, + (errmsg("timed out waiting for sync event from %s (lsn %s) on new node", + origin_node, lsn))); + zodan_sleep_ms(500); + } + pfree(progress_sql); +} + +/* + * Create a subscription. If on_dsn is NULL the subscription is created locally + * (on the new node) over SPI; otherwise it is created on the remote node over + * libpq. This mirrors the create_sub() helper in the old zodan.sql. + * + * The replication set list is read from provider_dsn rather than taken from a + * single cluster-wide list: see zodan_node_repset_list() for why it has to be + * the provider's. + */ +static void +zodan_create_sub(ZodanAddCtx *ctx, const char *on_dsn, const char *sub_name, + const char *provider_dsn, bool sync_structure, + bool sync_data, bool enabled) +{ + StringInfoData sql; + char *repsets = zodan_node_repset_list(provider_dsn); + + initStringInfo(&sql); + appendStringInfo(&sql, + "SELECT spock.sub_create(" + "subscription_name := %s, " + "provider_dsn := %s, " + "replication_sets := %s, " + "synchronize_structure := %s, " + "synchronize_data := %s, " + "forward_origins := '{}', " + "apply_delay := '0'::interval, " + "force_text_transfer := false, " + "enabled := %s)", + quote_literal_cstr(sub_name), + quote_literal_cstr(provider_dsn), + repsets, + sync_structure ? "true" : "false", + sync_data ? "true" : "false", + enabled ? "true" : "false"); + + if (on_dsn == NULL) + { + zodan_local_command(sql.data); + } + else + { + PGconn *conn = zodan_connect(on_dsn, "subcreate"); + + zodan_remote_command(conn, sql.data); + zodan_disconnect(conn); + } + pfree(sql.data); + pfree(repsets); +} + +/* + * Phase 3: create disabled subscriptions and replication slots for every + * "other" node (all cluster nodes except source and new). For a 2-node + * cluster this only records a sync event on the source node. + * + * The stored sync LSNs (for later enabling) are kept in a temp table on the + * new node, exactly as the old zodan.sql did, so the later phases can find + * them regardless of intervening commits. + */ +static void +zodan_create_disabled_subs_and_slots(ZodanAddCtx *ctx) +{ + int i; + + ZNOTICE("Phase 3: Creating disabled subscriptions and slots"); + + /* + * The temp table lives for the whole session, so a retry after a failed + * attach_node in the same session finds the previous attempt's rows. + * Start from an empty table rather than relying on every later read being + * for an origin this attempt has just refreshed. + */ + zodan_local_command( + "CREATE TEMP TABLE IF NOT EXISTS temp_sync_lsns (" + "origin_node text PRIMARY KEY, sync_lsn text NOT NULL, slot_lsn pg_lsn)"); + zodan_local_command("TRUNCATE temp_sync_lsns"); + + if (zodan_num_other_nodes(ctx) == 0) + { + /* 2-node scenario: capture a source sync event for later. */ + PGconn *conn = zodan_connect(ctx->src_dsn, "sync"); + char *lsn = zodan_remote_sync_event(conn, false); + + zodan_disconnect(conn); + if (lsn == NULL) + ereport(ERROR, + (errmsg("could not trigger sync event on source node %s", + ctx->src_node_name))); + + zodan_local_command(psprintf( + "INSERT INTO temp_sync_lsns (origin_node, sync_lsn) VALUES (%s, %s) " + "ON CONFLICT (origin_node) DO UPDATE SET sync_lsn = EXCLUDED.sync_lsn", + quote_literal_cstr(ctx->src_node_name), + quote_literal_cstr(lsn))); + ZNOTICE(" - 2-node scenario: stored source sync event %s", lsn); + pfree(lsn); + + /* Same durability point as the multi-node path below. */ + SPI_commit(); + return; + } + + for (i = 0; i < ctx->nnodes; i++) + { + ZNode *rec = &ctx->nodes[i]; + char *slot_name; + char *sub_name; + char *slot_lsn; + char *catchup_lsn; + PGconn *conn; + + if (strcmp(rec->name, ctx->src_node_name) == 0 || + strcmp(rec->name, ctx->new_node_name) == 0) + continue; + + sub_name = zodan_gen_sub_name(rec->name, ctx->new_node_name); + slot_name = zodan_gen_slot_name(rec->name, sub_name); + + /* Create the slot on the other node. */ + conn = zodan_connect(rec->dsn, "slot"); + slot_lsn = zodan_remote_create_slot(conn, slot_name); + if (slot_lsn != NULL) + ZNOTICE(" OK: created replication slot %s on node %s", + slot_name, rec->name); + else + ZNOTICE(" - replication slot %s already exists on node %s", + slot_name, rec->name); + + /* Anchor a real commit past the slot so the catch-up target exists. */ + catchup_lsn = zodan_remote_sync_event(conn, true); + zodan_disconnect(conn); + if (catchup_lsn == NULL) + ereport(ERROR, + (errmsg("could not trigger sync event on node %s", rec->name))); + + zodan_local_command(psprintf( + "INSERT INTO temp_sync_lsns (origin_node, sync_lsn, slot_lsn) " + "VALUES (%s, %s, %s) " + "ON CONFLICT (origin_node) DO UPDATE " + "SET sync_lsn = EXCLUDED.sync_lsn, slot_lsn = EXCLUDED.slot_lsn", + quote_literal_cstr(rec->name), + quote_literal_cstr(catchup_lsn), + slot_lsn ? quote_literal_cstr(slot_lsn) : "NULL")); + + /* Wait until the source has applied rec's changes up to catchup_lsn. */ + zodan_wait_source_caughtup(ctx, rec->name, catchup_lsn); + + /* + * Drop any stale origin so the sub starts clean, then create + * disabled. + */ + { + PGconn *nconn = zodan_connect(ctx->new_node_dsn, "origin"); + + zodan_remote_command(nconn, psprintf( + "DO $x$ BEGIN " + "IF EXISTS (SELECT 1 FROM pg_replication_origin WHERE roname = %s) THEN " + "PERFORM pg_replication_origin_drop(%s); END IF; END $x$", + quote_literal_cstr(slot_name), quote_literal_cstr(slot_name))); + zodan_disconnect(nconn); + } + + zodan_create_sub(ctx, NULL, sub_name, rec->dsn, false, false, false); + ZNOTICE(" OK: created disabled subscription %s (provider %s)", + sub_name, rec->name); + + if (catchup_lsn) + pfree(catchup_lsn); + if (slot_lsn) + pfree(slot_lsn); + } + + /* Make the disabled subs and stored LSNs durable before continuing. */ + SPI_commit(); +} + +/* + * Phase 4: for each other node, trigger a sync event on that node and wait for + * the source node to apply it. + */ +static void +zodan_sync_other_nodes_wait_source(ZodanAddCtx *ctx) +{ + int i; + + ZNOTICE("Phase 4: Triggering sync events on other nodes and waiting on source"); + + if (zodan_num_other_nodes(ctx) == 0) + { + ZNOTICE(" - No other nodes, skipping"); + return; + } + + for (i = 0; i < ctx->nnodes; i++) + { + ZNode *rec = &ctx->nodes[i]; + PGconn *conn; + PGconn *src_conn; + char *lsn; + + if (strcmp(rec->name, ctx->src_node_name) == 0 || + strcmp(rec->name, ctx->new_node_name) == 0) + continue; + + conn = zodan_connect(rec->dsn, "sync"); + lsn = zodan_remote_sync_event(conn, false); + zodan_disconnect(conn); + if (lsn == NULL) + ereport(ERROR, (errmsg("could not trigger sync event on node %s", rec->name))); + + src_conn = zodan_connect(ctx->src_dsn, "sync"); + zodan_remote_wait_for_sync_event(ctx, src_conn, rec->name, lsn); + zodan_disconnect(src_conn); + ZNOTICE(" OK: sync event from %s confirmed on source", rec->name); + pfree(lsn); + } +} + +/* + * Phase 5: create the enabled source->new subscription (with structure and + * data synchronization). + */ +static void +zodan_create_source_to_new_sub(ZodanAddCtx *ctx) +{ + char *sub_name = zodan_gen_sub_name(ctx->src_node_name, ctx->new_node_name); + + ZNOTICE("Phase 5: Creating source to new node subscription"); + zodan_create_sub(ctx, NULL, sub_name, ctx->src_dsn, true, true, true); + ZNOTICE(" OK: created subscription %s", sub_name); + SPI_commit(); +} + +/* + * Phase 6: trigger a sync event on the source node and wait for the new node + * (this node) to apply it. + */ +static void +zodan_source_sync_wait_new(ZodanAddCtx *ctx) +{ + PGconn *conn; + char *lsn; + + ZNOTICE("Phase 6: Triggering sync on source node and waiting on new node"); + + conn = zodan_connect(ctx->src_dsn, "sync"); + lsn = zodan_remote_sync_event(conn, false); + zodan_disconnect(conn); + if (lsn == NULL) + ereport(ERROR, + (errmsg("could not trigger sync event on source node %s", + ctx->src_node_name))); + + zodan_local_wait_for_sync_event(ctx, ctx->src_node_name, lsn); + ZNOTICE(" OK: sync event from source confirmed on new node"); + pfree(lsn); +} + +/* + * Wait until the source->new subscription has finished its initial COPY and is + * READY (no pending table sync). Bounded by ctx->timeout_sec. + */ +static void +zodan_wait_sub_ready(ZodanAddCtx *ctx, const char *sub_name) +{ + char *subid; + TimestampTz start = GetCurrentTimestamp(); + + subid = zodan_local_scalar(psprintf( + "SELECT sub_id::text FROM spock.subscription WHERE sub_name = %s", + quote_literal_cstr(sub_name))); + + /* + * The subscription was created and committed in phase 5, so it must be + * here. Continuing without it would silently skip the readiness wait and + * let attach_node proceed as if the initial COPY had finished. + */ + if (subid == NULL) + ereport(ERROR, + (errmsg("subscription %s not found on the new node", sub_name), + errdetail("It was created earlier in this attach_node run; something dropped it."))); + + for (;;) + { + int64 failed; + int64 pending; + + /* + * SYNC_STATUS_FAILED is not a transient state -- it will never become + * 'y'/'r' on its own -- so check it separately. Counting it as + * merely "pending" would spin out the whole timeout and then report a + * timeout for what is really a failure with a diagnosable cause. + */ + failed = zodan_local_count(psprintf( + "SELECT count(*) FROM spock.local_sync_status " + "WHERE sync_subid = %s AND sync_status = 'f'", + quote_literal_cstr(subid))); + if (failed > 0) + ereport(ERROR, + (errmsg("initial synchronization failed for subscription %s", + sub_name), + errdetail(INT64_FORMAT " table(s) are in the failed sync state.", + failed), + errhint("Check the subscriber and provider logs for the sync worker error, then retry with spock.detach_node() followed by spock.attach_node()."))); + + pending = zodan_local_count(psprintf( + "SELECT count(*) FROM spock.local_sync_status " + "WHERE sync_subid = %s AND sync_status NOT IN ('y','r','f')", + quote_literal_cstr(subid))); + + if (pending == 0) + { + ZNOTICE(" - subscription %s is READY", sub_name); + break; + } + if (TimestampDifferenceExceeds(start, GetCurrentTimestamp(), + ctx->timeout_sec * 1000)) + ereport(ERROR, + (errmsg("timed out after %d seconds waiting for subscription %s " + "to finish initial synchronization", + ctx->timeout_sec, sub_name))); + zodan_sleep_ms(1000); + } + pfree(subid); +} + +/* + * Phase 7: wait for the source->new subscription to become READY. Slot/origin + * advancement is handled by the apply worker for the active subscription; we + * only advance the inactive "other node" slots defensively. + */ +static void +zodan_wait_ready_and_advance(ZodanAddCtx *ctx) +{ + char *sub_name; + int i; + + ZNOTICE("Phase 7: Waiting for initial sync and advancing slots"); + + sub_name = zodan_gen_sub_name(ctx->src_node_name, ctx->new_node_name); + zodan_wait_sub_ready(ctx, sub_name); + + if (zodan_num_other_nodes(ctx) == 0) + return; + + /* Advance each other-node slot/origin to the resume point. */ + for (i = 0; i < ctx->nnodes; i++) + { + ZNode *rec = &ctx->nodes[i]; + char *slot_name; + char *cur_lsn; + char *target_lsn; + PGconn *conn; + + if (strcmp(rec->name, ctx->src_node_name) == 0 || + strcmp(rec->name, ctx->new_node_name) == 0) + continue; + + slot_name = zodan_gen_slot_name(rec->name, + zodan_gen_sub_name(rec->name, ctx->new_node_name)); + + conn = zodan_connect(rec->dsn, "advance"); + cur_lsn = zodan_remote_scalar(conn, psprintf( + "SELECT restart_lsn FROM pg_replication_slots WHERE slot_name = %s", + quote_literal_cstr(slot_name))); + if (cur_lsn == NULL) + { + zodan_disconnect(conn); + ZNOTICE(" - slot %s does not exist, skipping advance", slot_name); + continue; + } + + /* Advance to the last commit from rec that the source had applied. */ + target_lsn = zodan_local_scalar(psprintf( + "SELECT p.remote_commit_lsn::text " + "FROM spock.progress p JOIN spock.node n ON n.node_id = p.remote_node_id " + "WHERE p.node_id = (SELECT node_id FROM spock.node_info()) " + "AND n.node_name = %s", quote_literal_cstr(rec->name))); + + if (target_lsn != NULL) + { + if (zodan_parse_lsn(target_lsn) > zodan_parse_lsn(cur_lsn)) + { + zodan_remote_command(conn, psprintf( + "SELECT pg_replication_slot_advance(%s, %s::pg_lsn)", + quote_literal_cstr(slot_name), quote_literal_cstr(target_lsn))); + + /* Advance the origin locally (it lives on the new node). */ + zodan_local_command(psprintf( + "DO $x$ BEGIN " + "IF NOT EXISTS (SELECT 1 FROM pg_replication_origin WHERE roname = %s) THEN " + "PERFORM pg_replication_origin_create(%s); END IF; " + "PERFORM pg_replication_origin_advance(%s, %s::pg_lsn); END $x$", + quote_literal_cstr(slot_name), quote_literal_cstr(slot_name), + quote_literal_cstr(slot_name), quote_literal_cstr(target_lsn))); + ZNOTICE(" OK: advanced slot/origin %s to %s", + slot_name, target_lsn); + } + pfree(target_lsn); + } + else + ereport(WARNING, + (errmsg("no spock.progress entry for node %s, leaving slot %s at %s", + rec->name, slot_name, cur_lsn))); + zodan_disconnect(conn); + pfree(cur_lsn); + } + SPI_commit(); +} + +/* + * Phase 8: make the new node's replication sets an exact copy of the source + * node's: the set definitions, the table memberships (including column lists + * and row filters) and the sequence memberships. + * + * This has to run after the structure and data sync, because the tables must + * exist locally before they can be added to a set, and before the new node + * starts acting as a provider in phase 10. + * + * Nothing in Spock copies replication set membership. What actually populates + * the sets on a freshly joined node is AutoDDL firing while pg_restore replays + * the structure dump, which routes each table by policy (primary key or replica + * identity to 'default', otherwise 'default_insert_only') rather than by what + * the source says. That derived membership is wiped here and replaced by the + * source's, so a table the user deliberately removed from a set on the source + * does not reappear on the new node, and extension-owned tables such as + * lolor's (which AutoDDL skips entirely, because it ignores anything created by + * CREATE EXTENSION) no longer have to be added by hand. + */ +static void +zodan_sync_repsets_from_source(ZodanAddCtx *ctx) +{ + PGconn *conn; + PGresult *res; + MemoryContext mcxt; + ZPair *local; + int nlocal; + int i; + int cleared = 0; + int added = 0; + int added_seqs = 0; + int dropped = 0; + char *src_sets; + char *src_digest; + char *local_digest; + StringInfoData missing; + + ZNOTICE("Phase 8: Mirroring replication sets from the source node"); + + mcxt = AllocSetContextCreate(CurrentMemoryContext, "zodan_repsets", + ALLOCSET_DEFAULT_SIZES); + initStringInfo(&missing); + + /* + * 1. Drop the membership AutoDDL derived during the structure restore. + * The source's memberships are replayed below; keeping both would leave + * two sources of truth. include_partitions is false throughout so the + * source's rows are mirrored one for one: if the source has a partition + * in a set, it has its own row for it. + */ + local = zodan_local_pairs( + "SELECT rs.set_name, rst.set_reloid::text " + "FROM spock.replication_set_table rst " + "JOIN spock.replication_set rs ON rs.set_id = rst.set_id " + "JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id", + mcxt, &nlocal); + for (i = 0; i < nlocal; i++) + { + zodan_local_command(psprintf( + "SELECT spock.repset_remove_table(%s, %s::regclass, false)", + quote_literal_cstr(local[i].a), quote_literal_cstr(local[i].b))); + cleared++; + } + + local = zodan_local_pairs( + "SELECT rs.set_name, rss.set_seqoid::text " + "FROM spock.replication_set_seq rss " + "JOIN spock.replication_set rs ON rs.set_id = rss.set_id " + "JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id", + mcxt, &nlocal); + for (i = 0; i < nlocal; i++) + { + zodan_local_command(psprintf( + "SELECT spock.repset_remove_seq(%s, %s::regclass)", + quote_literal_cstr(local[i].a), quote_literal_cstr(local[i].b))); + cleared++; + } + if (cleared > 0) + ZNOTICE(" OK: cleared %d auto-derived replication set entries", cleared); + + /* + * 2. Replication set definitions. The three built-ins already exist here + * (spock.node_create() creates them), so those are altered into shape and + * the rest created. + */ + conn = zodan_connect(ctx->src_dsn, "repsetdefs"); + res = zodan_remote_query(conn, + "SELECT rs.set_name, rs.replicate_insert, " + "rs.replicate_update, rs.replicate_delete, " + "rs.replicate_truncate " + "FROM spock.replication_set rs " + "JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id"); + for (i = 0; i < PQntuples(res); i++) + { + const char *name = PQgetvalue(res, i, 0); + + /* libpq renders booleans as "t"/"f", which is not a SQL literal. */ + const char *ins = ZBOOL(res, i, 1); + const char *upd = ZBOOL(res, i, 2); + const char *del = ZBOOL(res, i, 3); + const char *trunc = ZBOOL(res, i, 4); + + zodan_local_command(psprintf( + "SELECT CASE WHEN EXISTS (SELECT 1 FROM spock.replication_set rs " + " JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id " + " WHERE rs.set_name = %s) " + " THEN spock.repset_alter(%s, %s, %s, %s, %s) " + " ELSE spock.repset_create(%s, %s, %s, %s, %s) END", + quote_literal_cstr(name), + quote_literal_cstr(name), ins, upd, del, trunc, + quote_literal_cstr(name), ins, upd, del, trunc)); + } + + /* + * Any set the source does not have should not exist here either. The + * comparison list comes from the result already in hand: re-reading the + * source would let a set created there in between be created above and + * then immediately dropped again. + */ + src_sets = zodan_repset_array(res); + zodan_clear(res); + zodan_disconnect(conn); + + local = zodan_local_pairs(psprintf( + "SELECT rs.set_name, NULL::text " + "FROM spock.replication_set rs " + "JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id " + "WHERE rs.set_name <> ALL (%s::name[])", src_sets), + mcxt, &nlocal); + for (i = 0; i < nlocal; i++) + { + zodan_local_command(psprintf("SELECT spock.repset_drop(%s, true)", + quote_literal_cstr(local[i].a))); + dropped++; + ZNOTICE(" OK: dropped replication set %s (absent on source)", local[i].a); + } + + /* 3. Table membership, with column lists and row filters. */ + conn = zodan_connect(ctx->src_dsn, "repsettables"); + res = zodan_remote_query(conn, + "SELECT rs.set_name, " + "quote_ident(n.nspname) || '.' || quote_ident(c.relname), " + "rst.set_att_list::text, " + "pg_get_expr(rst.set_row_filter, rst.set_reloid) " + "FROM spock.replication_set_table rst " + "JOIN spock.replication_set rs ON rs.set_id = rst.set_id " + "JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id " + "JOIN pg_catalog.pg_class c ON c.oid = rst.set_reloid " + "JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace"); + for (i = 0; i < PQntuples(res); i++) + { + const char *name = PQgetvalue(res, i, 0); + const char *rel = PQgetvalue(res, i, 1); + char *atts = PQgetisnull(res, i, 2) ? pstrdup("NULL") : + psprintf("%s::text[]", quote_literal_cstr(PQgetvalue(res, i, 2))); + char *filter = PQgetisnull(res, i, 3) ? pstrdup("NULL") : + quote_literal_cstr(PQgetvalue(res, i, 3)); + char *present; + + present = zodan_local_scalar(psprintf( + "SELECT to_regclass(%s) IS NOT NULL", quote_literal_cstr(rel))); + if (present == NULL || present[0] != 't') + { + appendStringInfo(&missing, "%s%s", missing.len > 0 ? ", " : "", rel); + continue; + } + pfree(present); + + zodan_local_command(psprintf( + "SELECT spock.repset_add_table(%s, %s::regclass, false, %s, %s, false)", + quote_literal_cstr(name), quote_literal_cstr(rel), atts, filter)); + added++; + } + zodan_clear(res); + zodan_disconnect(conn); + + /* 4. Sequence membership. */ + conn = zodan_connect(ctx->src_dsn, "repsetseqs"); + res = zodan_remote_query(conn, + "SELECT rs.set_name, " + "quote_ident(n.nspname) || '.' || quote_ident(c.relname) " + "FROM spock.replication_set_seq rss " + "JOIN spock.replication_set rs ON rs.set_id = rss.set_id " + "JOIN spock.local_node ln ON rs.set_nodeid = ln.node_id " + "JOIN pg_catalog.pg_class c ON c.oid = rss.set_seqoid " + "JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace"); + for (i = 0; i < PQntuples(res); i++) + { + const char *name = PQgetvalue(res, i, 0); + const char *rel = PQgetvalue(res, i, 1); + char *present; + + present = zodan_local_scalar(psprintf( + "SELECT to_regclass(%s) IS NOT NULL", quote_literal_cstr(rel))); + if (present == NULL || present[0] != 't') + { + appendStringInfo(&missing, "%s%s", missing.len > 0 ? ", " : "", rel); + continue; + } + pfree(present); + + zodan_local_command(psprintf( + "SELECT spock.repset_add_seq(%s, %s::regclass, false)", + quote_literal_cstr(name), quote_literal_cstr(rel))); + added_seqs++; + } + zodan_clear(res); + zodan_disconnect(conn); + + if (missing.len > 0) + ereport(ERROR, + (errmsg("the source node replicates relations that do not exist on \"%s\"", + ctx->new_node_name), + errdetail("Missing: %s", missing.data), + errhint("The new node cannot join until those relations are present."))); + + ZNOTICE(" OK: replication sets: %d dropped; membership: %d tables, %d sequences copied from source", + dropped, added, added_seqs); + + /* + * 5. The source is a moving target: somebody running repset_add_table() + * there while the join is in progress would leave the two nodes out of + * step. Re-read the source and fail loudly rather than finish with a node + * that quietly replicates a different set of tables. + * + * Comparing counts would miss a swap (one table removed, another added + * keeps the count the same) and would not notice a changed column list or + * row filter at all, so compare an ordered digest of everything that was + * copied: set definitions, table membership and sequences. + */ + conn = zodan_connect(ctx->src_dsn, "repsetcheck"); + res = zodan_remote_query(conn, ZODAN_REPSET_DIGEST_SQL); + src_digest = pstrdup(PQgetvalue(res, 0, 0)); + zodan_clear(res); + zodan_disconnect(conn); + + local_digest = zodan_local_scalar(ZODAN_REPSET_DIGEST_SQL); + + if (local_digest == NULL || strcmp(src_digest, local_digest) != 0) + ereport(ERROR, + (errmsg("the source node's replication sets changed while \"%s\" was joining", + ctx->new_node_name), + errdetail("Source digest %s, %s digest %s.", + src_digest, ctx->new_node_name, + local_digest ? local_digest : "(none)"), + errhint("Re-run attach_node once the source is quiescent."))); + + pfree(missing.data); + MemoryContextDelete(mcxt); + SPI_commit(); + + ZNOTICE(" OK: replication sets on %s match the source", ctx->new_node_name); +} + +/* + * Phase 9: enable the previously disabled subscriptions (source->new for the + * 2-node case, other->new for the multi-node case), waiting for each stored + * sync event first. + */ +static void +zodan_enable_disabled_subs(ZodanAddCtx *ctx) +{ + int i; + + ZNOTICE("Phase 9: Enabling disabled subscriptions"); + + if (zodan_num_other_nodes(ctx) == 0) + { + char *sub_name = zodan_gen_sub_name(ctx->src_node_name, ctx->new_node_name); + char *lsn; + + zodan_local_command(psprintf( + "SELECT spock.sub_enable(subscription_name := %s, immediate := true)", + quote_literal_cstr(sub_name))); + SPI_commit(); + + lsn = zodan_local_scalar(psprintf( + "SELECT sync_lsn FROM temp_sync_lsns WHERE origin_node = %s", + quote_literal_cstr(ctx->src_node_name))); + if (lsn != NULL) + { + zodan_local_wait_for_sync_event(ctx, ctx->src_node_name, lsn); + pfree(lsn); + } + ZNOTICE(" OK: enabled subscription %s", sub_name); + return; + } + + for (i = 0; i < ctx->nnodes; i++) + { + ZNode *rec = &ctx->nodes[i]; + char *sub_name; + char *lsn; + + if (strcmp(rec->name, ctx->src_node_name) == 0 || + strcmp(rec->name, ctx->new_node_name) == 0) + continue; + + sub_name = zodan_gen_sub_name(rec->name, ctx->new_node_name); + zodan_local_command(psprintf( + "SELECT spock.sub_enable(subscription_name := %s, immediate := true)", + quote_literal_cstr(sub_name))); + SPI_commit(); + + lsn = zodan_local_scalar(psprintf( + "SELECT sync_lsn FROM temp_sync_lsns WHERE origin_node = %s", + quote_literal_cstr(rec->name))); + if (lsn != NULL) + { + zodan_local_wait_for_sync_event(ctx, rec->name, lsn); + pfree(lsn); + } + ZNOTICE(" OK: enabled subscription %s", sub_name); + } +} + +/* + * Phase 10: create subscriptions from every "other" existing node to the new node + * (so the rest of the cluster receives the new node's changes). These are + * created on the remote nodes, with the new node as provider. + * + * The source node is deliberately skipped here and handled by phase 11, which + * reaches it over the caller-supplied src_dsn -- verified reachable in phase 0 -- + * rather than the if_dsn registered in the source's own spock.node_interface, + * which nothing in this orchestration ever verifies. Note that ctx->nodes is + * read from the source and does contain the source itself, so without this skip + * phase 10 would create the new->source subscription and make phase 11 dead code. + */ +static void +zodan_create_subs_to_new_node(ZodanAddCtx *ctx) +{ + int i; + + ZNOTICE("Phase 10: Creating subscriptions from other nodes to the new node"); + + for (i = 0; i < ctx->nnodes; i++) + { + ZNode *rec = &ctx->nodes[i]; + char *sub_name; + + if (strcmp(rec->name, ctx->new_node_name) == 0 || + strcmp(rec->name, ctx->src_node_name) == 0) + continue; + + sub_name = zodan_gen_sub_name(ctx->new_node_name, rec->name); + zodan_create_sub(ctx, rec->dsn, sub_name, ctx->new_node_dsn, false, false, true); + ZNOTICE(" OK: created subscription %s on node %s", sub_name, rec->name); + } +} + +/* + * Phase 11: create the enabled new->source subscription, completing + * bidirectional replication with the source node. Skipped if it somehow already + * exists, so a re-run does not fail at the very last step. + */ +static void +zodan_create_new_to_source_sub(ZodanAddCtx *ctx) +{ + char *sub_name = zodan_gen_sub_name(ctx->new_node_name, ctx->src_node_name); + int64 exists; + PGconn *conn; + + ZNOTICE("Phase 11: Creating new to source node subscription"); + + conn = zodan_connect(ctx->src_dsn, "subcheck"); + exists = 0; + { + char *c = zodan_remote_scalar(conn, psprintf( + "SELECT count(*) FROM spock.subscription WHERE sub_name = %s", + quote_literal_cstr(sub_name))); + + if (c != NULL) + { + exists = pg_strtoint64(c); + pfree(c); + } + } + zodan_disconnect(conn); + + if (exists > 0) + { + ZNOTICE(" - subscription %s already exists, skipping", sub_name); + return; + } + + zodan_create_sub(ctx, ctx->src_dsn, sub_name, ctx->new_node_dsn, false, false, true); + ZNOTICE(" OK: created subscription %s", sub_name); +} + +/* ------------------------------------------------------------------------ + * detach_node phases + * ------------------------------------------------------------------------ */ + +/* + * Drop the inbound subscriptions (target as provider) on every surviving node, + * then drop every subscription local to the target node. Slot and origin + * cleanup is implicit in spock.sub_drop(). Per-node errors are logged and + * tolerated, matching the old zodremove.sql behavior. + */ +static void +zodan_remove_subscriptions(const char *target_node_name) +{ + int i; + int nnodes; + ZNode *others; + MemoryContext mcxt; + MemoryContext old; + + /* Snapshot every node locally; the target is filtered out in C below. */ + if (SPI_execute( + "SELECT n.node_name, i.if_dsn FROM spock.node n " + "JOIN spock.node_interface i ON n.node_id = i.if_nodeid", + true, 0) != SPI_OK_SELECT) + elog(ERROR, "failed to enumerate cluster nodes"); + + nnodes = SPI_processed; + mcxt = AllocSetContextCreate(CurrentMemoryContext, "zodan_remove", + ALLOCSET_DEFAULT_SIZES); + old = MemoryContextSwitchTo(mcxt); + others = (ZNode *) palloc0(sizeof(ZNode) * Max(nnodes, 1)); + for (i = 0; i < nnodes; i++) + { + char *name = SPI_getvalue(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 1); + char *dsn = SPI_getvalue(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 2); + + others[i].name = name ? pstrdup(name) : NULL; + others[i].dsn = dsn ? pstrdup(dsn) : NULL; + } + MemoryContextSwitchTo(old); + + /* + * Remote: drop each surviving node's inbound subscription from the + * target. + * + * Each node is handled inside its own internal subtransaction so that a + * failure to reach or update one node (for example a node that is down) + * is tolerated and does not abort the whole removal, matching the old + * zodremove.sql behavior. Without the subtransaction, catching the error + * would leave the surrounding transaction in an aborted state. + */ + for (i = 0; i < nnodes; i++) + { + MemoryContext subctx = CurrentMemoryContext; + ResourceOwner subowner = CurrentResourceOwner; + + /* + * volatile: assigned inside PG_TRY and read in PG_CATCH, so the + * compiler must not keep it in a register clobbered by the longjmp. + */ + PGconn *volatile conn = NULL; + + if (others[i].name == NULL || others[i].dsn == NULL || + strcmp(others[i].name, target_node_name) == 0) + continue; + + BeginInternalSubTransaction(NULL); + MemoryContextSwitchTo(subctx); + + PG_TRY(); + { + char *sub_name; + + conn = zodan_connect(others[i].dsn, "rmsub"); + sub_name = zodan_remote_scalar(conn, psprintf( + "SELECT s.sub_name FROM spock.subscription s " + "JOIN spock.node n ON n.node_id = s.sub_origin " + "WHERE n.node_name = %s", quote_literal_cstr(target_node_name))); + if (sub_name != NULL) + { + zodan_remote_command(conn, psprintf( + "SELECT spock.sub_drop(%s, true)", quote_literal_cstr(sub_name))); + ZNOTICE(" OK: dropped subscription %s on node %s", + sub_name, others[i].name); + } + zodan_disconnect(conn); + conn = NULL; + + ReleaseCurrentSubTransaction(); + MemoryContextSwitchTo(subctx); + CurrentResourceOwner = subowner; + } + PG_CATCH(); + { + ErrorData *edata; + + MemoryContextSwitchTo(subctx); + /* Keep the cause: FlushErrorState() discards it. */ + edata = CopyErrorData(); + if (conn != NULL) + zodan_disconnect(conn); + RollbackAndReleaseCurrentSubTransaction(); + MemoryContextSwitchTo(subctx); + CurrentResourceOwner = subowner; + FlushErrorState(); + ereport(WARNING, + (errmsg("could not drop inbound subscription on node %s; continuing", + others[i].name), + errdetail("%s", edata->message))); + FreeErrorData(edata); + } + PG_END_TRY(); + } + + /* Local: drop every subscription on the target node. */ + if (SPI_execute("SELECT sub_name FROM spock.subscription", true, 0) != SPI_OK_SELECT) + elog(ERROR, "failed to enumerate local subscriptions"); + { + int nlocal = SPI_processed; + char **names = (char **) palloc0(sizeof(char *) * Max(nlocal, 1)); + + for (i = 0; i < nlocal; i++) + { + char *nm = SPI_getvalue(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 1); + + names[i] = nm ? pstrdup(nm) : NULL; + } + for (i = 0; i < nlocal; i++) + { + if (names[i] == NULL) + continue; + zodan_local_command(psprintf( + "SELECT spock.sub_drop(%s, true)", quote_literal_cstr(names[i]))); + ZNOTICE(" OK: dropped local subscription %s", names[i]); + } + } + + MemoryContextDelete(mcxt); +} + +/* ------------------------------------------------------------------------ + * SQL entry points + * ------------------------------------------------------------------------ */ + +/* + * spock.attach_node(src_node_name, src_dsn, new_node_name, new_node_dsn, + * verb, new_node_location, new_node_country, new_node_info, + * timeout_sec) + * + * Run on the NEW node. + */ +Datum +spock_attach_node(PG_FUNCTION_ARGS) +{ + ZodanAddCtx *ctx; + MemoryContext mcxt; + MemoryContext old; + bool nonatomic; + + /* We must be able to commit between phases: require a top-level CALL. */ + nonatomic = fcinfo->context != NULL && IsA(fcinfo->context, CallContext) && + !((CallContext *) fcinfo->context)->atomic; + if (!nonatomic) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TRANSACTION_TERMINATION), + errmsg("spock.attach_node() cannot run inside a transaction block"), + errhint("Invoke it with a top-level CALL, not inside BEGIN/COMMIT."))); + + /* + * LANGUAGE c procedures cannot be declared STRICT, so guard the required + * text arguments explicitly: a SQL NULL would otherwise reach + * text_to_cstring() and detoast a NULL pointer, crashing the backend. + */ + if (PG_ARGISNULL(0) || PG_ARGISNULL(1) || PG_ARGISNULL(2) || PG_ARGISNULL(3)) + ereport(ERROR, + (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("src_node_name, src_dsn, new_node_name and new_node_dsn must not be NULL"))); + + /* Every previous call must have drained its registries. */ + Assert(zodan_nconns == 0 && zodan_nresults == 0); + + /* + * Connect first, so nothing between the context creation and the PG_TRY + * below can throw and orphan the context. + */ + SPI_connect_ext(SPI_OPT_NONATOMIC); + + /* + * ctx and everything it points at must survive the SPI_commit() calls + * between phases, so the context cannot live under the SPI procedure + * context. PortalContext is the right parent: a non-atomic SPI + * connection parents its own contexts there (see spi.c), + * PreCommit_Portals leaves the active portal alone across the commits, + * and the portal's cleanup releases this context for us if argument + * parsing below or a later phase raises before the PG_FINALLY. + */ + mcxt = AllocSetContextCreate(PortalContext != NULL ? PortalContext + : TopMemoryContext, + "zodan_add", ALLOCSET_DEFAULT_SIZES); + old = MemoryContextSwitchTo(mcxt); + ctx = (ZodanAddCtx *) palloc0(sizeof(ZodanAddCtx)); + ctx->mcxt = mcxt; + ctx->src_node_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + ctx->src_dsn = text_to_cstring(PG_GETARG_TEXT_PP(1)); + ctx->new_node_name = text_to_cstring(PG_GETARG_TEXT_PP(2)); + ctx->new_node_dsn = text_to_cstring(PG_GETARG_TEXT_PP(3)); + ctx->verb = PG_ARGISNULL(4) ? false : PG_GETARG_BOOL(4); + ctx->new_node_location = PG_ARGISNULL(5) ? pstrdup("NY") : text_to_cstring(PG_GETARG_TEXT_PP(5)); + ctx->new_node_country = PG_ARGISNULL(6) ? pstrdup("USA") : text_to_cstring(PG_GETARG_TEXT_PP(6)); + ctx->new_node_info = PG_ARGISNULL(7) ? pstrdup("{}") : + DatumGetCString(DirectFunctionCall1(jsonb_out, PG_GETARG_DATUM(7))); + ctx->timeout_sec = PG_ARGISNULL(8) ? 180 : PG_GETARG_INT32(8); + if (ctx->timeout_sec <= 0) + ctx->timeout_sec = 180; + /* Cap so the "* 1000" millisecond conversions cannot overflow int. */ + if (ctx->timeout_sec > INT_MAX / 1000) + ctx->timeout_sec = INT_MAX / 1000; + MemoryContextSwitchTo(old); + + zodan_verbose = ctx->verb; + + PG_TRY(); + { + zodan_fetch_cluster_nodes(ctx); + + zodan_check_versions(ctx); + zodan_verify_prerequisites(ctx); + zodan_create_nodes(ctx); + /* Cluster snapshot may be stale after node creation; refresh. */ + zodan_fetch_cluster_nodes(ctx); + zodan_create_disabled_subs_and_slots(ctx); + zodan_sync_other_nodes_wait_source(ctx); + zodan_create_source_to_new_sub(ctx); + zodan_source_sync_wait_new(ctx); + zodan_wait_ready_and_advance(ctx); + zodan_sync_repsets_from_source(ctx); + zodan_enable_disabled_subs(ctx); + zodan_create_subs_to_new_node(ctx); + zodan_create_new_to_source_sub(ctx); + + ereport(NOTICE, + (errmsg("attach_node: node \"%s\" added to the cluster", + ctx->new_node_name))); + } + PG_FINALLY(); + { + /* + * Release anything a failed phase left open before unwinding: neither + * PGconn nor PGresult is owned by a resource owner. + */ + zodan_release_all(); + MemoryContextDelete(mcxt); + } + PG_END_TRY(); + + SPI_finish(); + PG_RETURN_VOID(); +} + +/* + * Verify that detach_node is running on the node it is being asked to remove, by + * comparing the local system identifier and database name against what + * target_node_dsn points at. This is the mirror of the check attach_node makes + * against new_node_dsn, and it is what target_node_dsn is for. + * + * The check matters because everything after it operates on the local node: the + * local subscriptions and replication sets are dropped wholesale. Run from the + * wrong node, detach_node would tear down that node's replication instead of the + * target's. + */ +static void +zodan_verify_detach_target(const char *target_node_name, const char *target_dsn) +{ + char *local_sysid; + char *local_dbname; + char *remote_sysid; + char *remote_dbname; + PGconn *conn; + + local_sysid = zodan_local_scalar("SELECT system_identifier::text FROM pg_control_system()"); + local_dbname = zodan_local_scalar("SELECT current_database()"); + + conn = zodan_connect(target_dsn, "detach"); + remote_sysid = zodan_remote_scalar(conn, + "SELECT system_identifier::text FROM pg_control_system()"); + remote_dbname = zodan_remote_scalar(conn, "SELECT current_database()"); + zodan_disconnect(conn); + + if (local_sysid == NULL || remote_sysid == NULL || + strcmp(local_sysid, remote_sysid) != 0 || + local_dbname == NULL || remote_dbname == NULL || + strcmp(local_dbname, remote_dbname) != 0) + ereport(ERROR, + (errmsg("detach_node must be run on the node being removed"), + errdetail("target_node_dsn (%s) does not match the current database connection", + target_dsn), + errhint("Connect to node %s and re-run detach_node.", + target_node_name))); + + ZNOTICE(" OK: detach_node is running on node %s", target_node_name); +} + +/* + * spock.detach_node(target_node_name, target_node_dsn, verbose_mode) + * + * Run on the node being removed. Order: subscriptions (incl. implicit slot and + * origin cleanup) -> replication sets -> node. + * + * Like attach_node, this is not atomic across the cluster; see the file header. + * Unlike attach_node, a node that cannot be reached is tolerated with a WARNING + * (see zodan_remove_subscriptions), so a surviving node that was down during the + * detach keeps a stale inbound subscription from the removed node and needs + * spock.sub_drop() run on it by hand. + */ +Datum +spock_detach_node(PG_FUNCTION_ARGS) +{ + char *target_node_name; + char *target_node_dsn; + bool verb = PG_ARGISNULL(2) ? true : PG_GETARG_BOOL(2); + bool nonatomic; + int64 exists; + + /* See spock_attach_node(): LANGUAGE c procedures cannot be STRICT. */ + if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) + ereport(ERROR, + (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("target_node_name and target_node_dsn must not be NULL"))); + + target_node_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); + target_node_dsn = text_to_cstring(PG_GETARG_TEXT_PP(1)); + + zodan_verbose = verb; + + nonatomic = fcinfo->context != NULL && IsA(fcinfo->context, CallContext) && + !((CallContext *) fcinfo->context)->atomic; + if (!nonatomic) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TRANSACTION_TERMINATION), + errmsg("spock.detach_node() cannot run inside a transaction block"), + errhint("Invoke it with a top-level CALL, not inside BEGIN/COMMIT."))); + + /* Every previous call must have drained its registries. */ + Assert(zodan_nconns == 0 && zodan_nresults == 0); + + SPI_connect_ext(SPI_OPT_NONATOMIC); + + PG_TRY(); + { + /* Phase 1: validate the target node exists and that we are it. */ + exists = zodan_local_count(psprintf( + "SELECT count(*) FROM spock.node WHERE node_name = %s", + quote_literal_cstr(target_node_name))); + if (exists == 0) + ereport(ERROR, (errmsg("node %s does not exist", target_node_name))); + + zodan_verify_detach_target(target_node_name, target_node_dsn); + + ZNOTICE("Removing node %s from the cluster", target_node_name); + + /* Phase 2/3: drop subscriptions (remote inbound + all local). */ + zodan_remove_subscriptions(target_node_name); + SPI_commit(); + + /* Phase 4: drop replication sets local to the target node. */ + { + int i; + int nsets; + char **sets; + + if (SPI_execute("SELECT set_name FROM spock.replication_set", true, 0) != SPI_OK_SELECT) + elog(ERROR, "failed to enumerate replication sets"); + nsets = SPI_processed; + sets = (char **) palloc0(sizeof(char *) * Max(nsets, 1)); + for (i = 0; i < nsets; i++) + { + char *nm = SPI_getvalue(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 1); + + sets[i] = nm ? pstrdup(nm) : NULL; + } + for (i = 0; i < nsets; i++) + { + if (sets[i] == NULL) + continue; + zodan_local_command(psprintf( + "SELECT spock.repset_drop(%s, true)", quote_literal_cstr(sets[i]))); + ZNOTICE(" OK: dropped replication set %s", sets[i]); + } + } + SPI_commit(); + + /* Phase 5: drop the node itself. */ + exists = zodan_local_count(psprintf( + "SELECT count(*) FROM spock.node WHERE node_name = %s", + quote_literal_cstr(target_node_name))); + if (exists > 0) + { + zodan_local_command(psprintf( + "SELECT spock.node_drop(%s, true)", quote_literal_cstr(target_node_name))); + ZNOTICE(" OK: dropped node %s", target_node_name); + } + SPI_commit(); + + ereport(NOTICE, + (errmsg("detach_node: node \"%s\" removed from the cluster", + target_node_name))); + } + PG_FINALLY(); + { + /* Same reasoning as spock_attach_node(). */ + zodan_release_all(); + } + PG_END_TRY(); + + SPI_finish(); + PG_RETURN_VOID(); +} diff --git a/tests/docker/entrypoint.sh b/tests/docker/entrypoint.sh index d3fa324ed..7c42b404b 100644 --- a/tests/docker/entrypoint.sh +++ b/tests/docker/entrypoint.sh @@ -131,10 +131,8 @@ EOF country := 'ESP', location := 'Madrid', info := '{\"tiebreaker\" : \"1\"}')" else - # Add node to the existing cluster using Z0DAN - psql -h /tmp -c "CREATE EXTENSION dblink" - psql -h /tmp -f ${SPOCK_SOURCE_DIR}/samples/Z0DAN/zodan.sql - psql -h /tmp -c "CALL spock.add_node( + # Add node to the existing cluster (attach_node ships with the extension) + psql -h /tmp -c "CALL spock.attach_node( src_node_name := 'n1', src_dsn := 'host=n1 port=${PGPORT} dbname=${PGDATABASE} user=${PGUSER}', new_node_name := '${HOSTNAME}', diff --git a/tests/tap/schedule b/tests/tap/schedule index e1ab217ac..d3809e10a 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -15,13 +15,18 @@ test: 004_non_default_repset # test: 005_daylight_savings test: 006_sync_during_write test: 008_rmgr +# ZODAN add/remove node: SQL/dblink orchestration (samples/Z0DAN/zodan.sql) test: 009_zodan_add_remove_nodes +# ZODAN add/remove node: in-core C orchestration (spock.attach_node/detach_node) +test: 034_attach_detach_node_incore test: 013_origin_change_restore test: 014_pgdump_restore_conflict # Tests, consuming too much time to be launched on each check: +# (SQL/dblink orchestration, and its in-core C counterpart) #test: 011_zodan_sync_third +#test: 035_attach_node_sync_third_incore # # Use GitHub Actions to launch them (see workflows/zodan_sync.yml for an example # Also, it may be run locally by a bash script like the following: @@ -53,7 +58,10 @@ test: 027_reserved_object test: 028_failover_slots_naptime_guc test: 030_autoddl_repset_stickiness test: 032_lolor_largeobject_repset +# ZODAN + lolor large objects: SQL/dblink orchestration, then its +# in-core C counterpart (spock.attach_node) test: 033_zodan_lolor_add_node +test: 040_zodan_lolor_attach_node_incore test: 034_reserved_object_ddl test: 037_wire_format_datestyle test: 038_reserved_schema_ddl_guard diff --git a/tests/tap/schedule-nightly b/tests/tap/schedule-nightly index adcc3dbb4..f130406c8 100644 --- a/tests/tap/schedule-nightly +++ b/tests/tap/schedule-nightly @@ -11,6 +11,9 @@ # test: 012_zodan_basics +# ZODAN basics/timeout: in-core C orchestration (spock.attach_node/detach_node) +test: 036_attach_node_basics_incore +test: 037_attach_node_3n_timeout_incore # test: 016_crash_recovery_progress test: 017_zodan_3n_timeout diff --git a/tests/tap/t/034_attach_detach_node_incore.pl b/tests/tap/t/034_attach_detach_node_incore.pl new file mode 100755 index 000000000..5d7e97e7b --- /dev/null +++ b/tests/tap/t/034_attach_detach_node_incore.pl @@ -0,0 +1,350 @@ +use strict; +use warnings; +use Test::More; +use lib '.'; +use lib 't'; +use SpockTest qw(create_cluster destroy_cluster system_or_bail command_ok get_test_config cross_wire system_maybe); + +# ============================================================================= +# Test: 034_attach_detach_node_incore.pl - Test Zodan Node Addition and Removal +# ============================================================================= +# This test follows the sequence: +# 1. Create 2-node cluster and cross-wire them +# 2. Create test data and replication sets +# 3. Load ZODAN procedures on n1 +# 4. Add node n3 using ZODAN attach_node procedure +# 5. Verify n3 is properly integrated +# 6. Load ZODREMOVE procedures on n3 +# 7. Remove node n3 using ZODREMOVE detach_node procedure +# 8. Verify n3 is properly removed +# 9. Clean up + +# Step 1: Create a 2-node cluster initially +create_cluster(2, 'Create initial 2-node Spock test cluster'); + +# Get cluster configuration +my $config = get_test_config(); +my $node_count = $config->{node_count}; +my $node_ports = $config->{node_ports}; +my $host = $config->{host}; +my $dbname = $config->{db_name}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; +my $pg_bin = $config->{pg_bin}; + +# Step 1a: Cross-wire the 2 nodes (n1 and n2) +cross_wire(2, ['n1', 'n2'], 'Cross-wire nodes n1 and n2'); + +pass('2-node cluster created and cross-wired'); + +# Step 2: Create test data and replication sets +pass('Creating test data and replication sets'); + +# Create test table on n1 +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', " + CREATE TABLE test_zodan_table ( + id SERIAL PRIMARY KEY, + name VARCHAR(50), + value INTEGER, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) +"; + +# Create a custom replication set on n1 +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', " + SELECT spock.repset_create('zodan_test_set', true, true, true, true) +"; + +# Add table to the custom replication set +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', " + SELECT spock.repset_add_table('zodan_test_set', 'test_zodan_table') +"; + +# Insert test data on n1 +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', " + INSERT INTO test_zodan_table (name, value) VALUES + ('test_data_1', 100), + ('test_data_2', 200), + ('test_data_3', 300) +"; + +pass('Test table created and data inserted on n1'); + +# Wait for replication to n2 +system_or_bail 'sleep', '2'; + +# Verify data replicated to n2 +my $data_on_n2 = `$pg_bin/psql -p $node_ports->[1] -d $dbname -t -c "SELECT COUNT(*) FROM test_zodan_table"`; +chomp($data_on_n2); +$data_on_n2 =~ s/\s+//g; +if ($data_on_n2 eq '3') { + pass('Data successfully replicated from n1 to n2 (3 rows)'); +} else { + fail("Data replication failed: expected 3 rows, got $data_on_n2"); +} + +# Step 3: Create n3 database instance +pass('Creating n3 database instance'); + +# Create n3 data directory and start PostgreSQL instance +my $n3_datadir = "/tmp/tmp_spock_node_3_datadir"; +my $n3_port = $node_ports->[0] + 2; # Use next available port + +# Clean up any existing n3 directory +system_or_bail 'rm', '-rf', $n3_datadir; + +# Initialize n3 data directory +system_or_bail "$pg_bin/initdb", '-A', 'trust', '-D', $n3_datadir; + +# Copy configuration files if they exist +if (-f 'regress-pg_hba.conf') { + system_or_bail 'cp', 'regress-pg_hba.conf', "$n3_datadir/pg_hba.conf"; +} + +# Create PostgreSQL configuration for n3 +open(my $conf, '>>', "$n3_datadir/postgresql.conf") or die "Cannot open config file: $!"; +print $conf "shared_buffers=1GB\n"; +print $conf "shared_preload_libraries='spock'\n"; +print $conf "wal_level=logical\n"; +print $conf "spock.enable_ddl_replication=on\n"; +print $conf "spock.include_ddl_repset=on\n"; +print $conf "spock.allow_ddl_from_functions=on\n"; +print $conf "spock.exception_behaviour=sub_disable\n"; +print $conf "spock.conflict_resolution=last_update_wins\n"; +print $conf "track_commit_timestamp=on\n"; +print $conf "spock.exception_replay_queue_size=1MB\n"; +print $conf "spock.enable_spill=on\n"; +print $conf "port=$n3_port\n"; +print $conf "listen_addresses='*'\n"; +print $conf "logging_collector=on\n"; +print $conf "log_directory='/tmp/logs'\n"; +print $conf "log_filename='00$n3_port.log'\n"; +close($conf); + +# Start n3 PostgreSQL instance +system("$pg_bin/postgres -D $n3_datadir >> '$config->{log_file}' 2>&1 &"); + +# Allow n3 to startup +system_or_bail 'sleep', '10'; + +# Create database and user for testing on n3 +system_or_bail "$pg_bin/psql", '-p', $n3_port, '-d', 'postgres', '-c', "CREATE DATABASE $dbname"; +system_or_bail "$pg_bin/psql", '-p', $n3_port, '-d', $dbname, '-c', "CREATE USER $db_user SUPERUSER"; +system_or_bail "$pg_bin/psql", '-p', $n3_port, '-d', $dbname, '-c', "CREATE USER super SUPERUSER"; + +# Install Spock extension on n3 +system_or_bail "$pg_bin/psql", '-p', $n3_port, '-d', $dbname, '-c', "CREATE EXTENSION IF NOT EXISTS spock"; +system_or_bail "$pg_bin/psql", '-p', $n3_port, '-d', $dbname, '-c', "ALTER EXTENSION spock UPDATE"; + +pass('n3 database instance created and configured'); + +# attach_node/detach_node ship with the spock extension; nothing to load. +my $pipe; + +print "=== STARTING ADD_NODE PROCEDURE ===\n"; + +my $attach_node_cmd = "$pg_bin/psql -p $n3_port -d $dbname -c \" + CALL spock.attach_node( + 'n1', + 'host=$host dbname=$dbname port=$node_ports->[0] user=$db_user password=$db_password', + 'n3', + 'host=$host dbname=$dbname port=$n3_port user=$db_user password=$db_password', + true, + 'CA', + 'USA', + '{}'::jsonb + ) +\""; + +print "Executing: $attach_node_cmd\n"; +print "---\n"; + +open($pipe, "$attach_node_cmd 2>&1 |") or die "Cannot open pipe: $!"; +while (my $line = <$pipe>) { + print $line; +} +close($pipe); +my $attach_node_result = $? >> 8; + +print "---\n"; +print "=== ADD_NODE PROCEDURE COMPLETED (exit code: $attach_node_result) ===\n"; + +if ($attach_node_result == 0) { + pass('attach_node procedure executed successfully'); +} else { + fail("attach_node procedure failed with exit code: $attach_node_result"); +} + +# Step 6: Verify n3 is properly integrated +pass('Verifying n3 integration'); + +# Wait for replication to complete +system_or_bail 'sleep', '3'; + +# Check if test table exists on n3 +my $table_exists_n3 = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'test_zodan_table')"`; +chomp($table_exists_n3); +$table_exists_n3 =~ s/\s+//g; +if ($table_exists_n3 eq 't') { + pass('Test table exists on n3'); + + # Check data count on n3 + my $data_on_n3 = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SELECT COUNT(*) FROM test_zodan_table"`; + chomp($data_on_n3); + $data_on_n3 =~ s/\s+//g; + if ($data_on_n3 eq '3') { + pass('Data successfully replicated to n3 (3 rows)'); + } else { + fail("Data replication to n3 failed: expected 3 rows, got $data_on_n3"); + } +} else { + fail('Test table does not exist on n3'); +} + +# Verify n3 has subscriptions that reference the default replication sets +my $sub_count_n3 = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SELECT COUNT(*) FROM spock.subscription"`; +chomp($sub_count_n3); +$sub_count_n3 =~ s/\s+//g; +if ($sub_count_n3 >= 2) { + pass('n3 has subscriptions referencing default replication sets (ZODAN behavior)'); +} else { + # Debug: List all subscriptions on n3 + my $debug_subs = `$pg_bin/psql -p $n3_port -d $dbname -c "SELECT sub_name FROM spock.subscription ORDER BY sub_name"`; + print "Debug: Subscriptions on n3:\n$debug_subs\n"; + fail('n3 does not have expected subscriptions'); +} + +# Verify n3 is in the node list on all nodes +my $node_count_n1 = `$pg_bin/psql -p $node_ports->[0] -d $dbname -t -c "SELECT COUNT(*) FROM spock.node"`; +chomp($node_count_n1); +$node_count_n1 =~ s/\s+//g; +if ($node_count_n1 eq '3') { + pass('n1 shows 3 nodes in cluster'); +} else { + fail("n1 node count incorrect: expected 3, got $node_count_n1"); +} + +my $node_count_n2 = `$pg_bin/psql -p $node_ports->[1] -d $dbname -t -c "SELECT COUNT(*) FROM spock.node"`; +chomp($node_count_n2); +$node_count_n2 =~ s/\s+//g; +if ($node_count_n2 eq '3') { + pass('n2 shows 3 nodes in cluster'); +} else { + fail("n2 node count incorrect: expected 3, got $node_count_n2"); +} + +my $node_count_n3 = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SELECT COUNT(*) FROM spock.node"`; +chomp($node_count_n3); +$node_count_n3 =~ s/\s+//g; +if ($node_count_n3 eq '3') { + pass('n3 shows 3 nodes in cluster'); +} else { + fail("n3 node count incorrect: expected 3, got $node_count_n3"); +} + +# Step 7: Remove node n3 using the in-core detach_node procedure +pass('Removing node n3 using detach_node procedure'); + +print "=== STARTING REMOVE_NODE PROCEDURE ===\n"; + +my $detach_node_cmd = "$pg_bin/psql -p $n3_port -d $dbname -c \" + CALL spock.detach_node( + 'n3', + 'host=$host dbname=$dbname port=$n3_port user=$db_user password=$db_password', + true + ) +\""; + +print "Executing: $detach_node_cmd\n"; +print "---\n"; + +open($pipe, "$detach_node_cmd 2>&1 |") or die "Cannot open pipe: $!"; +while (my $line = <$pipe>) { + print $line; +} +close($pipe); +my $detach_node_result = $? >> 8; + +print "---\n"; +print "=== REMOVE_NODE PROCEDURE COMPLETED (exit code: $detach_node_result) ===\n"; + +if ($detach_node_result == 0) { + pass('detach_node procedure executed successfully'); +} else { + fail("detach_node procedure failed with exit code: $detach_node_result"); +} + +# Step 9: Verify n3 is properly removed +pass('Verifying n3 removal'); + +# Wait for cleanup to complete +system_or_bail 'sleep', '2'; + +# Verify n3 is no longer in the node list on n1 and n2 +my $node_count_n1_after = `$pg_bin/psql -p $node_ports->[0] -d $dbname -t -c "SELECT COUNT(*) FROM spock.node"`; +chomp($node_count_n1_after); +$node_count_n1_after =~ s/\s+//g; +if ($node_count_n1_after eq '2') { + pass('n1 shows 2 nodes in cluster after removal'); +} else { + fail("n1 node count incorrect after removal: expected 2, got $node_count_n1_after"); +} + +my $node_count_n2_after = `$pg_bin/psql -p $node_ports->[1] -d $dbname -t -c "SELECT COUNT(*) FROM spock.node"`; +chomp($node_count_n2_after); +$node_count_n2_after =~ s/\s+//g; +if ($node_count_n2_after eq '2') { + pass('n2 shows 2 nodes in cluster after removal'); +} else { + fail("n2 node count incorrect after removal: expected 2, got $node_count_n2_after"); +} + +# Verify n3 no longer exists in node list +my $n3_exists_n1 = `$pg_bin/psql -p $node_ports->[0] -d $dbname -t -c "SELECT EXISTS (SELECT 1 FROM spock.node WHERE node_name = 'n3')"`; +chomp($n3_exists_n1); +$n3_exists_n1 =~ s/\s+//g; +if ($n3_exists_n1 eq 'f') { + pass('n3 no longer exists in n1 node list'); +} else { + fail('n3 still exists in n1 node list after removal'); +} + +my $n3_exists_n2 = `$pg_bin/psql -p $node_ports->[1] -d $dbname -t -c "SELECT EXISTS (SELECT 1 FROM spock.node WHERE node_name = 'n3')"`; +chomp($n3_exists_n2); +$n3_exists_n2 =~ s/\s+//g; +if ($n3_exists_n2 eq 'f') { + pass('n3 no longer exists in n2 node list'); +} else { + fail('n3 still exists in n2 node list after removal'); +} + +# Step 10: Clean up n3 +pass('Cleaning up n3'); + +# Stop n3 PostgreSQL instance +system("$pg_bin/pg_ctl stop -D $n3_datadir -m immediate >> '$config->{log_file}' 2>&1 &"); +system_or_bail 'sleep', '5'; +system_or_bail 'rm', '-rf', $n3_datadir; + +pass('n3 database instance cleaned up'); + +# Final verification +pass('ZODAN node addition and removal test completed successfully'); + +# Test summary +print "\n=== TEST SUMMARY ===\n"; +print "✓ Created 2-node cluster\n"; +print "✓ Cross-wired nodes n1 and n2\n"; +print "✓ Created test data and replication sets\n"; +print "✓ Added node n3 using ZODAN attach_node\n"; +print "✓ Verified n3 integration\n"; +print "✓ Removed node n3 using ZODREMOVE detach_node\n"; +print "✓ Verified n3 removal\n"; +print "✓ Cleaned up n3\n"; +print "========================\n"; + +# Tear down n1/n2 explicitly (n3 was already cleaned up above) so the +# destroy_cluster pass() is counted within the plan rather than firing from the +# END block after done_testing(). +destroy_cluster('Destroy ZODAN add/remove test cluster'); +done_testing(); diff --git a/tests/tap/t/035_attach_node_sync_third_incore.pl b/tests/tap/t/035_attach_node_sync_third_incore.pl new file mode 100755 index 000000000..25a3a6e43 --- /dev/null +++ b/tests/tap/t/035_attach_node_sync_third_incore.pl @@ -0,0 +1,450 @@ +use strict; +use warnings; +use Test::More tests => 30; +use IPC::Run; +use lib '.'; +use lib 't'; +use SpockTest qw(create_cluster destroy_cluster system_or_bail get_test_config cross_wire psql_or_bail scalar_query); + +# ============================================================================= +# Test: Add third node (N3) to the configuration of highly loaded (N1 and N2) +# by non-intersecting DMLs. +# ============================================================================= +# This test follows the sequence: +# 1. Create nodes N1 and N2 +# 2. Init pgbench database +# 3. CHECK: database replicated and we see a 'zero' lag +# 4. Load N1 and N2 with a custom non-intersecting UPDATE load +# 5. Call attach_node() on N3 +# 6. Check that pgbench load still exists after the end of the Z0DAN protocol +# 7. Wait for the end of the test and final data sync. +# 8. Check consistency of the data on each node. +# 9. Clean up + +create_cluster(3, 'Create initial 2-node Spock test cluster'); + + +my ($ret1, $ret2, $ret3, $lsn1, $lsn2, $lsn3); + +# Get cluster configuration +my $config = get_test_config(); +my $node_count = $config->{node_count}; +my $node_ports = $config->{node_ports}; +my $host = $config->{host}; +my $dbname = $config->{db_name}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; +my $pg_bin = $config->{pg_bin}; + +cross_wire(2, ['n1', 'n2'], 'Cross-wire nodes N1 and N2'); + +print STDERR "Install the helper functions and do other preparatory stuff\n"; +my $helper_sql = '../../samples/Z0DAN/wait_subscription.sql'; +psql_or_bail(1, "\\i $helper_sql"); +psql_or_bail(2, "\\i $helper_sql"); +psql_or_bail(3, "SELECT spock.node_drop('n3')"); + +# Reduce the logfile size +psql_or_bail(1, "ALTER SYSTEM SET log_min_messages TO LOG"); +psql_or_bail(1, "ALTER SYSTEM SET log_statement TO none"); +psql_or_bail(1, "ALTER SYSTEM SET log_checkpoints TO off"); +psql_or_bail(1, "ALTER SYSTEM SET log_connections TO off"); +psql_or_bail(1, "ALTER SYSTEM SET log_disconnections TO off"); +psql_or_bail(1, "ALTER SYSTEM SET log_lock_waits TO off"); +psql_or_bail(1, "ALTER SYSTEM SET log_statement_stats TO off"); +psql_or_bail(1, "SELECT pg_reload_conf()"); + +psql_or_bail(2, "ALTER SYSTEM SET log_min_messages TO LOG"); +psql_or_bail(2, "ALTER SYSTEM SET log_statement TO none"); +psql_or_bail(2, "ALTER SYSTEM SET log_checkpoints TO off"); +psql_or_bail(2, "ALTER SYSTEM SET log_connections TO off"); +psql_or_bail(2, "ALTER SYSTEM SET log_disconnections TO off"); +psql_or_bail(2, "ALTER SYSTEM SET log_lock_waits TO off"); +psql_or_bail(2, "ALTER SYSTEM SET log_statement_stats TO off"); +psql_or_bail(2, "SELECT pg_reload_conf()"); + +psql_or_bail(3, "ALTER SYSTEM SET log_min_messages TO LOG"); +psql_or_bail(3, "ALTER SYSTEM SET log_statement TO none"); +psql_or_bail(3, "ALTER SYSTEM SET log_checkpoints TO off"); +psql_or_bail(3, "ALTER SYSTEM SET log_connections TO off"); +psql_or_bail(3, "ALTER SYSTEM SET log_disconnections TO off"); +psql_or_bail(3, "ALTER SYSTEM SET log_lock_waits TO off"); +psql_or_bail(3, "ALTER SYSTEM SET log_statement_stats TO off"); +psql_or_bail(3, "SELECT pg_reload_conf()"); + +print STDERR "Initialize pgbench database and wait for initial sync on N1 and N2 ...\n"; +system_or_bail "$pg_bin/pgbench", '-i', '-s', 1, '-h', $host, + '-p', $node_ports->[0], '-U', $db_user, $dbname; +# Wait until tables and data will be sent to N2 +psql_or_bail(1, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +# Test 1: after the end of replication process we should be able to see a 'zero' +# lag between the nodes. +my $lag = scalar_query(2, "SELECT * FROM wait_subscription(remote_node_name := 'n1', + report_it := true, + timeout := '10 minutes', + delay := 1.)"); + +ok($lag <= 0, "Initial replication has been successful"); + +# Create non-intersecting load for nodes N1 and N2. +# Test duration should be enough to cover all the Z0DAN stages. We will kill +# pgbench immediately after the N3 is attached. +my $load1 = '../../samples/Z0DAN/n1.pgb'; +my $load2 = '../../samples/Z0DAN/n2.pgb'; +my $pgbench_stdout1=''; +my $pgbench_stderr1=''; +my $pgbench_stdout2=''; +my $pgbench_stderr2=''; +my $pgbench_handle1 = IPC::Run::start( + [ "$pg_bin/pgbench", '-n', '-f', $load1, '-T', 80, '-j', 3, '-c', 3, + '-h', $host, '-p', $node_ports->[0], '-U', $db_user, $dbname], + '>', \$pgbench_stdout1, '2>', \$pgbench_stderr1); +my $pgbench_handle2 = IPC::Run::start( + [ "$pg_bin/pgbench", '-n', '-f', $load2, '-T', 80, '-j', 3, '-c', 3, + '-h', $host, '-p', $node_ports->[1], '-U', $db_user, $dbname], + '>', \$pgbench_stdout2, '2>', \$pgbench_stderr2); +$pgbench_handle1->pump(); +$pgbench_handle2->pump(); + +# Warming up ... +print STDERR "warming up pgbench for 30s\n"; +sleep(30); +print STDERR "done warmup\n"; + +print STDERR "Add N3 into highly loaded configuration of N1 and N2 ...\n"; +# Use transdiscard on N3 so that any "row not found" errors during catch-up +# (from transactions whose effects are already in the COPY snapshot) are +# gracefully discarded instead of disabling the subscription. +psql_or_bail(3, "ALTER SYSTEM SET spock.exception_behaviour = 'transdiscard'"); +psql_or_bail(3, "SELECT pg_reload_conf()"); + +# Drain replication backlog before attach_node so apply workers are less busy +# during slot creation, increasing the chance of an idle inter-commit gap. +print STDERR "Draining replication before attach_node ...\n"; +psql_or_bail(1, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +psql_or_bail(2, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); + +psql_or_bail(3, + "CALL spock.attach_node(src_node_name := 'n1', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[0] user=$db_user', + new_node_name := 'n3', + new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user', + verb := false);"); + +# Wait for replication to stabilize after attach_node before checking. +# pgbench is still running so lag won't reach zero; just drain current backlog. +print STDERR "Waiting for replication to settle after attach_node ...\n"; +psql_or_bail(1, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +psql_or_bail(2, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +sleep(5); + +# Ensure that pgbench load lasts longer than the Z0DAN protocol. +my $pid = $pgbench_handle1->{KIDS}[0]{PID}; +my $alive = kill 0, $pid; +ok($alive eq 1, "pgbench load to N1 still exists"); +$pid = $pgbench_handle2->{KIDS}[0]{PID}; +$alive = kill 0, $pid; +ok($alive eq 1, "pgbench load to N2 still exists"); + +print STDERR "Kill pgbench process to reduce test time\n"; +$pgbench_handle1->pump(); +$pgbench_handle2->pump(); +$pgbench_handle1->kill_kill; +$pgbench_handle2->kill_kill; + +print STDERR "Check if pgbench finalised correctly\n"; +$pgbench_handle1->finish; +$pgbench_handle2->finish; +print STDERR "##### output of pgbench #####\n"; +print STDERR $pgbench_stdout1; +print STDERR $pgbench_stdout2; +print STDERR "##### end of output #####\n"; + +psql_or_bail(1, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +psql_or_bail(2, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); + +print STDERR "Wait until the end of replication ..\n"; +$lag = scalar_query(1, "SELECT * FROM wait_subscription(remote_node_name := 'n2', + report_it := true, + timeout := '10 minutes', + delay := 1.)"); +ok($lag <= 0, "Replication N2 => N1 has been finished successfully"); +$lag = scalar_query(2, "SELECT * FROM wait_subscription(remote_node_name := 'n1', + report_it := true, + timeout := '10 minutes', + delay := 1.)"); +ok($lag <= 0, "Replication N1 => N2 has been finished successfully"); +$lag = scalar_query(3, "SELECT * FROM wait_subscription(remote_node_name := 'n1', + report_it := true, + timeout := '10 minutes', + delay := 1.)"); +ok($lag <= 0, "Replication N1 => N3 has been finished successfully"); +$lag = scalar_query(3, "SELECT * FROM wait_subscription(remote_node_name := 'n2', + report_it := true, + timeout := '10 minutes', + delay := 1.)"); +ok($lag <= 0, "Replication N2 => N3 has been finished successfully"); + +print STDERR "Check the data consistency.\n"; +$ret1 = scalar_query(1, "SELECT sum(abalance), sum(aid), count(*) FROM pgbench_accounts"); +print STDERR "The N1's pgbench_accounts aggregates: $ret1\n"; +$ret2 = scalar_query(2, "SELECT sum(abalance), sum(aid), count(*) FROM pgbench_accounts"); +print STDERR "The N2's pgbench_accounts aggregates: $ret2\n"; +$ret3 = scalar_query(3, "SELECT sum(abalance), sum(aid), count(*) FROM pgbench_accounts"); + +print STDERR "The N3's pgbench_accounts aggregates: $ret3\n"; +ok($ret1 eq $ret3, "Equality of the data on N1 and N3 is confirmed"); +ok($ret2 eq $ret3, "Equality of the data on N2 and N3 is confirmed"); + +# Before we finish this test and destroy the cluster, we need to ensure that +# the nodes are not stuck in some work. N1 and N2 have done their job and are +# ready to switch off. However, after receiving a large amount of WAL during +# catch-up, N3 may be decoding WAL to determine a proper LSN for N3->N1 and +# N3->N2 replication. +# Being in this process, the walsender doesn't send anything valuable except +# a 'keepalive' message. Hence, we can't clearly detect the end of the process. +# So, nudge it, employing the sync_event machinery. +psql_or_bail(3, "SELECT spock.sync_event()"); +print STDERR "Wait for the end of N3->N1, N3->N2 decoding process that means the actual start of LR\n"; +psql_or_bail(3, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); + +# Nothing sensitive should be awaited here, just to be sure. +$lag = scalar_query(1, "SELECT * FROM wait_subscription(remote_node_name := 'n3', + report_it := true, + timeout := '10 minutes', + delay := 1.)"); +ok($lag <= 0, "Replication N3 => N1 has been finished successfully"); +$lag = scalar_query(2, "SELECT * FROM wait_subscription(remote_node_name := 'n3', + report_it := true, + timeout := '10 minutes', + delay := 1.)"); +ok($lag <= 0, "Replication N3 => N2 has been finished successfully"); +# 2n congiguration. With non-intersecting load we don't anticipate any issues +# with this test. It is written to prepare infrastructure and for demonstration +# purposes. +# +# ############################################################################## + +psql_or_bail(3, "CALL spock.detach_node( + target_node_name := 'n3', + target_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user', + verbose_mode := true)"); +system_or_bail "$pg_bin/pgbench", '-i', '-I', 'd', '-h', $host, '-p', $node_ports->[2], '-U', $db_user, $dbname; +psql_or_bail(3, 'DROP FUNCTION wait_subscription'); +psql_or_bail(3, 'VACUUM FULL'); + +# Let the cluster settle after detach_node before starting the next cycle. +print STDERR "Waiting for cluster to settle after detach_node ...\n"; +scalar_query(1, "SELECT * FROM wait_subscription(remote_node_name := 'n2', + timeout := '3 minutes', delay := 0.5)"); +scalar_query(2, "SELECT * FROM wait_subscription(remote_node_name := 'n1', + timeout := '3 minutes', delay := 0.5)"); + +# To improve TPS +psql_or_bail(1, "CREATE UNIQUE INDEX ON pgbench_accounts(abs(aid))"); +$lag = scalar_query(2, "SELECT * FROM wait_subscription(remote_node_name := 'n1', + report_it := true, + timeout := '10 minutes', + delay := 1.)"); +ok($lag <= 0, "Wait replication of the CREATE INDEX"); + +# Create non-intersecting load for nodes N1 and N2. +# Test duration should be enough to cover all the Z0DAN stages. We will kill +# pgbench immediately after the N3 is attached. +$load1 = '../../samples/Z0DAN/n1_1.pgb'; +$load2 = '../../samples/Z0DAN/n2_1.pgb'; +$pgbench_stdout1=''; +$pgbench_stderr1=''; +$pgbench_stdout2=''; +$pgbench_stderr2=''; +$pgbench_handle1 = IPC::Run::start( + [ "$pg_bin/pgbench", '-n', '-f', $load1, '-T', 80, '-j', 3, '-c', 3, + '-h', $host, '-p', $node_ports->[0], '-U', $db_user, $dbname], + '>', \$pgbench_stdout1, '2>', \$pgbench_stderr1); +$pgbench_handle2 = IPC::Run::start( + [ "$pg_bin/pgbench", '-n', '-f', $load2, '-T', 80, '-j', 3, '-c', 3, + '-h', $host, '-p', $node_ports->[1], '-U', $db_user, $dbname], + '>', \$pgbench_stdout2, '2>', \$pgbench_stderr2); +$pgbench_handle1->pump(); +$pgbench_handle2->pump(); + +# Warming up ... +print STDERR "warming up pgbench for 30s\n"; +sleep(30); +print STDERR "done warmup\n"; + +# Ensure that pgbench load lasts longer than the Z0DAN protocol. +$pid = $pgbench_handle1->{KIDS}[0]{PID}; +$alive = kill 0, $pid; +ok($alive eq 1, "pgbench load to N1 still exists"); +$pid = $pgbench_handle2->{KIDS}[0]{PID}; +$alive = kill 0, $pid; +ok($alive eq 1, "pgbench load to N2 still exists"); + +print STDERR "Kill pgbench process to reduce test time\n"; +$pgbench_handle1->pump(); +$pgbench_handle2->pump(); +$pgbench_handle1->kill_kill; +$pgbench_handle2->kill_kill; + +print STDERR "Check if pgbench finalised correctly\n"; +$pgbench_handle1->finish; +$pgbench_handle2->finish; +print STDERR "##### output of pgbench #####\n"; +print STDERR "$pgbench_stdout1"; +print STDERR "$pgbench_stdout2"; +print STDERR "##### end of output #####\n"; + +psql_or_bail(1, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +psql_or_bail(2, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); + +print STDERR "Wait until the end of replication ..\n"; +$lag = scalar_query(1, "SELECT * FROM wait_subscription(remote_node_name := 'n2', + report_it := true, + timeout := '10 minutes', + delay := 1.)"); +ok($lag <= 0, "Replication N2 => N1 has been finished successfully"); +$lag = scalar_query(2, "SELECT * FROM wait_subscription(remote_node_name := 'n1', + report_it := true, + timeout := '10 minutes', + delay := 1.)"); +ok($lag <= 0, "Replication N1 => N2 has been finished successfully"); + +print STDERR "Check the data consistency.\n"; +$ret1 = scalar_query(1, "SELECT sum(abalance), sum(aid), count(*) FROM pgbench_accounts"); +print STDERR "The N1's pgbench_accounts aggregates: $ret1\n"; +$ret2 = scalar_query(2, "SELECT sum(abalance), sum(aid), count(*) FROM pgbench_accounts"); +print STDERR "The N2's pgbench_accounts aggregates: $ret2\n"; + +ok($ret1 eq $ret2, "Equality of the data on N1 and N2 is confirmed"); + +# ############################################################################## +# +# Try to update an IDENTITY column in case of 3n configuration. +# It works precisely like the previous one, but node 3 should sync its state +# with loaded nodes in real time under the pgbench load. +# Here, we also inderectly test how the Z0DAN add/remove protocol works in case +# of multiple adding cycles. +# +# ############################################################################## + +$pgbench_stdout1=''; +$pgbench_stderr1=''; +$pgbench_stdout2=''; +$pgbench_stderr2=''; +$pgbench_handle1 = IPC::Run::start( + [ "$pg_bin/pgbench", '-n', '-f', $load1, '-T', 80, '-j', 3, '-c', 3, + '-h', $host, '-p', $node_ports->[0], '-U', $db_user, $dbname], + '>', \$pgbench_stdout1, '2>', \$pgbench_stderr1); +$pgbench_handle2 = IPC::Run::start( + [ "$pg_bin/pgbench", '-n', '-f', $load2, '-T', 80, '-j', 3, '-c', 3, + '-h', $host, '-p', $node_ports->[1], '-U', $db_user, $dbname], + '>', \$pgbench_stdout2, '2>', \$pgbench_stderr2); +$pgbench_handle1->pump(); +$pgbench_handle2->pump(); + +# Warming up ... +print STDERR "warming up pgbench for 30s\n"; +sleep(30); +print STDERR "done warmup\n"; + +print STDERR "Add N3 into highly loaded configuration of N1 and N2 ..."; +psql_or_bail(3, "ALTER SYSTEM SET spock.exception_behaviour = 'transdiscard'"); +psql_or_bail(3, "SELECT pg_reload_conf()"); + +# Drain replication backlog before second attach_node. +print STDERR "Draining replication before second attach_node ...\n"; +psql_or_bail(1, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +psql_or_bail(2, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); + +psql_or_bail(3, + "CALL spock.attach_node(src_node_name := 'n1', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[0] user=$db_user', + new_node_name := 'n3', + new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user', + verb := false);"); + +# Wait for replication to stabilize after second attach_node. +# pgbench is still running so lag won't reach zero; just drain current backlog. +print STDERR "Waiting for replication to settle after attach_node ...\n"; +psql_or_bail(1, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +psql_or_bail(2, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +sleep(5); + +# Ensure that pgbench load lasts longer than the Z0DAN protocol. +$pid = $pgbench_handle1->{KIDS}[0]{PID}; +$alive = kill 0, $pid; +ok($alive eq 1, "pgbench load to N1 still exists"); +$pid = $pgbench_handle2->{KIDS}[0]{PID}; +$alive = kill 0, $pid; +ok($alive eq 1, "pgbench load to N2 still exists"); + +print STDERR "Kill pgbench process to reduce test time\n"; +$pgbench_handle1->pump(); +$pgbench_handle2->pump(); +$pgbench_handle1->kill_kill; +$pgbench_handle2->kill_kill; + +print STDERR "Check if pgbench finalised correctly\n"; +$pgbench_handle1->finish; +$pgbench_handle2->finish; +print STDERR "##### output of pgbench #####\n"; +print STDERR $pgbench_stdout1; +print STDERR $pgbench_stdout2; +print STDERR "##### end of output #####\n"; + +# +# Wait for the end of apply process +# +print STDERR "Wait for the end of LR caused by the pgbench load\n"; +$lsn1 = scalar_query(1, "SELECT spock.sync_event()"); +$lsn2 = scalar_query(2, "SELECT spock.sync_event()"); +$lsn3 = scalar_query(3, "SELECT spock.sync_event()"); +print STDERR "DEBUGGING. LSNs: N1: $lsn1, N2: $lsn2, N3: $lsn3\n"; + +print STDERR "Wait for the N2 -> N1 sync message ...\n"; +psql_or_bail(1, "CALL spock.wait_for_sync_event(true, 'n2', '$lsn2'::pg_lsn, 1200, true)"); +print STDERR "Wait for the N1 -> N2 sync message ...\n"; +psql_or_bail(2, "CALL spock.wait_for_sync_event(true, 'n1', '$lsn1'::pg_lsn, 1200, true)"); +print STDERR "Wait for the N1 -> N3 sync message ...\n"; +psql_or_bail(3, "CALL spock.wait_for_sync_event(true, 'n1', '$lsn1'::pg_lsn, 1200, true)"); +print STDERR "Wait for the N2 -> N3 sync message ...\n"; +psql_or_bail(3, "CALL spock.wait_for_sync_event(true, 'n2', '$lsn2'::pg_lsn, 1200, true)"); +print STDERR "LR messages from active nodes has arrived to the new one\n"; + +print STDERR "Wait for the N3 -> N1 sync message ...\n"; +psql_or_bail(1, "CALL spock.wait_for_sync_event(true, 'n3', '$lsn3'::pg_lsn, 1200, true)"); +print STDERR "Wait for the N3 -> N2 sync message ...\n"; +psql_or_bail(2, "CALL spock.wait_for_sync_event(true, 'n3', '$lsn3'::pg_lsn, 1200, true)"); +print STDERR "First LR transaction has arrived from new node to the active ones\n"; + +# Wait for all replication directions to fully catch up. +psql_or_bail(1, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +psql_or_bail(2, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +psql_or_bail(3, 'SELECT spock.wait_slot_confirm_lsn(NULL, NULL)'); +scalar_query(1, "SELECT * FROM wait_subscription(remote_node_name := 'n2', + timeout := '3 minutes', delay := 0.5)"); +scalar_query(1, "SELECT * FROM wait_subscription(remote_node_name := 'n3', + timeout := '3 minutes', delay := 0.5)"); +scalar_query(2, "SELECT * FROM wait_subscription(remote_node_name := 'n1', + timeout := '3 minutes', delay := 0.5)"); +scalar_query(2, "SELECT * FROM wait_subscription(remote_node_name := 'n3', + timeout := '3 minutes', delay := 0.5)"); +scalar_query(3, "SELECT * FROM wait_subscription(remote_node_name := 'n1', + timeout := '3 minutes', delay := 0.5)"); +scalar_query(3, "SELECT * FROM wait_subscription(remote_node_name := 'n2', + timeout := '3 minutes', delay := 0.5)"); + +print STDERR "Check the data consistency.\n"; +$ret1 = scalar_query(1, "SELECT sum(abalance), sum(aid), count(*) FROM pgbench_accounts"); +print STDERR "The N1's pgbench_accounts aggregates: $ret1\n"; +$ret2 = scalar_query(2, "SELECT sum(abalance), sum(aid), count(*) FROM pgbench_accounts"); +print STDERR "The N2's pgbench_accounts aggregates: $ret2\n"; +$ret3 = scalar_query(3, "SELECT sum(abalance), sum(aid), count(*) FROM pgbench_accounts"); +print STDERR "The N3's pgbench_accounts aggregates: $ret3\n"; + +ok($ret1 eq $ret2, "Equality of the data on N1 and N2 is confirmed"); +ok($ret1 eq $ret3, "Equality of the data on N1 and N3 is confirmed"); + +# Cleanup will be handled by SpockTest.pm END block +# No need for done_testing() when using a test plan diff --git a/tests/tap/t/036_attach_node_basics_incore.pl b/tests/tap/t/036_attach_node_basics_incore.pl new file mode 100755 index 000000000..5a385ff75 --- /dev/null +++ b/tests/tap/t/036_attach_node_basics_incore.pl @@ -0,0 +1,119 @@ +use strict; +use warnings; +use Test::More; +use lib '.'; +use lib 't'; +use SpockTest qw(create_cluster destroy_cluster get_test_config psql_or_bail scalar_query); + +my ($result); + +create_cluster(3, 'Create basic Spock test cluster'); + +# Get cluster configuration +my $config = get_test_config(); +my $node_count = $config->{node_count}; +my $node_ports = $config->{node_ports}; +my $host = $config->{host}; +my $dbname = $config->{db_name}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; +my $pg_bin = $config->{pg_bin}; + +psql_or_bail(2, "SELECT spock.node_drop('n2')"); +psql_or_bail(3, "SELECT spock.node_drop('n3')"); +psql_or_bail(1, "CREATE EXTENSION amcheck"); +psql_or_bail(1, "CREATE TABLE test(x serial PRIMARY KEY)"); +psql_or_bail(1, "INSERT INTO test DEFAULT VALUES"); + +print STDERR "All supporting stuff has been installed successfully\n"; + +# ############################################################################## +# +# Basic check that Z0DAN correctly add node to the single-node cluster +# +# ############################################################################## + +print STDERR "Call Z0DAN: n2 => n1\n"; +psql_or_bail(2, " + CALL spock.attach_node( + src_node_name := 'n1', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[0] user=$db_user password=$db_password', + new_node_name := 'n2', + new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[1] user=$db_user password=$db_password', + verb := false + )"); +print STDERR "Z0DAN (n2 => n1) has finished the attach process\n"; +$result = scalar_query(2, "SELECT x FROM test"); +print STDERR "Check result: $result\n"; +ok($result eq '1', "Check state of the test table after the attachment"); + +psql_or_bail(1, "SELECT spock.sub_disable('sub_n2_n1')"); + +# ############################################################################## +# +# Z0DAN reject node addition if some subscriptions are disabled +# +# ############################################################################## + +print STDERR "Call Z0DAN: n3 => n2\n"; +scalar_query(3, " + CALL spock.attach_node( + src_node_name := 'n2', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[1] user=$db_user password=$db_password', + new_node_name := 'n3', new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user password=$db_password', + verb := false)"); + +$result = scalar_query(3, "SELECT count(*) FROM spock.local_node"); +ok($result eq '0', "N3 is not in the cluster yet"); +print STDERR "Z0DAN should fail because of a disabled subscription\n"; + +psql_or_bail(1, "SELECT spock.sub_enable('sub_n2_n1')"); +psql_or_bail(3, " + CALL spock.attach_node( + src_node_name := 'n2', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[1] user=$db_user password=$db_password', + new_node_name := 'n3', new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user password=$db_password', + verb := true)"); + +$result = scalar_query(3, "SELECT count(*) FROM spock.local_node"); +ok($result eq '1', "N3 is in the cluster"); +$result = scalar_query(3, "SELECT x FROM test"); +print STDERR "Check result: $result\n"; +ok($result eq '1', "Check state of the test table on N3 after the attachment"); +print STDERR "Z0DAN should add N3 to the cluster\n"; + +# ############################################################################## +# +# Test that Z0DAN correctly doesn't add node to the cluster if something happens +# during the SYNC process. +# +# ############################################################################## + +# Remove node from the cluster and data leftovers. +psql_or_bail(3, "CALL spock.detach_node(target_node_name := 'n3', + target_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user password=$db_password', + verbose_mode := true)"); +psql_or_bail(3, "DROP TABLE test"); + +psql_or_bail(1, "CREATE FUNCTION fake_fn() RETURNS integer LANGUAGE sql AS \$\$ SELECT 1\$\$"); +psql_or_bail(3, "CREATE FUNCTION fake_fn() RETURNS integer LANGUAGE sql AS \$\$ SELECT 1\$\$"); +scalar_query(3, " + CALL spock.attach_node( + src_node_name := 'n2', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[1] user=$db_user password=$db_password', + new_node_name := 'n3', new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user password=$db_password', + verb := true)"); + +# TODO: +# It seems that attach_node keeps remnants after unsuccessful execution. It is +# happened because we have commited some intermediate results before. +# It would be better to keep remote transaction opened until the end of the +# operation or just remove these remnants at the end pretending to be a +# distributed transaction. +# +# $result = scalar_query(3, "SELECT count(*) FROM spock.local_node"); +# ok($result eq '0', "N3 is not in the cluster"); + +# Clean up +destroy_cluster('Destroy test cluster'); +done_testing(); diff --git a/tests/tap/t/037_attach_node_3n_timeout_incore.pl b/tests/tap/t/037_attach_node_3n_timeout_incore.pl new file mode 100755 index 000000000..66567b49e --- /dev/null +++ b/tests/tap/t/037_attach_node_3n_timeout_incore.pl @@ -0,0 +1,197 @@ +use strict; +use warnings; +use Test::More; +use lib '.'; +use lib 't'; +use SpockTest qw(create_cluster destroy_cluster get_test_config psql_or_bail scalar_query); +use Time::HiRes qw(time); + +# ============================================================================= +# Test: Verify that attach_node fails immediately with an error when sync_event +# function is missing on the source node, rather than waiting for timeout. +# ============================================================================= +# attach_node raises an error (rather than looping until timeout) when a required +# helper such as sync_event is missing on the source node. The test: +# 1. Creates a 3-node cluster (n1, n2 active; n3 dropped and re-added) +# 2. Drops n3 from the cluster +# 3. Renames spock.sync_event() on node 1 to simulate it being missing +# 4. Attempts attach_node from n3 to join the cluster +# 5. Verifies the call fails quickly with an error (not timeout) + +create_cluster(3, 'Create 3-node Spock test cluster'); + +# Get cluster configuration +my $config = get_test_config(); +my $node_ports = $config->{node_ports}; +my $host = $config->{host}; +my $dbname = $config->{db_name}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; +my $pg_bin = $config->{pg_bin}; + +# Prepare node 2: drop from cluster (attach_node/detach_node ship with the extension) +print STDERR "Prepare N2: drop local node\n"; +psql_or_bail(2, "SELECT spock.node_drop('n2')"); +psql_or_bail(3, "SELECT spock.node_drop('n3')"); + +# Rename sync_event function on node 1 to simulate it being missing +print STDERR "Rename spock.sync_event() on N1 to simulate missing function\n"; +psql_or_bail(1, "ALTER FUNCTION spock.sync_event(boolean) RENAME TO sync_event_renamed"); + +# Attempt attach_node - this should fail quickly with an error, not timeout +print STDERR "Attempt attach_node from N2 to N1 (should fail quickly with error)\n"; +my $start_time = time(); + +# scalar_query uses backticks which don't throw exceptions - check $? for exit code +my $result = scalar_query(2, qq{ + CALL spock.attach_node( + src_node_name := 'n1', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[0] user=$db_user password=$db_password', + new_node_name := 'n2', + new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[1] user=$db_user password=$db_password', + verb := false, + timeout_sec := 30 + )}); +my $exit_code = $? >> 8; + +my $elapsed_time = time() - $start_time; +print STDERR "attach_node call completed in $elapsed_time seconds (exit code: $exit_code)\n"; + +# The call should fail quickly, well under the 30s timeout_sec above. The bound +# must stay safely below that timeout, otherwise a call that loops to the full +# timeout would still pass this assertion and defeat its purpose. +ok($elapsed_time < 25, "attach_node failed quickly (${elapsed_time}s < 25s), not waiting for timeout"); +ok($exit_code != 0, "attach_node failed as expected when sync_event is missing (exit code: $exit_code)"); + +# Restore sync_event function on N1 for cleanup +print STDERR "Restore spock.sync_event() on N1\n"; +psql_or_bail(1, "ALTER FUNCTION spock.sync_event_renamed(boolean) RENAME TO sync_event"); + +# Clean leftovers in the Spock cluster caused by unsuccessful addition +scalar_query(2, qq{ + CALL spock.detach_node( + target_node_name := 'n2', + target_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[1] user=$db_user password=$db_password', + verbose_mode := true) +}); + +print STDERR "Check: an error during pg_replication_slot_advance should stop node addition\n"; + +psql_or_bail(2, "ALTER FUNCTION pg_replication_slot_advance RENAME TO pg_replication_slot_advance_renamed"); + +# Should be OK, because of 2-n configuration, no advance needed. This join is +# expected to succeed and can take tens of seconds, so it must not be inside the +# window the "failed quickly" assertion below measures. +psql_or_bail(2, qq{ + CALL spock.attach_node( + src_node_name := 'n1', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[0] user=$db_user password=$db_password', + new_node_name := 'n2', + new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[1] user=$db_user password=$db_password', + verb := false + )}); + +# Time only the n3 attach, which is the call expected to fail quickly. +$start_time = time(); + +# Should fail quickly +$result = scalar_query(3, qq{ + CALL spock.attach_node( + src_node_name := 'n1', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[0] user=$db_user password=$db_password', + new_node_name := 'n3', + new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user password=$db_password', + verb := false, + timeout_sec := 30 + )}); +$exit_code = $? >> 8; + +$elapsed_time = time() - $start_time; +print STDERR "attach_node call completed in $elapsed_time seconds (exit code: $exit_code)\n"; + +ok($elapsed_time < 25, "attach_node on n3 failed quickly (${elapsed_time}s < 25s), not waiting for timeout"); +ok($exit_code != 0, "attach_node failed as expected when pg_replication_slot_advance is missing (exit code: $exit_code)"); + +psql_or_bail(2, "ALTER FUNCTION pg_replication_slot_advance_renamed RENAME TO pg_replication_slot_advance"); + +# Clean leftovers of node-3 in the Spock cluster caused by unsuccessful addition +scalar_query(3, qq{ + CALL spock.detach_node( + target_node_name := 'n3', + target_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user password=$db_password', + verbose_mode := true) +}); + +print STDERR "Check: quick fail if something happens during subscription creation on source node\n"; + +psql_or_bail(1, "ALTER FUNCTION spock.sub_create RENAME TO sub_create_renamed"); +$start_time = time(); + +$result = scalar_query(3, qq{ + CALL spock.attach_node( + src_node_name := 'n1', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[0] user=$db_user password=$db_password', + new_node_name := 'n3', + new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user password=$db_password', + verb := false, + timeout_sec := 30 + )}); +$exit_code = $? >> 8; + +$elapsed_time = time() - $start_time; +print STDERR "attach_node call completed in $elapsed_time seconds (exit code: $exit_code)\n"; + +ok($elapsed_time < 25, "attach_node on n3 failed quickly (${elapsed_time}s < 25s), not waiting for timeout"); +ok($exit_code != 0, "attach_node failed as expected when sub_create is missing (exit code: $exit_code)"); + +psql_or_bail(1, "ALTER FUNCTION spock.sub_create_renamed RENAME TO sub_create"); + +# Clean leftovers of node-3 in the Spock cluster caused by unsuccessful addition +scalar_query(2, qq{ + CALL spock.detach_node( + target_node_name := 'n2', + target_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[1] user=$db_user password=$db_password', + verbose_mode := true) +}); +scalar_query(3, qq{ + CALL spock.detach_node( + target_node_name := 'n3', + target_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user password=$db_password', + verbose_mode := true) +}); + +print STDERR "Final check: node-3 adds to the cluster successfully\n"; + +psql_or_bail(2, qq{ + CALL spock.attach_node( + src_node_name := 'n1', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[0] user=$db_user password=$db_password', + new_node_name := 'n2', + new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[1] user=$db_user password=$db_password', + verb := false + )}); +psql_or_bail(3, qq{ + CALL spock.attach_node( + src_node_name := 'n1', + src_dsn := 'host=$host dbname=$dbname port=$node_ports->[0] user=$db_user password=$db_password', + new_node_name := 'n3', + new_node_dsn := 'host=$host dbname=$dbname port=$node_ports->[2] user=$db_user password=$db_password', + verb := false + )}); + +# Verify cluster has 3 nodes on each node +for my $node (1, 2, 3) { + my $node_count = scalar_query($node, "SELECT count(*) FROM spock.node"); + ok($node_count == 3, "Node $node sees 3 nodes in cluster (got $node_count)"); +} + +# Verify each node has 2 non-disabled subscriptions +for my $node (1, 2, 3) { + my $sub_count = scalar_query($node, + "SELECT count(*) FROM spock.subscription WHERE sub_enabled = true"); + ok($sub_count == 2, "Node $node has 2 enabled subscriptions (got $sub_count)"); +} + +# Cleanup will be handled by SpockTest.pm END block +destroy_cluster('Destroy test cluster'); +done_testing(); diff --git a/tests/tap/t/040_zodan_lolor_attach_node_incore.pl b/tests/tap/t/040_zodan_lolor_attach_node_incore.pl new file mode 100644 index 000000000..67da795b0 --- /dev/null +++ b/tests/tap/t/040_zodan_lolor_attach_node_incore.pl @@ -0,0 +1,182 @@ +use strict; +use warnings; +use Test::More; +use lib '.'; +use lib 't'; +use SpockTest qw(create_cluster destroy_cluster system_maybe + get_test_config cross_wire scalar_query ensure_lolor); + +# Zodan attach_node with lolor large objects. The source cluster replicates the +# lolor tables (lolor.pg_largeobject, lolor.pg_largeobject_metadata) in the +# default replication set. Adding a node through zodan's attach_node() must: +# - reject a new node that lacks the lolor extension (data sync would fail), +# - reject a new node whose lolor tables already contain data, +# - accept a new node with lolor installed and empty, and copy the large +# object data from the source during the initial data sync, +# - stream large objects created after the join, +# - leave the lolor tables in the new node's own default replication set, +# so large objects created there replicate back to the existing nodes. + +my $cfg = get_test_config(); +my $PG = $cfg->{pg_bin}; +my $DB = $cfg->{db_name}; +my $USER = $cfg->{db_user}; +my $PASS = $cfg->{db_password}; +my $HOST = $cfg->{host}; + +plan skip_all => "lolor extension unavailable (clone/build failed)" + unless ensure_lolor(); + +# psql helper: 1-based node index, returns true on success (output to log). +sub psql_ok { + my ($node, $sql) = @_; + my $port = $cfg->{node_ports}[$node - 1]; + return system_maybe("$PG/psql", '-X', '-p', $port, '-d', $DB, + '-v', 'ON_ERROR_STOP=1', '-c', $sql); +} + +# Run SQL and capture combined stdout/stderr without going through a shell, +# so quoting inside the SQL is preserved. Returns (exit_code, output). +sub psql_capture { + my ($node, $sql) = @_; + my $port = $cfg->{node_ports}[$node - 1]; + my $pid = open(my $fh, '-|'); + die "fork failed: $!" unless defined $pid; + if ($pid == 0) { + open(STDERR, '>&', \*STDOUT) or die "cannot dup STDERR: $!"; + exec("$PG/psql", '-X', '-p', $port, '-d', $DB, + '-v', 'ON_ERROR_STOP=1', '-c', $sql); + exit 127; + } + my $out = do { local $/; <$fh> } // ''; + close($fh); + return ($? >> 8, $out); +} + +# Poll a scalar query on a node until it returns $expected. +sub wait_for_scalar { + my ($node, $sql, $expected, $timeout) = @_; + $timeout //= 60; + for (1 .. $timeout) { + my $v = scalar_query($node, $sql); + return 1 if defined $v && $v eq $expected; + sleep(1); + } + return 0; +} + +sub dsn { + my ($node) = @_; + my $port = $cfg->{node_ports}[$node - 1]; + return "host=$HOST dbname=$DB port=$port user=$USER password=$PASS"; +} + +# --- Cluster setup: n1/n2 cross-wired, n3 is a blank target for zodan ------- + +create_cluster(3, 'Create 3 instances for zodan lolor test'); +cross_wire(2, ['n1', 'n2'], 'Cross-wire nodes n1 and n2'); + +# create_cluster registered a spock node on n3; drop it so n3 looks like a +# freshly prepared instance (spock installed, no node/repsets). attach_node is a +# C-native procedure shipped with the extension, so no dblink or zodan.sql is +# needed on the new node. +ok(psql_ok(3, "SELECT spock.node_drop('n3')"), 'n3 spock node registration dropped'); + +# lolor on the source cluster, its tables in the default replication set. +# n1 and n2 are cross-wired with automatic DDL replication, so CREATE +# EXTENSION on n1 arrives on n2 by itself. +ok(psql_ok(1, "CREATE EXTENSION lolor"), 'lolor installed on n1'); +ok(wait_for_scalar(2, "SELECT count(*) FROM pg_extension WHERE extname = 'lolor'", '1'), + 'lolor arrived on n2 via DDL replication'); +for my $node (1, 2) { + ok(psql_ok($node, "SELECT spock.repset_add_table('default', 'lolor.pg_largeobject')"), + "lolor.pg_largeobject in default repset on n$node"); + ok(psql_ok($node, "SELECT spock.repset_add_table('default', 'lolor.pg_largeobject_metadata')"), + "lolor.pg_largeobject_metadata in default repset on n$node"); +} + +# Large object on n1; sanity-check it reaches n2 before involving zodan. +ok(psql_ok(1, "SET lolor.node=1; SELECT lo_from_bytea(0, '\\xdeadbeefcafe')"), + 'large object created on n1'); +ok(wait_for_scalar(2, "SELECT count(*) FROM lolor.pg_largeobject WHERE encode(data, 'hex') = 'deadbeefcafe'", '1'), + 'large object replicated from n1 to n2'); + +my $attach_node_sql = + "CALL spock.attach_node('n1', '" . dsn(1) . "', 'n3', '" . dsn(3) . "', " . + "true, 'CA', 'USA', '{}'::jsonb)"; + +# --- Negative: source replicates lolor but n3 has no lolor extension -------- + +my ($rc, $out) = psql_capture(3, $attach_node_sql); +ok($rc != 0, 'attach_node rejected while n3 lacks the lolor extension'); +like($out, qr/does not have the lolor extension installed/, + 'rejection message asks for CREATE EXTENSION lolor'); + +# --- Negative: n3 has lolor installed but with pre-existing data ------------ + +ok(psql_ok(3, "CREATE EXTENSION lolor"), 'lolor installed on n3'); +ok(psql_ok(3, "SET lolor.node=3; SELECT lo_from_bytea(0, '\\x0bad0bad')"), + 'pre-existing large object created on n3'); + +($rc, $out) = psql_capture(3, $attach_node_sql); +ok($rc != 0, 'attach_node rejected while n3 has pre-existing lolor data'); +like($out, qr/pre-existing large object data/, + 'rejection message mentions pre-existing large object data'); + +# --- Positive: empty lolor tables, attach_node copies the data -------------- + +ok(psql_ok(3, "DELETE FROM lolor.pg_largeobject; DELETE FROM lolor.pg_largeobject_metadata"), + 'pre-existing lolor data cleared on n3'); + +($rc, $out) = psql_capture(3, $attach_node_sql); +is($rc, 0, 'attach_node succeeded with lolor installed and empty') or diag($out); + +ok(wait_for_scalar(3, "SELECT count(*) FROM lolor.pg_largeobject WHERE encode(data, 'hex') = 'deadbeefcafe'", '1'), + 'existing large object data copied to n3 by initial sync'); +ok(wait_for_scalar(3, "SELECT count(*) FROM lolor.pg_largeobject_metadata", '1'), + 'large object metadata copied to n3'); + +# --- Streaming: a large object created after the join reaches n3 ------------ + +ok(psql_ok(1, "SET lolor.node=1; SELECT lo_from_bytea(0, '\\xfeedface')"), + 'second large object created on n1 after join'); +ok(wait_for_scalar(3, "SELECT count(*) FROM lolor.pg_largeobject WHERE encode(data, 'hex') = 'feedface'", '1'), + 'post-join large object streamed to n3'); + +# --- Outbound: a large object created on n3 must reach n1 and n2 ------------ +# +# Everything above tests the inbound direction, which rides on n1's +# replication sets. Outbound rides on n3's own sets, and nothing in the +# replication machinery copies set membership to a joining node. What +# populates the sets during a normal join is AutoDDL firing while pg_restore +# replays the structure dump, and that never sees the lolor tables: they are +# left out of the dump (reserved_object.exclude_from_dump), so they arrive via +# CREATE EXTENSION instead, and autoddl_can_proceed() returns false while +# creating_extension is set. So the lolor tables reach n3's default set only +# if attach_node mirrors the source node's sets onto the new node. + +for my $tbl ('pg_largeobject', 'pg_largeobject_metadata') { + is(scalar_query(3, + "SELECT set_name FROM spock.tables " . + "WHERE nspname = 'lolor' AND relname = '$tbl'"), + 'default', + "lolor.$tbl is in n3 default replication set after the join"); +} + +ok(psql_ok(3, "SET lolor.node=3; SELECT lo_from_bytea(0, '\\xc0ffee')"), + 'large object created on n3 after the join'); + +for my $node (1, 2) { + ok(wait_for_scalar($node, + "SELECT count(*) FROM lolor.pg_largeobject WHERE encode(data, 'hex') = 'c0ffee'", '1'), + "large object created on n3 replicated to n$node"); + ok(wait_for_scalar($node, + "SELECT count(*) FROM lolor.pg_largeobject_metadata m " . + "JOIN lolor.pg_largeobject l ON l.loid = m.oid " . + "WHERE encode(l.data, 'hex') = 'c0ffee'", '1'), + "metadata for the n3 large object replicated to n$node"); +} + +destroy_cluster('Destroy zodan lolor test cluster'); + +done_testing(); diff --git a/utils/pgindent/typedefs.list b/utils/pgindent/typedefs.list index fbcd665c4..d2d1c76b9 100644 --- a/utils/pgindent/typedefs.list +++ b/utils/pgindent/typedefs.list @@ -4517,6 +4517,9 @@ yy_size_t YYLTYPE yyscan_t YYSTYPE +ZNode +ZodanAddCtx +ZPair z_stream z_streamp ZSTD_CCtx