diff --git a/docs/docs/postgresql/source-database.md b/docs/docs/postgresql/source-database.md index 3048adf..21b2eb9 100644 --- a/docs/docs/postgresql/source-database.md +++ b/docs/docs/postgresql/source-database.md @@ -20,42 +20,85 @@ After that, you can enable selective tracking and pick which database tables you ## Permissions +### Existing credentials + You can specify the same regular database credentials you use to connect to PostgreSQL from your code. And that's it, everything should just work! -Alternatively, you can create read-only PostgreSQL database credentials to connect to the primary instance's WAL. +### New read-only credentials + +Alternatively, you can manually create read-only PostgreSQL database credentials to connect to the primary instance's WAL. Here is an example if you use AWS RDS: ```sql -/* Read-only user */ +/* Create read-only user */ CREATE ROLE [username] WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD '[password]'; -/* To grant access to WAL */ +/* Grant replication permission for reading changes from WAL */ GRANT rds_replication TO [username]; -/* To enable selective table tracking */ +/* Grant SELECT access to tables for selective tracking */ GRANT SELECT ON ALL TABLES IN SCHEMA public TO [username]; -/* Create "bemi" PUBLICATION to enable replication with dynamic selective tracking */ +/* Create "bemi" PUBLICATION to enable logical replication */ CREATE PUBLICATION bemi FOR ALL TABLES; -/* Enable REPLICA IDENTITY FULL for each table to track "before" state on each DB row change */ -DO $$ -DECLARE - current_tablename TEXT; +/* Create a procedure to set REPLICA IDENTITY FULL for tables to track the "before" state on DB row changes */ +CREATE OR REPLACE PROCEDURE _bemi_set_replica_identity() AS $$ DECLARE current_tablename TEXT; BEGIN - FOR current_tablename IN - SELECT tablename FROM pg_tables WHERE schemaname = 'public' - LOOP + FOR current_tablename IN SELECT tablename FROM pg_tables WHERE schemaname = 'public' LOOP EXECUTE format('ALTER TABLE %I REPLICA IDENTITY FULL', current_tablename); END LOOP; -END; -$$; +END $$ LANGUAGE plpgsql; +/* Call the created procedure */ +CALL _bemi_set_replica_identity(); +/* Create a trigger function that calls the created procedure */ +CREATE OR REPLACE FUNCTION _bemi_set_replica_identity_func() RETURNS event_trigger AS $$ +BEGIN CALL _bemi_set_replica_identity(); END $$ LANGUAGE plpgsql; +/* Create a trigger to set REPLICA IDENTITY FULL for all new created tables */ +CREATE EVENT TRIGGER _bemi_set_replica_identity_trigger ON ddl_command_end WHEN TAG IN ('CREATE TABLE') +EXECUTE FUNCTION _bemi_set_replica_identity_func(); ``` These commands are safe to execute without any downtime or performance issues: -* `CREATE PUBLICATION` creates a "channel" that we'll subscribe to to track changes made in database tables. +* `CREATE ROLE` creates a new read-only user for Bemi to read database changes. +* `CREATE PUBLICATION` creates a "channel" that we'll subscribe to and track changes in real-time. * `REPLICA IDENTITY FULL` enhances records stored in WAL to record the previous state (“before”) in addition to the tracked by default new state (“after”), transaction ID, position, timestamp, database name, table name, etc. +### New read-only credentials with manually managed permissions for each table + +If you want to isolate read access only to logical replication for certain tables and manage permissions manually instead of relying on our robust built-in selective tracking manageable through our UI, +you can run the following commands: + +```sql +/* Create read-only user */ +CREATE ROLE [username] WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION PASSWORD '[password]'; +/* Grant replication permission to allow using replication slots */ +GRANT rds_replication TO [username]; +/* Revoke all read access leaving only replication permission */ +REVOKE SELECT ON ALL TABLES IN SCHEMA public FROM [username]; + +/* Create "bemi" PUBLICATION to enable logical replication for selected tables */ +CREATE PUBLICATION bemi FOR TABLE [table1], [table2]; + +/* Set REPLICA IDENTITY FULL for tables to track the "before" state on DB row changes */ +ALTER TABLE [table1] REPLICA IDENTITY FULL; +ALTER TABLE [table2] REPLICA IDENTITY FULL; +``` + +To enable data change tracking for a new table: + +```sql +ALTER PUBLICATION bemi ADD TABLE [table3]; +ALTER TABLE [table3] REPLICA IDENTITY FULL; +``` + +To stop data change tracking for a table: + +```sql +ALTER PUBLICATION bemi DROP TABLE [table3]; +ALTER TABLE [table3] REPLICA IDENTITY DEFAULT; +``` + ## WAL Level Bemi relies on logical replication that allows ingesting changes row-by-row, unlike physical replication that sends disk block changes.