Compare commits

..

4 Commits

Author SHA1 Message Date
Michael Hoennig
8c8b82461e add type to twiddle 2024-08-27 20:36:32 +02:00
Michael Hoennig
9ea4ea41dd move historization to Liquibase 2024-08-27 20:34:39 +02:00
Michael Hoennig
38121ad3dd improve tx_history_txid checks 2024-08-27 20:13:27 +02:00
Michael Hoennig
2f198664fa extract tx_history_txid() into a CTE-query 2024-08-27 20:03:03 +02:00
5 changed files with 185 additions and 133 deletions

View File

@ -8,129 +8,9 @@ drop function if exists tx_historicize_tf();
drop type if exists tx_operation;
-- ========================================================
-- Historization
-- Historization twiddle
-- --------------------------------------------------------
-- FIXME: Liquibase-integration
create type "tx_operation" as enum ('INSERT', 'UPDATE', 'DELETE', 'TRUNCATE');
create or replace function tx_historicize_tf()
returns trigger
language plpgsql
strict as $$
declare
currentUser varchar(63);
currentTask varchar(127);
"row" record;
"alive" boolean;
"sql" varchar;
begin
-- determine user_id
begin
currentUser := current_setting('hsadminng.currentUser');
exception
when others then
currentUser := null;
end;
if (currentUser is null or currentUser = '') then
raise exception 'hsadminng.currentUser must be defined, please use "SET LOCAL ...;"';
end if;
raise notice 'currentUser: %', currentUser;
-- determine task
currentTask = current_setting('hsadminng.currentTask');
assert currentTask is not null and length(currentTask) >= 12,
format('hsadminng.currentTask (%s) must be defined and min 12 characters long, please use "SET LOCAL ...;"',
currentTask);
assert length(currentTask) <= 127,
format('hsadminng.currentTask (%s) must not be longer than 127 characters"', currentTask);
if (TG_OP = 'INSERT') or (TG_OP = 'UPDATE') then
"row" := NEW;
"alive" := true;
else -- DELETE or TRUNCATE
"row" := OLD;
"alive" := false;
end if;
sql := format('INSERT INTO %3$I_ex VALUES (DEFAULT, pg_current_xact_id(), %1$L, %2$L, $1.*)', TG_OP, alive, TG_TABLE_NAME);
raise notice 'sql: %', sql;
execute sql using "row";
return "row";
end; $$;
create or replace procedure tx_create_historicization(baseTable varchar)
language plpgsql as $$
declare
createHistTableSql varchar;
createTriggerSQL varchar;
viewName varchar;
exVersionsTable varchar;
createViewSQL varchar;
baseCols varchar;
begin
-- create the history table
createHistTableSql = '' ||
'CREATE TABLE ' || baseTable || '_ex (' ||
' version_id serial PRIMARY KEY,' ||
' txid xid8 NOT NULL REFERENCES tx_context(txid),' ||
' trigger_op tx_operation NOT NULL,' ||
' alive boolean not null,' ||
' LIKE ' || baseTable ||
' EXCLUDING CONSTRAINTS' ||
' EXCLUDING STATISTICS' ||
')';
raise notice 'sql: %', createHistTableSql;
execute createHistTableSql;
-- create the historical view
viewName = quote_ident(format('%s_hv', baseTable));
exVersionsTable = quote_ident(format('%s_ex', baseTable));
baseCols = (select string_agg(quote_ident(column_name), ', ')
from information_schema.columns
where table_schema = 'public'
and table_name = baseTable);
createViewSQL = format(
'CREATE OR REPLACE VIEW %1$s AS' ||
'(' ||
' SELECT %2$s' ||
' FROM %3$s' ||
' WHERE alive = TRUE' ||
' AND version_id IN' ||
' (' ||
' SELECT max(ex.version_id) AS history_id' ||
' FROM %3$s AS ex' ||
' JOIN tx_context as txc ON ex.txid = txc.txid' ||
' WHERE txc.txid = current_setting(''hsadminng.tx_history_txid'', true)::xid8' ||
' GROUP BY uuid' ||
' )' ||
')',
viewName, baseCols, exVersionsTable
);
raise notice 'sql: %', createViewSQL;
execute createViewSQL;
createTriggerSQL = 'CREATE TRIGGER ' || baseTable || '_tx_historicize_tg' ||
' AFTER INSERT OR DELETE OR UPDATE ON ' || baseTable ||
' FOR EACH ROW EXECUTE PROCEDURE tx_historicize_tf()';
raise notice 'sql: %', createTriggerSQL;
execute createTriggerSQL;
end; $$;
--- ==================================================
call tx_create_historicization('hs_hosting_asset'); -- FIXME: move to 7010-hosting-asset.sql
-- and expanded:
-- ===========================================================================================
rollback;
begin transaction;
call defineContext('historization testing', null, 'superuser-alex@hostsharing.net',
@ -138,19 +18,21 @@ call defineContext('historization testing', null, 'superuser-alex@hostsharing.ne
-- 'hs_booking_project#D-1000300-mihdefaultproject:ADMIN'); -- prod
-- 'hs_booking_project#D-1000300-mimdefaultproject:ADMIN'); -- test
-- update hs_hosting_asset set caption='lug00 b' where identifier = 'lug00' and type = 'MANAGED_WEBSPACE'; -- prod
update hs_hosting_asset set caption='hsh00 E ' || now()::text where identifier = 'hsh00' and type = 'MANAGED_WEBSPACE'; -- test
update hs_hosting_asset set caption='hsh00 F ' || now()::text where identifier = 'hsh00' and type = 'MANAGED_WEBSPACE'; -- test
update hs_hosting_asset set caption='hsh00 C ' || now()::text where identifier = 'hsh00' and type = 'MANAGED_WEBSPACE'; -- test
update hs_hosting_asset set caption='hsh00 D ' || now()::text where identifier = 'hsh00' and type = 'MANAGED_WEBSPACE'; -- test
commit;
-- single version at point in time
-- set hsadminng.tx_history_txid to (select max(txid) from tx_context where txtimestamp<='2024-08-27 12:13:13.450821');
SELECT set_config('hsadminng.tx_history_txid', (select max(txid)::Text from tx_context where txtimestamp<='2024-08-27 12:13:13.450821'), FALSE);
select uuid, version, identifier, caption from hs_hosting_asset_hv p where identifier in ('hsh00');
set hsadminng.tx_history_txid to '2604';
set hsadminng.tx_history_timestamp to '';
-- all versions
select tx_history_txid(), txc.txtimestamp, txc.currentUser, txc.currentTask, haex.*
from hs_hosting_asset_ex haex
join tx_context txc on haex.txid=txc.txid
where haex.identifier in ('hsh00', 'mim00');
select uuid, version, type, identifier, caption from hs_hosting_asset_hv p where identifier in ('hsh00', 'mim00');
select pg_current_xact_id();
-- all versions
select txc.txtimestamp, txc.currentUser, txc.currentTask, haex.*
from hs_hosting_asset_ex haex
join tx_context txc on haex.txid=txc.txid
where haex.identifier in ('hsh00');

View File

@ -0,0 +1,160 @@
--liquibase formatted sql
-- ============================================================================
--changeset hs-global-historization-tx-history-txid:1 endDelimiter:--//
-- ----------------------------------------------------------------------------
create or replace function tx_history_txid()
returns xid8 stable
language plpgsql as $$
declare
historicalTxIdSetting text;
historicalTimestampSetting text;
historicalTxId xid8;
historicalTimestamp timestamp;
begin
select coalesce(current_setting('hsadminng.tx_history_txid', true), '') into historicalTxIdSetting;
select coalesce(current_setting('hsadminng.tx_history_timestamp', true), '') into historicalTimestampSetting;
if historicalTxIdSetting > '' and historicalTimestampSetting > '' then
raise exception 'either hsadminng.tx_history_txid or hsadminng.tx_history_timestamp must be set, but both are set: (%, %)',
historicalTxIdSetting, historicalTimestampSetting;
end if;
if historicalTxIdSetting = '' and historicalTimestampSetting = '' then
raise exception 'either hsadminng.tx_history_txid or hsadminng.tx_history_timestamp must be set, but both are unset or empty: (%, %)',
historicalTxIdSetting, historicalTimestampSetting;
end if;
-- just for debugging / making sure the function is only called once per query
-- raise notice 'tx_history_txid() called with: (%, %)', historicalTxIdSetting, historicalTimestampSetting;
if historicalTxIdSetting is null or historicalTxIdSetting = '' then
select historicalTimestampSetting::timestamp into historicalTimestamp;
select max(txc.txid) from tx_context txc where txc.txtimestamp <= historicalTimestamp into historicalTxId;
else
historicalTxId = historicalTxIdSetting::xid8;
end if;
return historicalTxId;
end; $$;
--//
-- ============================================================================
--changeset hs-global-historization-tx-historicize-tf:1 endDelimiter:--//
-- ----------------------------------------------------------------------------
create type "tx_operation" as enum ('INSERT', 'UPDATE', 'DELETE', 'TRUNCATE');
create or replace function tx_historicize_tf()
returns trigger
language plpgsql
strict as $$
declare
currentUser varchar(63);
currentTask varchar(127);
"row" record;
"alive" boolean;
"sql" varchar;
begin
-- determine user_id
begin
currentUser := current_setting('hsadminng.currentUser');
exception
when others then
currentUser := null;
end;
if (currentUser is null or currentUser = '') then
raise exception 'hsadminng.currentUser must be defined, please use "SET LOCAL ...;"';
end if;
raise notice 'currentUser: %', currentUser;
-- determine task
currentTask = current_setting('hsadminng.currentTask');
assert currentTask is not null and length(currentTask) >= 12,
format('hsadminng.currentTask (%s) must be defined and min 12 characters long, please use "SET LOCAL ...;"',
currentTask);
assert length(currentTask) <= 127,
format('hsadminng.currentTask (%s) must not be longer than 127 characters"', currentTask);
if (TG_OP = 'INSERT') or (TG_OP = 'UPDATE') then
"row" := NEW;
"alive" := true;
else -- DELETE or TRUNCATE
"row" := OLD;
"alive" := false;
end if;
sql := format('INSERT INTO %3$I_ex VALUES (DEFAULT, pg_current_xact_id(), %1$L, %2$L, $1.*)', TG_OP, alive, TG_TABLE_NAME);
raise notice 'sql: %', sql;
execute sql using "row";
return "row";
end; $$;
--//
-- ============================================================================
--changeset hs-global-historization-tx-create-historicization:1 endDelimiter:--//
-- ----------------------------------------------------------------------------
create or replace procedure tx_create_historicization(baseTable varchar)
language plpgsql as $$
declare
createHistTableSql varchar;
createTriggerSQL varchar;
viewName varchar;
exVersionsTable varchar;
createViewSQL varchar;
baseCols varchar;
begin
-- create the history table
createHistTableSql = '' ||
'CREATE TABLE ' || baseTable || '_ex (' ||
' version_id serial PRIMARY KEY,' ||
' txid xid8 NOT NULL REFERENCES tx_context(txid),' ||
' trigger_op tx_operation NOT NULL,' ||
' alive boolean not null,' ||
' LIKE ' || baseTable ||
' EXCLUDING CONSTRAINTS' ||
' EXCLUDING STATISTICS' ||
')';
raise notice 'sql: %', createHistTableSql;
execute createHistTableSql;
-- create the historical view
viewName = quote_ident(format('%s_hv', baseTable));
exVersionsTable = quote_ident(format('%s_ex', baseTable));
baseCols = (select string_agg(quote_ident(column_name), ', ')
from information_schema.columns
where table_schema = 'public'
and table_name = baseTable);
createViewSQL = format(
'CREATE OR REPLACE VIEW %1$s AS' ||
'(' ||
-- make sure the function is only called once, not for every matching row in tx_context
' WITH txh AS (SELECT tx_history_txid() AS txid) ' ||
' SELECT %2$s' ||
' FROM %3$s' ||
' WHERE alive = TRUE' ||
' AND version_id IN' ||
' (' ||
' SELECT max(ex.version_id) AS history_id' ||
' FROM %3$s AS ex' ||
' JOIN tx_context as txc ON ex.txid = txc.txid' ||
' WHERE txc.txid <= (SELECT txid FROM txh)' ||
' GROUP BY uuid' ||
' )' ||
')',
viewName, baseCols, exVersionsTable
);
raise notice 'sql: %', createViewSQL;
execute createViewSQL;
createTriggerSQL = 'CREATE TRIGGER ' || baseTable || '_tx_historicize_tg' ||
' AFTER INSERT OR DELETE OR UPDATE ON ' || baseTable ||
' FOR EACH ROW EXECUTE PROCEDURE tx_historicize_tf()';
raise notice 'sql: %', createTriggerSQL;
execute createTriggerSQL;
end; $$;
--//

View File

@ -166,6 +166,14 @@ execute procedure hs_hosting_asset_booking_item_hierarchy_check_tf();
-- ============================================================================
--changeset hs-hosting-asset-MAIN-TABLE-JOURNAL:1 endDelimiter:--//
-- ----------------------------------------------------------------------------
call create_journal('hs_hosting_asset');
--//
-- ============================================================================
--changeset hs-hosting-asset-MAIN-TABLE-HISTORIZATION:1 endDelimiter:--//
-- ----------------------------------------------------------------------------
call tx_create_historicization('hs_hosting_asset');
--//

View File

@ -21,6 +21,8 @@ databaseChangeLog:
file: db/changelog/0-basis/010-context.sql
- include:
file: db/changelog/0-basis/020-audit-log.sql
- include:
file: db/changelog/0-basis/030-historization.sql
- include:
file: db/changelog/0-basis/090-log-slow-queries-extensions.sql
- include:
@ -152,4 +154,4 @@ databaseChangeLog:
- include:
file: db/changelog/7-hs-hosting/701-hosting-asset/7018-hs-hosting-asset-test-data.sql
- include:
file: db/changelog/9-hs-global/9000-statistics.sql
file: db/changelog/9-hs-global/9000-statistics.sql

View File

@ -249,7 +249,7 @@ public class CsvDataImport extends ContextBasedTest {
context(rbacSuperuser);
// TODO.perf: could we instead skip creating test-data based on an env var?
em.createNativeQuery("delete from hs_hosting_asset where true").executeUpdate();
// FIXME em.createNativeQuery("delete from hs_hosting_asset_ex where true").executeUpdate();
em.createNativeQuery("delete from hs_hosting_asset_ex where true").executeUpdate();
em.createNativeQuery("delete from hs_booking_item where true").executeUpdate();
em.createNativeQuery("delete from hs_booking_project where true").executeUpdate();
em.createNativeQuery("delete from hs_office_coopassetstransaction where true").executeUpdate();