hostsharing/hs.hsadmin.ng
6
0
Fork 0
Code Pull Requests 4 Activity

historic-view #92

Merged
hsh-michaelhoennig merged 19 commits from historic-view into master 2024-08-29 17:00:20 +02:00
Conversation 0 Commits 19 Files Changed 42 +400 -198
5 changed files with 400 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 36c21505a3 - Show all commits

37
.run/ImportHostingAssets into local.run.xml Normal file
View File

@ -0,0 +1,37 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="ImportHostingAssets into local" type="GradleRunConfiguration" factoryName="Gradle">
<ExternalSystemSettings>
<option name="env">
<map>
<entry key="HSADMINNG_POSTGRES_ADMIN_PASSWORD" value="password" />
<entry key="HSADMINNG_POSTGRES_ADMIN_USERNAME" value="postgres" />
<entry key="HSADMINNG_POSTGRES_JDBC_URL" value="jdbc:postgresql://localhost:5432/postgres" />
<entry key="HSADMINNG_POSTGRES_RESTRICTED_USERNAME" value="restricted" />
</map>
</option>
<option name="executionName" />
<option name="externalProjectPath" value="$PROJECT_DIR$" />
<option name="externalSystemIdString" value="GRADLE" />
<option name="scriptParameters" value="" />
<option name="taskDescriptions">
<list />
</option>
<option name="taskNames">
<list>
<option value=":importHostingAssets" />
<option value="--tests" />
<option value="&quot;net.hostsharing.hsadminng.hs.migration.ImportHostingAssets&quot;" />
</list>
</option>
<option name="vmOptions" />
</ExternalSystemSettings>
<ExternalSystemDebugServerProcess>false</ExternalSystemDebugServerProcess>
<ExternalSystemReattachDebugProcess>true</ExternalSystemReattachDebugProcess>
<EXTENSION ID="com.intellij.execution.ExternalSystemRunConfigurationJavaExtension">
<extension name="coverage" sample_coverage="false" />
</EXTENSION>
<DebugAllEnabled>false</DebugAllEnabled>
<RunAsTest>true</RunAsTest>
<method v="2" />
</configuration>
</component>

33
.run/ImportHostingAssets.run.xml
View File

@ -33,37 +33,4 @@
<RunAsTest>true</RunAsTest>
<method v="2" />
</configuration>
<configuration default="false" name="ImportHostingAssets" type="GradleRunConfiguration" factoryName="Gradle">
<ExternalSystemSettings>
<option name="env">
<map>
<entry key="HSADMINNG_POSTGRES_ADMIN_USERNAME" value="admin" />
<entry key="HSADMINNG_POSTGRES_RESTRICTED_USERNAME" value="restricted" />
</map>
</option>
<option name="executionName" />
<option name="externalProjectPath" value="$PROJECT_DIR$" />
<option name="externalSystemIdString" value="GRADLE" />
<option name="scriptParameters" value="" />
<option name="taskDescriptions">
<list />
</option>
<option name="taskNames">
<list>
<option value=":importHostingAssets" />
<option value="--tests" />
<option value="&quot;net.hostsharing.hsadminng.hs.migration.ImportHostingAssets&quot;" />
</list>
</option>
<option name="vmOptions" />
</ExternalSystemSettings>
<ExternalSystemDebugServerProcess>false</ExternalSystemDebugServerProcess>
<ExternalSystemReattachDebugProcess>true</ExternalSystemReattachDebugProcess>
<EXTENSION ID="com.intellij.execution.ExternalSystemRunConfigurationJavaExtension">
<extension name="coverage" sample_coverage="false" />
</EXTENSION>
<DebugAllEnabled>false</DebugAllEnabled>
<RunAsTest>true</RunAsTest>
<method v="2" />
</configuration>
</component>

53
sql/examples.sql
View File

@ -1,53 +0,0 @@
-- ========================================================
-- First Example Entity with History
-- --------------------------------------------------------
CREATE TABLE IF NOT EXISTS customer (
"id" SERIAL PRIMARY KEY,
"reference" int not null unique, -- 10000-99999
"prefix" character(3) unique
);
CALL create_historicization('customer');
-- ========================================================
-- Second Example Entity with History
-- --------------------------------------------------------
CREATE TABLE IF NOT EXISTS package_type (
"id" serial PRIMARY KEY,
"name" character varying(8)
);
CALL create_historicization('package_type');
-- ========================================================
-- Third Example Entity with History
-- --------------------------------------------------------
CREATE TABLE IF NOT EXISTS package (
"id" serial PRIMARY KEY,
"name" character varying(5),
"customer_id" INTEGER REFERENCES customer(id)
);
CALL create_historicization('package');
-- ========================================================
-- query historical data
-- --------------------------------------------------------
ABORT;
BEGIN TRANSACTION;
SET LOCAL hsadminng.currentUser TO 'mih42_customer_aaa';
SET LOCAL hsadminng.currentTask TO 'adding customer_aaa';
INSERT INTO package (customer_id, name) VALUES (10000, 'aaa00');
COMMIT;
-- Usage:
SET hsadminng.timestamp TO '2022-07-12 08:53:27.723315';
SET hsadminng.timestamp TO '2022-07-12 11:38:27.723315';
SELECT * FROM customer_hv p WHERE prefix = 'aaa';

174
sql/historization-ng.sql Normal file
View File

@ -0,0 +1,174 @@
-- ========================================================
-- Historization
-- --------------------------------------------------------
drop PROCEDURE if exists tx_create_historical_view;
drop procedure if exists tx_create_historicization;
drop view if exists hs_hosting_asset_hv;
drop table if exists hs_hosting_asset_ex;
drop trigger if exists hs_hosting_asset_historicize_tg on hs_hosting_asset;
drop procedure if exists tx_historicize_tf;
drop type "tx_operation";
CREATE TYPE "tx_operation" AS ENUM ('INSERT', 'UPDATE', 'DELETE');
CREATE OR REPLACE FUNCTION tx_historicize_tf()
RETURNS trigger
LANGUAGE plpgsql STRICT AS $$
DECLARE
currentTask VARCHAR(127);
curContextId bigint;
"row" RECORD;
"alive" BOOLEAN;
"sql" varchar;
BEGIN
-- determine task
currentTask = current_setting('hsadminng.currentTask');
assert currentTask IS NOT NULL AND length(currentTask) >= 12,
format('hsadminng.currentTask (%s) must be defined and min 12 characters long, please use "SET LOCAL ...;"', currentTask);
assert length(currentTask) <= 127,
format('hsadminng.currentTask (%s) must not be longer than 127 characters"', currentTask);
IF (TG_OP = 'INSERT') OR (TG_OP = 'UPDATE') THEN
"row" := NEW;
"alive" := TRUE;
ELSE -- DELETE or TRUNCATE
"row" := OLD;
"alive" := FALSE;
END IF;
curContextId := txid_current()+bigIntHash(currentTask);
sql := format('INSERT INTO %3$I_ex ' ||
' VALUES (DEFAULT, curContextId, %1$L, %2$L, $1.*)', TG_OP, alive, TG_TABLE_NAME);
RAISE NOTICE 'sql: %', sql;
EXECUTE sql USING "row";
RETURN "row";
END; $$;
-- ------
CREATE OR REPLACE PROCEDURE tx_create_historical_view(baseTable varchar)
LANGUAGE plpgsql AS $$
DECLARE
createTriggerSQL varchar;
viewName varchar;
versionsTable varchar;
createViewSQL varchar;
baseCols varchar;
BEGIN
viewName = quote_ident(format('%s_hv', baseTable));
versionsTable = quote_ident(format('%s_ex', baseTable));
baseCols = (SELECT string_agg(quote_ident(column_name), ', ')
FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = baseTable);
createViewSQL = format(
'CREATE OR REPLACE VIEW %1$s AS' ||
'(' ||
' SELECT %2$s' ||
' FROM %3$s' ||
' WHERE alive = TRUE' ||
' AND version_id IN' ||
' (' ||
' SELECT max(vt.version_id) AS history_id' ||
' FROM %3$s AS vt' ||
' JOIN tx_history as txh ON vt.tx_id = txh.tx_id' ||
' WHERE txh.tx_timestamp <= current_setting(''hsadminng.historical_timestamp'')::timestamp' ||
' GROUP BY id' ||
' )' ||
')',
viewName, baseCols, versionsTable
);
RAISE NOTICE 'sql: %', createViewSQL;
EXECUTE createViewSQL;
createTriggerSQL = 'CREATE TRIGGER ' || baseTable || '_historicize_tg' ||
' AFTER INSERT OR DELETE OR UPDATE ON ' || baseTable ||
' FOR EACH ROW EXECUTE PROCEDURE tx_historicize_tf()';
RAISE NOTICE 'sql: %', createTriggerSQL;
EXECUTE createTriggerSQL;
END; $$;
-- ------
CREATE OR REPLACE PROCEDURE tx_create_historicization(baseTable varchar)
LANGUAGE plpgsql AS $$
DECLARE
createHistTableSql varchar;
createTriggerSQL varchar;
viewName varchar;
versionsTable varchar;
createViewSQL varchar;
baseCols varchar;
BEGIN
-- create the history table
createHistTableSql = '' ||
'CREATE TABLE ' || baseTable || '_ex (' ||
' version_id serial PRIMARY KEY,' ||
' context_id bigint NOT NULL REFERENCES tx_context(contextid),' ||
' trigger_op tx_operation NOT NULL,' ||
' alive boolean not null,' ||
-- followed by all columns of the original table
' LIKE ' || baseTable ||
' EXCLUDING CONSTRAINTS' ||
' EXCLUDING STATISTICS' ||
')';
RAISE NOTICE 'sql: %', createHistTableSql;
EXECUTE createHistTableSql;
-- create the historical view
viewName = quote_ident(format('%s_hv', baseTable));
versionsTable = quote_ident(format('%s_ex', baseTable));
baseCols = (SELECT string_agg(quote_ident(column_name), ', ')
FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = baseTable);
createViewSQL = format(
'CREATE OR REPLACE VIEW %1$s AS' ||
'(' ||
' SELECT %2$s' ||
' FROM %3$s' ||
' WHERE alive = TRUE' ||
' AND version_id IN' ||
' (' ||
' SELECT max(vt.version_id) AS history_id' ||
' FROM %3$s AS vt' ||
' JOIN tx_context as txc ON vt.context_id = txc.contextid' ||
' WHERE txc.txtimestamp <= current_setting(''hsadminng.historical_timestamp'')::timestamp' ||
' GROUP BY context_id' ||
' )' ||
')',
viewName, baseCols, versionsTable
);
RAISE NOTICE 'sql: %', createViewSQL;
EXECUTE createViewSQL;
createTriggerSQL = 'CREATE TRIGGER ' || baseTable || '_historicize_tg' ||
' AFTER INSERT OR DELETE OR UPDATE ON ' || baseTable ||
' FOR EACH ROW EXECUTE PROCEDURE tx_historicize_tf()';
RAISE NOTICE 'sql: %', createTriggerSQL;
EXECUTE createTriggerSQL;
END; $$;
call tx_create_historicization('hs_hosting_asset');
ROLLBACK;
BEGIN TRANSACTION;
call defineContext('historization testing', null, 'superuser-alex@hostsharing.net',
'hs_booking_project#D-1000000-hshdefaultproject:ADMIN');
-- 'hs_booking_project#D-1000300-mihdefaultproject:ADMIN');
update hs_hosting_asset set caption='lug00 a' where identifier='lug00' and type='MANAGED_WEBSPACE';
COMMIT;
SET hsadminng.historical_timestamp TO '2022-07-12 11:38:27.723315';
SELECT * FROM hs_hosting_asset_ex p WHERE identifier = 'lug00';

227
sql/historization.sql
View File

@ -1,80 +1,95 @@
rollback;
drop table if exists hs_hosting_asset_versions;
drop table if exists hs_hosting_asset_ex;
drop view if exists hs_hosting_asset_hv;
drop procedure if exists tx_create_historicization;
drop view if exists tx_create_historical_view;
drop function if exists tx_historicize_tf();
drop table if exists tx_history;
drop type if exists tx_operation;
-- ========================================================
-- Historization
-- --------------------------------------------------------
CREATE TABLE "tx_history" (
"tx_id" BIGINT NOT NULL UNIQUE,
"tx_timestamp" TIMESTAMP NOT NULL,
"user" VARCHAR(64) NOT NULL, -- references postgres user
"task" VARCHAR NOT NULL
create table "tx_history"
(
"tx_id" bigint not null unique,
"tx_timestamp" timestamp not null,
"user" varchar(64) not null, -- references postgres user
"task" varchar not null
);
CREATE TYPE "operation" AS ENUM ('INSERT', 'UPDATE', 'DELETE', 'TRUNCATE');
create type "tx_operation" as enum ('INSERT', 'UPDATE', 'DELETE', 'TRUNCATE');
-- see https://www.postgresql.org/docs/current/plpgsql-trigger.html
CREATE OR REPLACE FUNCTION historicize()
RETURNS trigger
LANGUAGE plpgsql STRICT AS $$
DECLARE
currentUser VARCHAR(63);
currentTask VARCHAR(127);
"row" RECORD;
"alive" BOOLEAN;
create or replace function tx_historicize_tf()
returns trigger
language plpgsql
strict as $$
declare
currentUser varchar(63);
currentTask varchar(127);
"row" record;
"alive" boolean;
"sql" varchar;
BEGIN
begin
-- determine user_id
BEGIN
begin
currentUser := current_setting('hsadminng.currentUser');
EXCEPTION WHEN OTHERS THEN
currentUser := NULL;
END;
IF (currentUser IS NULL OR currentUser = '') THEN
RAISE EXCEPTION 'hsadminng.currentUser must be defined, please use "SET LOCAL ...;"';
END IF;
RAISE NOTICE 'currentUser: %', currentUser;
exception
when others then
currentUser := null;
end;
if (currentUser is null or currentUser = '') then
raise exception 'hsadminng.currentUser must be defined, please use "SET LOCAL ...;"';
end if;
raise notice 'currentUser: %', currentUser;
-- determine task
currentTask = current_setting('hsadminng.currentTask');
assert currentTask IS NOT NULL AND length(currentTask) >= 12,
format('hsadminng.currentTask (%s) must be defined and min 12 characters long, please use "SET LOCAL ...;"', currentTask);
assert currentTask is not null and length(currentTask) >= 12,
format('hsadminng.currentTask (%s) must be defined and min 12 characters long, please use "SET LOCAL ...;"',
currentTask);
assert length(currentTask) <= 127,
format('hsadminng.currentTask (%s) must not be longer than 127 characters"', currentTask);
IF (TG_OP = 'INSERT') OR (TG_OP = 'UPDATE') THEN
if (TG_OP = 'INSERT') or (TG_OP = 'UPDATE') then
"row" := NEW;
"alive" := TRUE;
ELSE -- DELETE or TRUNCATE
"alive" := true;
else -- DELETE or TRUNCATE
"row" := OLD;
"alive" := FALSE;
END IF;
"alive" := false;
end if;
sql := format('INSERT INTO tx_history VALUES (txid_current(), now(), %1L, %2L) ON CONFLICT DO NOTHING', currentUser, currentTask);
RAISE NOTICE 'sql: %', sql;
EXECUTE sql;
sql := format('INSERT INTO %3$I_versions VALUES (DEFAULT, txid_current(), %1$L, %2$L, $1.*)', TG_OP, alive, TG_TABLE_NAME);
RAISE NOTICE 'sql: %', sql;
EXECUTE sql USING "row";
sql := format('INSERT INTO tx_history VALUES (txid_current(), now(), %1L, %2L) ON CONFLICT DO NOTHING', currentUser,
currentTask);
raise notice 'sql: %', sql;
execute sql;
sql := format('INSERT INTO %3$I_ex VALUES (DEFAULT, txid_current(), %1$L, %2$L, $1.*)', TG_OP, alive, TG_TABLE_NAME);
raise notice 'sql: %', sql;
execute sql using "row";
RETURN "row";
END; $$;
return "row";
end; $$;
CREATE OR REPLACE PROCEDURE create_historical_view(baseTable varchar)
LANGUAGE plpgsql AS $$
DECLARE
createTriggerSQL varchar;
create or replace procedure tx_create_historical_view(baseTable varchar)
language plpgsql as $$
declare
createTriggerSQL varchar;
viewName varchar;
versionsTable varchar;
createViewSQL varchar;
baseCols varchar;
BEGIN
begin
viewName = quote_ident(format('%s_hv', baseTable));
versionsTable = quote_ident(format('%s_versions', baseTable));
baseCols = (SELECT string_agg(quote_ident(column_name), ', ')
FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = baseTable);
versionsTable = quote_ident(format('%s_ex', baseTable));
baseCols = (select string_agg(quote_ident(column_name), ', ')
from information_schema.columns
where table_schema = 'public'
and table_name = baseTable);
createViewSQL = format(
'CREATE OR REPLACE VIEW %1$s AS' ||
@ -88,54 +103,54 @@ BEGIN
' FROM %3$s AS vt' ||
' JOIN tx_history as txh ON vt.tx_id = txh.tx_id' ||
' WHERE txh.tx_timestamp <= current_setting(''hsadminng.timestamp'')::timestamp' ||
' GROUP BY id' ||
' GROUP BY uuid' ||
' )' ||
')',
viewName, baseCols, versionsTable
);
RAISE NOTICE 'sql: %', createViewSQL;
EXECUTE createViewSQL;
raise notice 'sql: %', createViewSQL;
execute createViewSQL;
createTriggerSQL = 'CREATE TRIGGER ' || baseTable || '_historicize' ||
createTriggerSQL = 'CREATE TRIGGER ' || baseTable || '_tx_historicize_tf' ||
' AFTER INSERT OR DELETE OR UPDATE ON ' || baseTable ||
' FOR EACH ROW EXECUTE PROCEDURE historicize()';
RAISE NOTICE 'sql: %', createTriggerSQL;
EXECUTE createTriggerSQL;
' FOR EACH ROW EXECUTE PROCEDURE tx_historicize_tf()';
raise notice 'sql: %', createTriggerSQL;
execute createTriggerSQL;
END; $$;
end; $$;
CREATE OR REPLACE PROCEDURE create_historicization(baseTable varchar)
LANGUAGE plpgsql AS $$
DECLARE
create or replace procedure tx_create_historicization(baseTable varchar)
language plpgsql as $$
declare
createHistTableSql varchar;
createTriggerSQL varchar;
viewName varchar;
versionsTable varchar;
createViewSQL varchar;
baseCols varchar;
BEGIN
begin
-- create the history table
createHistTableSql = '' ||
'CREATE TABLE ' || baseTable || '_versions (' ||
'CREATE TABLE ' || baseTable || '_ex (' ||
' version_id serial PRIMARY KEY,' ||
' tx_id bigint NOT NULL REFERENCES tx_history(tx_id),' ||
' trigger_op operation NOT NULL,' ||
' trigger_op tx_operation NOT NULL,' ||
' alive boolean not null,' ||
' LIKE ' || baseTable ||
' EXCLUDING CONSTRAINTS' ||
' EXCLUDING STATISTICS' ||
')';
RAISE NOTICE 'sql: %', createHistTableSql;
EXECUTE createHistTableSql;
raise notice 'sql: %', createHistTableSql;
execute createHistTableSql;
-- create the historical view
viewName = quote_ident(format('%s_hv', baseTable));
versionsTable = quote_ident(format('%s_versions', baseTable));
baseCols = (SELECT string_agg(quote_ident(column_name), ', ')
FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = baseTable);
versionsTable = quote_ident(format('%s_ex', baseTable));
baseCols = (select string_agg(quote_ident(column_name), ', ')
from information_schema.columns
where table_schema = 'public'
and table_name = baseTable);
createViewSQL = format(
'CREATE OR REPLACE VIEW %1$s AS' ||
@ -154,13 +169,75 @@ BEGIN
')',
viewName, baseCols, versionsTable
);
RAISE NOTICE 'sql: %', createViewSQL;
EXECUTE createViewSQL;
raise notice 'sql: %', createViewSQL;
execute createViewSQL;
createTriggerSQL = 'CREATE TRIGGER ' || baseTable || '_historicize' ||
createTriggerSQL = 'CREATE TRIGGER ' || baseTable || '_tx_historicize_tf' ||
' AFTER INSERT OR DELETE OR UPDATE ON ' || baseTable ||
' FOR EACH ROW EXECUTE PROCEDURE historicize()';
RAISE NOTICE 'sql: %', createTriggerSQL;
EXECUTE createTriggerSQL;
' FOR EACH ROW EXECUTE PROCEDURE tx_historicize_tf()';
raise notice 'sql: %', createTriggerSQL;
execute createTriggerSQL;
END; $$;
end; $$;
--- ==================================================
call tx_create_historicization('hs_hosting_asset');
-- and expanded:
create table hs_hosting_asset_ex
(
version_id serial primary key,
tx_id bigint not null references tx_history (tx_id),
trigger_op tx_operation not null,
alive boolean not null,
like hs_hosting_asset excluding constraints excluding statistics
);
create or replace view hs_hosting_asset_hv as
(
select uuid,
version,
bookingitemuuid,
type,
parentassetuuid,
assignedtoassetuuid,
identifier,
caption,
config,
alarmcontactuuid
from hs_hosting_asset_ex
where alive = true
and version_id in (select max(vt.version_id) as history_id
from hs_hosting_asset_ex as vt
join tx_history as txh on vt.tx_id = txh.tx_id
where txh.tx_timestamp <= current_setting('hsadminng.timestamp')::timestamp
group by uuid)
);
CREATE TRIGGER hs_hosting_asset_tx_historicize_tf
AFTER INSERT OR DELETE OR UPDATE ON hs_hosting_asset
FOR EACH ROW EXECUTE PROCEDURE tx_historicize_tf();
-- ===========================================================================================
rollback;
begin transaction;
call defineContext('historization testing', null, 'superuser-alex@hostsharing.net',
-- 'hs_booking_project#D-1000000-hshdefaultproject:ADMIN'); -- prod+test
-- 'hs_booking_project#D-1000300-mihdefaultproject:ADMIN'); -- prod
'hs_booking_project#D-1000300-mimdefaultproject:ADMIN'); -- test
-- update hs_hosting_asset set caption='lug00 b' where identifier = 'lug00' and type = 'MANAGED_WEBSPACE'; -- prod
update hs_hosting_asset set caption='mim00 d' where identifier = 'mim00' and type = 'MANAGED_WEBSPACE'; -- test
commit;
-- all versions
select txh.tx_timestamp, txh."user", txh.task, ha.*
from hs_hosting_asset_ex ha
join tx_history txh on ha.tx_id=txh.tx_id
where ha.identifier in ('lug00', 'mim00');
-- single version at point in time
set hsadminng.timestamp to '2024-08-27 04:15:00'; -- UTC
select uuid, version, identifier, caption from hs_hosting_asset_hv p where identifier in ('lug00', 'mim00');