From 3d00696985101ede861c304666e0df5b31a8dff6 Mon Sep 17 00:00:00 2001 From: Michael Hoennig Date: Thu, 1 Aug 2024 19:03:30 +0200 Subject: [PATCH] importing database-users and -databases and creating database-instances (WIP) --- .../hsadminng/hash/HashGenerator.java | 10 + .../hs/hosting/asset/HsHostingAssetType.java | 1 + ...sMariaDbDatabaseHostingAssetValidator.java | 4 +- .../HsMariaDbUserHostingAssetValidator.java | 4 +- ...stgreSqlDatabaseHostingAssetValidator.java | 4 +- ...HsPostgreSqlUserHostingAssetValidator.java | 12 +- .../hs/validation/HsEntityValidator.java | 14 +- .../hs/validation/PasswordProperty.java | 6 +- .../hs/validation/StringProperty.java | 8 + .../changelog/9-hs-global/9000-statistics.sql | 22 ++ .../db/changelog/db.changelog-master.yaml | 2 + ...DatabaseHostingAssetValidatorUnitTest.java | 8 +- ...iaDbUserHostingAssetValidatorUnitTest.java | 10 +- ...DatabaseHostingAssetValidatorUnitTest.java | 14 +- ...eSqlUserHostingAssetValidatorUnitTest.java | 10 +- .../hsadminng/hs/migration/CsvDataImport.java | 33 ++- .../hs/migration/ImportHostingAssets.java | 197 +++++++++++++++++- .../resources/migration/hosting/database.csv | 23 ++ .../migration/hosting/database_user.csv | 15 ++ 19 files changed, 362 insertions(+), 35 deletions(-) create mode 100644 src/main/resources/db/changelog/9-hs-global/9000-statistics.sql create mode 100644 src/test/resources/migration/hosting/database.csv create mode 100644 src/test/resources/migration/hosting/database_user.csv diff --git a/src/main/java/net/hostsharing/hsadminng/hash/HashGenerator.java b/src/main/java/net/hostsharing/hsadminng/hash/HashGenerator.java index 5bc09cc6..a577f3ce 100644 --- a/src/main/java/net/hostsharing/hsadminng/hash/HashGenerator.java +++ b/src/main/java/net/hostsharing/hsadminng/hash/HashGenerator.java @@ -59,6 +59,10 @@ public final class HashGenerator { this.algorithm = algorithm; } + public boolean couldBeHash(final String value) { + return value.startsWith(algorithm.prefix); + } + public String hash(final String plaintextPassword) { if (plaintextPassword == null) { throw new IllegalStateException("no password given"); @@ -67,6 +71,12 @@ public final class HashGenerator { return algorithm.implementation.apply(this, plaintextPassword); } + public String hashIfNotYetHashed(final String plaintextPasswordOrHash) { + return couldBeHash(plaintextPasswordOrHash) + ? plaintextPasswordOrHash + : hash(plaintextPasswordOrHash); + } + public static void nextSalt(final String salt) { predefinedSalts.add(salt); } diff --git a/src/main/java/net/hostsharing/hsadminng/hs/hosting/asset/HsHostingAssetType.java b/src/main/java/net/hostsharing/hsadminng/hs/hosting/asset/HsHostingAssetType.java index 4209a05e..f08248c4 100644 --- a/src/main/java/net/hostsharing/hsadminng/hs/hosting/asset/HsHostingAssetType.java +++ b/src/main/java/net/hostsharing/hsadminng/hs/hosting/asset/HsHostingAssetType.java @@ -50,6 +50,7 @@ public enum HsHostingAssetType implements Node { inGroup("Webspace"), requiredParent(MANAGED_WEBSPACE)), + // TODO.spec: do we really want to keep email aliases or migrate to unix users with .forward? EMAIL_ALIAS( // named e.g. xyz00-abc inGroup("Webspace"), requiredParent(MANAGED_WEBSPACE)), diff --git a/src/main/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsMariaDbDatabaseHostingAssetValidator.java b/src/main/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsMariaDbDatabaseHostingAssetValidator.java index 48618be3..823308ed 100644 --- a/src/main/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsMariaDbDatabaseHostingAssetValidator.java +++ b/src/main/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsMariaDbDatabaseHostingAssetValidator.java @@ -9,6 +9,8 @@ import static net.hostsharing.hsadminng.hs.validation.StringProperty.stringPrope class HsMariaDbDatabaseHostingAssetValidator extends HostingAssetEntityValidator { + final static String HEAD_REGEXP = "^MAD\\|"; + public HsMariaDbDatabaseHostingAssetValidator() { super( MARIADB_DATABASE, @@ -20,6 +22,6 @@ class HsMariaDbDatabaseHostingAssetValidator extends HostingAssetEntityValidator @Override protected Pattern identifierPattern(final HsHostingAsset assetEntity) { final var webspaceIdentifier = assetEntity.getParentAsset().getParentAsset().getIdentifier(); - return Pattern.compile("^"+webspaceIdentifier+"$|^"+webspaceIdentifier+"_[a-z0-9_]+$"); + return Pattern.compile(HEAD_REGEXP+webspaceIdentifier+"$|"+HEAD_REGEXP+webspaceIdentifier+"_[a-zA-Z0-9_]+$"); } } diff --git a/src/main/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsMariaDbUserHostingAssetValidator.java b/src/main/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsMariaDbUserHostingAssetValidator.java index 15ae0b45..58a33520 100644 --- a/src/main/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsMariaDbUserHostingAssetValidator.java +++ b/src/main/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsMariaDbUserHostingAssetValidator.java @@ -10,6 +10,8 @@ import static net.hostsharing.hsadminng.hs.validation.PasswordProperty.passwordP class HsMariaDbUserHostingAssetValidator extends HostingAssetEntityValidator { + final static String HEAD_REGEXP = "^MAU\\|"; + public HsMariaDbUserHostingAssetValidator() { super( MARIADB_USER, @@ -28,6 +30,6 @@ class HsMariaDbUserHostingAssetValidator extends HostingAssetEntityValidator { @Override protected Pattern identifierPattern(final HsHostingAsset assetEntity) { final var webspaceIdentifier = assetEntity.getParentAsset().getIdentifier(); - return Pattern.compile("^"+webspaceIdentifier+"$|^"+webspaceIdentifier+"_[a-z0-9_]+$"); + return Pattern.compile(HEAD_REGEXP+webspaceIdentifier+"$|"+HEAD_REGEXP+webspaceIdentifier+"_[a-zA-Z0-9_]+$"); } } diff --git a/src/main/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsPostgreSqlDatabaseHostingAssetValidator.java b/src/main/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsPostgreSqlDatabaseHostingAssetValidator.java index 57d302d0..830b2fbf 100644 --- a/src/main/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsPostgreSqlDatabaseHostingAssetValidator.java +++ b/src/main/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsPostgreSqlDatabaseHostingAssetValidator.java @@ -9,6 +9,8 @@ import static net.hostsharing.hsadminng.hs.validation.StringProperty.stringPrope class HsPostgreSqlDatabaseHostingAssetValidator extends HostingAssetEntityValidator { + final static String HEAD_REGEXP = "^PGD\\|"; + public HsPostgreSqlDatabaseHostingAssetValidator() { super( PGSQL_DATABASE, @@ -23,6 +25,6 @@ class HsPostgreSqlDatabaseHostingAssetValidator extends HostingAssetEntityValida @Override protected Pattern identifierPattern(final HsHostingAsset assetEntity) { final var webspaceIdentifier = assetEntity.getParentAsset().getParentAsset().getIdentifier(); - return Pattern.compile("^"+webspaceIdentifier+"$|^"+webspaceIdentifier+"_[a-z0-9_]+$"); + return Pattern.compile(HEAD_REGEXP+webspaceIdentifier+"$|"+HEAD_REGEXP+webspaceIdentifier+"_[a-zA-Z0-9_]+$"); } } diff --git a/src/main/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsPostgreSqlUserHostingAssetValidator.java b/src/main/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsPostgreSqlUserHostingAssetValidator.java index 7d527892..736648ff 100644 --- a/src/main/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsPostgreSqlUserHostingAssetValidator.java +++ b/src/main/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsPostgreSqlUserHostingAssetValidator.java @@ -3,6 +3,7 @@ package net.hostsharing.hsadminng.hs.hosting.asset.validators; import net.hostsharing.hsadminng.hash.HashGenerator; import net.hostsharing.hsadminng.hs.hosting.asset.HsHostingAsset; +import java.util.List; import java.util.regex.Pattern; import static net.hostsharing.hsadminng.hs.hosting.asset.HsHostingAssetType.PGSQL_USER; @@ -10,6 +11,8 @@ import static net.hostsharing.hsadminng.hs.validation.PasswordProperty.passwordP class HsPostgreSqlUserHostingAssetValidator extends HostingAssetEntityValidator { + final static String HEAD_REGEXP = "^PGU\\|"; + public HsPostgreSqlUserHostingAssetValidator() { super( PGSQL_USER, @@ -25,9 +28,16 @@ class HsPostgreSqlUserHostingAssetValidator extends HostingAssetEntityValidator passwordProperty("password").minLength(8).maxLength(40).hashedUsing(HashGenerator.Algorithm.SCRAM_SHA256).writeOnly()); } + // FIXME: remove method + @Override + public List validateEntity(final HsHostingAsset assetEntity) { + final var result = super.validateEntity(assetEntity); + return result; + } + @Override protected Pattern identifierPattern(final HsHostingAsset assetEntity) { final var webspaceIdentifier = assetEntity.getParentAsset().getIdentifier(); - return Pattern.compile("^"+webspaceIdentifier+"$|^"+webspaceIdentifier+"_[a-z0-9_]+$"); + return Pattern.compile(HEAD_REGEXP+webspaceIdentifier+"$|"+HEAD_REGEXP+webspaceIdentifier+"_[a-zA-Z0-9_]+$"); } } diff --git a/src/main/java/net/hostsharing/hsadminng/hs/validation/HsEntityValidator.java b/src/main/java/net/hostsharing/hsadminng/hs/validation/HsEntityValidator.java index 77cc2514..0ba440d2 100644 --- a/src/main/java/net/hostsharing/hsadminng/hs/validation/HsEntityValidator.java +++ b/src/main/java/net/hostsharing/hsadminng/hs/validation/HsEntityValidator.java @@ -5,6 +5,7 @@ package net.hostsharing.hsadminng.hs.validation; import jakarta.persistence.EntityManager; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -62,7 +63,18 @@ public abstract class HsEntityValidator { } protected ArrayList validateProperties(final PropertiesProvider propsProvider) { - final var result = new ArrayList(); + final var result = new ArrayList() { + + @Override + public boolean add(final String s) { + return super.add(s); + } + + @Override + public boolean addAll(final Collection c) { + return super.addAll(c); + } + }; // verify that all actually given properties are specified final var properties = propsProvider.directProps(); diff --git a/src/main/java/net/hostsharing/hsadminng/hs/validation/PasswordProperty.java b/src/main/java/net/hostsharing/hsadminng/hs/validation/PasswordProperty.java index 083e69ca..fa23b1d2 100644 --- a/src/main/java/net/hostsharing/hsadminng/hs/validation/PasswordProperty.java +++ b/src/main/java/net/hostsharing/hsadminng/hs/validation/PasswordProperty.java @@ -31,6 +31,10 @@ public class PasswordProperty extends StringProperty { @Override protected void validate(final List result, final String propValue, final PropertiesProvider propProvider) { + if (HashGenerator.using(hashedUsing).couldBeHash(propValue) && propValue.length() > this.maxLength()) { + // already hashed => do not validate + return; + } super.validate(result, propValue, propProvider); validatePassword(result, propValue); } @@ -40,7 +44,7 @@ public class PasswordProperty extends StringProperty { computedBy( ComputeMode.IN_PREP, (em, entity) -> ofNullable(entity.getDirectValue(propertyName, String.class)) - .map(password -> HashGenerator.using(algorithm).withRandomSalt().hash(password)) + .map(password -> HashGenerator.using(algorithm).withRandomSalt().hashIfNotYetHashed(password)) .orElse(null)); return self(); } diff --git a/src/main/java/net/hostsharing/hsadminng/hs/validation/StringProperty.java b/src/main/java/net/hostsharing/hsadminng/hs/validation/StringProperty.java index aa804916..7870ca87 100644 --- a/src/main/java/net/hostsharing/hsadminng/hs/validation/StringProperty.java +++ b/src/main/java/net/hostsharing/hsadminng/hs/validation/StringProperty.java @@ -41,11 +41,19 @@ public class StringProperty

> extends ValidatableProp return self(); } + public Integer minLength() { + return this.minLength; + } + public P maxLength(final int maxLength) { this.maxLength = maxLength; return self(); } + public Integer maxLength() { + return this.maxLength; + } + public P matchesRegEx(final String... regExPattern) { this.matchesRegEx = stream(regExPattern).map(Pattern::compile).toArray(Pattern[]::new); return self(); diff --git a/src/main/resources/db/changelog/9-hs-global/9000-statistics.sql b/src/main/resources/db/changelog/9-hs-global/9000-statistics.sql new file mode 100644 index 00000000..f4c841e9 --- /dev/null +++ b/src/main/resources/db/changelog/9-hs-global/9000-statistics.sql @@ -0,0 +1,22 @@ +--liquibase formatted sql + +-- ============================================================================ +--changeset hs-global-object-statistics:1 endDelimiter:--// +-- ---------------------------------------------------------------------------- +select * + from (select count, "table" as "rbac-table", '' as "hs-table", '' as "type" + from rbacstatisticsview + union all + select to_char(count(*)::int, '9 999 999 999') as "count", 'objects' as "rbac-table", objecttable as "hs-table", '' as "type" + from rbacobject + group by objecttable + union all + select to_char(count(*)::int, '9 999 999 999'), 'objects', 'hs_hosting_asset', type::text + from hs_hosting_asset + group by type + union all + select to_char(count(*)::int, '9 999 999 999'), 'objects', 'hs_booking_item', type::text + from hs_booking_item + group by type + ) as totals order by replace(count, ' ', '')::int desc; +--// diff --git a/src/main/resources/db/changelog/db.changelog-master.yaml b/src/main/resources/db/changelog/db.changelog-master.yaml index a9c6711d..8771ae81 100644 --- a/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/src/main/resources/db/changelog/db.changelog-master.yaml @@ -151,3 +151,5 @@ databaseChangeLog: file: db/changelog/7-hs-hosting/701-hosting-asset/7013-hs-hosting-asset-rbac.sql - include: file: db/changelog/7-hs-hosting/701-hosting-asset/7018-hs-hosting-asset-test-data.sql + - include: + file: db/changelog/9-hs-global/9000-statistics.sql diff --git a/src/test/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsMariaDbDatabaseHostingAssetValidatorUnitTest.java b/src/test/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsMariaDbDatabaseHostingAssetValidatorUnitTest.java index 37c8fb85..7e7c8b5b 100644 --- a/src/test/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsMariaDbDatabaseHostingAssetValidatorUnitTest.java +++ b/src/test/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsMariaDbDatabaseHostingAssetValidatorUnitTest.java @@ -40,7 +40,7 @@ class HsMariaDbDatabaseHostingAssetValidatorUnitTest { return HsHostingAssetEntity.builder() .type(MARIADB_DATABASE) .parentAsset(GIVEN_MARIADB_USER) - .identifier("xyz00_temp") + .identifier("MAD|xyz00_temp") .caption("some valid test MariaDB-Database") .config(new HashMap<>(ofEntries( entry("encoding", "latin1") @@ -93,8 +93,8 @@ class HsMariaDbDatabaseHostingAssetValidatorUnitTest { // then assertThat(result).containsExactlyInAnyOrder( - "'MARIADB_DATABASE:xyz00_temp.config.unknown' is not expected but is set to 'wrong'", - "'MARIADB_DATABASE:xyz00_temp.config.encoding' is expected to be of type String, but is of type Integer" + "'MARIADB_DATABASE:MAD|xyz00_temp.config.unknown' is not expected but is set to 'wrong'", + "'MARIADB_DATABASE:MAD|xyz00_temp.config.encoding' is expected to be of type String, but is of type Integer" ); } @@ -111,6 +111,6 @@ class HsMariaDbDatabaseHostingAssetValidatorUnitTest { // then assertThat(result).containsExactly( - "'identifier' expected to match '^xyz00$|^xyz00_[a-z0-9_]+$', but is 'xyz99-temp'"); + "'identifier' expected to match '^MAD\\|xyz00$|^MAD\\|xyz00_[a-zA-Z0-9_]+$', but is 'xyz99-temp'"); } } diff --git a/src/test/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsMariaDbUserHostingAssetValidatorUnitTest.java b/src/test/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsMariaDbUserHostingAssetValidatorUnitTest.java index 97c8429b..70b823c8 100644 --- a/src/test/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsMariaDbUserHostingAssetValidatorUnitTest.java +++ b/src/test/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsMariaDbUserHostingAssetValidatorUnitTest.java @@ -32,7 +32,7 @@ class HsMariaDbUserHostingAssetValidatorUnitTest { .type(MARIADB_USER) .parentAsset(TEST_MANAGED_WEBSPACE_HOSTING_ASSET) .assignedToAsset(GIVEN_MARIADB_INSTANCE) - .identifier("xyz00_temp") + .identifier("MAU|xyz00_temp") .caption("some valid test MariaDB-User") .config(new HashMap<>(ofEntries( entry("password", "Test1234") @@ -101,9 +101,9 @@ class HsMariaDbUserHostingAssetValidatorUnitTest { // then assertThat(result).containsExactlyInAnyOrder( - "'MARIADB_USER:xyz00_temp.config.unknown' is not expected but is set to '100'", - "'MARIADB_USER:xyz00_temp.config.password' length is expected to be at min 8 but length of provided value is 5", - "'MARIADB_USER:xyz00_temp.config.password' must contain at least one character of at least 3 of the following groups: upper case letters, lower case letters, digits, special characters" + "'MARIADB_USER:MAU|xyz00_temp.config.unknown' is not expected but is set to '100'", + "'MARIADB_USER:MAU|xyz00_temp.config.password' length is expected to be at min 8 but length of provided value is 5", + "'MARIADB_USER:MAU|xyz00_temp.config.password' must contain at least one character of at least 3 of the following groups: upper case letters, lower case letters, digits, special characters" ); } @@ -120,6 +120,6 @@ class HsMariaDbUserHostingAssetValidatorUnitTest { // then assertThat(result).containsExactly( - "'identifier' expected to match '^xyz00$|^xyz00_[a-z0-9_]+$', but is 'xyz99-temp'"); + "'identifier' expected to match '^MAU\\|xyz00$|^MAU\\|xyz00_[a-zA-Z0-9_]+$', but is 'xyz99-temp'"); } } diff --git a/src/test/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsPostgreSqlDatabaseHostingAssetValidatorUnitTest.java b/src/test/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsPostgreSqlDatabaseHostingAssetValidatorUnitTest.java index 35780466..78a59288 100644 --- a/src/test/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsPostgreSqlDatabaseHostingAssetValidatorUnitTest.java +++ b/src/test/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsPostgreSqlDatabaseHostingAssetValidatorUnitTest.java @@ -42,7 +42,7 @@ class HsPostgreSqlDatabaseHostingAssetValidatorUnitTest { return HsHostingAssetEntity.builder() .type(PGSQL_DATABASE) .parentAsset(GIVEN_PGSQL_USER) - .identifier("xyz00_db") + .identifier("PGD|xyz00_db") .caption("some valid test PgSql-Database") .config(new HashMap<>(ofEntries( entry("encoding", "LATIN1") @@ -94,9 +94,9 @@ class HsPostgreSqlDatabaseHostingAssetValidatorUnitTest { // then assertThat(result).containsExactlyInAnyOrder( - "'PGSQL_DATABASE:xyz00_db.bookingItem' must be null but is of type CLOUD_SERVER", - "'PGSQL_DATABASE:xyz00_db.parentAsset' must be of type PGSQL_USER but is of type PGSQL_INSTANCE", - "'PGSQL_DATABASE:xyz00_db.assignedToAsset' must be null but is of type PGSQL_INSTANCE" + "'PGSQL_DATABASE:PGD|xyz00_db.bookingItem' must be null but is of type CLOUD_SERVER", + "'PGSQL_DATABASE:PGD|xyz00_db.parentAsset' must be of type PGSQL_USER but is of type PGSQL_INSTANCE", + "'PGSQL_DATABASE:PGD|xyz00_db.assignedToAsset' must be null but is of type PGSQL_INSTANCE" ); } @@ -116,8 +116,8 @@ class HsPostgreSqlDatabaseHostingAssetValidatorUnitTest { // then assertThat(result).containsExactlyInAnyOrder( - "'PGSQL_DATABASE:xyz00_db.config.unknown' is not expected but is set to 'wrong'", - "'PGSQL_DATABASE:xyz00_db.config.encoding' is expected to be of type String, but is of type Integer" + "'PGSQL_DATABASE:PGD|xyz00_db.config.unknown' is not expected but is set to 'wrong'", + "'PGSQL_DATABASE:PGD|xyz00_db.config.encoding' is expected to be of type String, but is of type Integer" ); } @@ -134,6 +134,6 @@ class HsPostgreSqlDatabaseHostingAssetValidatorUnitTest { // then assertThat(result).containsExactly( - "'identifier' expected to match '^xyz00$|^xyz00_[a-z0-9_]+$', but is 'xyz99-temp'"); + "'identifier' expected to match '^PGD\\|xyz00$|^PGD\\|xyz00_[a-zA-Z0-9_]+$', but is 'xyz99-temp'"); } } diff --git a/src/test/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsPostgreSqlUserHostingAssetValidatorUnitTest.java b/src/test/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsPostgreSqlUserHostingAssetValidatorUnitTest.java index 588631c2..bb589a7b 100644 --- a/src/test/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsPostgreSqlUserHostingAssetValidatorUnitTest.java +++ b/src/test/java/net/hostsharing/hsadminng/hs/hosting/asset/validators/HsPostgreSqlUserHostingAssetValidatorUnitTest.java @@ -35,7 +35,7 @@ class HsPostgreSqlUserHostingAssetValidatorUnitTest { .type(PGSQL_USER) .parentAsset(TEST_MANAGED_WEBSPACE_HOSTING_ASSET) .assignedToAsset(GIVEN_PGSQL_INSTANCE) - .identifier("xyz00_temp") + .identifier("PGU|xyz00_temp") .caption("some valid test PgSql-User") .config(new HashMap<>(ofEntries( entry("password", "Test1234") @@ -104,9 +104,9 @@ class HsPostgreSqlUserHostingAssetValidatorUnitTest { // then assertThat(result).containsExactlyInAnyOrder( - "'PGSQL_USER:xyz00_temp.config.unknown' is not expected but is set to '100'", - "'PGSQL_USER:xyz00_temp.config.password' length is expected to be at min 8 but length of provided value is 5", - "'PGSQL_USER:xyz00_temp.config.password' must contain at least one character of at least 3 of the following groups: upper case letters, lower case letters, digits, special characters" + "'PGSQL_USER:PGU|xyz00_temp.config.unknown' is not expected but is set to '100'", + "'PGSQL_USER:PGU|xyz00_temp.config.password' length is expected to be at min 8 but length of provided value is 5", + "'PGSQL_USER:PGU|xyz00_temp.config.password' must contain at least one character of at least 3 of the following groups: upper case letters, lower case letters, digits, special characters" ); } @@ -123,6 +123,6 @@ class HsPostgreSqlUserHostingAssetValidatorUnitTest { // then assertThat(result).containsExactly( - "'identifier' expected to match '^xyz00$|^xyz00_[a-z0-9_]+$', but is 'xyz99-temp'"); + "'identifier' expected to match '^PGU\\|xyz00$|^PGU\\|xyz00_[a-zA-Z0-9_]+$', but is 'xyz99-temp'"); } } diff --git a/src/test/java/net/hostsharing/hsadminng/hs/migration/CsvDataImport.java b/src/test/java/net/hostsharing/hsadminng/hs/migration/CsvDataImport.java index 6405d543..70337bcb 100644 --- a/src/test/java/net/hostsharing/hsadminng/hs/migration/CsvDataImport.java +++ b/src/test/java/net/hostsharing/hsadminng/hs/migration/CsvDataImport.java @@ -48,6 +48,7 @@ import static net.hostsharing.hsadminng.mapper.Array.emptyArray; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assumptions.assumeThat; +import static org.junit.jupiter.api.Assertions.fail; public class CsvDataImport extends ContextBasedTest { @@ -281,6 +282,12 @@ public class CsvDataImport extends ContextBasedTest { }).assertSuccessful(); } + // makes it possible to fail when an expression is expected + T failWith(final String message) { + fail(message); + return null; + } + void logError(final Runnable assertion) { try { assertion.run(); @@ -290,7 +297,9 @@ public class CsvDataImport extends ContextBasedTest { } void logErrors() { - assertThat(errors).isEmpty(); + final var errorsToLog = new ArrayList<>(errors); + errors.clear(); + assertThat(errorsToLog).isEmpty(); } void expectErrors(final String... expectedErrors) { @@ -305,8 +314,16 @@ public class CsvDataImport extends ContextBasedTest { } public static void assertContainsExactlyInAnyOrderIgnoringWhitespace(final List expected, final List actual) { - final var sortedExpected = expected.stream().map(m -> m.replaceAll("\\s", "")).toList(); - final var sortedActual = actual.stream().map(m -> m.replaceAll("\\s", "")).toArray(String[]::new); + final var sortedExpected = expected.stream() + .map(m -> m.replaceAll("\\s+", " ")) + .map(m -> m.replaceAll("^ ", "")) + .map(m -> m.replaceAll(" $", "")) + .toList(); + final var sortedActual = actual.stream() + .map(m -> m.replaceAll("\\s+", " ")) + .map(m -> m.replaceAll("^ ", "")) + .map(m -> m.replaceAll(" $", "")) + .toArray(String[]::new); assertThat(sortedExpected).containsExactlyInAnyOrder(sortedActual); } @@ -324,6 +341,10 @@ class Columns { } int indexOf(final String columnName) { + return columnNames.indexOf(columnName); + } + + int indexOfOrFail(final String columnName) { int index = columnNames.indexOf(columnName); if (index < 0) { throw new RuntimeException("column name '" + columnName + "' not found in: " + columnNames); @@ -342,6 +363,12 @@ class Record { this.row = row; } + String getString(final String columnName, final String defaultValue) { + final var index = columns.indexOf(columnName); + final var value = index >= 0 && index < row.length ? row[index].trim() : null; + return value != null ? value : defaultValue; + } + String getString(final String columnName) { return row[columns.indexOf(columnName)].trim(); } diff --git a/src/test/java/net/hostsharing/hsadminng/hs/migration/ImportHostingAssets.java b/src/test/java/net/hostsharing/hsadminng/hs/migration/ImportHostingAssets.java index 3092dd85..17943479 100644 --- a/src/test/java/net/hostsharing/hsadminng/hs/migration/ImportHostingAssets.java +++ b/src/test/java/net/hostsharing/hsadminng/hs/migration/ImportHostingAssets.java @@ -1,6 +1,8 @@ package net.hostsharing.hsadminng.hs.migration; import net.hostsharing.hsadminng.context.Context; +import net.hostsharing.hsadminng.hash.HashGenerator; +import net.hostsharing.hsadminng.hash.HashGenerator.Algorithm; import net.hostsharing.hsadminng.hs.booking.debitor.HsBookingDebitorEntity; import net.hostsharing.hsadminng.hs.booking.item.HsBookingItemEntity; import net.hostsharing.hsadminng.hs.booking.item.HsBookingItemType; @@ -31,13 +33,21 @@ import java.util.function.Function; import static java.util.Arrays.stream; import static java.util.Map.entry; +import static java.util.Map.ofEntries; import static java.util.Optional.ofNullable; import static java.util.stream.Collectors.toMap; +import static java.util.stream.Collectors.toSet; import static net.hostsharing.hsadminng.hs.hosting.asset.HsHostingAssetType.CLOUD_SERVER; import static net.hostsharing.hsadminng.hs.hosting.asset.HsHostingAssetType.EMAIL_ALIAS; import static net.hostsharing.hsadminng.hs.hosting.asset.HsHostingAssetType.IPV4_NUMBER; import static net.hostsharing.hsadminng.hs.hosting.asset.HsHostingAssetType.MANAGED_SERVER; import static net.hostsharing.hsadminng.hs.hosting.asset.HsHostingAssetType.MANAGED_WEBSPACE; +import static net.hostsharing.hsadminng.hs.hosting.asset.HsHostingAssetType.MARIADB_DATABASE; +import static net.hostsharing.hsadminng.hs.hosting.asset.HsHostingAssetType.MARIADB_INSTANCE; +import static net.hostsharing.hsadminng.hs.hosting.asset.HsHostingAssetType.MARIADB_USER; +import static net.hostsharing.hsadminng.hs.hosting.asset.HsHostingAssetType.PGSQL_DATABASE; +import static net.hostsharing.hsadminng.hs.hosting.asset.HsHostingAssetType.PGSQL_INSTANCE; +import static net.hostsharing.hsadminng.hs.hosting.asset.HsHostingAssetType.PGSQL_USER; import static net.hostsharing.hsadminng.hs.hosting.asset.HsHostingAssetType.UNIX_USER; import static net.hostsharing.hsadminng.mapper.PostgresDateRange.toPostgresDateRange; import static org.assertj.core.api.Assertions.assertThat; @@ -97,6 +107,8 @@ public class ImportHostingAssets extends ImportOfficeData { static final Integer PACKET_ID_OFFSET = 3000000; static final Integer UNIXUSER_ID_OFFSET = 4000000; static final Integer EMAILALIAS_ID_OFFSET = 5000000; + static final Integer DBUSER_ID_OFFSET = 6000000; + static final Integer DB_ID_OFFSET = 7000000; record Hive(int hive_id, String hive_name, int inet_addr_id, AtomicReference serverRef) {} @@ -333,6 +345,87 @@ public class ImportHostingAssets extends ImportOfficeData { """); } + @Test + @Order(15000) + void createDatabaseInstances() { + // FIXME + } + + @Test + @Order(15009) + void verifyDatabaseINSTANCES() { + assumeThatWeAreImportingControlledTestData(); +// FIXME +// assertThat(firstOfEachType(5, PGSQL_INSTANCE, MARIADB_INSTANCE)).isEqualToIgnoringWhitespace(""" +// { +// } +// """); + } + + @Test + @Order(15010) + void importDatabaseUsers() { + try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "/hosting/database_user.csv")) { + final var lines = readAllLines(reader); + importDatabaseUsers(justHeader(lines), withoutHeader(lines)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Test + @Order(15019) + void verifyDatabaseUsers() { + assumeThatWeAreImportingControlledTestData(); + + assertThat(firstOfEachType(5, PGSQL_USER, MARIADB_USER)).isEqualToIgnoringWhitespace(""" + { + 6001858=HsHostingAssetRawEntity(PGSQL_USER, PGU|hsh00, hsh00, MANAGED_WEBSPACE:hsh00, PGSQL_INSTANCE:PGI|hsh00, { "password": "SCRAM-SHA-256$4096:Zml4ZWQgc2FsdA==$JDiZmaxU+O+ByArLY/CkYZ8HbOk0r/I8LyABnno5gQs=:NI3T500/63dzI1B07Jh3UtQGlukS6JxuS0XoxM/QgAc="}), + 6001860=HsHostingAssetRawEntity(PGSQL_USER, PGU|hsh00_hsadmin, hsh00_hsadmin, MANAGED_WEBSPACE:hsh00, PGSQL_INSTANCE:PGI|hsh00_hsadmin, { "password": "SCRAM-SHA-256$4096:Zml4ZWQgc2FsdA==$54Wh+OGx/GaIvAia+I3k78jHGhqmYwe4+iLssmH5zhk=:D4Gq1z2Li2BVSaZrz1azDrs6pwsIzhq4+suK1Hh6ZIg="}), + 6001861=HsHostingAssetRawEntity(PGSQL_USER, PGU|hsh00_hsadmin_ro, hsh00_hsadmin_ro, MANAGED_WEBSPACE:hsh00, PGSQL_INSTANCE:PGI|hsh00_hsadmin_ro, { "password": "SCRAM-SHA-256$4096:Zml4ZWQgc2FsdA==$UhJnJJhmKANbcaG+izWK3rz5bmhhluSuiCJFlUmDVI8=:6AC4mbLfJGiGlEOWhpz9BivvMODhLLHOnRnnktJPgn8="}), + 6004908=HsHostingAssetRawEntity(MARIADB_USER, MAU|hsh00_mantis, hsh00_mantis, MANAGED_WEBSPACE:hsh00, MARIADB_INSTANCE:MAI|hsh00_mantis, { "password": "*EA4C0889A22AAE66BBEBC88161E8CF862D73B44F"}), + 6004909=HsHostingAssetRawEntity(MARIADB_USER, MAU|hsh00_mantis_ro, hsh00_mantis_ro, MANAGED_WEBSPACE:hsh00, MARIADB_INSTANCE:MAI|hsh00_mantis_ro, { "password": "*B3BB6D0DA2EC01958616E9B3BCD2926FE8C38383"}), + 6004931=HsHostingAssetRawEntity(MARIADB_USER, MAU|hsh00_phpPgSqlAdmin, hsh00_phpPgSqlAdmin, MANAGED_WEBSPACE:hsh00, MARIADB_INSTANCE:MAI|hsh00_phpPgSqlAdmin, { "password": "*59067A36BA197AD0A47D74909296C5B002A0FB9F"}), + 6004932=HsHostingAssetRawEntity(MARIADB_USER, MAU|hsh00_phpMyAdmin, hsh00_phpMyAdmin, MANAGED_WEBSPACE:hsh00, MARIADB_INSTANCE:MAI|hsh00_phpMyAdmin, { "password": "*3188720B1889EF5447C722629765F296F40257C2"}), + 6007520=HsHostingAssetRawEntity(MARIADB_USER, MAU|lug00_wla, lug00_wla, MANAGED_WEBSPACE:lug00, MARIADB_INSTANCE:MAI|lug00_wla, { "password": "*11667C0EAC42BF8B0295ABEDC7D2868A835E4DB5"}), + 6007522=HsHostingAssetRawEntity(PGSQL_USER, PGU|lug00_ola, lug00_ola, MANAGED_WEBSPACE:lug00, PGSQL_INSTANCE:PGI|lug00_ola, { "password": "SCRAM-SHA-256$4096:Zml4ZWQgc2FsdA==$tir+cV3ZzOZeEWurwAJk+8qkvsTAWaBfwx846oYMOr4=:p4yk/4hHkfSMAFxSuTuh3RIrbSpHNBh7h6raVa3nt1c="}), + 6007605=HsHostingAssetRawEntity(PGSQL_USER, PGU|mim00_office, mim00_office, MANAGED_WEBSPACE:mim00, PGSQL_INSTANCE:PGI|mim00_office, { "password": "SCRAM-SHA-256$4096:Zml4ZWQgc2FsdA==$43jziwd1o+nkfjE0zFbks24Zy5GK+km87B7vzEQt4So=:xRQntZxBxdo1JJbhkegnUFKHT0T8MDW75hkQs2S3z6k="}) + } + """); + } + + @Test + @Order(15020) + void importDatabases() { + try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "/hosting/database.csv")) { + final var lines = readAllLines(reader); + importDatabases(justHeader(lines), withoutHeader(lines)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Test + @Order(15029) + void verifyDatabases() { + assumeThatWeAreImportingControlledTestData(); + + assertThat(firstOfEachType(5, PGSQL_DATABASE, MARIADB_DATABASE)).isEqualToIgnoringWhitespace(""" + { + 7000077=HsHostingAssetRawEntity(PGSQL_DATABASE, PGD|hsh00_vorstand, hsh00_vorstand, { "encoding": "LATIN1"}), + 7000786=HsHostingAssetRawEntity(MARIADB_DATABASE, MAD|hsh00_addr, hsh00_addr, { "encoding": "latin1"}), + 7000805=HsHostingAssetRawEntity(MARIADB_DATABASE, MAD|hsh00_db2, hsh00_db2, { "encoding": "latin1"}), + 7001858=HsHostingAssetRawEntity(PGSQL_DATABASE, PGD|hsh00, hsh00, { "encoding": "LATIN1"}), + 7001860=HsHostingAssetRawEntity(PGSQL_DATABASE, PGD|hsh00_hsadmin, hsh00_hsadmin, { "encoding": "UTF8"}), + 7004908=HsHostingAssetRawEntity(MARIADB_DATABASE, MAD|hsh00_mantis, hsh00_mantis, { "encoding": "utf8"}), + 7004931=HsHostingAssetRawEntity(PGSQL_DATABASE, PGD|hsh00_phpPgSqlAdmin, hsh00_phpPgSqlAdmin, { "encoding": "UTF8"}), + 7004932=HsHostingAssetRawEntity(PGSQL_DATABASE, PGD|hsh00_phpPgSqlAdmin_new, hsh00_phpPgSqlAdmin_new, { "encoding": "UTF8"}), + 7004941=HsHostingAssetRawEntity(MARIADB_DATABASE, MAD|hsh00_phpMyAdmin, hsh00_phpMyAdmin, { "encoding": "utf8"}), + 7004942=HsHostingAssetRawEntity(MARIADB_DATABASE, MAD|hsh00_phpMyAdmin_old, hsh00_phpMyAdmin_old, { "encoding": "utf8"}) + } + """); + } + // -------------------------------------------------------------------------------------------- @Test @@ -447,6 +540,30 @@ public class ImportHostingAssets extends ImportOfficeData { persistHostingAssetsOfType(EMAIL_ALIAS); } + @Test + @Order(19200) + @Commit + void persistDatabaseInstances() { + System.out.println("PERSISTING db-users to database '" + jdbcUrl + "' as user '" + postgresAdminUser + "'"); + persistHostingAssetsOfType(PGSQL_INSTANCE, MARIADB_INSTANCE); + } + + @Test + @Order(19210) + @Commit + void persistDatabaseUsers() { + System.out.println("PERSISTING db-users to database '" + jdbcUrl + "' as user '" + postgresAdminUser + "'"); + persistHostingAssetsOfType(PGSQL_USER, MARIADB_USER); + } + + @Test + @Order(19220) + @Commit + void persistDatabases() { + System.out.println("PERSISTING databases to database '" + jdbcUrl + "' as user '" + postgresAdminUser + "'"); + persistHostingAssetsOfType(PGSQL_DATABASE, MARIADB_DATABASE); + } + @Test @Order(19900) void verifyPersistedUnixUsersWithUserId() { @@ -483,7 +600,7 @@ public class ImportHostingAssets extends ImportOfficeData { @Order(19920) void verifyHostingAssetsAreActuallyPersisted() { final var haCount = (Integer) em.createNativeQuery("SELECT count(*) FROM hs_hosting_asset", Integer.class).getSingleResult(); - assertThat(haCount).isGreaterThan(isImportingControlledTestData() ? 20 : 10000); + assertThat(haCount).isGreaterThan(isImportingControlledTestData() ? 30 : 10000); } // ============================================================================================ @@ -517,11 +634,12 @@ public class ImportHostingAssets extends ImportOfficeData { // ============================================================================================ - private void persistHostingAssetsOfType(final HsHostingAssetType hsHostingAssetType) { + private void persistHostingAssetsOfType(final HsHostingAssetType... hsHostingAssetTypes) { + final var hsHostingAssetTypeSet = stream(hsHostingAssetTypes).collect(toSet()); jpaAttempt.transacted(() -> { hostingAssets.forEach((key, ha) -> { context(rbacSuperuser); - if (ha.getType() == hsHostingAssetType) { + if (hsHostingAssetTypeSet.contains(ha.getType())) { new HostingAssetEntitySaveProcessor(em, ha) .preprocessEntity() .validateEntityIgnoring("'EMAIL_ALIAS:.*\\.config\\.target' .*") @@ -750,7 +868,7 @@ public class ImportHostingAssets extends ImportOfficeData { .identifier(rec.getString("name")) .caption(rec.getString("comment")) .isLoaded(true) // avoid overwriting imported userids with generated ids - .config(new HashMap<>(Map.ofEntries( + .config(new HashMap<>(ofEntries( entry("shell", rec.getString("shell")), // entry("homedir", rec.getString("homedir")), do not import, it's calculated entry("locked", rec.getBoolean("locked")), @@ -806,7 +924,7 @@ public class ImportHostingAssets extends ImportOfficeData { .parentAsset(hostingAssets.get(PACKET_ID_OFFSET + packet_id)) .identifier(rec.getString("name")) .caption(rec.getString("name")) - .config(Map.ofEntries( + .config(ofEntries( entry("target", targets) )) .build(); @@ -814,6 +932,75 @@ public class ImportHostingAssets extends ImportOfficeData { }); } + private void importDatabaseUsers(final String[] header, final List records) { + final var columns = new Columns(header); + records.stream() + .map(this::trimAll) + .map(row -> new Record(columns, row)) + .forEach(rec -> { + final var dbuser_id = rec.getInteger("dbuser_id"); + final var packet_id = rec.getInteger("packet_id"); + final var engine = rec.getString("engine"); + final HsHostingAssetType dbUserAssetType = "mysql".equals(engine) ? MARIADB_USER + : "pgsql".equals(engine) ? PGSQL_USER + : failWith("unknown DB engine " + engine); + // FIXME: Create one instance for each managed server, not for each db-user! + final HsHostingAssetType dbInstanceAssetType = "mysql".equals(engine) ? MARIADB_INSTANCE + : "pgsql".equals(engine) ? PGSQL_INSTANCE + : failWith("unknown DB engine " + engine); + final var hash = dbUserAssetType == MARIADB_USER ? Algorithm.MYSQL_NATIVE : Algorithm.SCRAM_SHA256; + final var name = rec.getString("name"); + final var password_hash = rec.getString("password_hash", HashGenerator.using(hash).withSalt("fixed salt").hash("fake pw " + name)); + + final var dbInstanceAsset = HsHostingAssetRawEntity.builder() + .type(dbInstanceAssetType) + .parentAsset(hostingAssets.get(PACKET_ID_OFFSET + packet_id).getParentAsset()) + .identifier(dbUserAssetType.name().substring(0, 2) + "I|" + name) + .caption(name) + .build(); + + final var dbUserAsset = HsHostingAssetRawEntity.builder() + .type(dbUserAssetType) + .parentAsset(hostingAssets.get(PACKET_ID_OFFSET + packet_id)) + .assignedToAsset(dbInstanceAsset) + .identifier(dbUserAssetType.name().substring(0, 2) + "U|" + name) + .caption(name) + .config(new HashMap<>(ofEntries( + entry("password", password_hash) + ))) + .build(); + hostingAssets.put(DBUSER_ID_OFFSET + dbuser_id, dbUserAsset); + }); + } + + private void importDatabases(final String[] header, final List records) { + final var columns = new Columns(header); + records.stream() + .map(this::trimAll) + .map(row -> new Record(columns, row)) + .forEach(rec -> { + final var database_id = rec.getInteger("database_id"); + final var packet_id = rec.getInteger("packet_id"); + final var owning_dbuser_id = 0; // FIXME + final var engine = rec.getString("engine"); + final HsHostingAssetType type = "mysql".equals(engine) ? MARIADB_DATABASE + : "pgsql".equals(engine) ? PGSQL_DATABASE + : failWith("unknown DB engine " + engine); + final var name = rec.getString("name"); + final var encoding = rec.getString("encoding"); + final var dbUserAsset = HsHostingAssetRawEntity.builder() + .type(type) + .parentAsset(hostingAssets.get(DBUSER_ID_OFFSET + owning_dbuser_id)) + .identifier(type.name().substring(0, 2) + "D|" + name) + .caption(name) + .config(ofEntries( + entry("encoding", encoding) + )) + .build(); + hostingAssets.put(DB_ID_OFFSET + database_id, dbUserAsset); + }); + } + // ============================================================================================ V returning( diff --git a/src/test/resources/migration/hosting/database.csv b/src/test/resources/migration/hosting/database.csv new file mode 100644 index 00000000..d35bbda5 --- /dev/null +++ b/src/test/resources/migration/hosting/database.csv @@ -0,0 +1,23 @@ +database_id;engine;packet_id;name;owner;encoding + +77;pgsql;630;hsh00_vorstand;hsh00_vorstand;LATIN1 +786;mysql;630;hsh00_addr;hsh00;latin1 +805;mysql;630;hsh00_db2;hsh00;latin1 +291568;pgsql;1112;mih00_invoicing;mih00_invoicing;UTF8 + +1858;pgsql;630;hsh00;hsh00;LATIN1 +1860;pgsql;630;hsh00_hsadmin;hsh00_hsadmin;UTF8 + +4931;pgsql;630;hsh00_phpPgSqlAdmin;hsh00_phpPgSqlAdmin;UTF8 +4932;pgsql;630;hsh00_phpPgSqlAdmin_new;hsh00_phpPgSqlAdmin;UTF8 +4908;mysql;630;hsh00_mantis;hsh00_mantis;utf8 +4941;mysql;630;hsh00_phpMyAdmin;hsh00_phpMyAdmin;utf8 +4942;mysql;630;hsh00_phpMyAdmin_old;hsh00_phpMyAdmin;utf8 + +7520;mysql;1094;lug00_wla;lug00_wla;utf8 +7521;mysql;1094;lug00_wla_test;lug00_wla;utf8 +7522;pgsql;1094;lug00_ola;lug00_ola;UTF8 +7523;pgsql;1094;lug00_ola_Test;lug00_ola;UTF8 + +7604;mysql;1112;mim00_test;mim00_test;latin1 +7605;pgsql;1112;mim00_office;mim00_office;UTF8 diff --git a/src/test/resources/migration/hosting/database_user.csv b/src/test/resources/migration/hosting/database_user.csv new file mode 100644 index 00000000..5581740f --- /dev/null +++ b/src/test/resources/migration/hosting/database_user.csv @@ -0,0 +1,15 @@ +dbuser_id;engine;packet_id;name;password_hash + +1858;pgsql;630;hsh00 +1860;pgsql;630;hsh00_hsadmin +1861;pgsql;630;hsh00_hsadmin_ro +4931;mysql;630;hsh00_phpPgSqlAdmin +4908;mysql;630;hsh00_mantis +4909;mysql;630;hsh00_mantis_ro +4932;mysql;630;hsh00_phpMyAdmin + +7520;mysql;1094;lug00_wla +7522;pgsql;1094;lug00_ola + +7604;mysql;1112;mim00_test +7605;pgsql;1112;mim00_office