From 83e3ed142e66dabe0743a5898933924f1c10342c Mon Sep 17 00:00:00 2001 From: Timotheus Pokorra Date: Thu, 30 May 2024 16:12:50 +0200 Subject: [PATCH 1/5] workarounds for office data import --- .../hs/office/migration/ImportOfficeData.java | 41 ++++++++++++++++--- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportOfficeData.java b/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportOfficeData.java index 5d2b85c6..e2e5823f 100644 --- a/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportOfficeData.java +++ b/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportOfficeData.java @@ -134,6 +134,17 @@ public class ImportOfficeData extends ContextBasedTest { static int relationId = 2000000; + private static final List IGNORE_BUSINESS_PARTNERS = Arrays.asList( + 512167, // 11139, partner without contractual contact + 512170, // 11142, partner without contractual contact + -1 + ); + + private static final List IGNORE_CONTACTS = Arrays.asList( + 90547, // Kontakt hat keine Rolle + -1 + ); + @Value("${spring.datasource.url}") private String jdbcUrl; @@ -171,7 +182,7 @@ public class ImportOfficeData extends ContextBasedTest { @Order(1010) void importBusinessPartners() { - try (Reader reader = resourceReader("migration/business-partners.csv")) { + try (Reader reader = resourceReader("migration.data/business-partners.csv")) { final var lines = readAllLines(reader); importBusinessPartners(justHeader(lines), withoutHeader(lines)); } catch (Exception e) { @@ -217,7 +228,7 @@ public class ImportOfficeData extends ContextBasedTest { @Order(1020) void importContacts() { - try (Reader reader = resourceReader("migration/contacts.csv")) { + try (Reader reader = resourceReader("migration.data/contacts.csv")) { final var lines = readAllLines(reader); importContacts(justHeader(lines), withoutHeader(lines)); } catch (Exception e) { @@ -317,7 +328,7 @@ public class ImportOfficeData extends ContextBasedTest { @Order(1030) void importSepaMandates() { - try (Reader reader = resourceReader("migration/sepa-mandates.csv")) { + try (Reader reader = resourceReader("migration.data/sepa-mandates.csv")) { final var lines = readAllLines(reader); importSepaMandates(justHeader(lines), withoutHeader(lines)); } catch (Exception e) { @@ -349,7 +360,7 @@ public class ImportOfficeData extends ContextBasedTest { @Test @Order(1040) void importCoopShares() { - try (Reader reader = resourceReader("migration/share-transactions.csv")) { + try (Reader reader = resourceReader("migration.data/share-transactions.csv")) { final var lines = readAllLines(reader); importCoopShares(justHeader(lines), withoutHeader(lines)); } catch (Exception e) { @@ -376,7 +387,7 @@ public class ImportOfficeData extends ContextBasedTest { @Order(1050) void importCoopAssets() { - try (Reader reader = resourceReader("migration/asset-transactions.csv")) { + try (Reader reader = resourceReader("migration.data/asset-transactions.csv")) { final var lines = readAllLines(reader); importCoopAssets(justHeader(lines), withoutHeader(lines)); } catch (Exception e) { @@ -736,6 +747,10 @@ public class ImportOfficeData extends ContextBasedTest { .map(this::trimAll) .map(row -> new Record(columns, row)) .forEach(rec -> { + if (this.IGNORE_BUSINESS_PARTNERS.contains(rec.getInteger("bp_id"))) { + return; + } + final var person = HsOfficePersonEntity.builder().build(); final var partnerRel = addRelation( @@ -837,6 +852,11 @@ public class ImportOfficeData extends ContextBasedTest { .map(row -> new Record(columns, row)) .forEach(rec -> { final var bpId = rec.getInteger("bp_id"); + + if (this.IGNORE_BUSINESS_PARTNERS.contains(bpId)) { + return; + } + final var member = ofNullable(memberships.get(bpId)) .orElseGet(() -> createOnDemandMembership(bpId)); @@ -907,6 +927,10 @@ public class ImportOfficeData extends ContextBasedTest { .forEach(rec -> { final var debitor = debitors.get(rec.getInteger("bp_id")); + if (this.IGNORE_BUSINESS_PARTNERS.contains(rec.getInteger("bp_id"))) { + return; + } + final var sepaMandate = HsOfficeSepaMandateEntity.builder() .debitor(debitor) .bankAccount(HsOfficeBankAccountEntity.builder() @@ -938,6 +962,13 @@ public class ImportOfficeData extends ContextBasedTest { final var contactId = rec.getInteger("contact_id"); final var bpId = rec.getInteger("bp_id"); + if (this.IGNORE_CONTACTS.contains(contactId)) { + return; + } + if (this.IGNORE_BUSINESS_PARTNERS.contains(bpId)) { + return; + } + if (rec.getString("roles").isBlank()) { fail("empty roles assignment not allowed for contact_id: " + contactId); } -- 2.39.5 From ca7dbd651ecb2c7037d8ae5d57275834830816aa Mon Sep 17 00:00:00 2001 From: Timotheus Pokorra Date: Fri, 12 Jul 2024 06:59:14 +0200 Subject: [PATCH 2/5] dump hosting assets from legacy hsadmin --- src/test/resources/migration/dump.sh | 50 ++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/test/resources/migration/dump.sh b/src/test/resources/migration/dump.sh index e5183164..061bcdfa 100644 --- a/src/test/resources/migration/dump.sh +++ b/src/test/resources/migration/dump.sh @@ -6,6 +6,10 @@ dbname="hsh02_hsdb" username="hsh02_hsdb_readonly" target="/tmp" +if [ ! -z $DEST ]; +then + target=$DEST +fi dump() { sql="copy ($1) to stdout with csv header delimiter ';' quote '\"'" @@ -41,3 +45,49 @@ dump "select member_share_id, bp_id, date, action, quantity, comment WHERE bp_id NOT IN (511912) order by member_share_id" \ "share-transactions.csv" + +dump "select inet_addr_id, inet_addr, description + from inet_addr + order by inet_addr_id" \ + "inet_addr.csv" + +dump "select hive_id, hive_name, inet_addr_id, description + from hive + order by hive_id" \ + "hive.csv" + +dump "select packet_id, basepacket_code, packet_name, bp_id, hive_id, created, cancelled, cur_inet_addr_id, old_inet_addr_id, free + from packet + left join basepacket using (basepacket_id) + order by packet_id" \ + "packet.csv" + +dump "select packet_component_id, packet_id, quantity, basecomponent_code, created, cancelled + from packet_component + left join basecomponent using (basecomponent_id) + order by packet_component_id" \ + "packet_component.csv" + +dump "select unixuser_id, name, comment, shell, homedir, locked, packet_id, userid, quota_softlimit, quota_hardlimit, storage_softlimit, storage_hardlimit + from unixuser + order by unixuser_id" \ + "unixuser.csv" + +# weil das fehlt, muss group by komplett gesetzt werden: alter table domain add constraint PK_domain primary key (domain_id); +dump "select domain_id, domain_name, domain_since, domain_dns_master, domain_owner, valid_subdomain_names, passenger_python, passenger_nodejs, passenger_ruby, fcgi_php_bin, array_to_string(array_agg(domain_option_name), ',') as domainoptions + from domain + left join domain__domain_option using(domain_id) + left join domain_option using (domain_option_id) + group by domain.domain_id, domain.domain_name, domain_since, domain_dns_master, domain_owner, valid_subdomain_names, passenger_python, passenger_nodejs, passenger_ruby, fcgi_php_bin + order by domain.domain_id" \ + "domain.csv" + +dump "select emailaddr_id, domain_id, localpart, subdomain, target + from emailaddr + order by emailaddr_id" \ + "emailaddr.csv" + +dump "select emailalias_id, pac_id, name, target + from emailalias + order by emailalias_id" \ + "emailalias.csv" -- 2.39.5 From 94bda1df8a1122dc05ff43cf0a7cce2e8563a162 Mon Sep 17 00:00:00 2001 From: Timotheus Pokorra Date: Fri, 12 Jul 2024 08:34:27 +0200 Subject: [PATCH 3/5] Migration: also dump database and database user tables --- src/test/resources/migration/dump.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/test/resources/migration/dump.sh b/src/test/resources/migration/dump.sh index 061bcdfa..29318b89 100644 --- a/src/test/resources/migration/dump.sh +++ b/src/test/resources/migration/dump.sh @@ -91,3 +91,13 @@ dump "select emailalias_id, pac_id, name, target from emailalias order by emailalias_id" \ "emailalias.csv" + +dump "select dbuser_id, engine, packet_id, name + from database_user + order by dbuser_id" \ + "database_user.csv" + +dump "select database_id, engine, packet_id, name, owner, encoding + from database + order by database_id" \ + "database.csv" -- 2.39.5 From 9e77cefabfcbc1603b4378d7be5105304bd43b45 Mon Sep 17 00:00:00 2001 From: Michael Hoennig Date: Tue, 16 Jul 2024 07:17:12 +0200 Subject: [PATCH 4/5] read migration data path from env var HSADMINNG_MIGRATION_DATA_PATH --- .../hs/office/migration/ImportOfficeData.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportOfficeData.java b/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportOfficeData.java index c2ea4efe..9ecedc07 100644 --- a/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportOfficeData.java +++ b/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportOfficeData.java @@ -131,6 +131,7 @@ public class ImportOfficeData extends ContextBasedTest { // at least as the number of lines in business-partners.csv from test-data, but less than real data partner count public static final int MAX_NUMBER_OF_TEST_DATA_PARTNERS = 100; + public static final String MIGRATION_DATA_PATH = ofNullable(System.getenv("HSADMINNG_MIGRATION_DATA_PATH")).orElse("migration") + "/"; static int relationId = 2000000; @@ -182,7 +183,7 @@ public class ImportOfficeData extends ContextBasedTest { @Order(1010) void importBusinessPartners() { - try (Reader reader = resourceReader("migration.data/business-partners.csv")) { + try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "business-partners.csv")) { final var lines = readAllLines(reader); importBusinessPartners(justHeader(lines), withoutHeader(lines)); } catch (Exception e) { @@ -228,7 +229,7 @@ public class ImportOfficeData extends ContextBasedTest { @Order(1020) void importContacts() { - try (Reader reader = resourceReader("migration.data/contacts.csv")) { + try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "contacts.csv")) { final var lines = readAllLines(reader); importContacts(justHeader(lines), withoutHeader(lines)); } catch (Exception e) { @@ -328,7 +329,7 @@ public class ImportOfficeData extends ContextBasedTest { @Order(1030) void importSepaMandates() { - try (Reader reader = resourceReader("migration.data/sepa-mandates.csv")) { + try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "sepa-mandates.csv")) { final var lines = readAllLines(reader); importSepaMandates(justHeader(lines), withoutHeader(lines)); } catch (Exception e) { @@ -360,7 +361,7 @@ public class ImportOfficeData extends ContextBasedTest { @Test @Order(1040) void importCoopShares() { - try (Reader reader = resourceReader("migration.data/share-transactions.csv")) { + try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "share-transactions.csv")) { final var lines = readAllLines(reader); importCoopShares(justHeader(lines), withoutHeader(lines)); } catch (Exception e) { @@ -387,7 +388,7 @@ public class ImportOfficeData extends ContextBasedTest { @Order(1050) void importCoopAssets() { - try (Reader reader = resourceReader("migration.data/asset-transactions.csv")) { + try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "asset-transactions.csv")) { final var lines = readAllLines(reader); importCoopAssets(justHeader(lines), withoutHeader(lines)); } catch (Exception e) { -- 2.39.5 From df56e6def08808aee649c6e70a8bb8a8c9ec2d89 Mon Sep 17 00:00:00 2001 From: Michael Hoennig Date: Tue, 16 Jul 2024 10:04:00 +0200 Subject: [PATCH 5/5] amend text formatting to changed config formatting in prod code and extract .tc-environment --- .aliases | 15 +++----- .tc-environment | 5 +++ .../hs/office/migration/ImportOfficeData.java | 36 +++++++------------ 3 files changed, 21 insertions(+), 35 deletions(-) create mode 100644 .tc-environment diff --git a/.aliases b/.aliases index cb78c781..991f34de 100644 --- a/.aliases +++ b/.aliases @@ -1,9 +1,6 @@ -# For using the alias import-office-tables, # copy these exports to .environment (ignored by git) -# and amend them according to your external DB: -export HSADMINNG_POSTGRES_JDBC_URL=jdbc:tc:postgresql:15.5-bookworm:///spring_boot_testcontainers -export HSADMINNG_POSTGRES_ADMIN_USERNAME=admin -export HSADMINNG_POSTGRES_ADMIN_PASSWORD= -export HSADMINNG_POSTGRES_RESTRICTED_USERNAME=restricted +# For using the alias import-office-tables, +# copy the file .tc-environment to .environment (ignored by git) +# and amend them according to your external DB. gradleWrapper () { if [ ! -f gradlew ]; then @@ -46,11 +43,7 @@ postgresAutodoc () { alias postgres-autodoc=postgresAutodoc function importOfficeData() { - export HSADMINNG_POSTGRES_JDBC_URL=jdbc:tc:postgresql:15.5-bookworm:///spring_boot_testcontainers - export HSADMINNG_POSTGRES_ADMIN_USERNAME=admin - export HSADMINNG_POSTGRES_ADMIN_PASSWORD=password - export HSADMINNG_POSTGRES_RESTRICTED_USERNAME=restricted - export HSADMINNG_SUPERUSER=superuser-alex@hostsharing.net + source .tc-environment if [ -f .environment ]; then source .environment diff --git a/.tc-environment b/.tc-environment new file mode 100644 index 00000000..5c7b8d42 --- /dev/null +++ b/.tc-environment @@ -0,0 +1,5 @@ +export HSADMINNG_POSTGRES_JDBC_URL=jdbc:tc:postgresql:15.5-bookworm:///spring_boot_testcontainers +export HSADMINNG_POSTGRES_ADMIN_USERNAME=admin +export HSADMINNG_POSTGRES_ADMIN_PASSWORD= +export HSADMINNG_POSTGRES_RESTRICTED_USERNAME=restricted +export HSADMINNG_MIGRATION_DATA_PATH=migration diff --git a/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportOfficeData.java b/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportOfficeData.java index 9ecedc07..52188e79 100644 --- a/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportOfficeData.java +++ b/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportOfficeData.java @@ -92,13 +92,7 @@ import static org.assertj.core.api.Fail.fail; -- maybe something like that is needed for the 2nd user -- GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public to hsh99_restricted; - * Then copy this to a file named .environment (excluded from git) and fill in your specific values: - - export HSADMINNG_POSTGRES_JDBC_URL=jdbc:postgresql://localhost:6432/hsh99_hsadminng - export HSADMINNG_POSTGRES_ADMIN_USERNAME=hsh99_admin - export HSADMINNG_POSTGRES_ADMIN_PASSWORD=password - export HSADMINNG_POSTGRES_RESTRICTED_USERNAME=hsh99_restricted - export HSADMINNG_SUPERUSER=some-precreated-superuser@example.org + * Then copy the file .tc-environment to a file named .environment (excluded from git) and fill in your specific values. * To finally import the office data, run: * @@ -253,16 +247,16 @@ public class ImportOfficeData extends ContextBasedTest { """); assertThat(toFormattedString(contacts)).isEqualToIgnoringWhitespace(""" { - 1101=contact(caption='Herr Michael Mellies ', emailAddresses='{ main: mih@example.org }'), - 1200=contact(caption='JM e.K.', emailAddresses='{ main: jm-ex-partner@example.org }'), - 1201=contact(caption='Frau Dr. Jenny Meyer-Billing , JM GmbH', emailAddresses='{ main: jm-billing@example.org }'), - 1202=contact(caption='Herr Andrew Meyer-Operation , JM GmbH', emailAddresses='{ main: am-operation@example.org }'), - 1203=contact(caption='Herr Philip Meyer-Contract , JM GmbH', emailAddresses='{ main: pm-partner@example.org }'), - 1204=contact(caption='Frau Tammy Meyer-VIP , JM GmbH', emailAddresses='{ main: tm-vip@example.org }'), - 1301=contact(caption='Petra Schmidt , Test PS', emailAddresses='{ main: ps@example.com }'), - 1401=contact(caption='Frau Frauke Fanninga ', emailAddresses='{ main: ff@example.org }'), - 1501=contact(caption='Frau Cecilia Camus ', emailAddresses='{ main: cc@example.org }') - } + 1101=contact(caption='Herr Michael Mellies ', emailAddresses='{ "main": "mih@example.org"}'), + 1200=contact(caption='JM e.K.', emailAddresses='{ "main": "jm-ex-partner@example.org"}'), + 1201=contact(caption='Frau Dr. Jenny Meyer-Billing , JM GmbH', emailAddresses='{ "main": "jm-billing@example.org"}'), + 1202=contact(caption='Herr Andrew Meyer-Operation , JM GmbH', emailAddresses='{ "main": "am-operation@example.org"}'), + 1203=contact(caption='Herr Philip Meyer-Contract , JM GmbH', emailAddresses='{ "main": "pm-partner@example.org"}'), + 1204=contact(caption='Frau Tammy Meyer-VIP , JM GmbH', emailAddresses='{ "main": "tm-vip@example.org"}'), + 1301=contact(caption='Petra Schmidt , Test PS', emailAddresses='{ "main": "ps@example.com"}'), + 1401=contact(caption='Frau Frauke Fanninga ', emailAddresses='{ "main": "ff@example.org"}'), + 1501=contact(caption='Frau Cecilia Camus ', emailAddresses='{ "main": "cc@example.org"}') + } """); assertThat(toFormattedString(persons)).isEqualToIgnoringWhitespace(""" { @@ -1141,6 +1135,7 @@ public class ImportOfficeData extends ContextBasedTest { return "{\n" + map.keySet().stream() .map(id -> " " + id + "=" + map.get(id).toString()) + .map(e -> e.replaceAll("\n ", " ").replace("\n", "")) .collect(Collectors.joining(",\n")) + "\n}\n"; } @@ -1228,13 +1223,6 @@ public class ImportOfficeData extends ContextBasedTest { return new InputStreamReader(requireNonNull(getClass().getClassLoader().getResourceAsStream(resourcePath))); } - private Reader fileReader(@NotNull final Path filePath) throws IOException { - // Path path = Paths.get( - // ClassLoader.getSystemResource("csv/twoColumn.csv").toURI()) - // ); - return Files.newBufferedReader(filePath); - } - private static String[] justHeader(final List lines) { return stream(lines.getFirst()).map(String::trim).toArray(String[]::new); } -- 2.39.5