From 2c24a2f593b3a1181b470aa93ff6f4d68915233c Mon Sep 17 00:00:00 2001 From: Michael Hoennig Date: Wed, 17 Jul 2024 16:48:30 +0200 Subject: [PATCH] WIP: importing office data as base for hosting assets --- .../hs/office/migration/CsvDataImport.java | 6 +++- .../office/migration/ImportHostingAssets.java | 26 +++++++++------ .../hs/office/migration/ImportOfficeData.java | 33 +++++++++++++++++++ 3 files changed, 54 insertions(+), 11 deletions(-) diff --git a/src/test/java/net/hostsharing/hsadminng/hs/office/migration/CsvDataImport.java b/src/test/java/net/hostsharing/hsadminng/hs/office/migration/CsvDataImport.java index 291d473c..e2578e19 100644 --- a/src/test/java/net/hostsharing/hsadminng/hs/office/migration/CsvDataImport.java +++ b/src/test/java/net/hostsharing/hsadminng/hs/office/migration/CsvDataImport.java @@ -119,10 +119,14 @@ public class CsvDataImport extends ContextBasedTest { protected void persist(final Integer id, final RbacObject entity) { try { + final var asString = entity.toString(); + if ( asString.contains("'null null, null'") || asString.equals("person()")) { + return; + } //System.out.println("persisting #" + entity.hashCode() + ": " + entity); em.persist(entity); // uncomment for debugging purposes - // em.flush(); + em.flush(); // System.out.println("persisted #" + entity.hashCode() + " as " + entity.getUuid()); } catch (Exception exc) { System.err.println("failed to persist #" + entity.hashCode() + ": " + entity); diff --git a/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportHostingAssets.java b/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportHostingAssets.java index fdf7d0a0..11f5f859 100644 --- a/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportHostingAssets.java +++ b/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportHostingAssets.java @@ -7,6 +7,7 @@ import net.hostsharing.hsadminng.hs.booking.item.validators.HsBookingItemEntityV import net.hostsharing.hsadminng.hs.hosting.asset.HsHostingAssetEntity; import net.hostsharing.hsadminng.hs.hosting.asset.HsHostingAssetType; import net.hostsharing.hsadminng.hs.hosting.asset.validators.HostingAssetEntitySaveProcessor; +import net.hostsharing.hsadminng.hs.office.debitor.HsOfficeDebitorEntity; import net.hostsharing.hsadminng.rbac.test.JpaAttempt; import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.MethodOrderer; @@ -35,6 +36,7 @@ import static net.hostsharing.hsadminng.hs.hosting.asset.HsHostingAssetType.MANA import static net.hostsharing.hsadminng.hs.hosting.asset.HsHostingAssetType.MANAGED_WEBSPACE; import static net.hostsharing.hsadminng.mapper.PostgresDateRange.toPostgresDateRange; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; /* * This 'test' includes the complete legacy 'office' data import. @@ -83,7 +85,7 @@ import static org.assertj.core.api.Assertions.assertThat; @Import({ Context.class, JpaAttempt.class }) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @ExtendWith(OrderedDependedTestsExtension.class) -public class ImportHostingAssets extends CsvDataImport { +public class ImportHostingAssets extends ImportOfficeData { private static final Integer IP_NUMBER_ID_OFFSET = 1000000; private static final Integer HIVE_ID_OFFSET = 2000000; @@ -96,7 +98,7 @@ public class ImportHostingAssets extends CsvDataImport { private static Map hostingAssets = new WriteOnceMap<>(); @Test - @Order(1010) + @Order(2010) void importIpNumbers() { try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "inet_addr.csv")) { final var lines = readAllLines(reader); @@ -107,7 +109,7 @@ public class ImportHostingAssets extends CsvDataImport { } @Test - @Order(1019) + @Order(2019) void verifyIpNumbers() { assumeThatWeAreImportingControlledTestData(); @@ -124,7 +126,7 @@ public class ImportHostingAssets extends CsvDataImport { } @Test - @Order(1030) + @Order(2030) void importHives() { try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "hive.csv")) { final var lines = readAllLines(reader); @@ -135,7 +137,7 @@ public class ImportHostingAssets extends CsvDataImport { } @Test - @Order(1039) + @Order(2039) void verifyHives() { assumeThatWeAreImportingControlledTestData(); @@ -152,7 +154,7 @@ public class ImportHostingAssets extends CsvDataImport { } @Test - @Order(2000) + @Order(3000) void importPackets() { try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "packet.csv")) { final var lines = readAllLines(reader); @@ -163,7 +165,7 @@ public class ImportHostingAssets extends CsvDataImport { } @Test - @Order(2009) + @Order(3009) void verifyPackets() { assumeThatWeAreImportingControlledTestData(); @@ -200,7 +202,7 @@ public class ImportHostingAssets extends CsvDataImport { } @Test - @Order(2010) + @Order(3010) void importPacketComponents() { try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "packet_component.csv")) { final var lines = readAllLines(reader); @@ -211,7 +213,7 @@ public class ImportHostingAssets extends CsvDataImport { } @Test - @Order(2019) + @Order(3019) void verifyPacketComponents() { assumeThatWeAreImportingControlledTestData(); @@ -294,7 +296,6 @@ public class ImportHostingAssets extends CsvDataImport { }).assertSuccessful(); } - private void importIpNumbers(final String[] header, final List records) { final var columns = new Columns(header); records.stream() @@ -547,6 +548,11 @@ public class ImportHostingAssets extends CsvDataImport { .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); } + @Override + protected void assumeThatWeAreExplicitlyImportingOfficeData() { + assumeThat(false).isTrue(); + } + protected static boolean isImportingControlledTestData() { return MIGRATION_DATA_PATH.equals("migration"); } diff --git a/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportOfficeData.java b/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportOfficeData.java index 0d9b16e9..aa58efb0 100644 --- a/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportOfficeData.java +++ b/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportOfficeData.java @@ -111,6 +111,7 @@ public class ImportOfficeData extends CsvDataImport { private static final List IGNORE_BUSINESS_PARTNERS = Arrays.asList( 512167, // 11139, partner without contractual contact 512170, // 11142, partner without contractual contact + 511725, // 10764, partner without contractual contact -1 ); @@ -313,6 +314,8 @@ public class ImportOfficeData extends CsvDataImport { @Test @Order(1040) void importCoopShares() { + assumeThatWeAreExplicitlyImportingOfficeData(); + try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "share-transactions.csv")) { final var lines = readAllLines(reader); importCoopShares(justHeader(lines), withoutHeader(lines)); @@ -324,6 +327,7 @@ public class ImportOfficeData extends CsvDataImport { @Test @Order(1041) void verifyCoopShares() { + assumeThatWeAreExplicitlyImportingOfficeData(); assumeThatWeAreImportingControlledTestData(); assertThat(toFormattedString(coopShares)).isEqualToIgnoringWhitespace(""" @@ -339,6 +343,7 @@ public class ImportOfficeData extends CsvDataImport { @Test @Order(1050) void importCoopAssets() { + assumeThatWeAreExplicitlyImportingOfficeData(); try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "asset-transactions.csv")) { final var lines = readAllLines(reader); @@ -351,6 +356,7 @@ public class ImportOfficeData extends CsvDataImport { @Test @Order(1059) void verifyCoopAssets() { + assumeThatWeAreExplicitlyImportingOfficeData(); assumeThatWeAreImportingControlledTestData(); assertThat(toFormattedString(coopAssets)).isEqualToIgnoringWhitespace(""" @@ -372,7 +378,9 @@ public class ImportOfficeData extends CsvDataImport { @Test @Order(1099) void verifyMemberships() { + assumeThatWeAreExplicitlyImportingOfficeData(); assumeThatWeAreImportingControlledTestData(); + assertThat(toFormattedString(memberships)).isEqualToIgnoringWhitespace(""" { 17=Membership(M-1001700, P-10017, [2000-12-06,), ACTIVE), @@ -386,6 +394,8 @@ public class ImportOfficeData extends CsvDataImport { @Test @Order(2000) void verifyAllPartnersHavePersons() { + assumeThatWeAreExplicitlyImportingOfficeData(); + partners.forEach((id, p) -> { final var partnerRel = p.getPartnerRel(); assertThat(partnerRel).describedAs("partner " + id + " without partnerRel").isNotNull(); @@ -480,6 +490,23 @@ public class ImportOfficeData extends CsvDataImport { assertThat(idsToRemove.size()).isEqualTo(1); // only from partner #99 } + @Test + @Order(3005) + void removeEmptyPersons() { + // avoid a error when persisting the deliberately invalid partner entry #99 + final var idsToRemove = new HashSet(); + persons.forEach( (id, p) -> { + if ( p.getPersonType() == null || + (p.getFamilyName() == null && p.getGivenName() == null && p.getTradeName() == null) ) { + idsToRemove.add(id); + } + }); + idsToRemove.forEach(id -> persons.remove(id)); + + assumeThatWeAreImportingControlledTestData(); + assertThat(idsToRemove.size()).isEqualTo(0); + } + @Test @Order(9000) @Commit @@ -500,6 +527,8 @@ public class ImportOfficeData extends CsvDataImport { jpaAttempt.transacted(() -> { context(rbacSuperuser); persons.forEach(this::persist); + relations.forEach( (id, rel) -> this.persist(id, rel.getAnchor()) ); + relations.forEach( (id, rel) -> this.persist(id, rel.getHolder()) ); }).assertSuccessful(); jpaAttempt.transacted(() -> { @@ -560,6 +589,10 @@ public class ImportOfficeData extends CsvDataImport { } + protected void assumeThatWeAreExplicitlyImportingOfficeData() { + assumeThat(true).isFalse(); + } + private static boolean isImportingControlledTestData() { return partners.size() <= MAX_NUMBER_OF_TEST_DATA_PARTNERS; }