From ee26cb6476c1aa5213859f0943e4d4c3a322f116 Mon Sep 17 00:00:00 2001 From: Michael Hoennig Date: Fri, 9 Aug 2024 09:32:11 +0200 Subject: [PATCH] chunked persist --- .../hs/migration/ImportHostingAssets.java | 101 ++++++++++++++---- 1 file changed, 78 insertions(+), 23 deletions(-) diff --git a/src/test/java/net/hostsharing/hsadminng/hs/migration/ImportHostingAssets.java b/src/test/java/net/hostsharing/hsadminng/hs/migration/ImportHostingAssets.java index 46189ecf..7d59d58e 100644 --- a/src/test/java/net/hostsharing/hsadminng/hs/migration/ImportHostingAssets.java +++ b/src/test/java/net/hostsharing/hsadminng/hs/migration/ImportHostingAssets.java @@ -15,6 +15,7 @@ import net.hostsharing.hsadminng.hs.hosting.asset.validators.HostingAssetEntityS import net.hostsharing.hsadminng.hs.hosting.asset.validators.HostingAssetEntityValidatorRegistry; import net.hostsharing.hsadminng.hs.hosting.asset.validators.HsDomainDnsSetupHostingAssetValidator; import net.hostsharing.hsadminng.rbac.test.JpaAttempt; +import org.apache.commons.collections4.ListUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.junit.jupiter.api.MethodOrderer; @@ -489,10 +490,6 @@ public class ImportHostingAssets extends ImportOfficeData { }); } - private String vmName(final String zonenfileName) { - return zonenfileName.substring(zonenfileName.length() - "vm0000.json".length()).substring(0, 6); - } - @Test @Order(16029) void verifyDomains() { @@ -527,6 +524,40 @@ public class ImportHostingAssets extends ImportOfficeData { """); } + @Test + @Order(17010) + void importEmailAddresses() { + try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "/hosting/emailaddr.csv")) { + final var lines = readAllLines(reader); + importEmailAddresses(justHeader(lines), withoutHeader(lines)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Test + @Order(17029) + void verifyEmailAddresses() { + assumeThatWeAreImportingControlledTestData(); + + assertThat(firstOfEach(12, emailAddressAssets)).isEqualToIgnoringWhitespace(""" + { + 54745=HsHostingAssetRealEntity(EMAIL_ADDRESS, lugmaster@l-u-g.org, lugmaster@l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "lugmaster", "target": [ "nobody" ]}), + 54746=HsHostingAssetRealEntity(EMAIL_ADDRESS, abuse@l-u-g.org, abuse@l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "abuse", "target": [ "lug00" ]}), + 54747=HsHostingAssetRealEntity(EMAIL_ADDRESS, postmaster@l-u-g.org, postmaster@l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "postmaster", "target": [ "nobody" ]}), + 54748=HsHostingAssetRealEntity(EMAIL_ADDRESS, webmaster@l-u-g.org, webmaster@l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "webmaster", "target": [ "nobody" ]}), + 54749=HsHostingAssetRealEntity(EMAIL_ADDRESS, abuse@linuxfanboysngirls.de, abuse@linuxfanboysngirls.de, DOMAIN_MBOX_SETUP:linuxfanboysngirls.de|MBOX, {"local-part": "abuse", "target": [ "lug00-mars" ]}), + 54750=HsHostingAssetRealEntity(EMAIL_ADDRESS, postmaster@linuxfanboysngirls.de, postmaster@linuxfanboysngirls.de, DOMAIN_MBOX_SETUP:linuxfanboysngirls.de|MBOX, {"local-part": "postmaster", "target": [ "m.hinsel@example.org" ]}), + 54751=HsHostingAssetRealEntity(EMAIL_ADDRESS, webmaster@linuxfanboysngirls.de, webmaster@linuxfanboysngirls.de, DOMAIN_MBOX_SETUP:linuxfanboysngirls.de|MBOX, {"local-part": "webmaster", "target": [ "m.hinsel@example.org" ]}), + 54755=HsHostingAssetRealEntity(EMAIL_ADDRESS, abuse@lug-mars.de, abuse@lug-mars.de, DOMAIN_MBOX_SETUP:lug-mars.de|MBOX, {"local-part": "abuse", "target": [ "lug00-marl" ]}), + 54756=HsHostingAssetRealEntity(EMAIL_ADDRESS, postmaster@lug-mars.de, postmaster@lug-mars.de, DOMAIN_MBOX_SETUP:lug-mars.de|MBOX, {"local-part": "postmaster", "target": [ "m.hinsel@example.org" ]}), + 54757=HsHostingAssetRealEntity(EMAIL_ADDRESS, webmaster@lug-mars.de, webmaster@lug-mars.de, DOMAIN_MBOX_SETUP:lug-mars.de|MBOX, {"local-part": "webmaster", "target": [ "m.hinsel@example.org" ]}), + 54760=HsHostingAssetRealEntity(EMAIL_ADDRESS, info@hamburg-west.l-u-g.org, info@hamburg-west.l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "info", "sub-domain": "hamburg-west", "target": [ "peter.lottmann@example.com" ]}), + 54761=HsHostingAssetRealEntity(EMAIL_ADDRESS, lugmaster@hamburg-west.l-u-g.org, lugmaster@hamburg-west.l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "lugmaster", "sub-domain": "hamburg-west", "target": [ "raoul.lottmann@example.com" ]}) + } + """); + } + // -------------------------------------------------------------------------------------------- @Test @@ -761,6 +792,9 @@ public class ImportHostingAssets extends ImportOfficeData { final var haCount = (Integer) em.createNativeQuery("select count(*) from hs_hosting_asset", Integer.class) .getSingleResult(); assertThat(haCount).isGreaterThan(isImportingControlledTestData() ? 40 : 15000); + + verifyActuallyPersistedHostingAssetCount(EMAIL_ALIAS, 9, 1400); + verifyActuallyPersistedHostingAssetCount(EMAIL_ADDRESS, 70, 30000); } // ============================================================================================ @@ -780,6 +814,12 @@ public class ImportHostingAssets extends ImportOfficeData { assertNoErrors(); } + // ============================================================================================ + + private String vmName(final String zonenfileName) { + return zonenfileName.substring(zonenfileName.length() - "vm0000.json".length()).substring(0, 6); + } + private void persistRecursively(final Integer key, final HsBookingItemEntity bi) { if (bi.getParentItem() != null) { persistRecursively(key, HsBookingItemEntityValidatorRegistry.validated(bi.getParentItem())); @@ -787,28 +827,43 @@ public class ImportHostingAssets extends ImportOfficeData { persist(key, HsBookingItemEntityValidatorRegistry.validated(bi)); } - // ============================================================================================ - private void persistHostingAssets(final Map assets) { persistHostingAssets(assets, null); } private void persistHostingAssets(final Map assets, final HsHostingAssetType type) { - jpaAttempt.transacted(() -> - assets.forEach((key, ha) -> { - if (type == null || type == ha.getType()) { - context(rbacSuperuser); // if put only outside the loop, it seems to get lost after a while, no idea why - logError(() -> - new HostingAssetEntitySaveProcessor(em, ha) - .preprocessEntity() - .validateEntityIgnoring("'EMAIL_ALIAS:.*\\.config\\.target' .*") - .prepareForSave() - .saveUsing(entity -> persist(key, entity)) - .validateContext() - ); - } - }) - ).assertSuccessful(); + final var assetsOfType = assets.entrySet().stream() + .filter(entry -> type == null || type == entry.getValue().getType()) + .toList(); + final var chunkSize = isImportingControlledTestData() ? 10: 500; + ListUtils.partition(assetsOfType, chunkSize).forEach(chunk -> { + chunk.forEach(entry -> { + jpaAttempt.transacted(() -> { + context(rbacSuperuser); + logError(() -> + new HostingAssetEntitySaveProcessor(em, entry.getValue()) + .preprocessEntity() + .validateEntityIgnoring("'EMAIL_ALIAS:.*\\.config\\.target' .*") + .prepareForSave() + .saveUsing(entity -> persist(entry.getKey(), entity)) + .validateContext() + ); + }).assertSuccessful(); + }); + } + ); + } + + private void verifyActuallyPersistedHostingAssetCount( + final HsHostingAssetType assetType, + final int minExpectedInTestData, + final int minExpectedInProdData) { + final var q = em.createNativeQuery( + "select count(*) from hs_hosting_asset where type = cast(:type as HsHostingAssetType)", + Integer.class); + q.setParameter("type", assetType.name()); + final var count = (Integer) q.getSingleResult(); + assertThat(count).isGreaterThan(isImportingControlledTestData() ? minExpectedInTestData : minExpectedInProdData); } private void importIpNumbers(final String[] header, final List records) { @@ -1386,7 +1441,7 @@ public class ImportHostingAssets extends ImportOfficeData { final var domainMboxSetup = domainMBoxSetupAssets.get(domain_id); final var domainSetup = domainMboxSetup.getParentAsset(); final var emailAddress = localpart + "@" + - (subdomain != null && !subdomain.isBlank() ? subdomain + "." : "") + domainSetup.getIdentifier(); + (subdomain != null && !subdomain.isBlank() ? subdomain + "." : "") + domainSetup.getIdentifier(); final var emailAddressAsset = HsHostingAssetRealEntity.builder() .type(EMAIL_ADDRESS) .parentAsset(domainMboxSetup) @@ -1461,7 +1516,7 @@ public class ImportHostingAssets extends ImportOfficeData { final int maxCount, final Map assets) { return toJsonFormattedString(assets.entrySet().stream().limit(maxCount) - .collect(toMap(Map.Entry::getKey, Map.Entry::getValue, ImportHostingAssets::uniqueKeys, TreeMap::new))); + .collect(toMap(Map.Entry::getKey, Map.Entry::getValue, ImportHostingAssets::uniqueKeys, TreeMap::new))); } private String firstOfEach(