hostsharing/hs.hsadmin.ng
6
0
Fork 0
Code Pull Requests 4 Activity

chunked persist

Browse Source
This commit is contained in:
Michael Hoennig 2024-08-09 09:32:11 +02:00
parent 5ed92ccd6f
commit ee26cb6476
1 changed files with 78 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
src/test/java/net/hostsharing/hsadminng/hs/migration/ImportHostingAssets.java
View File

@ -15,6 +15,7 @@ import net.hostsharing.hsadminng.hs.hosting.asset.validators.HostingAssetEntityS
import net.hostsharing.hsadminng.hs.hosting.asset.validators.HostingAssetEntityValidatorRegistry; import net.hostsharing.hsadminng.hs.hosting.asset.validators.HostingAssetEntityValidatorRegistry;
import net.hostsharing.hsadminng.hs.hosting.asset.validators.HsDomainDnsSetupHostingAssetValidator; import net.hostsharing.hsadminng.hs.hosting.asset.validators.HsDomainDnsSetupHostingAssetValidator;
import net.hostsharing.hsadminng.rbac.test.JpaAttempt; import net.hostsharing.hsadminng.rbac.test.JpaAttempt;
import org.apache.commons.collections4.ListUtils;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.MethodOrderer;
@ -489,10 +490,6 @@ public class ImportHostingAssets extends ImportOfficeData {
}); });
} }
private String vmName(final String zonenfileName) {
return zonenfileName.substring(zonenfileName.length() - "vm0000.json".length()).substring(0, 6);
}
@Test @Test
@Order(16029) @Order(16029)
void verifyDomains() { void verifyDomains() {
@ -527,6 +524,40 @@ public class ImportHostingAssets extends ImportOfficeData {
"""); """);
} }
@Test
@Order(17010)
void importEmailAddresses() {
try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "/hosting/emailaddr.csv")) {
final var lines = readAllLines(reader);
importEmailAddresses(justHeader(lines), withoutHeader(lines));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Test
@Order(17029)
void verifyEmailAddresses() {
assumeThatWeAreImportingControlledTestData();
assertThat(firstOfEach(12, emailAddressAssets)).isEqualToIgnoringWhitespace("""
{
54745=HsHostingAssetRealEntity(EMAIL_ADDRESS, lugmaster@l-u-g.org, lugmaster@l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "lugmaster", "target": [ "nobody" ]}),
54746=HsHostingAssetRealEntity(EMAIL_ADDRESS, abuse@l-u-g.org, abuse@l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "abuse", "target": [ "lug00" ]}),
54747=HsHostingAssetRealEntity(EMAIL_ADDRESS, postmaster@l-u-g.org, postmaster@l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "postmaster", "target": [ "nobody" ]}),
54748=HsHostingAssetRealEntity(EMAIL_ADDRESS, webmaster@l-u-g.org, webmaster@l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "webmaster", "target": [ "nobody" ]}),
54749=HsHostingAssetRealEntity(EMAIL_ADDRESS, abuse@linuxfanboysngirls.de, abuse@linuxfanboysngirls.de, DOMAIN_MBOX_SETUP:linuxfanboysngirls.de|MBOX, {"local-part": "abuse", "target": [ "lug00-mars" ]}),
54750=HsHostingAssetRealEntity(EMAIL_ADDRESS, postmaster@linuxfanboysngirls.de, postmaster@linuxfanboysngirls.de, DOMAIN_MBOX_SETUP:linuxfanboysngirls.de|MBOX, {"local-part": "postmaster", "target": [ "m.hinsel@example.org" ]}),
54751=HsHostingAssetRealEntity(EMAIL_ADDRESS, webmaster@linuxfanboysngirls.de, webmaster@linuxfanboysngirls.de, DOMAIN_MBOX_SETUP:linuxfanboysngirls.de|MBOX, {"local-part": "webmaster", "target": [ "m.hinsel@example.org" ]}),
54755=HsHostingAssetRealEntity(EMAIL_ADDRESS, abuse@lug-mars.de, abuse@lug-mars.de, DOMAIN_MBOX_SETUP:lug-mars.de|MBOX, {"local-part": "abuse", "target": [ "lug00-marl" ]}),
54756=HsHostingAssetRealEntity(EMAIL_ADDRESS, postmaster@lug-mars.de, postmaster@lug-mars.de, DOMAIN_MBOX_SETUP:lug-mars.de|MBOX, {"local-part": "postmaster", "target": [ "m.hinsel@example.org" ]}),
54757=HsHostingAssetRealEntity(EMAIL_ADDRESS, webmaster@lug-mars.de, webmaster@lug-mars.de, DOMAIN_MBOX_SETUP:lug-mars.de|MBOX, {"local-part": "webmaster", "target": [ "m.hinsel@example.org" ]}),
54760=HsHostingAssetRealEntity(EMAIL_ADDRESS, info@hamburg-west.l-u-g.org, info@hamburg-west.l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "info", "sub-domain": "hamburg-west", "target": [ "peter.lottmann@example.com" ]}),
54761=HsHostingAssetRealEntity(EMAIL_ADDRESS, lugmaster@hamburg-west.l-u-g.org, lugmaster@hamburg-west.l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "lugmaster", "sub-domain": "hamburg-west", "target": [ "raoul.lottmann@example.com" ]})
}
""");
}
// -------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------
@Test @Test
@ -761,6 +792,9 @@ public class ImportHostingAssets extends ImportOfficeData {
final var haCount = (Integer) em.createNativeQuery("select count(*) from hs_hosting_asset", Integer.class) final var haCount = (Integer) em.createNativeQuery("select count(*) from hs_hosting_asset", Integer.class)
.getSingleResult(); .getSingleResult();
assertThat(haCount).isGreaterThan(isImportingControlledTestData() ? 40 : 15000); assertThat(haCount).isGreaterThan(isImportingControlledTestData() ? 40 : 15000);
verifyActuallyPersistedHostingAssetCount(EMAIL_ALIAS, 9, 1400);
verifyActuallyPersistedHostingAssetCount(EMAIL_ADDRESS, 70, 30000);
} }
// ============================================================================================ // ============================================================================================
@ -780,6 +814,12 @@ public class ImportHostingAssets extends ImportOfficeData {
assertNoErrors(); assertNoErrors();
} }
// ============================================================================================
private String vmName(final String zonenfileName) {
return zonenfileName.substring(zonenfileName.length() - "vm0000.json".length()).substring(0, 6);
}
private void persistRecursively(final Integer key, final HsBookingItemEntity bi) { private void persistRecursively(final Integer key, final HsBookingItemEntity bi) {
if (bi.getParentItem() != null) { if (bi.getParentItem() != null) {
persistRecursively(key, HsBookingItemEntityValidatorRegistry.validated(bi.getParentItem())); persistRecursively(key, HsBookingItemEntityValidatorRegistry.validated(bi.getParentItem()));
@ -787,28 +827,43 @@ public class ImportHostingAssets extends ImportOfficeData {
persist(key, HsBookingItemEntityValidatorRegistry.validated(bi)); persist(key, HsBookingItemEntityValidatorRegistry.validated(bi));
} }
// ============================================================================================
private void persistHostingAssets(final Map<Integer, HsHostingAssetRealEntity> assets) { private void persistHostingAssets(final Map<Integer, HsHostingAssetRealEntity> assets) {
persistHostingAssets(assets, null); persistHostingAssets(assets, null);
} }
private void persistHostingAssets(final Map<Integer, HsHostingAssetRealEntity> assets, final HsHostingAssetType type) { private void persistHostingAssets(final Map<Integer, HsHostingAssetRealEntity> assets, final HsHostingAssetType type) {
jpaAttempt.transacted(() -> final var assetsOfType = assets.entrySet().stream()
assets.forEach((key, ha) -> { .filter(entry -> type == null || type == entry.getValue().getType())
if (type == null || type == ha.getType()) { .toList();
context(rbacSuperuser); // if put only outside the loop, it seems to get lost after a while, no idea why final var chunkSize = isImportingControlledTestData() ? 10: 500;
logError(() -> ListUtils.partition(assetsOfType, chunkSize).forEach(chunk -> {
new HostingAssetEntitySaveProcessor(em, ha) chunk.forEach(entry -> {
.preprocessEntity() jpaAttempt.transacted(() -> {
.validateEntityIgnoring("'EMAIL_ALIAS:.*\\.config\\.target' .*") context(rbacSuperuser);
.prepareForSave() logError(() ->
.saveUsing(entity -> persist(key, entity)) new HostingAssetEntitySaveProcessor(em, entry.getValue())
.validateContext() .preprocessEntity()
); .validateEntityIgnoring("'EMAIL_ALIAS:.*\\.config\\.target' .*")
} .prepareForSave()
}) .saveUsing(entity -> persist(entry.getKey(), entity))
).assertSuccessful(); .validateContext()
);
}).assertSuccessful();
});
}
);
}
private void verifyActuallyPersistedHostingAssetCount(
final HsHostingAssetType assetType,
final int minExpectedInTestData,
final int minExpectedInProdData) {
final var q = em.createNativeQuery(
"select count(*) from hs_hosting_asset where type = cast(:type as HsHostingAssetType)",
Integer.class);
q.setParameter("type", assetType.name());
final var count = (Integer) q.getSingleResult();
assertThat(count).isGreaterThan(isImportingControlledTestData() ? minExpectedInTestData : minExpectedInProdData);
} }
private void importIpNumbers(final String[] header, final List<String[]> records) { private void importIpNumbers(final String[] header, final List<String[]> records) {
@ -1386,7 +1441,7 @@ public class ImportHostingAssets extends ImportOfficeData {
final var domainMboxSetup = domainMBoxSetupAssets.get(domain_id); final var domainMboxSetup = domainMBoxSetupAssets.get(domain_id);
final var domainSetup = domainMboxSetup.getParentAsset(); final var domainSetup = domainMboxSetup.getParentAsset();
final var emailAddress = localpart + "@" + final var emailAddress = localpart + "@" +
(subdomain != null && !subdomain.isBlank() ? subdomain + "." : "") + domainSetup.getIdentifier(); (subdomain != null && !subdomain.isBlank() ? subdomain + "." : "") + domainSetup.getIdentifier();
final var emailAddressAsset = HsHostingAssetRealEntity.builder() final var emailAddressAsset = HsHostingAssetRealEntity.builder()
.type(EMAIL_ADDRESS) .type(EMAIL_ADDRESS)
.parentAsset(domainMboxSetup) .parentAsset(domainMboxSetup)
@ -1461,7 +1516,7 @@ public class ImportHostingAssets extends ImportOfficeData {
final int maxCount, final int maxCount,
final Map<Integer, HsHostingAssetRealEntity> assets) { final Map<Integer, HsHostingAssetRealEntity> assets) {
return toJsonFormattedString(assets.entrySet().stream().limit(maxCount) return toJsonFormattedString(assets.entrySet().stream().limit(maxCount)
.collect(toMap(Map.Entry::getKey, Map.Entry::getValue, ImportHostingAssets::uniqueKeys, TreeMap::new))); .collect(toMap(Map.Entry::getKey, Map.Entry::getValue, ImportHostingAssets::uniqueKeys, TreeMap::new)));
} }
private String firstOfEach( private String firstOfEach(