chunked persist

This commit is contained in:
Michael Hoennig 2024-08-09 09:32:11 +02:00
parent 5ed92ccd6f
commit ee26cb6476

View File

@ -15,6 +15,7 @@ import net.hostsharing.hsadminng.hs.hosting.asset.validators.HostingAssetEntityS
import net.hostsharing.hsadminng.hs.hosting.asset.validators.HostingAssetEntityValidatorRegistry;
import net.hostsharing.hsadminng.hs.hosting.asset.validators.HsDomainDnsSetupHostingAssetValidator;
import net.hostsharing.hsadminng.rbac.test.JpaAttempt;
import org.apache.commons.collections4.ListUtils;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.junit.jupiter.api.MethodOrderer;
@ -489,10 +490,6 @@ public class ImportHostingAssets extends ImportOfficeData {
});
}
private String vmName(final String zonenfileName) {
return zonenfileName.substring(zonenfileName.length() - "vm0000.json".length()).substring(0, 6);
}
@Test
@Order(16029)
void verifyDomains() {
@ -527,6 +524,40 @@ public class ImportHostingAssets extends ImportOfficeData {
""");
}
@Test
@Order(17010)
void importEmailAddresses() {
try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "/hosting/emailaddr.csv")) {
final var lines = readAllLines(reader);
importEmailAddresses(justHeader(lines), withoutHeader(lines));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Test
@Order(17029)
void verifyEmailAddresses() {
assumeThatWeAreImportingControlledTestData();
assertThat(firstOfEach(12, emailAddressAssets)).isEqualToIgnoringWhitespace("""
{
54745=HsHostingAssetRealEntity(EMAIL_ADDRESS, lugmaster@l-u-g.org, lugmaster@l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "lugmaster", "target": [ "nobody" ]}),
54746=HsHostingAssetRealEntity(EMAIL_ADDRESS, abuse@l-u-g.org, abuse@l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "abuse", "target": [ "lug00" ]}),
54747=HsHostingAssetRealEntity(EMAIL_ADDRESS, postmaster@l-u-g.org, postmaster@l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "postmaster", "target": [ "nobody" ]}),
54748=HsHostingAssetRealEntity(EMAIL_ADDRESS, webmaster@l-u-g.org, webmaster@l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "webmaster", "target": [ "nobody" ]}),
54749=HsHostingAssetRealEntity(EMAIL_ADDRESS, abuse@linuxfanboysngirls.de, abuse@linuxfanboysngirls.de, DOMAIN_MBOX_SETUP:linuxfanboysngirls.de|MBOX, {"local-part": "abuse", "target": [ "lug00-mars" ]}),
54750=HsHostingAssetRealEntity(EMAIL_ADDRESS, postmaster@linuxfanboysngirls.de, postmaster@linuxfanboysngirls.de, DOMAIN_MBOX_SETUP:linuxfanboysngirls.de|MBOX, {"local-part": "postmaster", "target": [ "m.hinsel@example.org" ]}),
54751=HsHostingAssetRealEntity(EMAIL_ADDRESS, webmaster@linuxfanboysngirls.de, webmaster@linuxfanboysngirls.de, DOMAIN_MBOX_SETUP:linuxfanboysngirls.de|MBOX, {"local-part": "webmaster", "target": [ "m.hinsel@example.org" ]}),
54755=HsHostingAssetRealEntity(EMAIL_ADDRESS, abuse@lug-mars.de, abuse@lug-mars.de, DOMAIN_MBOX_SETUP:lug-mars.de|MBOX, {"local-part": "abuse", "target": [ "lug00-marl" ]}),
54756=HsHostingAssetRealEntity(EMAIL_ADDRESS, postmaster@lug-mars.de, postmaster@lug-mars.de, DOMAIN_MBOX_SETUP:lug-mars.de|MBOX, {"local-part": "postmaster", "target": [ "m.hinsel@example.org" ]}),
54757=HsHostingAssetRealEntity(EMAIL_ADDRESS, webmaster@lug-mars.de, webmaster@lug-mars.de, DOMAIN_MBOX_SETUP:lug-mars.de|MBOX, {"local-part": "webmaster", "target": [ "m.hinsel@example.org" ]}),
54760=HsHostingAssetRealEntity(EMAIL_ADDRESS, info@hamburg-west.l-u-g.org, info@hamburg-west.l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "info", "sub-domain": "hamburg-west", "target": [ "peter.lottmann@example.com" ]}),
54761=HsHostingAssetRealEntity(EMAIL_ADDRESS, lugmaster@hamburg-west.l-u-g.org, lugmaster@hamburg-west.l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "lugmaster", "sub-domain": "hamburg-west", "target": [ "raoul.lottmann@example.com" ]})
}
""");
}
// --------------------------------------------------------------------------------------------
@Test
@ -761,6 +792,9 @@ public class ImportHostingAssets extends ImportOfficeData {
final var haCount = (Integer) em.createNativeQuery("select count(*) from hs_hosting_asset", Integer.class)
.getSingleResult();
assertThat(haCount).isGreaterThan(isImportingControlledTestData() ? 40 : 15000);
verifyActuallyPersistedHostingAssetCount(EMAIL_ALIAS, 9, 1400);
verifyActuallyPersistedHostingAssetCount(EMAIL_ADDRESS, 70, 30000);
}
// ============================================================================================
@ -780,6 +814,12 @@ public class ImportHostingAssets extends ImportOfficeData {
assertNoErrors();
}
// ============================================================================================
private String vmName(final String zonenfileName) {
return zonenfileName.substring(zonenfileName.length() - "vm0000.json".length()).substring(0, 6);
}
private void persistRecursively(final Integer key, final HsBookingItemEntity bi) {
if (bi.getParentItem() != null) {
persistRecursively(key, HsBookingItemEntityValidatorRegistry.validated(bi.getParentItem()));
@ -787,28 +827,43 @@ public class ImportHostingAssets extends ImportOfficeData {
persist(key, HsBookingItemEntityValidatorRegistry.validated(bi));
}
// ============================================================================================
private void persistHostingAssets(final Map<Integer, HsHostingAssetRealEntity> assets) {
persistHostingAssets(assets, null);
}
private void persistHostingAssets(final Map<Integer, HsHostingAssetRealEntity> assets, final HsHostingAssetType type) {
jpaAttempt.transacted(() ->
assets.forEach((key, ha) -> {
if (type == null || type == ha.getType()) {
context(rbacSuperuser); // if put only outside the loop, it seems to get lost after a while, no idea why
final var assetsOfType = assets.entrySet().stream()
.filter(entry -> type == null || type == entry.getValue().getType())
.toList();
final var chunkSize = isImportingControlledTestData() ? 10: 500;
ListUtils.partition(assetsOfType, chunkSize).forEach(chunk -> {
chunk.forEach(entry -> {
jpaAttempt.transacted(() -> {
context(rbacSuperuser);
logError(() ->
new HostingAssetEntitySaveProcessor(em, ha)
new HostingAssetEntitySaveProcessor(em, entry.getValue())
.preprocessEntity()
.validateEntityIgnoring("'EMAIL_ALIAS:.*\\.config\\.target' .*")
.prepareForSave()
.saveUsing(entity -> persist(key, entity))
.saveUsing(entity -> persist(entry.getKey(), entity))
.validateContext()
);
}).assertSuccessful();
});
}
})
).assertSuccessful();
);
}
private void verifyActuallyPersistedHostingAssetCount(
final HsHostingAssetType assetType,
final int minExpectedInTestData,
final int minExpectedInProdData) {
final var q = em.createNativeQuery(
"select count(*) from hs_hosting_asset where type = cast(:type as HsHostingAssetType)",
Integer.class);
q.setParameter("type", assetType.name());
final var count = (Integer) q.getSingleResult();
assertThat(count).isGreaterThan(isImportingControlledTestData() ? minExpectedInTestData : minExpectedInProdData);
}
private void importIpNumbers(final String[] header, final List<String[]> records) {