chunked persist
This commit is contained in:
Michael Hoennig
parent
5ed92ccd6f
commit
ee26cb6476
@ -15,6 +15,7 @@ import net.hostsharing.hsadminng.hs.hosting.asset.validators.HostingAssetEntityS
|
||||
import net.hostsharing.hsadminng.hs.hosting.asset.validators.HostingAssetEntityValidatorRegistry;
|
||||
import net.hostsharing.hsadminng.hs.hosting.asset.validators.HsDomainDnsSetupHostingAssetValidator;
|
||||
import net.hostsharing.hsadminng.rbac.test.JpaAttempt;
|
||||
import org.apache.commons.collections4.ListUtils;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import org.junit.jupiter.api.MethodOrderer;
|
||||
@ -489,10 +490,6 @@ public class ImportHostingAssets extends ImportOfficeData {
|
||||
});
|
||||
}
|
||||
|
||||
private String vmName(final String zonenfileName) {
|
||||
return zonenfileName.substring(zonenfileName.length() - "vm0000.json".length()).substring(0, 6);
|
||||
}
|
||||
|
||||
@Test
|
||||
@Order(16029)
|
||||
void verifyDomains() {
|
||||
@ -527,6 +524,40 @@ public class ImportHostingAssets extends ImportOfficeData {
|
||||
""");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Order(17010)
|
||||
void importEmailAddresses() {
|
||||
try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "/hosting/emailaddr.csv")) {
|
||||
final var lines = readAllLines(reader);
|
||||
importEmailAddresses(justHeader(lines), withoutHeader(lines));
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@Order(17029)
|
||||
void verifyEmailAddresses() {
|
||||
assumeThatWeAreImportingControlledTestData();
|
||||
|
||||
assertThat(firstOfEach(12, emailAddressAssets)).isEqualToIgnoringWhitespace("""
|
||||
{
|
||||
54745=HsHostingAssetRealEntity(EMAIL_ADDRESS, lugmaster@l-u-g.org, lugmaster@l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "lugmaster", "target": [ "nobody" ]}),
|
||||
54746=HsHostingAssetRealEntity(EMAIL_ADDRESS, abuse@l-u-g.org, abuse@l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "abuse", "target": [ "lug00" ]}),
|
||||
54747=HsHostingAssetRealEntity(EMAIL_ADDRESS, postmaster@l-u-g.org, postmaster@l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "postmaster", "target": [ "nobody" ]}),
|
||||
54748=HsHostingAssetRealEntity(EMAIL_ADDRESS, webmaster@l-u-g.org, webmaster@l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "webmaster", "target": [ "nobody" ]}),
|
||||
54749=HsHostingAssetRealEntity(EMAIL_ADDRESS, abuse@linuxfanboysngirls.de, abuse@linuxfanboysngirls.de, DOMAIN_MBOX_SETUP:linuxfanboysngirls.de|MBOX, {"local-part": "abuse", "target": [ "lug00-mars" ]}),
|
||||
54750=HsHostingAssetRealEntity(EMAIL_ADDRESS, postmaster@linuxfanboysngirls.de, postmaster@linuxfanboysngirls.de, DOMAIN_MBOX_SETUP:linuxfanboysngirls.de|MBOX, {"local-part": "postmaster", "target": [ "m.hinsel@example.org" ]}),
|
||||
54751=HsHostingAssetRealEntity(EMAIL_ADDRESS, webmaster@linuxfanboysngirls.de, webmaster@linuxfanboysngirls.de, DOMAIN_MBOX_SETUP:linuxfanboysngirls.de|MBOX, {"local-part": "webmaster", "target": [ "m.hinsel@example.org" ]}),
|
||||
54755=HsHostingAssetRealEntity(EMAIL_ADDRESS, abuse@lug-mars.de, abuse@lug-mars.de, DOMAIN_MBOX_SETUP:lug-mars.de|MBOX, {"local-part": "abuse", "target": [ "lug00-marl" ]}),
|
||||
54756=HsHostingAssetRealEntity(EMAIL_ADDRESS, postmaster@lug-mars.de, postmaster@lug-mars.de, DOMAIN_MBOX_SETUP:lug-mars.de|MBOX, {"local-part": "postmaster", "target": [ "m.hinsel@example.org" ]}),
|
||||
54757=HsHostingAssetRealEntity(EMAIL_ADDRESS, webmaster@lug-mars.de, webmaster@lug-mars.de, DOMAIN_MBOX_SETUP:lug-mars.de|MBOX, {"local-part": "webmaster", "target": [ "m.hinsel@example.org" ]}),
|
||||
54760=HsHostingAssetRealEntity(EMAIL_ADDRESS, info@hamburg-west.l-u-g.org, info@hamburg-west.l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "info", "sub-domain": "hamburg-west", "target": [ "peter.lottmann@example.com" ]}),
|
||||
54761=HsHostingAssetRealEntity(EMAIL_ADDRESS, lugmaster@hamburg-west.l-u-g.org, lugmaster@hamburg-west.l-u-g.org, DOMAIN_MBOX_SETUP:l-u-g.org|MBOX, {"local-part": "lugmaster", "sub-domain": "hamburg-west", "target": [ "raoul.lottmann@example.com" ]})
|
||||
}
|
||||
""");
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------------------------
|
||||
|
||||
@Test
|
||||
@ -761,6 +792,9 @@ public class ImportHostingAssets extends ImportOfficeData {
|
||||
final var haCount = (Integer) em.createNativeQuery("select count(*) from hs_hosting_asset", Integer.class)
|
||||
.getSingleResult();
|
||||
assertThat(haCount).isGreaterThan(isImportingControlledTestData() ? 40 : 15000);
|
||||
|
||||
verifyActuallyPersistedHostingAssetCount(EMAIL_ALIAS, 9, 1400);
|
||||
verifyActuallyPersistedHostingAssetCount(EMAIL_ADDRESS, 70, 30000);
|
||||
}
|
||||
|
||||
// ============================================================================================
|
||||
@ -780,6 +814,12 @@ public class ImportHostingAssets extends ImportOfficeData {
|
||||
assertNoErrors();
|
||||
}
|
||||
|
||||
// ============================================================================================
|
||||
|
||||
private String vmName(final String zonenfileName) {
|
||||
return zonenfileName.substring(zonenfileName.length() - "vm0000.json".length()).substring(0, 6);
|
||||
}
|
||||
|
||||
private void persistRecursively(final Integer key, final HsBookingItemEntity bi) {
|
||||
if (bi.getParentItem() != null) {
|
||||
persistRecursively(key, HsBookingItemEntityValidatorRegistry.validated(bi.getParentItem()));
|
||||
@ -787,28 +827,43 @@ public class ImportHostingAssets extends ImportOfficeData {
|
||||
persist(key, HsBookingItemEntityValidatorRegistry.validated(bi));
|
||||
}
|
||||
|
||||
// ============================================================================================
|
||||
|
||||
private void persistHostingAssets(final Map<Integer, HsHostingAssetRealEntity> assets) {
|
||||
persistHostingAssets(assets, null);
|
||||
}
|
||||
|
||||
private void persistHostingAssets(final Map<Integer, HsHostingAssetRealEntity> assets, final HsHostingAssetType type) {
|
||||
jpaAttempt.transacted(() ->
|
||||
assets.forEach((key, ha) -> {
|
||||
if (type == null || type == ha.getType()) {
|
||||
context(rbacSuperuser); // if put only outside the loop, it seems to get lost after a while, no idea why
|
||||
logError(() ->
|
||||
new HostingAssetEntitySaveProcessor(em, ha)
|
||||
.preprocessEntity()
|
||||
.validateEntityIgnoring("'EMAIL_ALIAS:.*\\.config\\.target' .*")
|
||||
.prepareForSave()
|
||||
.saveUsing(entity -> persist(key, entity))
|
||||
.validateContext()
|
||||
);
|
||||
}
|
||||
})
|
||||
).assertSuccessful();
|
||||
final var assetsOfType = assets.entrySet().stream()
|
||||
.filter(entry -> type == null || type == entry.getValue().getType())
|
||||
.toList();
|
||||
final var chunkSize = isImportingControlledTestData() ? 10: 500;
|
||||
ListUtils.partition(assetsOfType, chunkSize).forEach(chunk -> {
|
||||
chunk.forEach(entry -> {
|
||||
jpaAttempt.transacted(() -> {
|
||||
context(rbacSuperuser);
|
||||
logError(() ->
|
||||
new HostingAssetEntitySaveProcessor(em, entry.getValue())
|
||||
.preprocessEntity()
|
||||
.validateEntityIgnoring("'EMAIL_ALIAS:.*\\.config\\.target' .*")
|
||||
.prepareForSave()
|
||||
.saveUsing(entity -> persist(entry.getKey(), entity))
|
||||
.validateContext()
|
||||
);
|
||||
}).assertSuccessful();
|
||||
});
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
private void verifyActuallyPersistedHostingAssetCount(
|
||||
final HsHostingAssetType assetType,
|
||||
final int minExpectedInTestData,
|
||||
final int minExpectedInProdData) {
|
||||
final var q = em.createNativeQuery(
|
||||
"select count(*) from hs_hosting_asset where type = cast(:type as HsHostingAssetType)",
|
||||
Integer.class);
|
||||
q.setParameter("type", assetType.name());
|
||||
final var count = (Integer) q.getSingleResult();
|
||||
assertThat(count).isGreaterThan(isImportingControlledTestData() ? minExpectedInTestData : minExpectedInProdData);
|
||||
}
|
||||
|
||||
private void importIpNumbers(final String[] header, final List<String[]> records) {
|
||||
@ -1386,7 +1441,7 @@ public class ImportHostingAssets extends ImportOfficeData {
|
||||
final var domainMboxSetup = domainMBoxSetupAssets.get(domain_id);
|
||||
final var domainSetup = domainMboxSetup.getParentAsset();
|
||||
final var emailAddress = localpart + "@" +
|
||||
(subdomain != null && !subdomain.isBlank() ? subdomain + "." : "") + domainSetup.getIdentifier();
|
||||
(subdomain != null && !subdomain.isBlank() ? subdomain + "." : "") + domainSetup.getIdentifier();
|
||||
final var emailAddressAsset = HsHostingAssetRealEntity.builder()
|
||||
.type(EMAIL_ADDRESS)
|
||||
.parentAsset(domainMboxSetup)
|
||||
@ -1461,7 +1516,7 @@ public class ImportHostingAssets extends ImportOfficeData {
|
||||
final int maxCount,
|
||||
final Map<Integer, HsHostingAssetRealEntity> assets) {
|
||||
return toJsonFormattedString(assets.entrySet().stream().limit(maxCount)
|
||||
.collect(toMap(Map.Entry::getKey, Map.Entry::getValue, ImportHostingAssets::uniqueKeys, TreeMap::new)));
|
||||
.collect(toMap(Map.Entry::getKey, Map.Entry::getValue, ImportHostingAssets::uniqueKeys, TreeMap::new)));
|
||||
}
|
||||
|
||||
private String firstOfEach(
|
||||
|
||||
Loading…
Reference in New Issue
Block a user