Compare commits

..

No commits in common. "c136064ff78967e365443ced58625b0c9868a9a8" and "6ddecbbfec18208e7cbd0ba435205921469f1a7a" have entirely different histories.

8 changed files with 15 additions and 51 deletions

View File

@ -84,9 +84,3 @@ alias fp='grep -r '@Accepts' src | sed -e 's/^.*@/@/g' | sort -u | wc -l'
alias gw-spotless='./gradlew spotlessApply -x pitest -x test -x :processResources' alias gw-spotless='./gradlew spotlessApply -x pitest -x test -x :processResources'
alias gw-test='. .aliases; ./gradlew test importOfficeData' alias gw-test='. .aliases; ./gradlew test importOfficeData'
alias gw-check='. .aliases; gw test importOfficeData check -x pitest -x :dependencyCheckAnalyze' alias gw-check='. .aliases; gw test importOfficeData check -x pitest -x :dependencyCheckAnalyze'
# etc/docker-compose.yml limits CPUs+MEM and includes a PostgreSQL config for analysing slow queries
alias gw-importOfficeData-in-docker-compose='
docker-compose -f etc/docker-compose.yml down &&
docker-compose -f etc/docker-compose.yml up -d && sleep 10 &&
time gw-importHostingAssets'

View File

@ -7,7 +7,7 @@ services:
environment: environment:
POSTGRES_PASSWORD: password POSTGRES_PASSWORD: password
volumes: volumes:
- ./postgresql-log-slow-queries.conf:/etc/postgresql/postgresql.conf - /home/mi/Projekte/Hostsharing/hsadmin-ng/etc/postgresql-log-slow-queries.conf:/etc/postgresql/postgresql.conf
ports: ports:
- "5432:5432" - "5432:5432"
command: command:
@ -17,11 +17,3 @@ services:
apt-get update && apt-get update &&
apt-get install -y postgresql-contrib && apt-get install -y postgresql-contrib &&
docker-entrypoint.sh postgres -c config_file=/etc/postgresql/postgresql.conf docker-entrypoint.sh postgres -c config_file=/etc/postgresql/postgresql.conf
deploy:
resources:
limits:
cpus: '4'
memory: 4G
reservations:
cpus: '2'
memory: 2G

View File

@ -6,10 +6,8 @@ import net.hostsharing.hsadminng.hs.hosting.generated.api.v1.model.HsHostingAsse
import net.hostsharing.hsadminng.hs.validation.HsEntityValidator; import net.hostsharing.hsadminng.hs.validation.HsEntityValidator;
import jakarta.persistence.EntityManager; import jakarta.persistence.EntityManager;
import java.util.Arrays;
import java.util.Map; import java.util.Map;
import java.util.function.Function; import java.util.function.Function;
import java.util.regex.Pattern;
/** /**
* Wraps the steps of the pararation, validation, mapping and revamp around saving of a HsHostingAsset into a readable API. * Wraps the steps of the pararation, validation, mapping and revamp around saving of a HsHostingAsset into a readable API.
@ -42,14 +40,12 @@ public class HostingAssetEntitySaveProcessor {
return this; return this;
} }
// TODO.impl: remove once the migration of legacy data is done
/// validates the entity itself including its properties, but ignoring some error messages for import of legacy data /// validates the entity itself including its properties, but ignoring some error messages for import of legacy data
public HostingAssetEntitySaveProcessor validateEntityIgnoring(final String... ignoreRegExp) { public HostingAssetEntitySaveProcessor validateEntityIgnoring(final String ignoreRegExp) {
step("validateEntity", "prepareForSave"); step("validateEntity", "prepareForSave");
final var ignoreRegExpPatterns = Arrays.stream(ignoreRegExp).map(Pattern::compile).toList();
MultiValidationException.throwIfNotEmpty( MultiValidationException.throwIfNotEmpty(
validator.validateEntity(entity).stream() validator.validateEntity(entity).stream()
.filter(error -> ignoreRegExpPatterns.stream().noneMatch(p -> p.matcher(error).matches() )) .filter(errorMsg -> !errorMsg.matches(ignoreRegExp))
.toList() .toList()
); );
return this; return this;

View File

@ -11,12 +11,11 @@ import static net.hostsharing.hsadminng.hs.validation.StringProperty.stringPrope
class HsEMailAddressHostingAssetValidator extends HostingAssetEntityValidator { class HsEMailAddressHostingAssetValidator extends HostingAssetEntityValidator {
private static final String TARGET_MAILBOX_REGEX = "^[a-z][a-z0-9]{2}[0-9]{2}(-[a-z0-9][a-z0-9\\.+_-]*)?$"; // also accepts legacy pac-names private static final String UNIX_USER_REGEX = "^[a-z][a-z0-9]{2}[0-9]{2}(-[a-z0-9][a-z0-9\\._-]*)?$"; // also accepts legacy pac-names
private static final String EMAIL_ADDRESS_LOCAL_PART_REGEX = "[a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+"; // RFC 5322 private static final String EMAIL_ADDRESS_LOCAL_PART_REGEX = "[a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+"; // RFC 5322
private static final String EMAIL_ADDRESS_DOMAIN_PART_REGEX = "[a-zA-Z0-9.-]+"; private static final String EMAIL_ADDRESS_DOMAIN_PART_REGEX = "[a-zA-Z0-9.-]+";
private static final String EMAIL_ADDRESS_FULL_REGEX = "^(" + EMAIL_ADDRESS_LOCAL_PART_REGEX + ")?@" + EMAIL_ADDRESS_DOMAIN_PART_REGEX + "$"; private static final String EMAIL_ADDRESS_FULL_REGEX = "^(" + EMAIL_ADDRESS_LOCAL_PART_REGEX + ")?@" + EMAIL_ADDRESS_DOMAIN_PART_REGEX + "$";
private static final String NOBODY_REGEX = "^nobody$"; private static final String NOBODY_REGEX = "^nobody$";
private static final String DEVNULL_REGEX = "^/dev/null$";
public static final int EMAIL_ADDRESS_MAX_LENGTH = 320; // according to RFC 5321 and RFC 5322 public static final int EMAIL_ADDRESS_MAX_LENGTH = 320; // according to RFC 5321 and RFC 5322
HsEMailAddressHostingAssetValidator() { HsEMailAddressHostingAssetValidator() {
@ -26,7 +25,7 @@ class HsEMailAddressHostingAssetValidator extends HostingAssetEntityValidator {
stringProperty("local-part").matchesRegEx("^" + EMAIL_ADDRESS_LOCAL_PART_REGEX + "$").writeOnce().optional(), stringProperty("local-part").matchesRegEx("^" + EMAIL_ADDRESS_LOCAL_PART_REGEX + "$").writeOnce().optional(),
stringProperty("sub-domain").matchesRegEx("^" + EMAIL_ADDRESS_LOCAL_PART_REGEX + "$").writeOnce().optional(), stringProperty("sub-domain").matchesRegEx("^" + EMAIL_ADDRESS_LOCAL_PART_REGEX + "$").writeOnce().optional(),
arrayOf( arrayOf(
stringProperty("target").maxLength(EMAIL_ADDRESS_MAX_LENGTH).matchesRegEx(TARGET_MAILBOX_REGEX, EMAIL_ADDRESS_FULL_REGEX, NOBODY_REGEX, DEVNULL_REGEX) stringProperty("target").maxLength(EMAIL_ADDRESS_MAX_LENGTH).matchesRegEx(UNIX_USER_REGEX, EMAIL_ADDRESS_FULL_REGEX, NOBODY_REGEX)
).required().minLength(1)); ).required().minLength(1));
} }

View File

@ -33,13 +33,7 @@ class HsEMailAddressHostingAssetValidatorUnitTest {
.identifier("old-local-part@example.org") .identifier("old-local-part@example.org")
.config(new HashMap<>(ofEntries( .config(new HashMap<>(ofEntries(
entry("local-part", "old-local-part"), entry("local-part", "old-local-part"),
entry("target", Array.of( entry("target", Array.of("xyz00", "xyz00-abc", "office@example.com"))
"xyz00",
"xyz00-abc",
"xyz00-xyz+list",
"office@example.com",
"/dev/null"
))
))); )));
} }
@ -52,7 +46,7 @@ class HsEMailAddressHostingAssetValidatorUnitTest {
assertThat(validator.properties()).map(Map::toString).containsExactlyInAnyOrder( assertThat(validator.properties()).map(Map::toString).containsExactlyInAnyOrder(
"{type=string, propertyName=local-part, matchesRegEx=[^[a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+$], writeOnce=true}", "{type=string, propertyName=local-part, matchesRegEx=[^[a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+$], writeOnce=true}",
"{type=string, propertyName=sub-domain, matchesRegEx=[^[a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+$], writeOnce=true}", "{type=string, propertyName=sub-domain, matchesRegEx=[^[a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+$], writeOnce=true}",
"{type=string[], propertyName=target, elementsOf={type=string, propertyName=target, matchesRegEx=[^[a-z][a-z0-9]{2}[0-9]{2}(-[a-z0-9][a-z0-9\\.+_-]*)?$, ^([a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+)?@[a-zA-Z0-9.-]+$, ^nobody$, ^/dev/null$], maxLength=320}, required=true, minLength=1}"); "{type=string[], propertyName=target, elementsOf={type=string, propertyName=target, matchesRegEx=[^[a-z][a-z0-9]{2}[0-9]{2}(-[a-z0-9][a-z0-9\\._-]*)?$, ^([a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+)?@[a-zA-Z0-9.-]+$, ^nobody$], maxLength=320}, required=true, minLength=1}");
} }
@Test @Test
@ -75,11 +69,7 @@ class HsEMailAddressHostingAssetValidatorUnitTest {
.config(new HashMap<>(ofEntries( .config(new HashMap<>(ofEntries(
entry("local-part", "no@allowed"), entry("local-part", "no@allowed"),
entry("sub-domain", "no@allowedeither"), entry("sub-domain", "no@allowedeither"),
entry("target", Array.of( entry("target", Array.of("xyz00", "xyz00-abc", "garbage", "office@example.com")))))
"xyz00",
"xyz00-abc",
"garbage",
"office@example.com")))))
.build(); .build();
final var validator = HostingAssetEntityValidatorRegistry.forType(emailAddressHostingAssetEntity.getType()); final var validator = HostingAssetEntityValidatorRegistry.forType(emailAddressHostingAssetEntity.getType());
@ -90,7 +80,7 @@ class HsEMailAddressHostingAssetValidatorUnitTest {
assertThat(result).containsExactlyInAnyOrder( assertThat(result).containsExactlyInAnyOrder(
"'EMAIL_ADDRESS:old-local-part@example.org.config.local-part' is expected to match any of [^[a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+$] but 'no@allowed' does not match", "'EMAIL_ADDRESS:old-local-part@example.org.config.local-part' is expected to match any of [^[a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+$] but 'no@allowed' does not match",
"'EMAIL_ADDRESS:old-local-part@example.org.config.sub-domain' is expected to match any of [^[a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+$] but 'no@allowedeither' does not match", "'EMAIL_ADDRESS:old-local-part@example.org.config.sub-domain' is expected to match any of [^[a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+$] but 'no@allowedeither' does not match",
"'EMAIL_ADDRESS:old-local-part@example.org.config.target' is expected to match any of [^[a-z][a-z0-9]{2}[0-9]{2}(-[a-z0-9][a-z0-9\\.+_-]*)?$, ^([a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+)?@[a-zA-Z0-9.-]+$, ^nobody$, ^/dev/null$] but 'garbage' does not match any"); "'EMAIL_ADDRESS:old-local-part@example.org.config.target' is expected to match any of [^[a-z][a-z0-9]{2}[0-9]{2}(-[a-z0-9][a-z0-9\\._-]*)?$, ^([a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+)?@[a-zA-Z0-9.-]+$, ^nobody$] but 'garbage' does not match any");
} }
@Test @Test

View File

@ -20,7 +20,6 @@ import org.springframework.transaction.support.TransactionTemplate;
import jakarta.persistence.EntityManager; import jakarta.persistence.EntityManager;
import jakarta.persistence.PersistenceContext; import jakarta.persistence.PersistenceContext;
import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletRequest;
import jakarta.validation.ValidationException;
import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.NotNull;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.IOException; import java.io.IOException;
@ -310,7 +309,7 @@ public class CsvDataImport extends ContextBasedTest {
void logError(final Runnable assertion) { void logError(final Runnable assertion) {
try { try {
assertion.run(); assertion.run();
} catch (final AssertionError | ValidationException exc) { } catch (final AssertionError exc) {
logError(exc.getMessage()); logError(exc.getMessage());
} }
} }

View File

@ -668,9 +668,6 @@ public class ImportHostingAssets extends ImportOfficeData {
if (isImportingControlledTestData()) { if (isImportingControlledTestData()) {
expectError("zonedata dom_owner of mellis.de is old00 but expected to be mim00"); expectError("zonedata dom_owner of mellis.de is old00 but expected to be mim00");
expectError("\nexpected: \"vm1068\"\n but was: \"vm1093\""); expectError("\nexpected: \"vm1068\"\n but was: \"vm1093\"");
expectError("['EMAIL_ADDRESS:webmaster@hamburg-west.l-u-g.org.config.target' is expected to match any of [^[a-z][a-z0-9]{2}[0-9]{2}(-[a-z0-9][a-z0-9\\.+_-]*)?$, ^([a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+)?@[a-zA-Z0-9.-]+$, ^nobody$, ^/dev/null$] but 'raoul.lottmann@example.com peter.lottmann@example.com' does not match any]");
expectError("['EMAIL_ADDRESS:abuse@mellis.de.config.target' length is expected to be at min 1 but length of [[]] is 0]");
expectError("['EMAIL_ADDRESS:abuse@ist-im-netz.de.config.target' length is expected to be at min 1 but length of [[]] is 0]");
} }
this.assertNoErrors(); this.assertNoErrors();
} }
@ -923,10 +920,7 @@ public class ImportHostingAssets extends ImportOfficeData {
logError(() -> logError(() ->
new HostingAssetEntitySaveProcessor(em, entry.getValue()) new HostingAssetEntitySaveProcessor(em, entry.getValue())
.preprocessEntity() .preprocessEntity()
.validateEntityIgnoring( .validateEntityIgnoring("'EMAIL_ALIAS:.*\\.config\\.target' .*")
"'EMAIL_ALIAS:.*\\.config\\.target' .*",
"'EMAIL_ADDRESS:.*\\.config\\.target' .*"
)
.prepareForSave() .prepareForSave()
.saveUsing(entity -> persist(entry.getKey(), entity)) .saveUsing(entity -> persist(entry.getKey(), entity))
.validateContext() .validateContext()

View File

@ -12,7 +12,7 @@ emailaddr_id;domain_id;localpart;subdomain;target
54760;4531;info;hamburg-west;peter.lottmann@example.com 54760;4531;info;hamburg-west;peter.lottmann@example.com
54761;4531;lugmaster;hamburg-west;raoul.lottmann@example.com 54761;4531;lugmaster;hamburg-west;raoul.lottmann@example.com
54762;4531;postmaster;hamburg-west;raoul.lottmann@example.com 54762;4531;postmaster;hamburg-west;raoul.lottmann@example.com
54763;4531;webmaster;hamburg-west;raoul.lottmann@example.com peter.lottmann@example.com 54763;4531;webmaster;hamburg-west;raoul.lottmann@example.com
54764;4531;;eliza;eliza@example.net 54764;4531;;eliza;eliza@example.net
54765;4531;;;lug00 54765;4531;;;lug00
54766;4532;;;nomail 54766;4532;;;nomail
@ -22,10 +22,10 @@ emailaddr_id;domain_id;localpart;subdomain;target
54963;4581;abuse;;mim00 54963;4581;abuse;;mim00
54964;4581;postmaster;;mim00 54964;4581;postmaster;;mim00
54965;4581;webmaster;;mim00 54965;4581;webmaster;;mim00
54981;4587;abuse;; 54981;4587;abuse;;mim00
54982;4587;postmaster;;/dev/null 54982;4587;postmaster;;mim00
54983;4587;webmaster;;mim00 54983;4587;webmaster;;mim00
54987;4589;abuse;;"" 54987;4589;abuse;;mim00
54988;4589;postmaster;;mim00 54988;4589;postmaster;;mim00
54989;4589;webmaster;;mim00 54989;4589;webmaster;;mim00
55020;4600;abuse;;mim00 55020;4600;abuse;;mim00

1 emailaddr_id domain_id localpart subdomain target
12 54760 4531 info hamburg-west peter.lottmann@example.com
13 54761 4531 lugmaster hamburg-west raoul.lottmann@example.com
14 54762 4531 postmaster hamburg-west raoul.lottmann@example.com
15 54763 4531 webmaster hamburg-west raoul.lottmann@example.com peter.lottmann@example.com raoul.lottmann@example.com
16 54764 4531 eliza eliza@example.net
17 54765 4531 lug00
18 54766 4532 nomail
22 54963 4581 abuse mim00
23 54964 4581 postmaster mim00
24 54965 4581 webmaster mim00
25 54981 4587 abuse mim00
26 54982 4587 postmaster /dev/null mim00
27 54983 4587 webmaster mim00
28 54987 4589 abuse mim00
29 54988 4589 postmaster mim00
30 54989 4589 webmaster mim00
31 55020 4600 abuse mim00