Compare commits

...

2 Commits

8 changed files with 51 additions and 15 deletions

View File

@ -84,3 +84,9 @@ alias fp='grep -r '@Accepts' src | sed -e 's/^.*@/@/g' | sort -u | wc -l'
alias gw-spotless='./gradlew spotlessApply -x pitest -x test -x :processResources'
alias gw-test='. .aliases; ./gradlew test importOfficeData'
alias gw-check='. .aliases; gw test importOfficeData check -x pitest -x :dependencyCheckAnalyze'
# etc/docker-compose.yml limits CPUs+MEM and includes a PostgreSQL config for analysing slow queries
alias gw-importOfficeData-in-docker-compose='
docker-compose -f etc/docker-compose.yml down &&
docker-compose -f etc/docker-compose.yml up -d && sleep 10 &&
time gw-importHostingAssets'

View File

@ -7,7 +7,7 @@ services:
environment:
POSTGRES_PASSWORD: password
volumes:
- /home/mi/Projekte/Hostsharing/hsadmin-ng/etc/postgresql-log-slow-queries.conf:/etc/postgresql/postgresql.conf
- ./postgresql-log-slow-queries.conf:/etc/postgresql/postgresql.conf
ports:
- "5432:5432"
command:
@ -17,3 +17,11 @@ services:
apt-get update &&
apt-get install -y postgresql-contrib &&
docker-entrypoint.sh postgres -c config_file=/etc/postgresql/postgresql.conf
deploy:
resources:
limits:
cpus: '4'
memory: 4G
reservations:
cpus: '2'
memory: 2G

View File

@ -6,8 +6,10 @@ import net.hostsharing.hsadminng.hs.hosting.generated.api.v1.model.HsHostingAsse
import net.hostsharing.hsadminng.hs.validation.HsEntityValidator;
import jakarta.persistence.EntityManager;
import java.util.Arrays;
import java.util.Map;
import java.util.function.Function;
import java.util.regex.Pattern;
/**
* Wraps the steps of the pararation, validation, mapping and revamp around saving of a HsHostingAsset into a readable API.
@ -40,12 +42,14 @@ public class HostingAssetEntitySaveProcessor {
return this;
}
// TODO.impl: remove once the migration of legacy data is done
/// validates the entity itself including its properties, but ignoring some error messages for import of legacy data
public HostingAssetEntitySaveProcessor validateEntityIgnoring(final String ignoreRegExp) {
public HostingAssetEntitySaveProcessor validateEntityIgnoring(final String... ignoreRegExp) {
step("validateEntity", "prepareForSave");
final var ignoreRegExpPatterns = Arrays.stream(ignoreRegExp).map(Pattern::compile).toList();
MultiValidationException.throwIfNotEmpty(
validator.validateEntity(entity).stream()
.filter(errorMsg -> !errorMsg.matches(ignoreRegExp))
.filter(error -> ignoreRegExpPatterns.stream().noneMatch(p -> p.matcher(error).matches() ))
.toList()
);
return this;

View File

@ -11,11 +11,12 @@ import static net.hostsharing.hsadminng.hs.validation.StringProperty.stringPrope
class HsEMailAddressHostingAssetValidator extends HostingAssetEntityValidator {
private static final String UNIX_USER_REGEX = "^[a-z][a-z0-9]{2}[0-9]{2}(-[a-z0-9][a-z0-9\\._-]*)?$"; // also accepts legacy pac-names
private static final String TARGET_MAILBOX_REGEX = "^[a-z][a-z0-9]{2}[0-9]{2}(-[a-z0-9][a-z0-9\\.+_-]*)?$"; // also accepts legacy pac-names
private static final String EMAIL_ADDRESS_LOCAL_PART_REGEX = "[a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+"; // RFC 5322
private static final String EMAIL_ADDRESS_DOMAIN_PART_REGEX = "[a-zA-Z0-9.-]+";
private static final String EMAIL_ADDRESS_FULL_REGEX = "^(" + EMAIL_ADDRESS_LOCAL_PART_REGEX + ")?@" + EMAIL_ADDRESS_DOMAIN_PART_REGEX + "$";
private static final String NOBODY_REGEX = "^nobody$";
private static final String DEVNULL_REGEX = "^/dev/null$";
public static final int EMAIL_ADDRESS_MAX_LENGTH = 320; // according to RFC 5321 and RFC 5322
HsEMailAddressHostingAssetValidator() {
@ -25,7 +26,7 @@ class HsEMailAddressHostingAssetValidator extends HostingAssetEntityValidator {
stringProperty("local-part").matchesRegEx("^" + EMAIL_ADDRESS_LOCAL_PART_REGEX + "$").writeOnce().optional(),
stringProperty("sub-domain").matchesRegEx("^" + EMAIL_ADDRESS_LOCAL_PART_REGEX + "$").writeOnce().optional(),
arrayOf(
stringProperty("target").maxLength(EMAIL_ADDRESS_MAX_LENGTH).matchesRegEx(UNIX_USER_REGEX, EMAIL_ADDRESS_FULL_REGEX, NOBODY_REGEX)
stringProperty("target").maxLength(EMAIL_ADDRESS_MAX_LENGTH).matchesRegEx(TARGET_MAILBOX_REGEX, EMAIL_ADDRESS_FULL_REGEX, NOBODY_REGEX, DEVNULL_REGEX)
).required().minLength(1));
}

View File

@ -33,7 +33,13 @@ class HsEMailAddressHostingAssetValidatorUnitTest {
.identifier("old-local-part@example.org")
.config(new HashMap<>(ofEntries(
entry("local-part", "old-local-part"),
entry("target", Array.of("xyz00", "xyz00-abc", "office@example.com"))
entry("target", Array.of(
"xyz00",
"xyz00-abc",
"xyz00-xyz+list",
"office@example.com",
"/dev/null"
))
)));
}
@ -46,7 +52,7 @@ class HsEMailAddressHostingAssetValidatorUnitTest {
assertThat(validator.properties()).map(Map::toString).containsExactlyInAnyOrder(
"{type=string, propertyName=local-part, matchesRegEx=[^[a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+$], writeOnce=true}",
"{type=string, propertyName=sub-domain, matchesRegEx=[^[a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+$], writeOnce=true}",
"{type=string[], propertyName=target, elementsOf={type=string, propertyName=target, matchesRegEx=[^[a-z][a-z0-9]{2}[0-9]{2}(-[a-z0-9][a-z0-9\\._-]*)?$, ^([a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+)?@[a-zA-Z0-9.-]+$, ^nobody$], maxLength=320}, required=true, minLength=1}");
"{type=string[], propertyName=target, elementsOf={type=string, propertyName=target, matchesRegEx=[^[a-z][a-z0-9]{2}[0-9]{2}(-[a-z0-9][a-z0-9\\.+_-]*)?$, ^([a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+)?@[a-zA-Z0-9.-]+$, ^nobody$, ^/dev/null$], maxLength=320}, required=true, minLength=1}");
}
@Test
@ -69,7 +75,11 @@ class HsEMailAddressHostingAssetValidatorUnitTest {
.config(new HashMap<>(ofEntries(
entry("local-part", "no@allowed"),
entry("sub-domain", "no@allowedeither"),
entry("target", Array.of("xyz00", "xyz00-abc", "garbage", "office@example.com")))))
entry("target", Array.of(
"xyz00",
"xyz00-abc",
"garbage",
"office@example.com")))))
.build();
final var validator = HostingAssetEntityValidatorRegistry.forType(emailAddressHostingAssetEntity.getType());
@ -80,7 +90,7 @@ class HsEMailAddressHostingAssetValidatorUnitTest {
assertThat(result).containsExactlyInAnyOrder(
"'EMAIL_ADDRESS:old-local-part@example.org.config.local-part' is expected to match any of [^[a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+$] but 'no@allowed' does not match",
"'EMAIL_ADDRESS:old-local-part@example.org.config.sub-domain' is expected to match any of [^[a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+$] but 'no@allowedeither' does not match",
"'EMAIL_ADDRESS:old-local-part@example.org.config.target' is expected to match any of [^[a-z][a-z0-9]{2}[0-9]{2}(-[a-z0-9][a-z0-9\\._-]*)?$, ^([a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+)?@[a-zA-Z0-9.-]+$, ^nobody$] but 'garbage' does not match any");
"'EMAIL_ADDRESS:old-local-part@example.org.config.target' is expected to match any of [^[a-z][a-z0-9]{2}[0-9]{2}(-[a-z0-9][a-z0-9\\.+_-]*)?$, ^([a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+)?@[a-zA-Z0-9.-]+$, ^nobody$, ^/dev/null$] but 'garbage' does not match any");
}
@Test

View File

@ -20,6 +20,7 @@ import org.springframework.transaction.support.TransactionTemplate;
import jakarta.persistence.EntityManager;
import jakarta.persistence.PersistenceContext;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.validation.ValidationException;
import jakarta.validation.constraints.NotNull;
import java.io.BufferedReader;
import java.io.IOException;
@ -309,7 +310,7 @@ public class CsvDataImport extends ContextBasedTest {
void logError(final Runnable assertion) {
try {
assertion.run();
} catch (final AssertionError exc) {
} catch (final AssertionError | ValidationException exc) {
logError(exc.getMessage());
}
}

View File

@ -668,6 +668,9 @@ public class ImportHostingAssets extends ImportOfficeData {
if (isImportingControlledTestData()) {
expectError("zonedata dom_owner of mellis.de is old00 but expected to be mim00");
expectError("\nexpected: \"vm1068\"\n but was: \"vm1093\"");
expectError("['EMAIL_ADDRESS:webmaster@hamburg-west.l-u-g.org.config.target' is expected to match any of [^[a-z][a-z0-9]{2}[0-9]{2}(-[a-z0-9][a-z0-9\\.+_-]*)?$, ^([a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+)?@[a-zA-Z0-9.-]+$, ^nobody$, ^/dev/null$] but 'raoul.lottmann@example.com peter.lottmann@example.com' does not match any]");
expectError("['EMAIL_ADDRESS:abuse@mellis.de.config.target' length is expected to be at min 1 but length of [[]] is 0]");
expectError("['EMAIL_ADDRESS:abuse@ist-im-netz.de.config.target' length is expected to be at min 1 but length of [[]] is 0]");
}
this.assertNoErrors();
}
@ -920,7 +923,10 @@ public class ImportHostingAssets extends ImportOfficeData {
logError(() ->
new HostingAssetEntitySaveProcessor(em, entry.getValue())
.preprocessEntity()
.validateEntityIgnoring("'EMAIL_ALIAS:.*\\.config\\.target' .*")
.validateEntityIgnoring(
"'EMAIL_ALIAS:.*\\.config\\.target' .*",
"'EMAIL_ADDRESS:.*\\.config\\.target' .*"
)
.prepareForSave()
.saveUsing(entity -> persist(entry.getKey(), entity))
.validateContext()

View File

@ -12,7 +12,7 @@ emailaddr_id;domain_id;localpart;subdomain;target
54760;4531;info;hamburg-west;peter.lottmann@example.com
54761;4531;lugmaster;hamburg-west;raoul.lottmann@example.com
54762;4531;postmaster;hamburg-west;raoul.lottmann@example.com
54763;4531;webmaster;hamburg-west;raoul.lottmann@example.com
54763;4531;webmaster;hamburg-west;raoul.lottmann@example.com peter.lottmann@example.com
54764;4531;;eliza;eliza@example.net
54765;4531;;;lug00
54766;4532;;;nomail
@ -22,10 +22,10 @@ emailaddr_id;domain_id;localpart;subdomain;target
54963;4581;abuse;;mim00
54964;4581;postmaster;;mim00
54965;4581;webmaster;;mim00
54981;4587;abuse;;mim00
54982;4587;postmaster;;mim00
54981;4587;abuse;;
54982;4587;postmaster;;/dev/null
54983;4587;webmaster;;mim00
54987;4589;abuse;;mim00
54987;4589;abuse;;""
54988;4589;postmaster;;mim00
54989;4589;webmaster;;mim00
55020;4600;abuse;;mim00

1 emailaddr_id domain_id localpart subdomain target
12 54760 4531 info hamburg-west peter.lottmann@example.com
13 54761 4531 lugmaster hamburg-west raoul.lottmann@example.com
14 54762 4531 postmaster hamburg-west raoul.lottmann@example.com
15 54763 4531 webmaster hamburg-west raoul.lottmann@example.com raoul.lottmann@example.com peter.lottmann@example.com
16 54764 4531 eliza eliza@example.net
17 54765 4531 lug00
18 54766 4532 nomail
22 54963 4581 abuse mim00
23 54964 4581 postmaster mim00
24 54965 4581 webmaster mim00
25 54981 4587 abuse mim00
26 54982 4587 postmaster mim00 /dev/null
27 54983 4587 webmaster mim00
28 54987 4589 abuse mim00
29 54988 4589 postmaster mim00
30 54989 4589 webmaster mim00
31 55020 4600 abuse mim00