add-ipnumber-validatation #77
15
.aliases
15
.aliases
@ -1,9 +1,6 @@
|
|||||||
# For using the alias import-office-tables, # copy these exports to .environment (ignored by git)
|
# For using the alias import-office-tables,
|
||||||
# and amend them according to your external DB:
|
# copy the file .tc-environment to .environment (ignored by git)
|
||||||
export HSADMINNG_POSTGRES_JDBC_URL=jdbc:tc:postgresql:15.5-bookworm:///spring_boot_testcontainers
|
# and amend them according to your external DB.
|
||||||
export HSADMINNG_POSTGRES_ADMIN_USERNAME=admin
|
|
||||||
export HSADMINNG_POSTGRES_ADMIN_PASSWORD=
|
|
||||||
export HSADMINNG_POSTGRES_RESTRICTED_USERNAME=restricted
|
|
||||||
|
|
||||||
gradleWrapper () {
|
gradleWrapper () {
|
||||||
if [ ! -f gradlew ]; then
|
if [ ! -f gradlew ]; then
|
||||||
@ -46,11 +43,7 @@ postgresAutodoc () {
|
|||||||
alias postgres-autodoc=postgresAutodoc
|
alias postgres-autodoc=postgresAutodoc
|
||||||
|
|
||||||
function importOfficeData() {
|
function importOfficeData() {
|
||||||
export HSADMINNG_POSTGRES_JDBC_URL=jdbc:tc:postgresql:15.5-bookworm:///spring_boot_testcontainers
|
source .tc-environment
|
||||||
export HSADMINNG_POSTGRES_ADMIN_USERNAME=admin
|
|
||||||
export HSADMINNG_POSTGRES_ADMIN_PASSWORD=password
|
|
||||||
export HSADMINNG_POSTGRES_RESTRICTED_USERNAME=restricted
|
|
||||||
export HSADMINNG_SUPERUSER=superuser-alex@hostsharing.net
|
|
||||||
|
|
||||||
if [ -f .environment ]; then
|
if [ -f .environment ]; then
|
||||||
source .environment
|
source .environment
|
||||||
|
5
.tc-environment
Normal file
5
.tc-environment
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
export HSADMINNG_POSTGRES_JDBC_URL=jdbc:tc:postgresql:15.5-bookworm:///spring_boot_testcontainers
|
||||||
|
export HSADMINNG_POSTGRES_ADMIN_USERNAME=admin
|
||||||
|
export HSADMINNG_POSTGRES_ADMIN_PASSWORD=
|
||||||
|
export HSADMINNG_POSTGRES_RESTRICTED_USERNAME=restricted
|
||||||
|
export HSADMINNG_MIGRATION_DATA_PATH=migration
|
@ -92,13 +92,7 @@ import static org.assertj.core.api.Fail.fail;
|
|||||||
-- maybe something like that is needed for the 2nd user
|
-- maybe something like that is needed for the 2nd user
|
||||||
-- GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public to hsh99_restricted;
|
-- GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public to hsh99_restricted;
|
||||||
|
|
||||||
* Then copy this to a file named .environment (excluded from git) and fill in your specific values:
|
* Then copy the file .tc-environment to a file named .environment (excluded from git) and fill in your specific values.
|
||||||
|
|
||||||
export HSADMINNG_POSTGRES_JDBC_URL=jdbc:postgresql://localhost:6432/hsh99_hsadminng
|
|
||||||
export HSADMINNG_POSTGRES_ADMIN_USERNAME=hsh99_admin
|
|
||||||
export HSADMINNG_POSTGRES_ADMIN_PASSWORD=password
|
|
||||||
export HSADMINNG_POSTGRES_RESTRICTED_USERNAME=hsh99_restricted
|
|
||||||
export HSADMINNG_SUPERUSER=some-precreated-superuser@example.org
|
|
||||||
|
|
||||||
* To finally import the office data, run:
|
* To finally import the office data, run:
|
||||||
*
|
*
|
||||||
@ -131,9 +125,21 @@ public class ImportOfficeData extends ContextBasedTest {
|
|||||||
|
|
||||||
// at least as the number of lines in business-partners.csv from test-data, but less than real data partner count
|
// at least as the number of lines in business-partners.csv from test-data, but less than real data partner count
|
||||||
public static final int MAX_NUMBER_OF_TEST_DATA_PARTNERS = 100;
|
public static final int MAX_NUMBER_OF_TEST_DATA_PARTNERS = 100;
|
||||||
|
public static final String MIGRATION_DATA_PATH = ofNullable(System.getenv("HSADMINNG_MIGRATION_DATA_PATH")).orElse("migration") + "/";
|
||||||
|
|
||||||
static int relationId = 2000000;
|
static int relationId = 2000000;
|
||||||
|
|
||||||
|
private static final List<Integer> IGNORE_BUSINESS_PARTNERS = Arrays.asList(
|
||||||
|
512167, // 11139, partner without contractual contact
|
||||||
|
512170, // 11142, partner without contractual contact
|
||||||
|
-1
|
||||||
|
);
|
||||||
|
|
||||||
|
private static final List<Integer> IGNORE_CONTACTS = Arrays.asList(
|
||||||
|
90547, // Kontakt hat keine Rolle
|
||||||
|
-1
|
||||||
|
);
|
||||||
|
|
||||||
@Value("${spring.datasource.url}")
|
@Value("${spring.datasource.url}")
|
||||||
private String jdbcUrl;
|
private String jdbcUrl;
|
||||||
|
|
||||||
@ -171,7 +177,7 @@ public class ImportOfficeData extends ContextBasedTest {
|
|||||||
@Order(1010)
|
@Order(1010)
|
||||||
void importBusinessPartners() {
|
void importBusinessPartners() {
|
||||||
|
|
||||||
try (Reader reader = resourceReader("migration/business-partners.csv")) {
|
try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "business-partners.csv")) {
|
||||||
final var lines = readAllLines(reader);
|
final var lines = readAllLines(reader);
|
||||||
importBusinessPartners(justHeader(lines), withoutHeader(lines));
|
importBusinessPartners(justHeader(lines), withoutHeader(lines));
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
@ -217,7 +223,7 @@ public class ImportOfficeData extends ContextBasedTest {
|
|||||||
@Order(1020)
|
@Order(1020)
|
||||||
void importContacts() {
|
void importContacts() {
|
||||||
|
|
||||||
try (Reader reader = resourceReader("migration/contacts.csv")) {
|
try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "contacts.csv")) {
|
||||||
final var lines = readAllLines(reader);
|
final var lines = readAllLines(reader);
|
||||||
importContacts(justHeader(lines), withoutHeader(lines));
|
importContacts(justHeader(lines), withoutHeader(lines));
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
@ -241,16 +247,16 @@ public class ImportOfficeData extends ContextBasedTest {
|
|||||||
""");
|
""");
|
||||||
assertThat(toFormattedString(contacts)).isEqualToIgnoringWhitespace("""
|
assertThat(toFormattedString(contacts)).isEqualToIgnoringWhitespace("""
|
||||||
{
|
{
|
||||||
1101=contact(caption='Herr Michael Mellies ', emailAddresses='{ main: mih@example.org }'),
|
1101=contact(caption='Herr Michael Mellies ', emailAddresses='{ "main": "mih@example.org"}'),
|
||||||
1200=contact(caption='JM e.K.', emailAddresses='{ main: jm-ex-partner@example.org }'),
|
1200=contact(caption='JM e.K.', emailAddresses='{ "main": "jm-ex-partner@example.org"}'),
|
||||||
1201=contact(caption='Frau Dr. Jenny Meyer-Billing , JM GmbH', emailAddresses='{ main: jm-billing@example.org }'),
|
1201=contact(caption='Frau Dr. Jenny Meyer-Billing , JM GmbH', emailAddresses='{ "main": "jm-billing@example.org"}'),
|
||||||
1202=contact(caption='Herr Andrew Meyer-Operation , JM GmbH', emailAddresses='{ main: am-operation@example.org }'),
|
1202=contact(caption='Herr Andrew Meyer-Operation , JM GmbH', emailAddresses='{ "main": "am-operation@example.org"}'),
|
||||||
1203=contact(caption='Herr Philip Meyer-Contract , JM GmbH', emailAddresses='{ main: pm-partner@example.org }'),
|
1203=contact(caption='Herr Philip Meyer-Contract , JM GmbH', emailAddresses='{ "main": "pm-partner@example.org"}'),
|
||||||
1204=contact(caption='Frau Tammy Meyer-VIP , JM GmbH', emailAddresses='{ main: tm-vip@example.org }'),
|
1204=contact(caption='Frau Tammy Meyer-VIP , JM GmbH', emailAddresses='{ "main": "tm-vip@example.org"}'),
|
||||||
1301=contact(caption='Petra Schmidt , Test PS', emailAddresses='{ main: ps@example.com }'),
|
1301=contact(caption='Petra Schmidt , Test PS', emailAddresses='{ "main": "ps@example.com"}'),
|
||||||
1401=contact(caption='Frau Frauke Fanninga ', emailAddresses='{ main: ff@example.org }'),
|
1401=contact(caption='Frau Frauke Fanninga ', emailAddresses='{ "main": "ff@example.org"}'),
|
||||||
1501=contact(caption='Frau Cecilia Camus ', emailAddresses='{ main: cc@example.org }')
|
1501=contact(caption='Frau Cecilia Camus ', emailAddresses='{ "main": "cc@example.org"}')
|
||||||
}
|
}
|
||||||
""");
|
""");
|
||||||
assertThat(toFormattedString(persons)).isEqualToIgnoringWhitespace("""
|
assertThat(toFormattedString(persons)).isEqualToIgnoringWhitespace("""
|
||||||
{
|
{
|
||||||
@ -317,7 +323,7 @@ public class ImportOfficeData extends ContextBasedTest {
|
|||||||
@Order(1030)
|
@Order(1030)
|
||||||
void importSepaMandates() {
|
void importSepaMandates() {
|
||||||
|
|
||||||
try (Reader reader = resourceReader("migration/sepa-mandates.csv")) {
|
try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "sepa-mandates.csv")) {
|
||||||
final var lines = readAllLines(reader);
|
final var lines = readAllLines(reader);
|
||||||
importSepaMandates(justHeader(lines), withoutHeader(lines));
|
importSepaMandates(justHeader(lines), withoutHeader(lines));
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
@ -349,7 +355,7 @@ public class ImportOfficeData extends ContextBasedTest {
|
|||||||
@Test
|
@Test
|
||||||
@Order(1040)
|
@Order(1040)
|
||||||
void importCoopShares() {
|
void importCoopShares() {
|
||||||
try (Reader reader = resourceReader("migration/share-transactions.csv")) {
|
try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "share-transactions.csv")) {
|
||||||
final var lines = readAllLines(reader);
|
final var lines = readAllLines(reader);
|
||||||
importCoopShares(justHeader(lines), withoutHeader(lines));
|
importCoopShares(justHeader(lines), withoutHeader(lines));
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
@ -376,7 +382,7 @@ public class ImportOfficeData extends ContextBasedTest {
|
|||||||
@Order(1050)
|
@Order(1050)
|
||||||
void importCoopAssets() {
|
void importCoopAssets() {
|
||||||
|
|
||||||
try (Reader reader = resourceReader("migration/asset-transactions.csv")) {
|
try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "asset-transactions.csv")) {
|
||||||
final var lines = readAllLines(reader);
|
final var lines = readAllLines(reader);
|
||||||
importCoopAssets(justHeader(lines), withoutHeader(lines));
|
importCoopAssets(justHeader(lines), withoutHeader(lines));
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
@ -737,6 +743,10 @@ public class ImportOfficeData extends ContextBasedTest {
|
|||||||
.map(this::trimAll)
|
.map(this::trimAll)
|
||||||
.map(row -> new Record(columns, row))
|
.map(row -> new Record(columns, row))
|
||||||
.forEach(rec -> {
|
.forEach(rec -> {
|
||||||
|
if (this.IGNORE_BUSINESS_PARTNERS.contains(rec.getInteger("bp_id"))) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
final var person = HsOfficePersonEntity.builder().build();
|
final var person = HsOfficePersonEntity.builder().build();
|
||||||
|
|
||||||
final var partnerRel = addRelation(
|
final var partnerRel = addRelation(
|
||||||
@ -838,6 +848,11 @@ public class ImportOfficeData extends ContextBasedTest {
|
|||||||
.map(row -> new Record(columns, row))
|
.map(row -> new Record(columns, row))
|
||||||
.forEach(rec -> {
|
.forEach(rec -> {
|
||||||
final var bpId = rec.getInteger("bp_id");
|
final var bpId = rec.getInteger("bp_id");
|
||||||
|
|
||||||
|
if (this.IGNORE_BUSINESS_PARTNERS.contains(bpId)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
final var member = ofNullable(memberships.get(bpId))
|
final var member = ofNullable(memberships.get(bpId))
|
||||||
.orElseGet(() -> createOnDemandMembership(bpId));
|
.orElseGet(() -> createOnDemandMembership(bpId));
|
||||||
|
|
||||||
@ -908,6 +923,10 @@ public class ImportOfficeData extends ContextBasedTest {
|
|||||||
.forEach(rec -> {
|
.forEach(rec -> {
|
||||||
final var debitor = debitors.get(rec.getInteger("bp_id"));
|
final var debitor = debitors.get(rec.getInteger("bp_id"));
|
||||||
|
|
||||||
|
if (this.IGNORE_BUSINESS_PARTNERS.contains(rec.getInteger("bp_id"))) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
final var sepaMandate = HsOfficeSepaMandateEntity.builder()
|
final var sepaMandate = HsOfficeSepaMandateEntity.builder()
|
||||||
.debitor(debitor)
|
.debitor(debitor)
|
||||||
.bankAccount(HsOfficeBankAccountEntity.builder()
|
.bankAccount(HsOfficeBankAccountEntity.builder()
|
||||||
@ -939,6 +958,13 @@ public class ImportOfficeData extends ContextBasedTest {
|
|||||||
final var contactId = rec.getInteger("contact_id");
|
final var contactId = rec.getInteger("contact_id");
|
||||||
final var bpId = rec.getInteger("bp_id");
|
final var bpId = rec.getInteger("bp_id");
|
||||||
|
|
||||||
|
if (this.IGNORE_CONTACTS.contains(contactId)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (this.IGNORE_BUSINESS_PARTNERS.contains(bpId)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (rec.getString("roles").isBlank()) {
|
if (rec.getString("roles").isBlank()) {
|
||||||
fail("empty roles assignment not allowed for contact_id: " + contactId);
|
fail("empty roles assignment not allowed for contact_id: " + contactId);
|
||||||
}
|
}
|
||||||
@ -1109,6 +1135,7 @@ public class ImportOfficeData extends ContextBasedTest {
|
|||||||
return "{\n" +
|
return "{\n" +
|
||||||
map.keySet().stream()
|
map.keySet().stream()
|
||||||
.map(id -> " " + id + "=" + map.get(id).toString())
|
.map(id -> " " + id + "=" + map.get(id).toString())
|
||||||
|
.map(e -> e.replaceAll("\n ", " ").replace("\n", ""))
|
||||||
.collect(Collectors.joining(",\n")) +
|
.collect(Collectors.joining(",\n")) +
|
||||||
"\n}\n";
|
"\n}\n";
|
||||||
}
|
}
|
||||||
@ -1196,13 +1223,6 @@ public class ImportOfficeData extends ContextBasedTest {
|
|||||||
return new InputStreamReader(requireNonNull(getClass().getClassLoader().getResourceAsStream(resourcePath)));
|
return new InputStreamReader(requireNonNull(getClass().getClassLoader().getResourceAsStream(resourcePath)));
|
||||||
}
|
}
|
||||||
|
|
||||||
private Reader fileReader(@NotNull final Path filePath) throws IOException {
|
|
||||||
// Path path = Paths.get(
|
|
||||||
// ClassLoader.getSystemResource("csv/twoColumn.csv").toURI())
|
|
||||||
// );
|
|
||||||
return Files.newBufferedReader(filePath);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String[] justHeader(final List<String[]> lines) {
|
private static String[] justHeader(final List<String[]> lines) {
|
||||||
return stream(lines.getFirst()).map(String::trim).toArray(String[]::new);
|
return stream(lines.getFirst()).map(String::trim).toArray(String[]::new);
|
||||||
}
|
}
|
||||||
|
@ -6,6 +6,10 @@ dbname="hsh02_hsdb"
|
|||||||
username="hsh02_hsdb_readonly"
|
username="hsh02_hsdb_readonly"
|
||||||
|
|
||||||
target="/tmp"
|
target="/tmp"
|
||||||
|
if [ ! -z $DEST ];
|
||||||
|
then
|
||||||
|
target=$DEST
|
||||||
|
fi
|
||||||
|
|
||||||
dump() {
|
dump() {
|
||||||
sql="copy ($1) to stdout with csv header delimiter ';' quote '\"'"
|
sql="copy ($1) to stdout with csv header delimiter ';' quote '\"'"
|
||||||
@ -41,3 +45,59 @@ dump "select member_share_id, bp_id, date, action, quantity, comment
|
|||||||
WHERE bp_id NOT IN (511912)
|
WHERE bp_id NOT IN (511912)
|
||||||
order by member_share_id" \
|
order by member_share_id" \
|
||||||
"share-transactions.csv"
|
"share-transactions.csv"
|
||||||
|
|
||||||
|
dump "select inet_addr_id, inet_addr, description
|
||||||
|
from inet_addr
|
||||||
|
order by inet_addr_id" \
|
||||||
|
"inet_addr.csv"
|
||||||
|
|
||||||
|
dump "select hive_id, hive_name, inet_addr_id, description
|
||||||
|
from hive
|
||||||
|
order by hive_id" \
|
||||||
|
"hive.csv"
|
||||||
|
|
||||||
|
dump "select packet_id, basepacket_code, packet_name, bp_id, hive_id, created, cancelled, cur_inet_addr_id, old_inet_addr_id, free
|
||||||
|
from packet
|
||||||
|
left join basepacket using (basepacket_id)
|
||||||
|
order by packet_id" \
|
||||||
|
"packet.csv"
|
||||||
|
|
||||||
|
dump "select packet_component_id, packet_id, quantity, basecomponent_code, created, cancelled
|
||||||
|
from packet_component
|
||||||
|
left join basecomponent using (basecomponent_id)
|
||||||
|
order by packet_component_id" \
|
||||||
|
"packet_component.csv"
|
||||||
|
|
||||||
|
dump "select unixuser_id, name, comment, shell, homedir, locked, packet_id, userid, quota_softlimit, quota_hardlimit, storage_softlimit, storage_hardlimit
|
||||||
|
from unixuser
|
||||||
|
order by unixuser_id" \
|
||||||
|
"unixuser.csv"
|
||||||
|
|
||||||
|
# weil das fehlt, muss group by komplett gesetzt werden: alter table domain add constraint PK_domain primary key (domain_id);
|
||||||
|
dump "select domain_id, domain_name, domain_since, domain_dns_master, domain_owner, valid_subdomain_names, passenger_python, passenger_nodejs, passenger_ruby, fcgi_php_bin, array_to_string(array_agg(domain_option_name), ',') as domainoptions
|
||||||
|
from domain
|
||||||
|
left join domain__domain_option using(domain_id)
|
||||||
|
left join domain_option using (domain_option_id)
|
||||||
|
group by domain.domain_id, domain.domain_name, domain_since, domain_dns_master, domain_owner, valid_subdomain_names, passenger_python, passenger_nodejs, passenger_ruby, fcgi_php_bin
|
||||||
|
order by domain.domain_id" \
|
||||||
|
"domain.csv"
|
||||||
|
|
||||||
|
dump "select emailaddr_id, domain_id, localpart, subdomain, target
|
||||||
|
from emailaddr
|
||||||
|
order by emailaddr_id" \
|
||||||
|
"emailaddr.csv"
|
||||||
|
|
||||||
|
dump "select emailalias_id, pac_id, name, target
|
||||||
|
from emailalias
|
||||||
|
order by emailalias_id" \
|
||||||
|
"emailalias.csv"
|
||||||
|
|
||||||
|
dump "select dbuser_id, engine, packet_id, name
|
||||||
|
from database_user
|
||||||
|
order by dbuser_id" \
|
||||||
|
"database_user.csv"
|
||||||
|
|
||||||
|
dump "select database_id, engine, packet_id, name, owner, encoding
|
||||||
|
from database
|
||||||
|
order by database_id" \
|
||||||
|
"database.csv"
|
||||||
|
Loading…
Reference in New Issue
Block a user