Merge remote-tracking branch 'refs/remotes/origin/master' into add-ipnumber-validatation

Co-authored-by: Timotheus Pokorra <timotheus.pokorra@hostsharing.net>
Co-authored-by: Timotheus Pokorra <timotheus.pokorra@solidcharity.com>
Co-authored-by: Dev und Test fuer hsadminng <hsh03-hsngdev@h50.hostsharing.net>
Co-authored-by: Michael Hoennig <michael@hoennig.de>
Reviewed-on: #78

.aliases

@@ -1,9 +1,6 @@
-# For using the alias import-office-tables, # copy these exports to .environment (ignored by git) 
-# and amend them according to your external DB:
-export HSADMINNG_POSTGRES_JDBC_URL=jdbc:tc:postgresql:15.5-bookworm:///spring_boot_testcontainers
-export HSADMINNG_POSTGRES_ADMIN_USERNAME=admin
-export HSADMINNG_POSTGRES_ADMIN_PASSWORD=
-export HSADMINNG_POSTGRES_RESTRICTED_USERNAME=restricted
+# For using the alias import-office-tables,
+# copy the file .tc-environment to .environment (ignored by git)
+# and amend them according to your external DB.
 
 gradleWrapper () {
     if [ ! -f gradlew ]; then
@@ -46,11 +43,7 @@ postgresAutodoc () {
 alias postgres-autodoc=postgresAutodoc
 
 function importOfficeData() {
-	export HSADMINNG_POSTGRES_JDBC_URL=jdbc:tc:postgresql:15.5-bookworm:///spring_boot_testcontainers
-	export HSADMINNG_POSTGRES_ADMIN_USERNAME=admin
-	export HSADMINNG_POSTGRES_ADMIN_PASSWORD=password
-	export HSADMINNG_POSTGRES_RESTRICTED_USERNAME=restricted
-	export HSADMINNG_SUPERUSER=superuser-alex@hostsharing.net
+	source .tc-environment
 	
 	if [ -f .environment ]; then
 		source .environment

.tc-environment

@@ -0,0 +1,5 @@
+export HSADMINNG_POSTGRES_JDBC_URL=jdbc:tc:postgresql:15.5-bookworm:///spring_boot_testcontainers
+export HSADMINNG_POSTGRES_ADMIN_USERNAME=admin
+export HSADMINNG_POSTGRES_ADMIN_PASSWORD=
+export HSADMINNG_POSTGRES_RESTRICTED_USERNAME=restricted
+export HSADMINNG_MIGRATION_DATA_PATH=migration

doc/hs-hosting-asset-type-structure.md

@@ -1,7 +1,7 @@
 ## HostingAsset Type Structure
 
 
-### Webspace+Server
+### Server+Webspace
 
 ```plantuml
 @startuml
@@ -22,6 +22,12 @@ package Hosting #feb28c{
         entity HA_IPV6_NUMBER
     }
 
+    package Webspace #99bcdb {
+        entity HA_MANAGED_WEBSPACE
+        entity HA_UNIX_USER
+        entity HA_EMAIL_ALIAS
+    }
+
 }
 
 BI_CLOUD_SERVER *--> BI_PRIVATE_CLOUD
@@ -30,10 +36,16 @@ BI_MANAGED_WEBSPACE *--> BI_MANAGED_SERVER
 
 HA_CLOUD_SERVER *==> BI_CLOUD_SERVER
 HA_MANAGED_SERVER *==> BI_MANAGED_SERVER
+HA_MANAGED_WEBSPACE *==> BI_MANAGED_WEBSPACE
+HA_MANAGED_WEBSPACE o..> HA_MANAGED_SERVER
+HA_UNIX_USER *==> HA_MANAGED_WEBSPACE
+HA_EMAIL_ALIAS *==> HA_MANAGED_WEBSPACE
 HA_IPV4_NUMBER o..> HA_CLOUD_SERVER
 HA_IPV4_NUMBER o..> HA_MANAGED_SERVER
+HA_IPV4_NUMBER o..> HA_MANAGED_WEBSPACE
 HA_IPV6_NUMBER o..> HA_CLOUD_SERVER
 HA_IPV6_NUMBER o..> HA_MANAGED_SERVER
+HA_IPV6_NUMBER o..> HA_MANAGED_WEBSPACE
 
 package Legend #white {
     SUB_ENTITY1 *--> REQUIRED_PARENT_ENTITY

src/main/java/net/hostsharing/hsadminng/hs/booking/item/HsBookingItemType.java

@@ -30,7 +30,7 @@ public enum HsBookingItemType implements Node {
 
     @Override
     public boolean belongsToAny(final Set<String> groups) {
-        return true;
+        return true; // we currently do not filter booking item types
     }
 
     @Override

src/main/java/net/hostsharing/hsadminng/hs/hosting/asset/HsHostingAssetType.java

@@ -316,7 +316,7 @@ public enum HsHostingAssetType implements Node {
                 .collect(toSet()));
 
         markdown
-                .append(renderAsPlantUML("Webspace+Server", Set.of("Server")))
+                .append(renderAsPlantUML("Server+Webspace", Set.of("Server", "Webspace")))
                 .append(renderAsPlantUML("Domain", Set.of("Domain", "Webspace")))
                 .append(renderAsPlantUML("MariaDB", Set.of("MariaDB", "Webspace")))
                 .append(renderAsPlantUML("PostgreSQL", Set.of("PostgreSQL", "Webspace")));

src/test/java/net/hostsharing/hsadminng/hs/hosting/asset/HsHostingAssetControllerRestTest.java

@@ -511,7 +511,7 @@ public class HsHostingAssetControllerRestTest {
         IPV6_NUMBER(
                 List.of(
                         HsHostingAssetEntity.builder()
-                                .type(HsHostingAssetType.IPV4_NUMBER)
+                                .type(HsHostingAssetType.IPV6_NUMBER)
                                 .assignedToAsset(TEST_MANAGED_SERVER_HOSTING_ASSET)
                                 .identifier("2001:db8:3333:4444:5555:6666:7777:8888")
                                 .caption("some fake IPv6 number")
@@ -519,7 +519,7 @@ public class HsHostingAssetControllerRestTest {
                 """
                     [
                         {
-                            "type": "IPV4_NUMBER",
+                            "type": "IPV6_NUMBER",
                             "identifier": "2001:db8:3333:4444:5555:6666:7777:8888",
                             "caption": "some fake IPv6 number",
                             "alarmContact": null,

src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportOfficeData.java

@@ -92,13 +92,7 @@ import static org.assertj.core.api.Fail.fail;
     -- maybe something like that is needed for the 2nd user
     -- GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public to hsh99_restricted;
 
- * Then copy this to a file named .environment (excluded from git) and fill in your specific values:
-
-   export HSADMINNG_POSTGRES_JDBC_URL=jdbc:postgresql://localhost:6432/hsh99_hsadminng
-   export HSADMINNG_POSTGRES_ADMIN_USERNAME=hsh99_admin
-   export HSADMINNG_POSTGRES_ADMIN_PASSWORD=password
-   export HSADMINNG_POSTGRES_RESTRICTED_USERNAME=hsh99_restricted
-   export HSADMINNG_SUPERUSER=some-precreated-superuser@example.org
+ * Then copy the file .tc-environment to a file named .environment (excluded from git) and fill in your specific values.
 
  * To finally import the office data, run:
  *
@@ -131,9 +125,21 @@ public class ImportOfficeData extends ContextBasedTest {
 
     // at least as the number of lines in business-partners.csv from test-data, but less than real data partner count
     public static final int MAX_NUMBER_OF_TEST_DATA_PARTNERS = 100;
+    public static final String MIGRATION_DATA_PATH = ofNullable(System.getenv("HSADMINNG_MIGRATION_DATA_PATH")).orElse("migration") + "/";
 
     static int relationId = 2000000;
 
+    private static final List<Integer> IGNORE_BUSINESS_PARTNERS = Arrays.asList(
+        512167, // 11139, partner without contractual contact
+        512170, // 11142, partner without contractual contact
+        -1
+    );
+
+    private static final List<Integer> IGNORE_CONTACTS = Arrays.asList(
+        90547, // Kontakt hat keine Rolle
+        -1
+    );
+
     @Value("${spring.datasource.url}")
     private String jdbcUrl;
 
@@ -171,7 +177,7 @@ public class ImportOfficeData extends ContextBasedTest {
     @Order(1010)
     void importBusinessPartners() {
 
-        try (Reader reader = resourceReader("migration/business-partners.csv")) {
+        try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "business-partners.csv")) {
             final var lines = readAllLines(reader);
             importBusinessPartners(justHeader(lines), withoutHeader(lines));
         } catch (Exception e) {
@@ -217,7 +223,7 @@ public class ImportOfficeData extends ContextBasedTest {
     @Order(1020)
     void importContacts() {
 
-        try (Reader reader = resourceReader("migration/contacts.csv")) {
+        try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "contacts.csv")) {
             final var lines = readAllLines(reader);
             importContacts(justHeader(lines), withoutHeader(lines));
         } catch (Exception e) {
@@ -241,15 +247,15 @@ public class ImportOfficeData extends ContextBasedTest {
                 """);
         assertThat(toFormattedString(contacts)).isEqualToIgnoringWhitespace("""
                 {
-                    1101=contact(caption='Herr Michael Mellies ', emailAddresses='{ main: mih@example.org }'),
-                    1200=contact(caption='JM e.K.', emailAddresses='{ main: jm-ex-partner@example.org }'),
-                    1201=contact(caption='Frau Dr. Jenny Meyer-Billing , JM GmbH', emailAddresses='{ main: jm-billing@example.org }'),
-                    1202=contact(caption='Herr Andrew Meyer-Operation , JM GmbH', emailAddresses='{ main: am-operation@example.org }'),
-                    1203=contact(caption='Herr Philip Meyer-Contract , JM GmbH', emailAddresses='{ main: pm-partner@example.org }'),
-                    1204=contact(caption='Frau Tammy Meyer-VIP , JM GmbH', emailAddresses='{ main: tm-vip@example.org }'),
-                    1301=contact(caption='Petra Schmidt , Test PS', emailAddresses='{ main: ps@example.com }'),
-                    1401=contact(caption='Frau Frauke Fanninga ', emailAddresses='{ main: ff@example.org }'),
-                    1501=contact(caption='Frau Cecilia Camus ', emailAddresses='{ main: cc@example.org }')
+                    1101=contact(caption='Herr Michael Mellies ', emailAddresses='{ "main": "mih@example.org"}'),
+                    1200=contact(caption='JM e.K.', emailAddresses='{ "main": "jm-ex-partner@example.org"}'),
+                    1201=contact(caption='Frau Dr. Jenny Meyer-Billing , JM GmbH', emailAddresses='{ "main": "jm-billing@example.org"}'),
+                    1202=contact(caption='Herr Andrew Meyer-Operation , JM GmbH', emailAddresses='{ "main": "am-operation@example.org"}'),
+                    1203=contact(caption='Herr Philip Meyer-Contract , JM GmbH', emailAddresses='{ "main": "pm-partner@example.org"}'),
+                    1204=contact(caption='Frau Tammy Meyer-VIP , JM GmbH', emailAddresses='{ "main": "tm-vip@example.org"}'),
+                    1301=contact(caption='Petra Schmidt , Test PS', emailAddresses='{ "main": "ps@example.com"}'),
+                    1401=contact(caption='Frau Frauke Fanninga ', emailAddresses='{ "main": "ff@example.org"}'),
+                    1501=contact(caption='Frau Cecilia Camus ', emailAddresses='{ "main": "cc@example.org"}')
                  }
                 """);
         assertThat(toFormattedString(persons)).isEqualToIgnoringWhitespace("""
@@ -317,7 +323,7 @@ public class ImportOfficeData extends ContextBasedTest {
     @Order(1030)
     void importSepaMandates() {
 
-        try (Reader reader = resourceReader("migration/sepa-mandates.csv")) {
+        try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "sepa-mandates.csv")) {
             final var lines = readAllLines(reader);
             importSepaMandates(justHeader(lines), withoutHeader(lines));
         } catch (Exception e) {
@@ -349,7 +355,7 @@ public class ImportOfficeData extends ContextBasedTest {
     @Test
     @Order(1040)
     void importCoopShares() {
-        try (Reader reader = resourceReader("migration/share-transactions.csv")) {
+        try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "share-transactions.csv")) {
             final var lines = readAllLines(reader);
             importCoopShares(justHeader(lines), withoutHeader(lines));
         } catch (Exception e) {
@@ -376,7 +382,7 @@ public class ImportOfficeData extends ContextBasedTest {
     @Order(1050)
     void importCoopAssets() {
 
-        try (Reader reader = resourceReader("migration/asset-transactions.csv")) {
+        try (Reader reader = resourceReader(MIGRATION_DATA_PATH + "asset-transactions.csv")) {
             final var lines = readAllLines(reader);
             importCoopAssets(justHeader(lines), withoutHeader(lines));
         } catch (Exception e) {
@@ -737,6 +743,10 @@ public class ImportOfficeData extends ContextBasedTest {
                 .map(this::trimAll)
                 .map(row -> new Record(columns, row))
                 .forEach(rec -> {
+                    if (this.IGNORE_BUSINESS_PARTNERS.contains(rec.getInteger("bp_id"))) {
+                        return;
+                    }
+
                     final var person = HsOfficePersonEntity.builder().build();
 
                     final var partnerRel = addRelation(
@@ -838,6 +848,11 @@ public class ImportOfficeData extends ContextBasedTest {
                 .map(row -> new Record(columns, row))
                 .forEach(rec -> {
                     final var bpId = rec.getInteger("bp_id");
+
+                    if (this.IGNORE_BUSINESS_PARTNERS.contains(bpId)) {
+                        return;
+                    }
+
                     final var member = ofNullable(memberships.get(bpId))
                             .orElseGet(() -> createOnDemandMembership(bpId));
 
@@ -908,6 +923,10 @@ public class ImportOfficeData extends ContextBasedTest {
                 .forEach(rec -> {
                     final var debitor = debitors.get(rec.getInteger("bp_id"));
 
+                    if (this.IGNORE_BUSINESS_PARTNERS.contains(rec.getInteger("bp_id"))) {
+                        return;
+                    }
+
                     final var sepaMandate = HsOfficeSepaMandateEntity.builder()
                             .debitor(debitor)
                             .bankAccount(HsOfficeBankAccountEntity.builder()
@@ -939,6 +958,13 @@ public class ImportOfficeData extends ContextBasedTest {
                     final var contactId = rec.getInteger("contact_id");
                     final var bpId = rec.getInteger("bp_id");
 
+                    if (this.IGNORE_CONTACTS.contains(contactId)) {
+                        return;
+                    }
+                    if (this.IGNORE_BUSINESS_PARTNERS.contains(bpId)) {
+                        return;
+                    }
+
                     if (rec.getString("roles").isBlank()) {
                         fail("empty roles assignment not allowed for contact_id: " + contactId);
                     }
@@ -1109,6 +1135,7 @@ public class ImportOfficeData extends ContextBasedTest {
         return "{\n" +
                 map.keySet().stream()
                         .map(id -> "   " + id + "=" + map.get(id).toString())
+                        .map(e -> e.replaceAll("\n    ", " ").replace("\n", ""))
                         .collect(Collectors.joining(",\n")) +
                 "\n}\n";
     }
@@ -1196,13 +1223,6 @@ public class ImportOfficeData extends ContextBasedTest {
         return new InputStreamReader(requireNonNull(getClass().getClassLoader().getResourceAsStream(resourcePath)));
     }
 
-    private Reader fileReader(@NotNull final Path filePath) throws IOException {
-        //        Path path = Paths.get(
-        //                ClassLoader.getSystemResource("csv/twoColumn.csv").toURI())
-        //    );
-        return Files.newBufferedReader(filePath);
-    }
-
     private static String[] justHeader(final List<String[]> lines) {
         return stream(lines.getFirst()).map(String::trim).toArray(String[]::new);
     }

src/test/resources/migration/dump.sh

@@ -6,6 +6,10 @@ dbname="hsh02_hsdb"
 username="hsh02_hsdb_readonly"
 
 target="/tmp"
+if [ ! -z $DEST ];
+then
+    target=$DEST
+fi
 
 dump() {
     sql="copy ($1) to stdout with csv header delimiter ';' quote '\"'"
@@ -41,3 +45,59 @@ dump "select member_share_id, bp_id, date, action, quantity, comment
      WHERE bp_id NOT IN (511912)
       order by member_share_id" \
      "share-transactions.csv"
+
+dump "select inet_addr_id, inet_addr, description
+      from inet_addr
+      order by inet_addr_id" \
+     "inet_addr.csv"
+
+dump "select hive_id, hive_name, inet_addr_id, description
+      from hive
+      order by hive_id" \
+     "hive.csv"
+
+dump "select packet_id, basepacket_code, packet_name, bp_id, hive_id, created, cancelled, cur_inet_addr_id, old_inet_addr_id, free
+      from packet
+      left join basepacket using (basepacket_id)
+      order by packet_id" \
+     "packet.csv"
+
+dump "select packet_component_id, packet_id, quantity, basecomponent_code, created, cancelled
+      from packet_component
+      left join basecomponent using (basecomponent_id)
+      order by packet_component_id" \
+     "packet_component.csv"
+
+dump "select unixuser_id, name, comment, shell, homedir, locked, packet_id, userid, quota_softlimit, quota_hardlimit, storage_softlimit, storage_hardlimit
+      from unixuser
+      order by unixuser_id" \
+     "unixuser.csv"
+
+# weil das fehlt, muss group by komplett gesetzt werden: alter table domain add constraint PK_domain primary key (domain_id);
+dump "select domain_id, domain_name, domain_since, domain_dns_master, domain_owner, valid_subdomain_names, passenger_python, passenger_nodejs, passenger_ruby, fcgi_php_bin, array_to_string(array_agg(domain_option_name), ',') as domainoptions
+      from domain
+      left join domain__domain_option using(domain_id)
+      left join domain_option using (domain_option_id)
+      group by domain.domain_id, domain.domain_name, domain_since, domain_dns_master, domain_owner, valid_subdomain_names, passenger_python, passenger_nodejs, passenger_ruby, fcgi_php_bin
+      order by domain.domain_id" \
+     "domain.csv"
+
+dump "select emailaddr_id, domain_id, localpart, subdomain, target
+      from emailaddr
+      order by emailaddr_id" \
+     "emailaddr.csv"
+
+dump "select emailalias_id, pac_id, name, target
+      from emailalias
+      order by emailalias_id" \
+     "emailalias.csv"
+
+dump "select dbuser_id, engine, packet_id, name
+      from database_user
+      order by dbuser_id" \
+     "database_user.csv"
+
+dump "select database_id, engine, packet_id, name, owner, encoding
+      from database
+      order by database_id" \
+     "database.csv"