add ex-partner to Relationship-Type and properly extract distinct persons from contacts.csv

UML object diagram for new target data model for business partners
2024-01-18 14:03:06 +01:00 · 2024-01-18 13:57:46 +01:00
11 changed files with 282 additions and 224 deletions
--- a/doc/hs-office-data-structure.md
+++ b/doc/hs-office-data-structure.md
@ -4,46 +4,51 @@
 classDiagram
    direction TD

-    namespace Geschäftspartner {
+    namespace Partner {
        
       class partner-MeierGmbH
-       class contact-MeierGmbH
+       class personDetails-MeierGmbH
+       class contactData-MeierGmbH
       class person-MeierGmbH
       
       class person-FrankMeier
-       class contact-FrankMeier
+       class contactData-FrankMeier
       class role-MeierGmbH-FrankMeier

       class person-SabineMeier
-       class contact-SabineMeier
+       class contactData-SabineMeier
       class role-MeierGmbH-SabineMeier
       
        class debitor-MeierGmbH
-        class contact-MeierGmbH-Buha
+        class contactData-MeierGmbH-Buha
        class role-MeierGmbH-Buha
    }

    class partner-MeierGmbH {
+        +partnerNumber: 12345
        +person
-        +contact
-        +details
    }
    partner-MeierGmbH o-- person-MeierGmbH
-    partner-MeierGmbH o-- contact-MeierGmbH

    class person-MeierGmbH {
        +personType: LEGAL
        +tradeName: Meier GmbH
        +familyName
        +givenName
+    }
+    person-MeierGmbH *-- personDetails-MeierGmbH : optiona
+
+    class personDetails-MeierGmbH {
+        +personContact
        +registrationOffice: AG Hamburg
        +registrationNumber: ABC123434
        +birthName
        +birthPlace
        +dateOfDeath
    }
+    personDetails-MeierGmbH o-- contactData-MeierGmbH

-    class contact-MeierGmbH {
+    class contactData-MeierGmbH {
        +postalAddress:     Hauptstraße 5, 22345 Hamburg
        +phoneNumbers:      +49 40 12345-00
        +emailAddresses:    office@meier-gmbh.de
@ -62,23 +67,23 @@ classDiagram
        +String     defaultPrefix:          mei
    }
    debitor-MeierGmbH o-- partner-MeierGmbH
-    debitor-MeierGmbH o-- role-MeierGmbH-Buha
+    debitor-MeierGmbH *-- role-MeierGmbH-Buha

-    class contact-MeierGmbH-Buha {
+    class contactData-MeierGmbH-Buha {
        +postalAddress:     Hauptstraße 5, 22345 Hamburg
        +phoneNumbers:      +49 40 12345-05
        +emailAddresses:    buha@meier-gmbh.de
    }

    class role-MeierGmbH-Buha {
-        +RelType relType ACCOUNTING
-        +Person relAnchor
-        +Person relHolder
-        +Contact contact
+        +RoleType RoleType ACCOUNTING
+        +Person anchor
+        +Person holder
+        +Contact roleContact
    }
-    role-MeierGmbH-Buha o-- person-MeierGmbH : relAnchor
-    role-MeierGmbH-Buha o-- person-MeierGmbH : relHolder
-    role-MeierGmbH-Buha o-- contact-MeierGmbH-Buha
+    role-MeierGmbH-Buha o-- person-MeierGmbH : anchor
+    role-MeierGmbH-Buha o-- person-MeierGmbH : holder
+    role-MeierGmbH-Buha o-- contactData-MeierGmbH-Buha

    class person-FrankMeier {
        + personType: NATURAL
@ -87,21 +92,21 @@ classDiagram
        + givenName: Frank
    }

-    class contact-FrankMeier {
+    class contactData-FrankMeier {
        +postalAddress
        +phoneNumbers: +49 40 12345-22
        +emailAddresses: frank.meier@meier-gmbh.de
    }

    class role-MeierGmbH-FrankMeier {
-        +RelType relType REPRESENTATIVE
-        +Person relAnchor
-        +Person relHolder
-        +Contact contact
+        +RoleType RoleType REPRESENTATIVE
+        +Person anchor
+        +Person holder
+        +Contact roleContact
    }
-    role-MeierGmbH-FrankMeier o-- person-MeierGmbH : relAnchor
-    role-MeierGmbH-FrankMeier o-- person-FrankMeier : relHolder
-    role-MeierGmbH-FrankMeier o-- contact-FrankMeier
+    role-MeierGmbH-FrankMeier o-- person-MeierGmbH : anchor
+    role-MeierGmbH-FrankMeier o-- person-FrankMeier : holder
+    role-MeierGmbH-FrankMeier o-- contactData-FrankMeier

    class person-SabineMeier {
        +personType: NATURAL
@ -110,25 +115,25 @@ classDiagram
        +givenName: Sabine
    }

-    class contact-SabineMeier {
+    class contactData-SabineMeier {
        +postalAddress
        +phoneNumbers: +49 40 12345-22
        +emailAddresses: sabine.meier@meier-gmbh.de
    }

    class role-MeierGmbH-SabineMeier {
-        +RelType relType OPERATIONAL
-        +Person relAnchor
-        +Person relHolder
-        +Contact contact
+        +RoleType RoleType OPERATIONAL
+        +Person anchor
+        +Person holder
+        +Contact roleContact
    }
-    role-MeierGmbH-SabineMeier o-- person-MeierGmbH : relAnchor
-    role-MeierGmbH-SabineMeier o-- person-SabineMeier : relHolder
-    role-MeierGmbH-SabineMeier o-- contact-SabineMeier
+    role-MeierGmbH-SabineMeier o-- person-MeierGmbH : anchor
+    role-MeierGmbH-SabineMeier o-- person-SabineMeier : holder
+    role-MeierGmbH-SabineMeier o-- contactData-SabineMeier

    namespace Enums {

-        class RelType {
+        class RoleType {
            <<enumeration>>
            UNKNOWN
            REPRESENTATIVE
--- a/src/main/java/net/hostsharing/hsadminng/hs/office/membership/HsOfficeMembershipEntity.java
+++ b/src/main/java/net/hostsharing/hsadminng/hs/office/membership/HsOfficeMembershipEntity.java
@ -53,7 +53,7 @@ public class HsOfficeMembershipEntity implements HasUuid, Stringifyable {
    private HsOfficeDebitorEntity mainDebitor;

    @Column(name = "membernumber")
-    private int memberNumber;
+    private int memberNumber; // TODO: migrate to suffix, like debitorNumberSuffix

    @Column(name = "validity", columnDefinition = "daterange")
    @Type(PostgreSQLRangeType.class)
--- a/src/main/java/net/hostsharing/hsadminng/hs/office/relationship/HsOfficeRelationshipType.java
+++ b/src/main/java/net/hostsharing/hsadminng/hs/office/relationship/HsOfficeRelationshipType.java
@ -2,6 +2,7 @@ package net.hostsharing.hsadminng.hs.office.relationship;

 public enum HsOfficeRelationshipType {
    UNKNOWN,
+    EX_PARTNER,
    REPRESENTATIVE,
    ACCOUNTING,
    OPERATIONS
--- a/src/main/resources/db/changelog/213-hs-office-person-rbac.sql
+++ b/src/main/resources/db/changelog/213-hs-office-person-rbac.sql
@ -37,6 +37,7 @@ begin
            grantedByRole => globalAdmin()
        );

+    -- TODO: who is admin? the person itself? is it allowed for the person itself or a representative to edit the data?
    perform createRoleWithGrants(
            hsOfficePersonAdmin(NEW),
            permissions => array['edit'],
--- a/src/main/resources/db/changelog/230-hs-office-relationship.sql
+++ b/src/main/resources/db/changelog/230-hs-office-relationship.sql
@ -4,7 +4,7 @@
 --changeset hs-office-relationship-MAIN-TABLE:1 endDelimiter:--//
 -- ----------------------------------------------------------------------------

-CREATE TYPE HsOfficeRelationshipType AS ENUM ('UNKNOWN', 'REPRESENTATIVE', 'ACCOUNTING', 'OPERATIONS');
+CREATE TYPE HsOfficeRelationshipType AS ENUM ('UNKNOWN', 'EX-PARTNER', 'REPRESENTATIVE', 'ACCOUNTING', 'OPERATIONS');

 CREATE CAST (character varying as HsOfficeRelationshipType) WITH INOUT AS IMPLICIT;

--- a/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportOfficeTables.java
+++ b/src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportOfficeTables.java
@ -22,11 +22,12 @@ import net.hostsharing.hsadminng.hs.office.relationship.HsOfficeRelationshipEnti
 import net.hostsharing.hsadminng.hs.office.relationship.HsOfficeRelationshipType;
 import net.hostsharing.hsadminng.hs.office.sepamandate.HsOfficeSepaMandateEntity;
 import net.hostsharing.test.JpaAttempt;
+import org.apache.commons.lang3.StringUtils;
 import org.junit.jupiter.api.*;
 import org.junit.jupiter.api.extension.BeforeEachCallback;
-import org.junit.jupiter.api.extension.TestWatcher;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.TestWatcher;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest;
@ -45,6 +46,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.LocalDate;
 import java.util.*;
+import java.util.stream.Collectors;

 import static java.util.Arrays.stream;
 import static java.util.Objects.requireNonNull;
@ -85,22 +87,24 @@ import static org.assertj.core.api.Fail.fail;
    -- maybe something like that is needed for the 2nd user
    -- GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public to hsh99_restricted;

- * Then copy this to a file named .environment (excluded from git):
+ * Then copy this to a file named .environment (excluded from git) and fill in your specific values:

   export HSADMINNG_POSTGRES_JDBC_URL=jdbc:postgresql://localhost:6432/hsh99_hsadminng
   export HSADMINNG_POSTGRES_ADMIN_USERNAME=hsh99_admin
   export HSADMINNG_POSTGRES_ADMIN_PASSWORD=password
   export HSADMINNG_POSTGRES_RESTRICTED_USERNAME=hsh99_restricted
+   export HSADMINNG_SUPERUSER=some-precreated-superuser@example.org

- * Then, import the office data, uncomment the @Diabled and then run:
+ * To finally import the office data, run:
 *
- *   import-office-tables # comes from .aliases file
+ *   import-office-tables # comes from .aliases file and uses .environment
 */
@Tag("import")
@DataJpaTest(properties = {
        "spring.datasource.url=${HSADMINNG_POSTGRES_JDBC_URL:jdbc:tc:postgresql:15.5-bookworm:///spring_boot_testcontainers}",
        "spring.datasource.username=${HSADMINNG_POSTGRES_ADMIN_USERNAME:admin}",
-        "spring.datasource.password=${HSADMINNG_POSTGRES_ADMIN_PASSWORD:password}"
+        "spring.datasource.password=${HSADMINNG_POSTGRES_ADMIN_PASSWORD:password}",
+        "hsadminng.superuser=${HSADMINNG_SUPERUSER:superuser-alex@hostsharing.net}"
 })
@Import({ Context.class, JpaAttempt.class })
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
@ -110,8 +114,8 @@ public class ImportOfficeTables extends ContextBasedTest {
    @Value("${spring.datasource.username}")
    private String postgresAdminUser;

-    // TODO: use real rbacSuperuser for actual import
-    private static final String rbacSuperuser = "superuser-alex@hostsharing.net";
+    @Value("${hsadminng.superuser}")
+    private String rbacSuperuser;

    private static NavigableMap<Integer, HsOfficeContactEntity> contacts = new TreeMap<>();
    private static NavigableMap<Integer, HsOfficePersonEntity> persons = new TreeMap<>();
@ -150,32 +154,30 @@ public class ImportOfficeTables extends ContextBasedTest {
            throw new RuntimeException(e);
        }

-        if ( !"admin".equals(postgresAdminUser) ) {
-            return;
-        }
+        assumeThat(postgresAdminUser).isEqualTo("admin");

        // no contacts yet => mostly null values
-        assertThat(partners.toString()).isEqualToIgnoringWhitespace("""
-            {
-                 7=partner(UNKNOWN null, null),
-                10=partner(UNKNOWN null, null),
-                12=partner(UNKNOWN null, null)
-            }
-            """);
-        assertThat(contacts.toString()).isEqualTo("{}");
-        assertThat(debitors.toString()).isEqualToIgnoringWhitespace("""
+        assertThat(toFormattedString(partners)).isEqualToIgnoringWhitespace("""
                {
-                     7=debitor(1000700: UNKNOWN null, null: mih),
-                    10=debitor(1001000: UNKNOWN null, null: xyz),
-                    12=debitor(1101200: UNKNOWN null, null: xxx)}
+                    17=partner(null null, null),
+                    20=partner(null null, null),
+                    22=partner(null null, null)
+                }
+                """);
+        assertThat(toFormattedString(contacts)).isEqualTo("{}");
+        assertThat(toFormattedString(debitors)).isEqualToIgnoringWhitespace("""
+                {
+                    17=debitor(1001700: null null, null: mih),
+                    20=debitor(1002000: null null, null: xyz),
+                    22=debitor(1102200: null null, null: xxx)}
+                """);
+        assertThat(toFormattedString(memberships)).isEqualToIgnoringWhitespace("""
+                {
+                    17=Membership(10017, null null, null, 1001700, [2000-12-06,), NONE),
+                    20=Membership(10020, null null, null, 1002000, [2000-12-06,2016-01-01), UNKNOWN), 
+                    22=Membership(11022, null null, null, 1102200, [2021-04-01,), NONE)
+                }
                """);
-        assertThat(memberships.toString()).isEqualToIgnoringWhitespace("""
-            {
-                 7=Membership(10007, UNKNOWN null, null, 1000700, [2000-12-06,), NONE),
-                10=Membership(10010, UNKNOWN null, null, 1001000, [2000-12-06,2016-01-01), UNKNOWN), 
-                12=Membership(11012, UNKNOWN null, null, 1101200, [2021-04-01,), NONE)
-            }
-            """);
    }

    @Test
@ -189,48 +191,48 @@ public class ImportOfficeTables extends ContextBasedTest {
            throw new RuntimeException(e);
        }

-        if ( !"admin".equals(postgresAdminUser) ) {
-            return;
-        }
+        assumeThat(postgresAdminUser).isEqualTo("admin");

-        assertThat(partners.toString()).isEqualToIgnoringWhitespace("""
+        assertThat(toFormattedString(partners)).isEqualToIgnoringWhitespace("""
                {
-                     7=partner(NATURAL Mellies, Michael: Herr Michael Mellies ),
-                    10=partner(LEGAL JM e.K.: Herr Philip Meyer-Contract , JM e.K.),
-                    12=partner(LEGAL Test PS: Petra Schmidt , Test PS)
+                    17=partner(NATURAL Mellies, Michael: Herr Michael Mellies ),
+                    20=partner(LEGAL JM e.K.: Herr Philip Meyer-Contract , JM e.K.),
+                    22=partner(LEGAL Test PS: Petra Schmidt , Test PS)
                }
                """);
-        assertThat(contacts.toString()).isEqualToIgnoringWhitespace("""
+        assertThat(toFormattedString(contacts)).isEqualToIgnoringWhitespace("""
                {
-                   1101=contact(label='Herr Michael Mellies ', emailAddresses='mih@example.org'),
-                   1201=contact(label='Frau Dr. Jenny Meyer-Billing , JM e.K.', emailAddresses='jm-billing@example.org'),
-                   1202=contact(label='Herr Andrew Meyer-Operation , JM e.K.', emailAddresses='am-operation@example.org'),
-                   1203=contact(label='Herr Philip Meyer-Contract , JM e.K.', emailAddresses='pm-partner@example.org'),
-                   1301=contact(label='Petra Schmidt , Test PS', emailAddresses='ps@example.com')
+                    1101=contact(label='Herr Michael Mellies ', emailAddresses='mih@example.org'),
+                    1201=contact(label='Frau Dr. Jenny Meyer-Billing , JM e.K.', emailAddresses='jm-billing@example.org'),
+                    1202=contact(label='Herr Andrew Meyer-Operation , JM e.K.', emailAddresses='am-operation@example.org'),
+                    1203=contact(label='Herr Philip Meyer-Contract , JM e.K.', emailAddresses='pm-partner@example.org'),
+                    1301=contact(label='Petra Schmidt , Test PS', emailAddresses='ps@example.com')
                }
                """);
-        assertThat(persons.toString()).isEqualToIgnoringWhitespace("""
+        assertThat(toFormattedString(persons)).isEqualToIgnoringWhitespace("""
                {
-                    900000=person(personType='NATURAL', tradeName='', familyName='Mellies', givenName='Michael'),
-                    900001=person(personType='LEGAL', tradeName='JM e.K.', familyName='Meyer-Contract', givenName='Philip'),
-                    900002=person(personType='LEGAL', tradeName='Test PS', familyName='Schmidt', givenName='Petra')
+                    1101=person(personType='NATURAL', tradeName='', familyName='Mellies', givenName='Michael'),
+                    1201=person(personType='LEGAL', tradeName='JM e.K.', familyName='Meyer-Billing', givenName='Jenny'),
+                    1202=person(personType='LEGAL', tradeName='JM e.K.', familyName='Meyer-Operation', givenName='Andrew'),
+                    1203=person(personType='LEGAL', tradeName='JM e.K.', familyName='Meyer-Contract', givenName='Philip'),
+                    1301=person(personType='LEGAL', tradeName='Test PS', familyName='Schmidt', givenName='Petra')
                }
                """);
-        assertThat(debitors.toString()).isEqualToIgnoringWhitespace("""
+        assertThat(toFormattedString(debitors)).isEqualToIgnoringWhitespace("""
                {
-                     7=debitor(1000700: NATURAL Mellies, Michael: mih), 
-                    10=debitor(1001000: LEGAL JM e.K.: xyz), 
-                    12=debitor(1101200: LEGAL Test PS: xxx)
+                    17=debitor(1001700: NATURAL Mellies, Michael: mih), 
+                    20=debitor(1002000: LEGAL JM e.K.: xyz), 
+                    22=debitor(1102200: LEGAL Test PS: xxx)
                }
                """);
-        assertThat(memberships.toString()).isEqualToIgnoringWhitespace("""
+        assertThat(toFormattedString(memberships)).isEqualToIgnoringWhitespace("""
                {
-                     7=Membership(10007, NATURAL Mellies, Michael, 1000700, [2000-12-06,), NONE),
-                    10=Membership(10010, LEGAL JM e.K., 1001000, [2000-12-06,2016-01-01), UNKNOWN),
-                    12=Membership(11012, LEGAL Test PS, 1101200, [2021-04-01,), NONE)
+                    17=Membership(10017, NATURAL Mellies, Michael, 1001700, [2000-12-06,), NONE),
+                    20=Membership(10020, LEGAL JM e.K., 1002000, [2000-12-06,2016-01-01), UNKNOWN),
+                    22=Membership(11022, LEGAL Test PS, 1102200, [2021-04-01,), NONE)
                }
                """);
-        assertThat(relationships.toString()).isEqualToIgnoringWhitespace("""
+        assertThat(toFormattedString(relationships)).isEqualToIgnoringWhitespace("""
                {
                   1101=rel(relAnchor='NATURAL Mellies, Michael', relType='OPERATIONS', relHolder='NATURAL Mellies, Michael', contact='Herr Michael Mellies '),
                   1202=rel(relAnchor='LEGAL JM e.K.', relType='OPERATIONS', relHolder='LEGAL JM e.K.', contact='Herr Andrew Meyer-Operation , JM e.K.'),
@ -251,9 +253,7 @@ public class ImportOfficeTables extends ContextBasedTest {
            throw new RuntimeException(e);
        }

-        if ( !"admin".equals(postgresAdminUser) ) {
-            return;
-        }
+        assumeThat(postgresAdminUser).isEqualTo("admin");

        assertThat(bankAccounts.toString()).isEqualToIgnoringWhitespace("""
                {
@ -280,16 +280,14 @@ public class ImportOfficeTables extends ContextBasedTest {
            throw new RuntimeException(e);
        }

-        if ( !"admin".equals(postgresAdminUser) ) {
-            return;
-        }
+        assumeThat(postgresAdminUser).isEqualTo("admin");

        assertThat(coopShares.toString()).isEqualToIgnoringWhitespace("""
                {
-                    33443=CoopShareTransaction(10007, 2000-12-06, SUBSCRIPTION, 20, initial share subscription),
-                    33451=CoopShareTransaction(10010, 2000-12-06, SUBSCRIPTION, 2, initial share subscription),
-                    33701=CoopShareTransaction(10007, 2005-01-10, SUBSCRIPTION, 40, increase),
-                    33810=CoopShareTransaction(10010, 2016-12-31, CANCELLATION, 22, membership ended)
+                    33443=CoopShareTransaction(10017, 2000-12-06, SUBSCRIPTION, 20, initial share subscription),
+                    33451=CoopShareTransaction(10020, 2000-12-06, SUBSCRIPTION, 2, initial share subscription),
+                    33701=CoopShareTransaction(10017, 2005-01-10, SUBSCRIPTION, 40, increase),
+                    33810=CoopShareTransaction(10020, 2016-12-31, CANCELLATION, 22, membership ended)
                }
                """);
    }
@ -305,20 +303,18 @@ public class ImportOfficeTables extends ContextBasedTest {
            throw new RuntimeException(e);
        }

-        if ( !"admin".equals(postgresAdminUser) ) {
-            return;
-        }
+        assumeThat(postgresAdminUser).isEqualTo("admin");

        assertThat(coopAssets.toString()).isEqualToIgnoringWhitespace("""
                {
-                    30000=CoopAssetsTransaction(10007, 2000-12-06, DEPOSIT, 1280.00, for subscription A),
-                    31000=CoopAssetsTransaction(10010, 2000-12-06, DEPOSIT, 128.00, for subscription B),
-                    32000=CoopAssetsTransaction(10007, 2005-01-10, DEPOSIT, 2560.00, for subscription C),
-                    33001=CoopAssetsTransaction(10007, 2005-01-10, TRANSFER, -512.00, for transfer to 10),
-                    33002=CoopAssetsTransaction(10010, 2005-01-10, ADOPTION, 512.00, for transfer from 7),
-                    34001=CoopAssetsTransaction(10010, 2016-12-31, CLEARING, -8.00, for cancellation D),
-                    34002=CoopAssetsTransaction(10010, 2016-12-31, DISBURSAL, -100.00, for cancellation D),
-                    34003=CoopAssetsTransaction(10010, 2016-12-31, LOSS, -20.00, for cancellation D)
+                    30000=CoopAssetsTransaction(10017, 2000-12-06, DEPOSIT, 1280.00, for subscription A),
+                    31000=CoopAssetsTransaction(10020, 2000-12-06, DEPOSIT, 128.00, for subscription B),
+                    32000=CoopAssetsTransaction(10017, 2005-01-10, DEPOSIT, 2560.00, for subscription C),
+                    33001=CoopAssetsTransaction(10017, 2005-01-10, TRANSFER, -512.00, for transfer to 10),
+                    33002=CoopAssetsTransaction(10020, 2005-01-10, ADOPTION, 512.00, for transfer from 7),
+                    34001=CoopAssetsTransaction(10020, 2016-12-31, CLEARING, -8.00, for cancellation D),
+                    34002=CoopAssetsTransaction(10020, 2016-12-31, DISBURSAL, -100.00, for cancellation D),
+                    34003=CoopAssetsTransaction(10020, 2016-12-31, LOSS, -20.00, for cancellation D)
                }
                """);
    }
@ -411,60 +407,66 @@ public class ImportOfficeTables extends ContextBasedTest {
    private void deleteTestDataFromHsOfficeTables() {
        jpaAttempt.transacted(() -> {
            context(rbacSuperuser);
-            em.createNativeQuery("DELETE FROM hs_office_relationship WHERE true").executeUpdate();
-            em.createNativeQuery("DELETE FROM hs_office_coopassetstransaction WHERE true").executeUpdate();
-            em.createNativeQuery("DELETE FROM hs_office_coopassetstransaction_legacy_id WHERE true").executeUpdate();
-            em.createNativeQuery("DELETE FROM hs_office_coopsharestransaction WHERE true").executeUpdate();
-            em.createNativeQuery("DELETE FROM hs_office_coopsharestransaction_legacy_id WHERE true").executeUpdate();
-            em.createNativeQuery("DELETE FROM hs_office_membership WHERE true").executeUpdate();
-            em.createNativeQuery("DELETE FROM hs_office_sepamandate WHERE true").executeUpdate();
-            em.createNativeQuery("DELETE FROM hs_office_sepamandate_legacy_id WHERE true").executeUpdate();
-            em.createNativeQuery("DELETE FROM hs_office_debitor WHERE true").executeUpdate();
-            em.createNativeQuery("DELETE FROM hs_office_bankaccount WHERE true").executeUpdate();
-            em.createNativeQuery("DELETE FROM hs_office_partner WHERE true").executeUpdate();
-            em.createNativeQuery("DELETE FROM hs_office_partner_details WHERE true").executeUpdate();
-            em.createNativeQuery("DELETE FROM hs_office_contact WHERE true").executeUpdate();
-            em.createNativeQuery("DELETE FROM hs_office_person WHERE true").executeUpdate();
+            em.createNativeQuery("delete from hs_office_relationship where true").executeUpdate();
+            em.createNativeQuery("delete from hs_office_coopassetstransaction where true").executeUpdate();
+            em.createNativeQuery("delete from hs_office_coopassetstransaction_legacy_id where true").executeUpdate();
+            em.createNativeQuery("delete from hs_office_coopsharestransaction where true").executeUpdate();
+            em.createNativeQuery("delete from hs_office_coopsharestransaction_legacy_id where true").executeUpdate();
+            em.createNativeQuery("delete from hs_office_membership where true").executeUpdate();
+            em.createNativeQuery("delete from hs_office_sepamandate where true").executeUpdate();
+            em.createNativeQuery("delete from hs_office_sepamandate_legacy_id where true").executeUpdate();
+            em.createNativeQuery("delete from hs_office_debitor where true").executeUpdate();
+            em.createNativeQuery("delete from hs_office_bankaccount where true").executeUpdate();
+            em.createNativeQuery("delete from hs_office_partner where true").executeUpdate();
+            em.createNativeQuery("delete from hs_office_partner_details where true").executeUpdate();
+            em.createNativeQuery("delete from hs_office_contact where true").executeUpdate();
+            em.createNativeQuery("delete from hs_office_person where true").executeUpdate();
        }).assertSuccessful();
    }

    private void resetFromHsOfficeSequences() {
        jpaAttempt.transacted(() -> {
            context(rbacSuperuser);
-            em.createNativeQuery("ALTER SEQUENCE hs_office_contact_legacy_id_seq RESTART WITH 1000000000;").executeUpdate();
-            em.createNativeQuery("ALTER SEQUENCE hs_office_coopassetstransaction_legacy_id_seq RESTART WITH 1000000000;").executeUpdate();
-            em.createNativeQuery("ALTER SEQUENCE public.hs_office_coopsharestransaction_legacy_id_seq RESTART WITH 1000000000;").executeUpdate();
-            em.createNativeQuery("ALTER SEQUENCE public.hs_office_partner_legacy_id_seq RESTART WITH 1000000000;").executeUpdate();
-            em.createNativeQuery("ALTER SEQUENCE public.hs_office_sepamandate_legacy_id_seq RESTART WITH 1000000000;").executeUpdate();
+            em.createNativeQuery("alter sequence hs_office_contact_legacy_id_seq restart with 1000000000;").executeUpdate();
+            em.createNativeQuery("alter sequence hs_office_coopassetstransaction_legacy_id_seq restart with 1000000000;")
+                    .executeUpdate();
+            em.createNativeQuery("alter sequence public.hs_office_coopsharestransaction_legacy_id_seq restart with 1000000000;")
+                    .executeUpdate();
+            em.createNativeQuery("alter sequence public.hs_office_partner_legacy_id_seq restart with 1000000000;")
+                    .executeUpdate();
+            em.createNativeQuery("alter sequence public.hs_office_sepamandate_legacy_id_seq restart with 1000000000;")
+                    .executeUpdate();
        });
    }

    private void deleteFromTestTables() {
        jpaAttempt.transacted(() -> {
            context(rbacSuperuser);
-            em.createNativeQuery("DELETE FROM test_domain WHERE true").executeUpdate();
-            em.createNativeQuery("DELETE FROM test_package WHERE true").executeUpdate();
-            em.createNativeQuery("DELETE FROM test_customer WHERE true").executeUpdate();
+            em.createNativeQuery("delete from test_domain where true").executeUpdate();
+            em.createNativeQuery("delete from test_package where true").executeUpdate();
+            em.createNativeQuery("delete from test_customer where true").executeUpdate();
        }).assertSuccessful();
    }

    private void deleteFromRbacTables() {
        jpaAttempt.transacted(() -> {
            context(rbacSuperuser);
-            final var usersNotToDelete = em.createNativeQuery("SELECT name FROM rbacuser WHERE name like 'superuser-%'").getResultList();
-            final var usersToDelete = em.createNativeQuery("SELECT name FROM rbacuser WHERE name not like 'superuser-%'").getResultList();
+            final var usersNotToDelete = em.createNativeQuery("select name from rbacuser where name like 'superuser-%'")
+                    .getResultList();
+            final var usersToDelete = em.createNativeQuery("select name from rbacuser where name not like 'superuser-%'")
+                    .getResultList();
            System.getenv();
        });

        jpaAttempt.transacted(() -> {
            context(rbacSuperuser);
-//            em.createNativeQuery("DELETE FROM rbacobject WHERE objecttable like 'hs_%'").executeUpdate();
-//            em.createNativeQuery("DELETE FROM rbacgrants WHERE true").executeUpdate();
-//            em.createNativeQuery("DELETE FROM rbacpermission WHERE true").executeUpdate();
-//            em.createNativeQuery("DELETE FROM rbacreference WHERE true").executeUpdate();
-            em.createNativeQuery("DELETE FROM rbacuser_rv WHERE name not like 'superuser-%'").executeUpdate();
-            em.createNativeQuery("DELETE FROM tx_journal WHERE true").executeUpdate();
-            em.createNativeQuery("DELETE FROM tx_context WHERE true").executeUpdate();
+            //            em.createNativeQuery("DELETE FROM rbacobject WHERE objecttable like 'hs_%'").executeUpdate();
+            //            em.createNativeQuery("DELETE FROM rbacgrants WHERE true").executeUpdate();
+            //            em.createNativeQuery("DELETE FROM rbacpermission WHERE true").executeUpdate();
+            //            em.createNativeQuery("DELETE FROM rbacreference WHERE true").executeUpdate();
+            em.createNativeQuery("delete from rbacuser_rv where name not like 'superuser-%'").executeUpdate();
+            em.createNativeQuery("delete from tx_journal where true").executeUpdate();
+            em.createNativeQuery("delete from tx_context where true").executeUpdate();
        }).assertSuccessful();
    }

@ -499,6 +501,7 @@ public class ImportOfficeTables extends ContextBasedTest {
            return csvReader.readAll();
        }
    }
+
    public static Reader skippingEmptyAndCommentLines(Reader reader) throws IOException {
        try (var bufferedReader = new BufferedReader(reader);
             StringWriter writer = new StringWriter()) {
@ -523,10 +526,7 @@ public class ImportOfficeTables extends ContextBasedTest {
                .map(this::trimAll)
                .map(row -> new Record(columns, row))
                .forEach(rec -> {
-                    final var person = HsOfficePersonEntity.builder()
-                            .personType(HsOfficePersonType.UNKNOWN) // TODO
-                            .build();
-                    persons.put(personId++, person);
+                    final var person = HsOfficePersonEntity.builder().build();

                    final var partner = HsOfficePartnerEntity.builder()
                            .debitorNumberPrefix(rec.getInteger("member_id"))
@ -678,91 +678,141 @@ public class ImportOfficeTables extends ContextBasedTest {
                .map(this::trimAll)
                .map(row -> new Record(columns, row))
                .forEach(rec -> {
-                        final var contactId = rec.getInteger("contact_id");
+                    final var contactId = rec.getInteger("contact_id");

-                        if (rec.getString("roles").isBlank()) {
-                            fail("empty roles assignment not allowed for contact_id: " + contactId);
-                        }
+                    if (rec.getString("roles").isBlank()) {
+                        fail("empty roles assignment not allowed for contact_id: " + contactId);
+                    }

-                        final var partner = partners.get(rec.getInteger("bp_id"));
-                        final var debitor = debitors.get(rec.getInteger("bp_id"));
+                    final var partner = partners.get(rec.getInteger("bp_id"));
+                    final var debitor = debitors.get(rec.getInteger("bp_id"));

-                        final var person = partner.getPerson();
-                        person.setTradeName(rec.getString("firma"));
-                        determinePersonType(person);
-                        // TODO: title+salutation: add to person
-                        person.setGivenName(rec.getString("first_name"));
-                        person.setFamilyName(rec.getString("last_name"));
+                    final var partnerPerson = partner.getPerson();
+                    if (rec.getString("roles").contains("partner")) {
+                        initPerson(partner.getPerson(), rec);
+                    }

-                        final var contact = HsOfficeContactEntity.builder().build();
-                        contacts.put(contactId, initContact(contact, rec));
+                    HsOfficePersonEntity contactPerson = partnerPerson;
+                    if (!StringUtils.equals(rec.getString("firma"), partnerPerson.getTradeName()) ||
+                            !StringUtils.equals(rec.getString("first_name"), partnerPerson.getGivenName()) ||
+                            !StringUtils.equals(rec.getString("last_name"), partnerPerson.getFamilyName())) {
+                        contactPerson = initPerson(HsOfficePersonEntity.builder().build(), rec);
+                    }

-                        var imported = false;
-                        if (rec.getString("roles").contains("partner")) {
-                            assertThat(partner.getContact()).isNull();
-                            partner.setContact(contact);
-                            imported = true;
-                        }
-                        if (rec.getString("roles").contains("billing")) {
-                            assertThat(debitor.getBillingContact()).isNull();
-                            debitor.setBillingContact(contact);
-                            imported = true;
-                        }
-                        if (rec.getString("roles").contains("operation")) {
-                            final var rel = HsOfficeRelationshipEntity.builder()
-                                    .relAnchor(partner.getPerson())
-                                    .relHolder(person)
-                                    .contact(contact)
-                                    .relType(HsOfficeRelationshipType.OPERATIONS)
-                                    .build();
-                            relationships.put(contactId, rel);
-                            imported = true;
-                        }
+                    final var contact = HsOfficeContactEntity.builder().build();
+                    initContact(contact, rec);
+
+                    if (rec.getString("roles").contains("partner")) {
+                        assertThat(partner.getContact()).isNull();
+                        partner.setContact(contact);
+                    }
+                    if (rec.getString("roles").contains("billing")) {
+                        assertThat(debitor.getBillingContact()).isNull();
+                        debitor.setBillingContact(contact);
+                    }
+                    if (rec.getString("roles").contains("operation")) {
+                        final var rel = HsOfficeRelationshipEntity.builder()
+                                .relAnchor(partnerPerson)
+                                .relHolder(contactPerson)
+                                .contact(contact)
+                                .relType(HsOfficeRelationshipType.OPERATIONS)
+                                .build();
+                        relationships.put(contactId, rel);
+                    }
                    if (rec.getString("roles").contains("contractual")) {
                        final var rel = HsOfficeRelationshipEntity.builder()
-                                .relAnchor(partner.getPerson())
-                                .relHolder(person)
+                                .relAnchor(partnerPerson)
+                                .relHolder(contactPerson)
                                .contact(contact)
                                .relType(HsOfficeRelationshipType.REPRESENTATIVE)
                                .build();
                        relationships.put(contactId, rel);
-                        imported = true;
                    }
-                    if (!imported) {
+                    if (rec.getString("roles").contains("ex-partner")) {
                        final var rel = HsOfficeRelationshipEntity.builder()
-                                .relAnchor(partner.getPerson())
-                                .relHolder(person)
+                                .relAnchor(partnerPerson)
+                                .relHolder(contactPerson)
                                .contact(contact)
-                                .relType(HsOfficeRelationshipType.UNKNOWN)
+                                .relType(HsOfficeRelationshipType.EX_PARTNER)
                                .build();
                        relationships.put(contactId, rel);
                    }
+                    verifyContainsOnly(rec.getString("roles"), "partner", "ex-partner", "billing", "contractual", "operation");
                });
    }

-    private void determinePersonType(final HsOfficePersonEntity person) {
+    private HsOfficePersonEntity initPerson(final HsOfficePersonEntity person, final Record contactRecord) {
+        // TODO: title+salutation: add to person
+        person.setGivenName(contactRecord.getString("first_name"));
+        person.setFamilyName(contactRecord.getString("last_name"));
+        person.setTradeName(contactRecord.getString("firma"));
+        determinePersonType(person, contactRecord.getString("roles"));
+
+        persons.put(contactRecord.getInteger("contact_id"), person);
+        return person;
+    }
+
+    private static void determinePersonType(final HsOfficePersonEntity person, final String roles) {
        if (person.getTradeName().isBlank()) {
            person.setPersonType(HsOfficePersonType.NATURAL);
+        } else if (roles.contains("partner")) {
+            person.setPersonType(HsOfficePersonType.LEGAL);
+        } else if (roles.contains("contractual") &&
+                !person.getFamilyName().isBlank() && !person.getGivenName().isBlank()) {
+            person.setPersonType(HsOfficePersonType.NATURAL);
+        } else if ( endsWithWord(person.getTradeName(), "e.K.", "e.G.", "eG", "GmbH", "AG")  ) {
+            person.setPersonType(HsOfficePersonType.LEGAL);
        } else {
-            person.setPersonType(HsOfficePersonType.LEGAL); // TODO: add rules if we distinguish
+            // TODO: detect the other person types as soon as we've switche to the new person types
+            person.setPersonType(HsOfficePersonType.UNKNOWN);
        }
    }

-    private HsOfficeContactEntity initContact(final HsOfficeContactEntity contact, final Record rec) {
-        contacts.put(rec.getInteger("contact_id"), contact);
+    private static boolean endsWithWord(final String value, final String... endings) {
+        final var lowerCaseValue = value.toLowerCase();
+        for( String ending: endings ) {
+            if (lowerCaseValue.endsWith(" " + ending.toLowerCase())) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private void verifyContainsOnly(final String roles, final String... allowedRoles) {
+        final var givenRolesSet = stream(roles.replace(" ", "").split(",")).collect(Collectors.toSet());
+        final var allowedRolesSet = stream(allowedRoles).collect(Collectors.toSet());
+        final var unexpectedRolesSet = new HashSet<>(givenRolesSet);
+        unexpectedRolesSet.removeAll(allowedRolesSet);
+        assertThat(unexpectedRolesSet).isEmpty();
+    }
+
+    private HsOfficeContactEntity initContact(final HsOfficeContactEntity contact, final Record contactRecord) {

        contact.setLabel(toLabel(
-                rec.getString("salut"),
-                rec.getString("title"),
-                rec.getString("first_name"),
-                rec.getString("last_name"),
-                rec.getString("firma")));
-        contact.setEmailAddresses(rec.getString("email"));
-        contact.setPostalAddress(toAddress(rec));
-        contact.setPhoneNumbers(toPhoneNumbers(rec));
+                contactRecord.getString("salut"),
+                contactRecord.getString("title"),
+                contactRecord.getString("first_name"),
+                contactRecord.getString("last_name"),
+                contactRecord.getString("firma")));
+        contact.setEmailAddresses(contactRecord.getString("email"));
+        contact.setPostalAddress(toAddress(contactRecord));
+        contact.setPhoneNumbers(toPhoneNumbers(contactRecord));
+
+        contacts.put(contactRecord.getInteger("contact_id"), contact);
        return contact;
    }

+    private <E> String toFormattedString(final Map<Integer, E> map) {
+        if ( map.isEmpty() ) {
+            return "{}";
+        }
+        return "{\n" +
+                map.keySet().stream()
+                        .map(id -> "   " + id + "=" + map.get(id).toString())
+                        .collect(Collectors.joining(",\n")) +
+                "\n}\n";
+    }
+
    private String[] trimAll(final String[] record) {
        for (int i = 0; i < record.length; ++i) {
            if (record[i] != null) {
@ -932,6 +982,7 @@ class Record {
 }

 class OrderedDependedTestsExtension implements TestWatcher, BeforeEachCallback {
+
    private static boolean previousTestsPassed = true;

    public void testAborted(ExtensionContext context, Throwable cause) {
--- a/src/test/resources/migration/asset-transactions.csv
+++ b/src/test/resources/migration/asset-transactions.csv
@ -1,9 +1,9 @@
 member_asset_id;	bp_id;	date;		action;		amount;	comment
-30000;		7;	2000-12-06;	PAYMENT;	1280.00; for subscription A
-31000;		10;	2000-12-06;	PAYMENT;	128.00; for subscription B
-32000;		7;	2005-01-10;	PAYMENT;	2560.00; for subscription C
-33001;		7;	2005-01-10;	HANDOVER;	-512.00; for transfer to 10
-33002;		10;	2005-01-10;	ADOPTION;	512.00; for transfer from 7
-34001;		10;	2016-12-31;	CLEARING;	-8.00; for cancellation D
-34002;		10;	2016-12-31;	PAYBACK;	-100.00; for cancellation D
-34003;		10;	2016-12-31;	LOSS;	-20.00; for cancellation D
+30000;		17;	2000-12-06;	PAYMENT;	1280.00; for subscription A
+31000;		20;	2000-12-06;	PAYMENT;	128.00; for subscription B
+32000;		17;	2005-01-10;	PAYMENT;	2560.00; for subscription C
+33001;		17;	2005-01-10;	HANDOVER;	-512.00; for transfer to 10
+33002;		20;	2005-01-10;	ADOPTION;	512.00; for transfer from 7
+34001;		20;	2016-12-31;	CLEARING;	-8.00; for cancellation D
+34002;		20;	2016-12-31;	PAYBACK;	-100.00; for cancellation D
+34003;		20;	2016-12-31;	LOSS;	-20.00; for cancellation D
--- a/src/test/resources/migration/business-partners.csv
+++ b/src/test/resources/migration/business-partners.csv
@ -1,4 +1,4 @@
 bp_id;member_id;member_code;member_since;member_until;member_role;author_contract;nondisc_contract;free;exempt_vat;indicator_vat;uid_vat
-7;10007;hsh00-mih;2000-12-06;;Aufsichtsrat;2006-10-15;2001-10-15;false;false;NET;DE-VAT-007
-10;10010;hsh00-xyz;2000-12-06;2015-12-31;;;;false;false;GROSS;
-12;11012;hsh00-xxx;2021-04-01;;;;;true;true;GROSS;
+17;10017;hsh00-mih;2000-12-06;;Aufsichtsrat;2006-10-15;2001-10-15;false;false;NET;DE-VAT-007
+20;10020;hsh00-xyz;2000-12-06;2015-12-31;;;;false;false;GROSS;
+22;11022;hsh00-xxx;2021-04-01;;;;;true;true;GROSS;
--- a/src/test/resources/migration/contacts.csv
+++ b/src/test/resources/migration/contacts.csv
@ -1,12 +1,12 @@
 contact_id;	bp_id;	salut;	first_name;	last_name;	title;	firma;	co;	street;			zipcode;city;	country;	phone_private;		phone_office;		phone_mobile;	fax;		email; roles

 # eine natürliche Person
-1101;		7;	Herr;	Michael;	Mellies;		;	;	;	Kleine Freiheit 50;	26524;	Hage;	DE;		;			+49 4931 123456;	+49 1522 123456;;		mih@example.org; partner,billing,operation
+1101;		17;	Herr;	Michael;	Mellies;		;	;	;	Kleine Freiheit 50;	26524;	Hage;	DE;		;			+49 4931 123456;	+49 1522 123456;;		mih@example.org; partner,billing,operation

 # eine juristische Person mit drei separaten Ansprechpartnern
-1201;		10;	Frau;	Jenny;		Meyer-Billing;		Dr.;	JM e.K.;;	Waldweg 5;		11001;	Berlin; DE;		+49 30 7777777;		+49 30 1111111;		;		+49 30 2222222; jm-billing@example.org; billing
-1202;		10;	Herr;	Andrew;		Meyer-Operation;		;	JM e.K.;;	Waldweg 5;		11001;	Berlin; DE;		+49 30 6666666;		+49 30 3333333;		;		+49 30 4444444; am-operation@example.org; operation
-1203;		10;	Herr;	Philip;		Meyer-Contract;		;	JM e.K.;;	Waldweg 5;		11001;	Berlin; DE;		+49 30 6666666;		+49 30 5555555;		;		+49 30 6666666; pm-partner@example.org; partner,contractual
+1201;		20;	Frau;	Jenny;		Meyer-Billing;		Dr.;	JM e.K.;;	Waldweg 5;		11001;	Berlin; DE;		+49 30 7777777;		+49 30 1111111;		;		+49 30 2222222; jm-billing@example.org; billing
+1202;		20;	Herr;	Andrew;		Meyer-Operation;		;	JM e.K.;;	Waldweg 5;		11001;	Berlin; DE;		+49 30 6666666;		+49 30 3333333;		;		+49 30 4444444; am-operation@example.org; operation
+1203;		20;	Herr;	Philip;		Meyer-Contract;		;	JM e.K.;;	Waldweg 5;		11001;	Berlin; DE;		+49 30 6666666;		+49 30 5555555;		;		+49 30 6666666; pm-partner@example.org; partner,contractual

 # eine juristische Person mit nur einem Ansprechpartner
-1301;		12;	;	Petra;		Schmidt;	;	Test PS;;	;			;	;	;		;			;			;		;		ps@example.com; partner,billing,contractual,operation
+1301;		22;	;	Petra;		Schmidt;	;	Test PS;;	;			;	;	;		;			;			;		;		ps@example.com; partner,billing,contractual,operation
--- a/src/test/resources/migration/sepa-mandates.csv
+++ b/src/test/resources/migration/sepa-mandates.csv
@ -1,3 +1,3 @@
 sepa_mandat_id;	bp_id;	bank_customer;	bank_name;	bank_iban;		bank_bic;	mandat_ref;	mandat_signed;	mandat_since;	mandat_until;	mandat_used
-234234;		7;	Michael Mellies;	ING Bank AG;	DE37500105177419788228;	INGDDEFFXXX;	MH12345;	2004-06-12;	2004-06-15;	;		2022-10-20
-235662;		10;	JM e.K.;	ING Bank AG;	DE49500105174516484892;	INGDDEFFXXX;	JM33344;	2005-06-28;	2005-07-01;	;		2016-01-18
+234234;		17;	Michael Mellies;	ING Bank AG;	DE37500105177419788228;	INGDDEFFXXX;	MH12345;	2004-06-12;	2004-06-15;	;		2022-10-20
+235662;		20;	JM e.K.;	ING Bank AG;	DE49500105174516484892;	INGDDEFFXXX;	JM33344;	2005-06-28;	2005-07-01;	;		2016-01-18
--- a/src/test/resources/migration/share-transactions.csv
+++ b/src/test/resources/migration/share-transactions.csv
@ -1,5 +1,5 @@
 member_share_id;bp_id;	date;		action;		quantity;	comment
-33443;		7;	2000-12-06;	SUBSCRIPTION;	20;		initial share subscription
-33451;		10;	2000-12-06;	SUBSCRIPTION;	2;		initial share subscription
-33701;		7;	2005-01-10;	SUBSCRIPTION;	40;		increase
-33810;		10;	2016-12-31;	UNSUBSCRIPTION;	22;		membership ended
+33443;		17;	2000-12-06;	SUBSCRIPTION;	20;		initial share subscription
+33451;		20;	2000-12-06;	SUBSCRIPTION;	2;		initial share subscription
+33701;		17;	2005-01-10;	SUBSCRIPTION;	40;		increase
+33810;		20;	2016-12-31;	UNSUBSCRIPTION;	22;		membership ended
Author	SHA1	Message	Date
Michael Hoennig	007e1522b7	add ex-partner to Relationship-Type and properly extract distinct persons from contacts.csv	2024-01-18 14:03:06 +01:00
Michael Hoennig	d03fb70465	UML object diagram for new target data model for business partners	2024-01-18 13:57:46 +01:00