hostsharing/hs.hsadmin.ng
6
0
Fork 0
Code Pull Requests 1 Activity

feature/remove-office-data-import #155

Merged
hsh-michaelhoennig merged 14 commits from feature/remove-office-data-import into master 2025-02-05 09:29:44 +01:00
Conversation 0 Commits 14 Files Changed 6 +7242 -10139
6 changed files with 7242 additions and 10139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 0b31c6aa2f - Show all commits

94
src/test/java/net/hostsharing/hsadminng/hs/migration/BaseOfficeDataImport.java
View File

@ -115,11 +115,18 @@ public abstract class BaseOfficeDataImport extends CsvDataImport {
@Test @Test
@Order(1) @Order(1)
void verifyInitialDatabase() { void verifyInitialDatabaseHasNoTestData() {
// SQL DELETE for thousands of records takes too long, so we make sure, we only start with initial or test data assertThat((Integer) em.createNativeQuery(
final var contactCount = (Integer) em.createNativeQuery("select count(*) from hs_office.contact", Integer.class) "select count(*) from hs_office.contact",
.getSingleResult(); Integer.class)
assertThat(contactCount).isLessThan(20); .getSingleResult()).isEqualTo(0);
assertThat((Integer) em.createNativeQuery(
"""
SELECT count(*) FROM information_schema.tables
WHERE table_schema = 'rbactest' AND table_name = 'customer'
""",
Integer.class)
.getSingleResult()).isEqualTo(0);
} }
@Test @Test
@ -624,15 +631,13 @@ public abstract class BaseOfficeDataImport extends CsvDataImport {
void persistOfficeEntities() { void persistOfficeEntities() {
System.out.println("PERSISTING office data to database '" + jdbcUrl + "' as user '" + postgresAdminUser + "'"); System.out.println("PERSISTING office data to database '" + jdbcUrl + "' as user '" + postgresAdminUser + "'");
deleteTestDataFromHsOfficeTables(); makeSureThatTheImportAdminUserExists();
resetHsOfficeSequences();
deleteFromTestTables();
deleteFromCommonTables();
assertEmptyTable("hs_office.contact");
jpaAttempt.transacted(() -> { jpaAttempt.transacted(() -> {
context(rbacSuperuser); context(rbacSuperuser);
contacts.forEach(this::persist); contacts.forEach(this::persist);
updateLegacyIds(contacts, "hs_office.contact_legacy_id", "contact_id"); updateLegacyIds(contacts, "hs_office.contact_legacy_id", "contact_id");
}).assertSuccessful(); }).assertSuccessful();
jpaAttempt.transacted(() -> { jpaAttempt.transacted(() -> {
@ -646,6 +651,7 @@ public abstract class BaseOfficeDataImport extends CsvDataImport {
}).assertSuccessful(); }).assertSuccessful();
System.out.println("persisting " + partners.size() + " partners"); System.out.println("persisting " + partners.size() + " partners");
assertEmptyTable("hs_office.partner");
jpaAttempt.transacted(() -> { jpaAttempt.transacted(() -> {
context(rbacSuperuser); context(rbacSuperuser);
partners.forEach((id, partner) -> { partners.forEach((id, partner) -> {
@ -697,6 +703,12 @@ public abstract class BaseOfficeDataImport extends CsvDataImport {
}).assertSuccessful(); }).assertSuccessful();
} }
private void assertEmptyTable(final String qualifiedTableName) {
assertThat((Integer) em.createNativeQuery(
"select count(*) from " + qualifiedTableName,
Integer.class)
.getSingleResult()).describedAs("expected empty " + qualifiedTableName).isEqualTo(0);
}
@Test @Test
@Order(9190) @Order(9190)
@ -883,7 +895,6 @@ public abstract class BaseOfficeDataImport extends CsvDataImport {
coopAssets.put(rec.getInteger("member_asset_id"), assetTransaction); coopAssets.put(rec.getInteger("member_asset_id"), assetTransaction);
}); });
coopAssets.entrySet().forEach(entry -> { coopAssets.entrySet().forEach(entry -> {
final var legacyId = entry.getKey(); final var legacyId = entry.getKey();
final var assetTransaction = entry.getValue(); final var assetTransaction = entry.getValue();
@ -896,7 +907,9 @@ public abstract class BaseOfficeDataImport extends CsvDataImport {
}); });
} }
private static void connectToRelatedRevertedAssetTx(final int legacyId, final HsOfficeCoopAssetsTransactionEntity assetTransaction) { private static void connectToRelatedRevertedAssetTx(
final int legacyId,
final HsOfficeCoopAssetsTransactionEntity assetTransaction) {
final var negativeValue = assetTransaction.getAssetValue().negate(); final var negativeValue = assetTransaction.getAssetValue().negate();
final var revertedAssetTx = coopAssets.values().stream().filter(a -> final var revertedAssetTx = coopAssets.values().stream().filter(a ->
a.getTransactionType() != HsOfficeCoopAssetsTransactionType.REVERSAL && a.getTransactionType() != HsOfficeCoopAssetsTransactionType.REVERSAL &&
@ -909,11 +922,14 @@ public abstract class BaseOfficeDataImport extends CsvDataImport {
//revertedAssetTx.setAssetReversalTx(assetTransaction); //revertedAssetTx.setAssetReversalTx(assetTransaction);
} }
private static void connectToRelatedAdoptionAssetTx(final int legacyId, final HsOfficeCoopAssetsTransactionEntity assetTransaction) { private static void connectToRelatedAdoptionAssetTx(
final int legacyId,
final HsOfficeCoopAssetsTransactionEntity assetTransaction) {
final var negativeValue = assetTransaction.getAssetValue().negate(); final var negativeValue = assetTransaction.getAssetValue().negate();
final var adoptionAssetTx = coopAssets.values().stream().filter(a -> final var adoptionAssetTx = coopAssets.values().stream().filter(a ->
a.getTransactionType() == HsOfficeCoopAssetsTransactionType.ADOPTION && a.getTransactionType() == HsOfficeCoopAssetsTransactionType.ADOPTION &&
(!a.getValueDate().equals(LocalDate.of( 2014 , 12 , 31)) || a.getComment().contains(Integer.toString(assetTransaction.getMembership().getMemberNumber()/100))) && (!a.getValueDate().equals(LocalDate.of(2014, 12, 31)) || a.getComment()
.contains(Integer.toString(assetTransaction.getMembership().getMemberNumber() / 100))) &&
a.getMembership() != assetTransaction.getMembership() && a.getMembership() != assetTransaction.getMembership() &&
a.getValueDate().equals(assetTransaction.getValueDate()) && a.getValueDate().equals(assetTransaction.getValueDate()) &&
a.getAssetValue().equals(negativeValue)) a.getAssetValue().equals(negativeValue))
@ -1131,14 +1147,14 @@ public abstract class BaseOfficeDataImport extends CsvDataImport {
final var personKey = ( final var personKey = (
person.getPersonType() + "|" + person.getPersonType() + "|" +
person.getSalutation() + "|" + person.getSalutation() + "|" +
person.getTradeName() + "|" + person.getTradeName() + "|" +
person.getTitle() + "|" + person.getTitle() + "|" +
person.getGivenName() + "|" + person.getGivenName() + "|" +
person.getFamilyName() person.getFamilyName()
).toLowerCase(); ).toLowerCase();
if ( !distinctPersons.containsKey(personKey) ) { if (!distinctPersons.containsKey(personKey)) {
distinctPersons.put(personKey, person); distinctPersons.put(personKey, person);
} }
return distinctPersons.get(personKey); return distinctPersons.get(personKey);
@ -1164,24 +1180,24 @@ public abstract class BaseOfficeDataImport extends CsvDataImport {
if (endsWithWord(tradeName, "OHG", "GbR", "KG", "UG", "PartGmbB", "mbB")) { if (endsWithWord(tradeName, "OHG", "GbR", "KG", "UG", "PartGmbB", "mbB")) {
return HsOfficePersonType.INCORPORATED_FIRM; // Personengesellschaft. Gesellschafter haften persönlich. return HsOfficePersonType.INCORPORATED_FIRM; // Personengesellschaft. Gesellschafter haften persönlich.
} else if (containsWord(tradeName, "e.K.", "e.G.", "eG", "gGmbH", "GmbH", "mbH", "AG", "e.V.", "eV", "e.V") } else if (containsWord(tradeName, "e.K.", "e.G.", "eG", "gGmbH", "GmbH", "mbH", "AG", "e.V.", "eV", "e.V")
|| tradeName.toLowerCase().contains("haftungsbeschränkt") || tradeName.toLowerCase().contains("haftungsbeschränkt")
|| tradeName.toLowerCase().contains("stiftung") || tradeName.toLowerCase().contains("stiftung")
|| tradeName.toLowerCase().contains("stichting") || tradeName.toLowerCase().contains("stichting")
|| tradeName.toLowerCase().contains("foundation") || tradeName.toLowerCase().contains("foundation")
|| tradeName.toLowerCase().contains("schule") || tradeName.toLowerCase().contains("schule")
|| tradeName.toLowerCase().contains("verein") || tradeName.toLowerCase().contains("verein")
|| tradeName.toLowerCase().contains("gewerkschaft") || tradeName.toLowerCase().contains("gewerkschaft")
|| tradeName.toLowerCase().contains("gesellschaft") || tradeName.toLowerCase().contains("gesellschaft")
|| tradeName.toLowerCase().contains("kirche") || tradeName.toLowerCase().contains("kirche")
|| tradeName.toLowerCase().contains("fraktion") || tradeName.toLowerCase().contains("fraktion")
|| tradeName.toLowerCase().contains("landkreis") || tradeName.toLowerCase().contains("landkreis")
|| tradeName.toLowerCase().contains("behörde") || tradeName.toLowerCase().contains("behörde")
|| tradeName.toLowerCase().contains("bundesamt") || tradeName.toLowerCase().contains("bundesamt")
|| tradeName.toLowerCase().contains("bezirksamt") || tradeName.toLowerCase().contains("bezirksamt")
) { ) {
return HsOfficePersonType.LEGAL_PERSON; // Haftungsbeschränkt return HsOfficePersonType.LEGAL_PERSON; // Haftungsbeschränkt
} else if (roles.contains("contractual") && !roles.contains("partner") && } else if (roles.contains("contractual") && !roles.contains("partner") &&
!familyName.isBlank() && !givenName.isBlank()) { !familyName.isBlank() && !givenName.isBlank()) {
// REPRESENTATIVES are always natural persons // REPRESENTATIVES are always natural persons
return HsOfficePersonType.NATURAL_PERSON; return HsOfficePersonType.NATURAL_PERSON;
} else { } else {
@ -1203,9 +1219,9 @@ public abstract class BaseOfficeDataImport extends CsvDataImport {
final var lowerCaseValue = value.toLowerCase(); final var lowerCaseValue = value.toLowerCase();
for (String ending : endings) { for (String ending : endings) {
if (lowerCaseValue.equals(ending.toLowerCase()) || if (lowerCaseValue.equals(ending.toLowerCase()) ||
lowerCaseValue.startsWith(ending.toLowerCase() + " ") || lowerCaseValue.startsWith(ending.toLowerCase() + " ") ||
lowerCaseValue.contains(" " + ending.toLowerCase() + " ") || lowerCaseValue.contains(" " + ending.toLowerCase() + " ") ||
lowerCaseValue.endsWith(" " + ending.toLowerCase())) { lowerCaseValue.endsWith(" " + ending.toLowerCase())) {
return true; return true;
} }
} }

59
src/test/java/net/hostsharing/hsadminng/hs/migration/CsvDataImport.java
View File

@ -248,51 +248,22 @@ public class CsvDataImport extends ContextBasedTest {
return json; return json;
} }
protected void deleteTestDataFromHsOfficeTables() { protected void makeSureThatTheImportAdminUserExists() {
jpaAttempt.transacted(() -> { jpaAttempt.transacted(() -> {
context(rbacSuperuser); context(null);
// TODO.perf: could we instead skip creating test-data based on an env var? em.createNativeQuery("""
em.createNativeQuery("delete from hs_office.coopassettx where true").executeUpdate(); do language plpgsql $$
em.createNativeQuery("delete from hs_office.coopassettx_legacy_id where true").executeUpdate(); declare
em.createNativeQuery("delete from hs_office.coopsharetx where true").executeUpdate(); admins uuid;
em.createNativeQuery("delete from hs_office.coopsharetx_legacy_id where true").executeUpdate(); begin
em.createNativeQuery("delete from hs_office.membership where true").executeUpdate(); if not exists (select 1 from rbac.subject where name = '${rbacSuperuser}') then
em.createNativeQuery("delete from hs_office.sepamandate where true").executeUpdate(); admins = rbac.findRoleId(rbac.global_ADMIN());
em.createNativeQuery("delete from hs_office.sepamandate_legacy_id where true").executeUpdate(); call rbac.grantRoleToSubjectUnchecked(admins, admins, rbac.create_subject('${rbacSuperuser}'));
em.createNativeQuery("delete from hs_office.debitor where true").executeUpdate(); end if;
em.createNativeQuery("delete from hs_office.bankaccount where true").executeUpdate(); end;
em.createNativeQuery("delete from hs_office.partner where true").executeUpdate(); $$;
em.createNativeQuery("delete from hs_office.partner_details where true").executeUpdate(); """.replace("${rbacSuperuser}", rbacSuperuser))
em.createNativeQuery("delete from hs_office.relation where true").executeUpdate(); .executeUpdate();
em.createNativeQuery("delete from hs_office.contact where true").executeUpdate();
em.createNativeQuery("delete from hs_office.person where true").executeUpdate();
}).assertSuccessful();
}
protected void resetHsOfficeSequences() {
jpaAttempt.transacted(() -> {
context(rbacSuperuser);
em.createNativeQuery("alter sequence hs_office.contact_legacy_id_seq restart with 1000000000;").executeUpdate();
em.createNativeQuery("alter sequence hs_office.coopassettx_legacy_id_seq restart with 1000000000;")
.executeUpdate();
});
}
protected void deleteFromTestTables() {
jpaAttempt.transacted(() -> {
context(rbacSuperuser);
em.createNativeQuery("delete from rbactest.domain where true").executeUpdate();
em.createNativeQuery("delete from rbactest.package where true").executeUpdate();
em.createNativeQuery("delete from rbactest.customer where true").executeUpdate();
}).assertSuccessful();
}
protected void deleteFromCommonTables() {
jpaAttempt.transacted(() -> {
context(rbacSuperuser);
em.createNativeQuery("delete from rbac.subject_rv where name not like 'superuser-%'").executeUpdate();
em.createNativeQuery("delete from base.tx_journal where true").executeUpdate();
em.createNativeQuery("delete from base.tx_context where true").executeUpdate();
}).assertSuccessful(); }).assertSuccessful();
} }

11
src/test/java/net/hostsharing/hsadminng/hs/migration/ImportHostingAssets.java
View File

@ -87,7 +87,7 @@ import static org.springframework.test.context.jdbc.Sql.ExecutionPhase.BEFORE_TE
"spring.datasource.url=${HSADMINNG_POSTGRES_JDBC_URL:jdbc:tc:postgresql:15.5-bookworm:///importHostingAssetsTC}", "spring.datasource.url=${HSADMINNG_POSTGRES_JDBC_URL:jdbc:tc:postgresql:15.5-bookworm:///importHostingAssetsTC}",
"spring.datasource.username=${HSADMINNG_POSTGRES_ADMIN_USERNAME:ADMIN}", "spring.datasource.username=${HSADMINNG_POSTGRES_ADMIN_USERNAME:ADMIN}",
"spring.datasource.password=${HSADMINNG_POSTGRES_ADMIN_PASSWORD:password}", "spring.datasource.password=${HSADMINNG_POSTGRES_ADMIN_PASSWORD:password}",
"hsadminng.superuser=${HSADMINNG_SUPERUSER:superuser-alex@hostsharing.net}", "hsadminng.superuser=${HSADMINNG_SUPERUSER:import-superuser@hostsharing.net}",
"spring.liquibase.enabled=false" // @Sql should go first, Liquibase will be initialized programmatically "spring.liquibase.enabled=false" // @Sql should go first, Liquibase will be initialized programmatically
}) })
@DirtiesContext @DirtiesContext
@ -748,6 +748,13 @@ public class ImportHostingAssets extends CsvDataImport {
@Test @Test
@Order(19000) @Order(19000)
@Commit @Commit
void prepareDataBase() {
makeSureThatTheImportAdminUserExists();
}
@Test
@Order(19100)
@Commit
void persistBookingProjects() { void persistBookingProjects() {
System.out.println("PERSISTING booking-projects to database '" + jdbcUrl + "' as user '" + postgresAdminUser + "'"); System.out.println("PERSISTING booking-projects to database '" + jdbcUrl + "' as user '" + postgresAdminUser + "'");
@ -759,7 +766,7 @@ public class ImportHostingAssets extends CsvDataImport {
} }
@Test @Test
@Order(19010) @Order(19110)
@Commit @Commit
void persistBookingItems() { void persistBookingItems() {

6
src/test/java/net/hostsharing/hsadminng/hs/migration/ImportOfficeData.java
View File

@ -53,10 +53,10 @@ import java.io.File;
"spring.datasource.url=${HSADMINNG_POSTGRES_JDBC_URL:jdbc:tc:postgresql:15.5-bookworm:///importOfficeDataTC}", "spring.datasource.url=${HSADMINNG_POSTGRES_JDBC_URL:jdbc:tc:postgresql:15.5-bookworm:///importOfficeDataTC}",
"spring.datasource.username=${HSADMINNG_POSTGRES_ADMIN_USERNAME:ADMIN}", "spring.datasource.username=${HSADMINNG_POSTGRES_ADMIN_USERNAME:ADMIN}",
"spring.datasource.password=${HSADMINNG_POSTGRES_ADMIN_PASSWORD:password}", "spring.datasource.password=${HSADMINNG_POSTGRES_ADMIN_PASSWORD:password}",
"hsadminng.superuser=${HSADMINNG_SUPERUSER:superuser-alex@hostsharing.net}", "hsadminng.superuser=${HSADMINNG_SUPERUSER:import-superuser@hostsharing.net}",
"spring.liquibase.contexts=only-office" "spring.liquibase.contexts=only-office,without-test-data"
}) })
@ActiveProfiles("without-test-data") @ActiveProfiles({"without-test-data"})
@DirtiesContext @DirtiesContext
@Import({ Context.class, JpaAttempt.class }) @Import({ Context.class, JpaAttempt.class })
@TestMethodOrder(MethodOrderer.OrderAnnotation.class) @TestMethodOrder(MethodOrderer.OrderAnnotation.class)

2
src/test/java/net/hostsharing/hsadminng/hs/migration/LiquibaseConfig.java
View File

@ -11,7 +11,7 @@ import jakarta.persistence.PersistenceContext;
import javax.sql.DataSource; import javax.sql.DataSource;
@Configuration @Configuration
@Profile("liquibase-migration-test") @Profile({"liquibase-migration", "liquibase-migration-test"})
public class LiquibaseConfig { public class LiquibaseConfig {
@PersistenceContext @PersistenceContext

17209
src/test/resources/db/released-only-office-schema-with-import-test-data.sql
View File

File diff suppressed because it is too large Load Diff