hostsharing/hs.hsadmin.ng
6
0
Fork 0
Code Pull Requests 1 Activity

replace office-data-import by db-restore #154

Merged
hsh-michaelhoennig merged 13 commits from feature/replace-office-data-import-by-db-restore into master 2025-02-04 09:56:00 +01:00
Conversation 1 Commits 13 Files Changed 16 +7242 -10139
6 changed files with 7242 additions and 10139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 0b31c6aa2f - Show all commits

94
src/test/java/net/hostsharing/hsadminng/hs/migration/BaseOfficeDataImport.java
View File

@ -115,11 +115,18 @@ public abstract class BaseOfficeDataImport extends CsvDataImport {
@Test
@Order(1)
void verifyInitialDatabase() {
// SQL DELETE for thousands of records takes too long, so we make sure, we only start with initial or test data
final var contactCount = (Integer) em.createNativeQuery("select count(*) from hs_office.contact", Integer.class)
.getSingleResult();
assertThat(contactCount).isLessThan(20);
void verifyInitialDatabaseHasNoTestData() {
assertThat((Integer) em.createNativeQuery(
"select count(*) from hs_office.contact",
Integer.class)
.getSingleResult()).isEqualTo(0);
assertThat((Integer) em.createNativeQuery(
"""
SELECT count(*) FROM information_schema.tables
WHERE table_schema = 'rbactest' AND table_name = 'customer'
""",
Integer.class)
.getSingleResult()).isEqualTo(0);
}
@Test
@ -624,15 +631,13 @@ public abstract class BaseOfficeDataImport extends CsvDataImport {
void persistOfficeEntities() {
System.out.println("PERSISTING office data to database '" + jdbcUrl + "' as user '" + postgresAdminUser + "'");
deleteTestDataFromHsOfficeTables();
resetHsOfficeSequences();
deleteFromTestTables();
deleteFromCommonTables();
makeSureThatTheImportAdminUserExists();
assertEmptyTable("hs_office.contact");
jpaAttempt.transacted(() -> {
context(rbacSuperuser);
contacts.forEach(this::persist);
updateLegacyIds(contacts, "hs_office.contact_legacy_id", "contact_id");
updateLegacyIds(contacts, "hs_office.contact_legacy_id", "contact_id");
}).assertSuccessful();
jpaAttempt.transacted(() -> {
@ -646,6 +651,7 @@ public abstract class BaseOfficeDataImport extends CsvDataImport {
}).assertSuccessful();
System.out.println("persisting " + partners.size() + " partners");
assertEmptyTable("hs_office.partner");
jpaAttempt.transacted(() -> {
context(rbacSuperuser);
partners.forEach((id, partner) -> {
@ -697,6 +703,12 @@ public abstract class BaseOfficeDataImport extends CsvDataImport {
}).assertSuccessful();
}
private void assertEmptyTable(final String qualifiedTableName) {
assertThat((Integer) em.createNativeQuery(
"select count(*) from " + qualifiedTableName,
Integer.class)
.getSingleResult()).describedAs("expected empty " + qualifiedTableName).isEqualTo(0);
}
@Test
@Order(9190)
@ -883,7 +895,6 @@ public abstract class BaseOfficeDataImport extends CsvDataImport {
coopAssets.put(rec.getInteger("member_asset_id"), assetTransaction);
});
coopAssets.entrySet().forEach(entry -> {
final var legacyId = entry.getKey();
final var assetTransaction = entry.getValue();
@ -896,7 +907,9 @@ public abstract class BaseOfficeDataImport extends CsvDataImport {
});
}
private static void connectToRelatedRevertedAssetTx(final int legacyId, final HsOfficeCoopAssetsTransactionEntity assetTransaction) {
private static void connectToRelatedRevertedAssetTx(
final int legacyId,
final HsOfficeCoopAssetsTransactionEntity assetTransaction) {
final var negativeValue = assetTransaction.getAssetValue().negate();
final var revertedAssetTx = coopAssets.values().stream().filter(a ->
a.getTransactionType() != HsOfficeCoopAssetsTransactionType.REVERSAL &&
@ -909,11 +922,14 @@ public abstract class BaseOfficeDataImport extends CsvDataImport {
//revertedAssetTx.setAssetReversalTx(assetTransaction);
}
private static void connectToRelatedAdoptionAssetTx(final int legacyId, final HsOfficeCoopAssetsTransactionEntity assetTransaction) {
private static void connectToRelatedAdoptionAssetTx(
final int legacyId,
final HsOfficeCoopAssetsTransactionEntity assetTransaction) {
final var negativeValue = assetTransaction.getAssetValue().negate();
final var adoptionAssetTx = coopAssets.values().stream().filter(a ->
a.getTransactionType() == HsOfficeCoopAssetsTransactionType.ADOPTION &&
(!a.getValueDate().equals(LocalDate.of( 2014 , 12 , 31)) || a.getComment().contains(Integer.toString(assetTransaction.getMembership().getMemberNumber()/100))) &&
(!a.getValueDate().equals(LocalDate.of(2014, 12, 31)) || a.getComment()
.contains(Integer.toString(assetTransaction.getMembership().getMemberNumber() / 100))) &&
a.getMembership() != assetTransaction.getMembership() &&
a.getValueDate().equals(assetTransaction.getValueDate()) &&
a.getAssetValue().equals(negativeValue))
@ -1131,14 +1147,14 @@ public abstract class BaseOfficeDataImport extends CsvDataImport {
final var personKey = (
person.getPersonType() + "|" +
person.getSalutation() + "|" +
person.getTradeName() + "|" +
person.getTitle() + "|" +
person.getGivenName() + "|" +
person.getFamilyName()
person.getSalutation() + "|" +
person.getTradeName() + "|" +
person.getTitle() + "|" +
person.getGivenName() + "|" +
person.getFamilyName()
).toLowerCase();
if ( !distinctPersons.containsKey(personKey) ) {
if (!distinctPersons.containsKey(personKey)) {
distinctPersons.put(personKey, person);
}
return distinctPersons.get(personKey);
@ -1164,24 +1180,24 @@ public abstract class BaseOfficeDataImport extends CsvDataImport {
if (endsWithWord(tradeName, "OHG", "GbR", "KG", "UG", "PartGmbB", "mbB")) {
return HsOfficePersonType.INCORPORATED_FIRM; // Personengesellschaft. Gesellschafter haften persönlich.
} else if (containsWord(tradeName, "e.K.", "e.G.", "eG", "gGmbH", "GmbH", "mbH", "AG", "e.V.", "eV", "e.V")
|| tradeName.toLowerCase().contains("haftungsbeschränkt")
|| tradeName.toLowerCase().contains("stiftung")
|| tradeName.toLowerCase().contains("stichting")
|| tradeName.toLowerCase().contains("foundation")
|| tradeName.toLowerCase().contains("schule")
|| tradeName.toLowerCase().contains("verein")
|| tradeName.toLowerCase().contains("gewerkschaft")
|| tradeName.toLowerCase().contains("gesellschaft")
|| tradeName.toLowerCase().contains("kirche")
|| tradeName.toLowerCase().contains("fraktion")
|| tradeName.toLowerCase().contains("landkreis")
|| tradeName.toLowerCase().contains("behörde")
|| tradeName.toLowerCase().contains("bundesamt")
|| tradeName.toLowerCase().contains("bezirksamt")
) {
|| tradeName.toLowerCase().contains("haftungsbeschränkt")
|| tradeName.toLowerCase().contains("stiftung")
|| tradeName.toLowerCase().contains("stichting")
|| tradeName.toLowerCase().contains("foundation")
|| tradeName.toLowerCase().contains("schule")
|| tradeName.toLowerCase().contains("verein")
|| tradeName.toLowerCase().contains("gewerkschaft")
|| tradeName.toLowerCase().contains("gesellschaft")
|| tradeName.toLowerCase().contains("kirche")
|| tradeName.toLowerCase().contains("fraktion")
|| tradeName.toLowerCase().contains("landkreis")
|| tradeName.toLowerCase().contains("behörde")
|| tradeName.toLowerCase().contains("bundesamt")
|| tradeName.toLowerCase().contains("bezirksamt")
) {
return HsOfficePersonType.LEGAL_PERSON; // Haftungsbeschränkt
} else if (roles.contains("contractual") && !roles.contains("partner") &&
!familyName.isBlank() && !givenName.isBlank()) {
!familyName.isBlank() && !givenName.isBlank()) {
// REPRESENTATIVES are always natural persons
return HsOfficePersonType.NATURAL_PERSON;
} else {
@ -1203,9 +1219,9 @@ public abstract class BaseOfficeDataImport extends CsvDataImport {
final var lowerCaseValue = value.toLowerCase();
for (String ending : endings) {
if (lowerCaseValue.equals(ending.toLowerCase()) ||
lowerCaseValue.startsWith(ending.toLowerCase() + " ") ||
lowerCaseValue.contains(" " + ending.toLowerCase() + " ") ||
lowerCaseValue.endsWith(" " + ending.toLowerCase())) {
lowerCaseValue.startsWith(ending.toLowerCase() + " ") ||
lowerCaseValue.contains(" " + ending.toLowerCase() + " ") ||
lowerCaseValue.endsWith(" " + ending.toLowerCase())) {
return true;
}
}

59
src/test/java/net/hostsharing/hsadminng/hs/migration/CsvDataImport.java
View File

@ -248,51 +248,22 @@ public class CsvDataImport extends ContextBasedTest {
return json;
}
protected void deleteTestDataFromHsOfficeTables() {
protected void makeSureThatTheImportAdminUserExists() {
jpaAttempt.transacted(() -> {
context(rbacSuperuser);
// TODO.perf: could we instead skip creating test-data based on an env var?
em.createNativeQuery("delete from hs_office.coopassettx where true").executeUpdate();
em.createNativeQuery("delete from hs_office.coopassettx_legacy_id where true").executeUpdate();
em.createNativeQuery("delete from hs_office.coopsharetx where true").executeUpdate();
em.createNativeQuery("delete from hs_office.coopsharetx_legacy_id where true").executeUpdate();
em.createNativeQuery("delete from hs_office.membership where true").executeUpdate();
em.createNativeQuery("delete from hs_office.sepamandate where true").executeUpdate();
em.createNativeQuery("delete from hs_office.sepamandate_legacy_id where true").executeUpdate();
em.createNativeQuery("delete from hs_office.debitor where true").executeUpdate();
em.createNativeQuery("delete from hs_office.bankaccount where true").executeUpdate();
em.createNativeQuery("delete from hs_office.partner where true").executeUpdate();
em.createNativeQuery("delete from hs_office.partner_details where true").executeUpdate();
em.createNativeQuery("delete from hs_office.relation where true").executeUpdate();
em.createNativeQuery("delete from hs_office.contact where true").executeUpdate();
em.createNativeQuery("delete from hs_office.person where true").executeUpdate();
}).assertSuccessful();
}
protected void resetHsOfficeSequences() {
jpaAttempt.transacted(() -> {
context(rbacSuperuser);
em.createNativeQuery("alter sequence hs_office.contact_legacy_id_seq restart with 1000000000;").executeUpdate();
em.createNativeQuery("alter sequence hs_office.coopassettx_legacy_id_seq restart with 1000000000;")
.executeUpdate();
});
}
protected void deleteFromTestTables() {
jpaAttempt.transacted(() -> {
context(rbacSuperuser);
em.createNativeQuery("delete from rbactest.domain where true").executeUpdate();
em.createNativeQuery("delete from rbactest.package where true").executeUpdate();
em.createNativeQuery("delete from rbactest.customer where true").executeUpdate();
}).assertSuccessful();
}
protected void deleteFromCommonTables() {
jpaAttempt.transacted(() -> {
context(rbacSuperuser);
em.createNativeQuery("delete from rbac.subject_rv where name not like 'superuser-%'").executeUpdate();
em.createNativeQuery("delete from base.tx_journal where true").executeUpdate();
em.createNativeQuery("delete from base.tx_context where true").executeUpdate();
context(null);
em.createNativeQuery("""
do language plpgsql $$
declare
admins uuid;
begin
if not exists (select 1 from rbac.subject where name = '${rbacSuperuser}') then
admins = rbac.findRoleId(rbac.global_ADMIN());
call rbac.grantRoleToSubjectUnchecked(admins, admins, rbac.create_subject('${rbacSuperuser}'));
end if;
end;
$$;
""".replace("${rbacSuperuser}", rbacSuperuser))
.executeUpdate();
}).assertSuccessful();
}

11
src/test/java/net/hostsharing/hsadminng/hs/migration/ImportHostingAssets.java
View File

@ -87,7 +87,7 @@ import static org.springframework.test.context.jdbc.Sql.ExecutionPhase.BEFORE_TE
"spring.datasource.url=${HSADMINNG_POSTGRES_JDBC_URL:jdbc:tc:postgresql:15.5-bookworm:///importHostingAssetsTC}",
"spring.datasource.username=${HSADMINNG_POSTGRES_ADMIN_USERNAME:ADMIN}",
"spring.datasource.password=${HSADMINNG_POSTGRES_ADMIN_PASSWORD:password}",
"hsadminng.superuser=${HSADMINNG_SUPERUSER:superuser-alex@hostsharing.net}",
"hsadminng.superuser=${HSADMINNG_SUPERUSER:import-superuser@hostsharing.net}",
"spring.liquibase.enabled=false" // @Sql should go first, Liquibase will be initialized programmatically
})
@DirtiesContext
@ -748,6 +748,13 @@ public class ImportHostingAssets extends CsvDataImport {
@Test
@Order(19000)
@Commit
void prepareDataBase() {
makeSureThatTheImportAdminUserExists();
}
@Test
@Order(19100)
@Commit
void persistBookingProjects() {
System.out.println("PERSISTING booking-projects to database '" + jdbcUrl + "' as user '" + postgresAdminUser + "'");
@ -759,7 +766,7 @@ public class ImportHostingAssets extends CsvDataImport {
}
@Test
@Order(19010)
@Order(19110)
@Commit
void persistBookingItems() {

6
src/test/java/net/hostsharing/hsadminng/hs/migration/ImportOfficeData.java
View File

@ -53,10 +53,10 @@ import java.io.File;
"spring.datasource.url=${HSADMINNG_POSTGRES_JDBC_URL:jdbc:tc:postgresql:15.5-bookworm:///importOfficeDataTC}",
"spring.datasource.username=${HSADMINNG_POSTGRES_ADMIN_USERNAME:ADMIN}",
"spring.datasource.password=${HSADMINNG_POSTGRES_ADMIN_PASSWORD:password}",
"hsadminng.superuser=${HSADMINNG_SUPERUSER:superuser-alex@hostsharing.net}",
"spring.liquibase.contexts=only-office"
"hsadminng.superuser=${HSADMINNG_SUPERUSER:import-superuser@hostsharing.net}",
"spring.liquibase.contexts=only-office,without-test-data"
})
@ActiveProfiles("without-test-data")
@ActiveProfiles({"without-test-data"})
@DirtiesContext
@Import({ Context.class, JpaAttempt.class })
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)

2
src/test/java/net/hostsharing/hsadminng/hs/migration/LiquibaseConfig.java
View File

@ -11,7 +11,7 @@ import jakarta.persistence.PersistenceContext;
import javax.sql.DataSource;
@Configuration
@Profile("liquibase-migration-test")
@Profile({"liquibase-migration", "liquibase-migration-test"})
public class LiquibaseConfig {
@PersistenceContext

17209
src/test/resources/db/released-only-office-schema-with-import-test-data.sql
View File

File diff suppressed because it is too large Load Diff