hostsharing/hs.hsadmin.ng
6
0
Fork 0
Code Pull Requests 4 Activity

hosting-asset-data-migration #79

Merged
hsh-michaelhoennig merged 22 commits from hosting-asset-data-migration into master 2024-07-22 11:30:34 +02:00
Conversation 0 Commits 22 Files Changed 51 +157 -140
2 changed files with 157 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit a54a3cd955 - Show all commits

156
src/test/java/net/hostsharing/hsadminng/hs/office/migration/CsvDataImport.java Normal file
View File

@ -0,0 +1,156 @@
package net.hostsharing.hsadminng.hs.office.migration;
import net.hostsharing.hsadminng.rbac.context.ContextBasedTest;
import net.hostsharing.hsadminng.rbac.test.JpaAttempt;
import org.junit.jupiter.api.extension.BeforeEachCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.api.extension.TestWatcher;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.transaction.support.TransactionTemplate;
import jakarta.persistence.EntityManager;
import jakarta.persistence.PersistenceContext;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.validation.constraints.NotNull;
import java.io.InputStreamReader;
import java.io.Reader;
import java.math.BigDecimal;
import java.time.LocalDate;
import java.util.List;
import java.util.TreeMap;
import static java.lang.Boolean.parseBoolean;
import static java.util.Arrays.stream;
import static java.util.Objects.requireNonNull;
import static java.util.Optional.ofNullable;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assumptions.assumeThat;
public class CsvDataImport extends ContextBasedTest {
public static final String MIGRATION_DATA_PATH = ofNullable(System.getenv("HSADMINNG_MIGRATION_DATA_PATH")).orElse("migration") + "/";
@Value("${spring.datasource.url}")
protected String jdbcUrl;
@Value("${spring.datasource.username}")
protected String postgresAdminUser;
@Value("${hsadminng.superuser}")
protected String rbacSuperuser;
@PersistenceContext
EntityManager em;
@Autowired
TransactionTemplate txTemplate;
@Autowired
JpaAttempt jpaAttempt;
@MockBean
HttpServletRequest request;
protected static String[] justHeader(final List<String[]> lines) {
return stream(lines.getFirst()).map(String::trim).toArray(String[]::new);
}
protected Reader resourceReader(@NotNull final String resourcePath) {
return new InputStreamReader(requireNonNull(getClass().getClassLoader().getResourceAsStream(resourcePath)));
}
protected List<String[]> withoutHeader(final List<String[]> records) {
return records.subList(1, records.size());
}
}
class Columns {
private final List<String> columnNames;
public Columns(final String[] header) {
columnNames = List.of(header);
}
int indexOf(final String columnName) {
int index = columnNames.indexOf(columnName);
if (index < 0) {
throw new RuntimeException("column name '" + columnName + "' not found in: " + columnNames);
}
return index;
}
}
class Record {
private final Columns columns;
private final String[] row;
public Record(final Columns columns, final String[] row) {
this.columns = columns;
this.row = row;
}
String getString(final String columnName) {
return row[columns.indexOf(columnName)];
}
boolean isEmpty(final String columnName) {
final String value = getString(columnName);
return value == null || value.isBlank();
}
boolean getBoolean(final String columnName) {
final String value = getString(columnName);
return isNotBlank(value) &&
( parseBoolean(value.trim()) || value.trim().startsWith("t"));
}
Integer getInteger(final String columnName) {
final String value = getString(columnName);
return isNotBlank(value) ? Integer.parseInt(value.trim()) : 0;
}
BigDecimal getBigDecimal(final String columnName) {
final String value = getString(columnName);
if (isNotBlank(value)) {
return new BigDecimal(value);
}
return null;
}
LocalDate getLocalDate(final String columnName) {
final String dateString = getString(columnName);
if (isNotBlank(dateString)) {
return LocalDate.parse(dateString);
}
return null;
}
}
class OrderedDependedTestsExtension implements TestWatcher, BeforeEachCallback {
private static boolean previousTestsPassed = true;
public void testFailed(ExtensionContext context, Throwable cause) {
previousTestsPassed = false;
}
@Override
public void beforeEach(final ExtensionContext extensionContext) {
assumeThat(previousTestsPassed).isTrue();
}
}
class WriteOnceMap<K, V> extends TreeMap<K, V> {
@Override
public V put(final K k, final V v) {
assertThat(containsKey(k)).describedAs("overwriting " + get(k) + " index " + k + " with " + v).isFalse();
return super.put(k, v);
}
}

141
src/test/java/net/hostsharing/hsadminng/hs/office/migration/ImportOfficeData.java
View File

@ -4,7 +4,6 @@ import com.opencsv.CSVParserBuilder;
import com.opencsv.CSVReader; import com.opencsv.CSVReader;
import com.opencsv.CSVReaderBuilder; import com.opencsv.CSVReaderBuilder;
import net.hostsharing.hsadminng.context.Context; import net.hostsharing.hsadminng.context.Context;
import net.hostsharing.hsadminng.rbac.context.ContextBasedTest;
import net.hostsharing.hsadminng.hs.office.bankaccount.HsOfficeBankAccountEntity; import net.hostsharing.hsadminng.hs.office.bankaccount.HsOfficeBankAccountEntity;
import net.hostsharing.hsadminng.hs.office.contact.HsOfficeContactEntity; import net.hostsharing.hsadminng.hs.office.contact.HsOfficeContactEntity;
import net.hostsharing.hsadminng.hs.office.coopassets.HsOfficeCoopAssetsTransactionEntity; import net.hostsharing.hsadminng.hs.office.coopassets.HsOfficeCoopAssetsTransactionEntity;
@ -26,34 +25,18 @@ import net.hostsharing.hsadminng.rbac.test.JpaAttempt;
import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.junit.jupiter.api.*; import org.junit.jupiter.api.*;
import org.junit.jupiter.api.extension.BeforeEachCallback;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.api.extension.TestWatcher;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest; import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Import;
import org.springframework.test.annotation.Commit; import org.springframework.test.annotation.Commit;
import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.annotation.DirtiesContext;
import org.springframework.transaction.support.TransactionTemplate;
import jakarta.persistence.EntityManager;
import jakarta.persistence.PersistenceContext;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.validation.constraints.NotNull;
import java.io.*; import java.io.*;
import java.math.BigDecimal;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDate; import java.time.LocalDate;
import java.util.*; import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static java.lang.Boolean.parseBoolean;
import static java.util.Arrays.stream; import static java.util.Arrays.stream;
import static java.util.Objects.requireNonNull;
import static java.util.Optional.ofNullable; import static java.util.Optional.ofNullable;
import static net.hostsharing.hsadminng.mapper.PostgresDateRange.toPostgresDateRange; import static net.hostsharing.hsadminng.mapper.PostgresDateRange.toPostgresDateRange;
import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isBlank;
@ -109,7 +92,7 @@ import static org.assertj.core.api.Fail.fail;
@Import({ Context.class, JpaAttempt.class }) @Import({ Context.class, JpaAttempt.class })
@TestMethodOrder(MethodOrderer.OrderAnnotation.class) @TestMethodOrder(MethodOrderer.OrderAnnotation.class)
@ExtendWith(OrderedDependedTestsExtension.class) @ExtendWith(OrderedDependedTestsExtension.class)
public class ImportOfficeData extends ContextBasedTest { public class ImportOfficeData extends CsvDataImport {
private static final String[] SUBSCRIBER_ROLES = new String[] { private static final String[] SUBSCRIBER_ROLES = new String[] {
"subscriber:operations-discussion", "subscriber:operations-discussion",
@ -125,7 +108,6 @@ public class ImportOfficeData extends ContextBasedTest {
// at least as the number of lines in business-partners.csv from test-data, but less than real data partner count // at least as the number of lines in business-partners.csv from test-data, but less than real data partner count
public static final int MAX_NUMBER_OF_TEST_DATA_PARTNERS = 100; public static final int MAX_NUMBER_OF_TEST_DATA_PARTNERS = 100;
public static final String MIGRATION_DATA_PATH = ofNullable(System.getenv("HSADMINNG_MIGRATION_DATA_PATH")).orElse("migration") + "/";
static int relationId = 2000000; static int relationId = 2000000;
@ -140,15 +122,6 @@ public class ImportOfficeData extends ContextBasedTest {
-1 -1
); );
@Value("${spring.datasource.url}")
private String jdbcUrl;
@Value("${spring.datasource.username}")
private String postgresAdminUser;
@Value("${hsadminng.superuser}")
private String rbacSuperuser;
private static Map<Integer, HsOfficeContactEntity> contacts = new WriteOnceMap<>(); private static Map<Integer, HsOfficeContactEntity> contacts = new WriteOnceMap<>();
private static Map<Integer, HsOfficePersonEntity> persons = new WriteOnceMap<>(); private static Map<Integer, HsOfficePersonEntity> persons = new WriteOnceMap<>();
private static Map<Integer, HsOfficePartnerEntity> partners = new WriteOnceMap<>(); private static Map<Integer, HsOfficePartnerEntity> partners = new WriteOnceMap<>();
@ -161,18 +134,6 @@ public class ImportOfficeData extends ContextBasedTest {
private static Map<Integer, HsOfficeCoopSharesTransactionEntity> coopShares = new WriteOnceMap<>(); private static Map<Integer, HsOfficeCoopSharesTransactionEntity> coopShares = new WriteOnceMap<>();
private static Map<Integer, HsOfficeCoopAssetsTransactionEntity> coopAssets = new WriteOnceMap<>(); private static Map<Integer, HsOfficeCoopAssetsTransactionEntity> coopAssets = new WriteOnceMap<>();
@PersistenceContext
EntityManager em;
@Autowired
TransactionTemplate txTemplate;
@Autowired
JpaAttempt jpaAttempt;
@MockBean
HttpServletRequest request;
@Test @Test
@Order(1010) @Order(1010)
void importBusinessPartners() { void importBusinessPartners() {
@ -1218,104 +1179,4 @@ public class ImportOfficeData extends ContextBasedTest {
private String toName(final String salut, final String title, final String firstname, final String lastname) { private String toName(final String salut, final String title, final String firstname, final String lastname) {
return toCaption(salut, title, firstname, lastname, null); return toCaption(salut, title, firstname, lastname, null);
} }
private Reader resourceReader(@NotNull final String resourcePath) {
return new InputStreamReader(requireNonNull(getClass().getClassLoader().getResourceAsStream(resourcePath)));
}
private static String[] justHeader(final List<String[]> lines) {
return stream(lines.getFirst()).map(String::trim).toArray(String[]::new);
}
private List<String[]> withoutHeader(final List<String[]> records) {
return records.subList(1, records.size());
}
}
class Columns {
private final List<String> columnNames;
public Columns(final String[] header) {
columnNames = List.of(header);
}
int indexOf(final String columnName) {
int index = columnNames.indexOf(columnName);
if (index < 0) {
throw new RuntimeException("column name '" + columnName + "' not found in: " + columnNames);
}
return index;
}
}
class Record {
private final Columns columns;
private final String[] row;
public Record(final Columns columns, final String[] row) {
this.columns = columns;
this.row = row;
}
String getString(final String columnName) {
return row[columns.indexOf(columnName)];
}
boolean isEmpty(final String columnName) {
final String value = getString(columnName);
return value == null || value.isBlank();
}
boolean getBoolean(final String columnName) {
final String value = getString(columnName);
return isNotBlank(value) &&
( parseBoolean(value.trim()) || value.trim().startsWith("t"));
}
Integer getInteger(final String columnName) {
final String value = getString(columnName);
return isNotBlank(value) ? Integer.parseInt(value.trim()) : 0;
}
BigDecimal getBigDecimal(final String columnName) {
final String value = getString(columnName);
if (isNotBlank(value)) {
return new BigDecimal(value);
}
return null;
}
LocalDate getLocalDate(final String columnName) {
final String dateString = getString(columnName);
if (isNotBlank(dateString)) {
return LocalDate.parse(dateString);
}
return null;
}
}
class OrderedDependedTestsExtension implements TestWatcher, BeforeEachCallback {
private static boolean previousTestsPassed = true;
public void testFailed(ExtensionContext context, Throwable cause) {
previousTestsPassed = false;
}
@Override
public void beforeEach(final ExtensionContext extensionContext) {
assumeThat(previousTestsPassed).isTrue();
}
}
class WriteOnceMap<K, V> extends TreeMap<K, V> {
@Override
public V put(final K k, final V v) {
assertThat(containsKey(k)).describedAs("overwriting " + get(k) + " index " + k + " with " + v).isFalse();
return super.put(k, v);
}
} }