hostsharing/hs.hsadmin.ng
6
0
Fork 0
Code Pull Requests 1 Activity

replace office-data-import by db-restore #154

Merged
hsh-michaelhoennig merged 13 commits from feature/replace-office-data-import-by-db-restore into master 2025-02-04 09:56:00 +01:00
Conversation 1 Commits 13 Files Changed 16 +154 -130
4 changed files with 154 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit aef3c9c546 - Show all commits

132
src/test/java/net/hostsharing/hsadminng/hs/migration/LiquibaseCompatibilityIntegrationTest.java
View File

@ -1,7 +1,5 @@
package net.hostsharing.hsadminng.hs.migration; package net.hostsharing.hsadminng.hs.migration;
import liquibase.Liquibase;
import lombok.SneakyThrows;
import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@ -10,24 +8,9 @@ import org.springframework.context.annotation.Import;
import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.jdbc.Sql; import org.springframework.test.context.jdbc.Sql;
import org.testcontainers.containers.JdbcDatabaseContainer;
import org.testcontainers.jdbc.ContainerDatabaseDriver;
import jakarta.persistence.EntityManager;
import jakarta.persistence.PersistenceContext;
import javax.sql.DataSource;
import java.io.BufferedReader;
import java.io.File; import java.io.File;
import java.io.InputStreamReader;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.commons.io.FileUtils.readFileToString;
import static org.apache.commons.io.FileUtils.write;
import static org.apache.commons.io.FileUtils.writeStringToFile;
import static org.assertj.core.api.Assertions.assertThat;
import static org.springframework.test.context.jdbc.Sql.ExecutionPhase.BEFORE_TEST_CLASS; import static org.springframework.test.context.jdbc.Sql.ExecutionPhase.BEFORE_TEST_CLASS;
// BLOG: Liquibase-migration-test (not before the reference-SQL-dump-generation is simplified) // BLOG: Liquibase-migration-test (not before the reference-SQL-dump-generation is simplified)
@ -58,123 +41,28 @@ import static org.springframework.test.context.jdbc.Sql.ExecutionPhase.BEFORE_TE
@DirtiesContext @DirtiesContext
@ActiveProfiles("liquibase-migration-test") @ActiveProfiles("liquibase-migration-test")
@Import(LiquibaseConfig.class) @Import(LiquibaseConfig.class)
@Sql(value = "/db/prod-only-office-schema-with-test-data.sql", executionPhase = BEFORE_TEST_CLASS) @Sql(value = "/db/prod-only-office-schema-with-test-data.sql", executionPhase = BEFORE_TEST_CLASS) // reads prod-schema-dump
public class LiquibaseCompatibilityIntegrationTest { public class LiquibaseCompatibilityIntegrationTest {
private static final String EXPECTED_CHANGESET_ONLY_AFTER_NEW_MIGRATION = "hs-global-liquibase-migration-test"; private static final String EXPECTED_CHANGESET_ONLY_AFTER_NEW_MIGRATION = "hs-global-liquibase-migration-test";
public static final int EXPECTED_LIQUIBASE_CHANGELOGS_IN_PROD_SCHEMA_DUMP = 287;
@Autowired @Autowired
private DataSource dataSource; private LiquibaseMigration liquibase;
@Autowired
private Liquibase liquibase;
@PersistenceContext
private EntityManager em;
@Test @Test
void migrationWorksBasedOnAPreviouslyPopulatedSchema() { void migrationWorksBasedOnAPreviouslyPopulatedSchema() {
// check the initial status from the @Sql-annotation // check the initial status from the @Sql-annotation
final var initialChangeSetCount = assertProdReferenceStatusAfterRestore(); final var initialChangeSetCount = liquibase.assertReferenceStatusAfterRestore(
EXPECTED_LIQUIBASE_CHANGELOGS_IN_PROD_SCHEMA_DUMP, EXPECTED_CHANGESET_ONLY_AFTER_NEW_MIGRATION);
// run the current migrations and dump the result to the build-directory // run the current migrations and dump the result to the build-directory
runLiquibaseMigrationsWithContexts("only-office", "with-test-data"); liquibase.runWithContexts("only-office", "with-test-data");
dumpTo(new File("build/db/prod-only-office-schema-with-test-data.sql")); PostgresTestcontainer.dumpTo(new File("build/db/prod-only-office-schema-with-test-data.sql"));
// then add another migration and assert if it was applied // then add another migration and assert if it was applied
runLiquibaseMigrationsWithContexts("liquibase-migration-test"); liquibase.runWithContexts("liquibase-migration-test");
assertThatCurrentMigrationsGotApplied(initialChangeSetCount); liquibase.assertThatCurrentMigrationsGotApplied(
} initialChangeSetCount, EXPECTED_CHANGESET_ONLY_AFTER_NEW_MIGRATION);
private int assertProdReferenceStatusAfterRestore() {
final var schemas = singleColumnSqlQuery("SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname='public'");
assertThat(schemas).containsExactly("databasechangelog", "databasechangeloglock");
final var liquibaseScripts1 = singleColumnSqlQuery("SELECT * FROM public.databasechangelog");
assertThat(liquibaseScripts1).hasSizeGreaterThan(285);
assertThat(liquibaseScripts1).doesNotContain(EXPECTED_CHANGESET_ONLY_AFTER_NEW_MIGRATION);
final var initialChangeSetCount = liquibaseScripts1.size();
return initialChangeSetCount;
}
private void assertThatCurrentMigrationsGotApplied(final int initialChangeSetCount) {
final var liquibaseScripts = singleColumnSqlQuery("SELECT id FROM public.databasechangelog");
assertThat(liquibaseScripts).hasSizeGreaterThan(initialChangeSetCount);
assertThat(liquibaseScripts).contains(EXPECTED_CHANGESET_ONLY_AFTER_NEW_MIGRATION);
}
@SneakyThrows
private void dumpTo(final File targetFileName) {
makeDir(targetFileName.getParentFile());
final var jdbcDatabaseContainer = getJdbcDatabaseContainer();
final var sqlDumpFile = new File(targetFileName.getParent(), "." + targetFileName.getName());
final var pb = new ProcessBuilder(
"pg_dump", "--column-inserts", "--disable-dollar-quoting",
"--host=" + jdbcDatabaseContainer.getHost(),
"--port=" + jdbcDatabaseContainer.getFirstMappedPort(),
"--username=" + jdbcDatabaseContainer.getUsername() ,
"--dbname=" + jdbcDatabaseContainer.getDatabaseName(),
"--file=" + sqlDumpFile.getCanonicalPath()
);
pb.environment().put("PGPASSWORD", jdbcDatabaseContainer.getPassword());
final var process = pb.start();
int exitCode = process.waitFor();
final var stderr = new BufferedReader(new InputStreamReader(process.getErrorStream()))
.lines().collect(Collectors.joining("\n"));
assertThat(exitCode).describedAs(stderr).isEqualTo(0);
final var header = """
-- =================================================================================
-- Generated reference-SQL-dump (hopefully of latest prod-release).
-- See: net.hostsharing.hsadminng.hs.migration.LiquibaseCompatibilityIntegrationTest
-- ---------------------------------------------------------------------------------
--
-- Explicit pre-initialization because we cannot use `pg_dump --create ...`
-- because the database is already created by Testcontainers.
--
CREATE ROLE postgres;
CREATE ROLE admin;
CREATE ROLE restricted;
""";
writeStringToFile(targetFileName, header, UTF_8, false); // false = overwrite
write(targetFileName, readFileToString(sqlDumpFile, UTF_8), UTF_8, true);
assertThat(sqlDumpFile.delete()).describedAs(sqlDumpFile + " cannot be deleted");
}
private void makeDir(final File dir) {
assertThat(!dir.exists() || dir.isDirectory()).describedAs(dir + " does exist, but is not a directory").isTrue();
assertThat(dir.isDirectory() || dir.mkdirs()).describedAs(dir + " cannot be created").isTrue();
}
@SneakyThrows
private void runLiquibaseMigrationsWithContexts(final String... contexts) {
liquibase.update(
new liquibase.Contexts(contexts),
new liquibase.LabelExpression());
}
private List<String> singleColumnSqlQuery(final String sql) {
//noinspection unchecked
final var rows = (List<Object>) em.createNativeQuery(sql).getResultList();
return rows.stream().map(Objects::toString).toList();
}
@SneakyThrows
private static JdbcDatabaseContainer<?> getJdbcDatabaseContainer() {
final var getContainerMethod = ContainerDatabaseDriver.class.getDeclaredMethod("getContainer", String.class);
getContainerMethod.setAccessible(true);
@SuppressWarnings("rawtypes")
final var container = (JdbcDatabaseContainer) getContainerMethod.invoke(null,
"jdbc:tc:postgresql:15.5-bookworm:///liquibaseMigrationTestTC");
return container;
} }
} }

15
src/test/java/net/hostsharing/hsadminng/hs/migration/LiquibaseConfig.java
View File

@ -1,28 +1,27 @@
package net.hostsharing.hsadminng.hs.migration; package net.hostsharing.hsadminng.hs.migration;
import liquibase.Liquibase;
import liquibase.database.DatabaseFactory; import liquibase.database.DatabaseFactory;
import liquibase.database.jvm.JdbcConnection; import liquibase.database.jvm.JdbcConnection;
import liquibase.resource.ClassLoaderResourceAccessor;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile; import org.springframework.context.annotation.Profile;
import jakarta.persistence.EntityManager;
import jakarta.persistence.PersistenceContext;
import javax.sql.DataSource; import javax.sql.DataSource;
@Configuration @Configuration
@Profile("liquibase-migration-test") @Profile("liquibase-migration-test")
public class LiquibaseConfig { public class LiquibaseConfig {
@PersistenceContext
private EntityManager em;
@Bean @Bean
public Liquibase liquibase(DataSource dataSource) throws Exception { public LiquibaseMigration liquibase(DataSource dataSource) throws Exception {
final var connection = dataSource.getConnection(); final var connection = dataSource.getConnection();
final var database = DatabaseFactory.getInstance() final var database = DatabaseFactory.getInstance()
.findCorrectDatabaseImplementation(new JdbcConnection(connection)); .findCorrectDatabaseImplementation(new JdbcConnection(connection));
return new Liquibase( return new LiquibaseMigration(em, "db/changelog/db.changelog-master.yaml", database);
"db/changelog/db.changelog-master.yaml", // Path to your Liquibase changelog
new ClassLoaderResourceAccessor(),
database
);
} }
} }

55
src/test/java/net/hostsharing/hsadminng/hs/migration/LiquibaseMigration.java Normal file
View File

@ -0,0 +1,55 @@
package net.hostsharing.hsadminng.hs.migration;
import liquibase.Liquibase;
import liquibase.database.Database;
import liquibase.resource.ClassLoaderResourceAccessor;
import lombok.SneakyThrows;
import jakarta.persistence.EntityManager;
import java.util.List;
import java.util.Objects;
import static org.assertj.core.api.Assertions.assertThat;
public class LiquibaseMigration extends Liquibase {
private final EntityManager em;
public LiquibaseMigration(final EntityManager em, final String changeLogFile, final Database db) {
super(changeLogFile, new ClassLoaderResourceAccessor(), db);
this.em = em;
}
@SneakyThrows
public void runWithContexts(final String... contexts) {
update(
new liquibase.Contexts(contexts),
new liquibase.LabelExpression());
}
public int assertReferenceStatusAfterRestore(
final int minExpectedLiquibaseChangelogs,
final String expectedChangesetOnlyAfterNewMigration) {
final var schemas = singleColumnSqlQuery("SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname='public'");
assertThat(schemas).containsExactly("databasechangelog", "databasechangeloglock");
final var liquibaseScripts = singleColumnSqlQuery("SELECT id FROM public.databasechangelog");
assertThat(liquibaseScripts).hasSize(minExpectedLiquibaseChangelogs);
assertThat(liquibaseScripts).doesNotContain(expectedChangesetOnlyAfterNewMigration);
return liquibaseScripts.size();
}
public void assertThatCurrentMigrationsGotApplied(
final int initialChangeSetCount,
final String expectedChangesetOnlyAfterNewMigration) {
final var liquibaseScripts = singleColumnSqlQuery("SELECT id FROM public.databasechangelog");
assertThat(liquibaseScripts).hasSizeGreaterThan(initialChangeSetCount);
assertThat(liquibaseScripts).contains(expectedChangesetOnlyAfterNewMigration);
}
private List<String> singleColumnSqlQuery(final String sql) {
//noinspection unchecked
final var rows = (List<Object>) em.createNativeQuery(sql).getResultList();
return rows.stream().map(Objects::toString).toList();
}
}

82
src/test/java/net/hostsharing/hsadminng/hs/migration/PostgresTestcontainer.java Normal file
View File

@ -0,0 +1,82 @@
package net.hostsharing.hsadminng.hs.migration;
import lombok.SneakyThrows;
import org.testcontainers.containers.JdbcDatabaseContainer;
import org.testcontainers.jdbc.ContainerDatabaseDriver;
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
import java.util.stream.Collectors;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.commons.io.FileUtils.readFileToString;
import static org.apache.commons.io.FileUtils.write;
import static org.apache.commons.io.FileUtils.writeStringToFile;
import static org.assertj.core.api.Assertions.assertThat;
public class PostgresTestcontainer {
@SneakyThrows
public static void dumpTo(final File targetFileName) {
makeDir(targetFileName.getParentFile());
final var jdbcDatabaseContainer = getJdbcDatabaseContainer();
final var sqlDumpFile = new File(targetFileName.getParent(), "." + targetFileName.getName());
final var pb = new ProcessBuilder(
"pg_dump", "--column-inserts", "--disable-dollar-quoting",
"--host=" + jdbcDatabaseContainer.getHost(),
"--port=" + jdbcDatabaseContainer.getFirstMappedPort(),
"--username=" + jdbcDatabaseContainer.getUsername() ,
"--dbname=" + jdbcDatabaseContainer.getDatabaseName(),
"--file=" + sqlDumpFile.getCanonicalPath()
);
pb.environment().put("PGPASSWORD", jdbcDatabaseContainer.getPassword());
final var process = pb.start();
int exitCode = process.waitFor();
final var stderr = new BufferedReader(new InputStreamReader(process.getErrorStream()))
.lines().collect(Collectors.joining("\n"));
assertThat(exitCode).describedAs(stderr).isEqualTo(0);
final var header = """
-- =================================================================================
-- Generated reference-SQL-dump (hopefully of latest prod-release).
-- See: net.hostsharing.hsadminng.hs.migration.LiquibaseCompatibilityIntegrationTest
-- ---------------------------------------------------------------------------------
--
-- Explicit pre-initialization because we cannot use `pg_dump --create ...`
-- because the database is already created by Testcontainers.
--
CREATE ROLE postgres;
CREATE ROLE admin;
CREATE ROLE restricted;
""";
writeStringToFile(targetFileName, header, UTF_8, false); // false = overwrite
write(targetFileName, readFileToString(sqlDumpFile, UTF_8), UTF_8, true);
assertThat(sqlDumpFile.delete()).describedAs(sqlDumpFile + " cannot be deleted");
}
private static void makeDir(final File dir) {
assertThat(!dir.exists() || dir.isDirectory()).describedAs(dir + " does exist, but is not a directory").isTrue();
assertThat(dir.isDirectory() || dir.mkdirs()).describedAs(dir + " cannot be created").isTrue();
}
@SneakyThrows
private static JdbcDatabaseContainer<?> getJdbcDatabaseContainer() {
// TODO.test: check if, in the future, there is a better way to access auto-created Testcontainers
final var getContainerMethod = ContainerDatabaseDriver.class.getDeclaredMethod("getContainer", String.class);
getContainerMethod.setAccessible(true);
@SuppressWarnings("rawtypes")
final var container = (JdbcDatabaseContainer) getContainerMethod.invoke(null,
"jdbc:tc:postgresql:15.5-bookworm:///liquibaseMigrationTestTC");
return container;
}
}