hostsharing/hs.hsadmin.ng
6
0
Fork 0
Code Pull Requests 1 Activity

feature/remove-office-data-import (#155)

Browse Source 
Co-authored-by: Michael Hoennig <michael@hoennig.de>
Reviewed-on: #155
Reviewed-by: Marc Sandlus <marc.sandlus@hostsharing.net>
This commit is contained in:
Michael Hoennig 2025-02-05 09:29:27 +01:00
parent a1a753e00a
commit abafd64813
6 changed files with 10 additions and 1512 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.aliases
View File

@ -1,4 +1,4 @@
# For using the alias gw-importOfficeData or gw-importHostingAssets, # For using the alias gw-importHostingAssets,
# copy the file .tc-environment to .environment (ignored by git) # copy the file .tc-environment to .environment (ignored by git)
# and amend them according to your external DB. # and amend them according to your external DB.
@ -71,7 +71,6 @@ function importLegacyData() {
./gradlew $target --rerun ./gradlew $target --rerun
fi fi
} }
alias gw-importOfficeData='importLegacyData importOfficeData'
alias gw-importHostingAssets='importLegacyData importHostingAssets' alias gw-importHostingAssets='importLegacyData importHostingAssets'
alias podman-start='systemctl --user enable --now podman.socket && systemctl --user status podman.socket && ls -la /run/user/$UID/podman/podman.sock' alias podman-start='systemctl --user enable --now podman.socket && systemctl --user status podman.socket && ls -la /run/user/$UID/podman/podman.sock'
@ -92,8 +91,8 @@ alias fp='grep -r '@Accepts' src | sed -e 's/^.*@/@/g' | sort -u | wc -l'
alias gw-spotless='./gradlew spotlessApply -x pitest -x test -x :processResources' alias gw-spotless='./gradlew spotlessApply -x pitest -x test -x :processResources'
alias gw-check='. .aliases; . .tc-environment; gw test check -x pitest' alias gw-check='. .aliases; . .tc-environment; gw test check -x pitest'
# HOWTO: run all 'normal' tests (no scenario+import-tests): `gw-test` # HOWTO: run all 'normal' tests (by default without scenario+import-tests): `gw-test`
# You can also mention specific targets: `gw-test importOfficeData`. # You can also mention specific targets: `gw-test importHostingAssets`, in that case only these tests are executed.
# This will always use the environment from `.tc-environment`. # This will always use the environment from `.tc-environment`.
# #
# HOWTO: re-run tests even if no changed can be detected: `gw-test --rerun` # HOWTO: re-run tests even if no changed can be detected: `gw-test --rerun`
@ -123,7 +122,7 @@ function _gwTest() {
time (_gwTest1 unitTest "$@" && time (_gwTest1 unitTest "$@" &&
_gwTest1 officeIntegrationTest bookingIntegrationTest hostingIntegrationTest "$@" && _gwTest1 officeIntegrationTest bookingIntegrationTest hostingIntegrationTest "$@" &&
_gwTest1 scenarioTest "$@" && _gwTest1 scenarioTest "$@" &&
_gwTest1 importOfficeData importHostingAssets "$@"); _gwTest1 importHostingAssets "$@");
elif [ $# -eq 0 ] || [[ $1 == -* ]]; then elif [ $# -eq 0 ] || [[ $1 == -* ]]; then
time _gwTest1 test "$@"; time _gwTest1 test "$@";
else else
@ -137,7 +136,7 @@ alias howto=bin/howto
alias cas-curl=bin/cas-curl alias cas-curl=bin/cas-curl
# etc/docker-compose.yml limits CPUs+MEM and includes a PostgreSQL config for analysing slow queries # etc/docker-compose.yml limits CPUs+MEM and includes a PostgreSQL config for analysing slow queries
alias gw-importOfficeData-in-docker-compose=' alias gw-importHostingAssets-in-docker-compose='
docker-compose -f etc/docker-compose.yml down && docker-compose -f etc/docker-compose.yml down &&
docker-compose -f etc/docker-compose.yml up -d && sleep 10 && docker-compose -f etc/docker-compose.yml up -d && sleep 10 &&
time gw-importHostingAssets' time gw-importHostingAssets'

103
.run/ImportOfficeData.run.xml
View File

@ -1,103 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="ImportOfficeData" type="GradleRunConfiguration" factoryName="Gradle">
<ExternalSystemSettings>
<option name="env">
<map>
<entry key="HSADMINNG_MIGRATION_DATA_PATH" value="migration" />
<entry key="HSADMINNG_POSTGRES_ADMIN_USERNAME" value="admin" />
<entry key="HSADMINNG_POSTGRES_RESTRICTED_USERNAME" value="restricted" />
</map>
</option>
<option name="executionName" />
<option name="externalProjectPath" value="$PROJECT_DIR$" />
<option name="externalSystemIdString" value="GRADLE" />
<option name="scriptParameters" value="" />
<option name="taskDescriptions">
<list />
</option>
<option name="taskNames">
<list>
<option value=":importOfficeData" />
<option value="--tests" />
<option value="&quot;net.hostsharing.hsadminng.hs.migration.ImportOfficeData&quot;" />
</list>
</option>
<option name="vmOptions" />
</ExternalSystemSettings>
<ExternalSystemDebugServerProcess>false</ExternalSystemDebugServerProcess>
<ExternalSystemReattachDebugProcess>true</ExternalSystemReattachDebugProcess>
<EXTENSION ID="com.intellij.execution.ExternalSystemRunConfigurationJavaExtension">
<extension name="coverage" sample_coverage="false" />
</EXTENSION>
<DebugAllEnabled>false</DebugAllEnabled>
<RunAsTest>true</RunAsTest>
<method v="2" />
</configuration>
<configuration default="false" name="ImportOfficeData" type="GradleRunConfiguration" factoryName="Gradle">
<ExternalSystemSettings>
<option name="env">
<map>
<entry key="HSADMINNG_MIGRATION_DATA_PATH" value="migration" />
<entry key="HSADMINNG_POSTGRES_ADMIN_USERNAME" value="admin" />
<entry key="HSADMINNG_POSTGRES_RESTRICTED_USERNAME" value="restricted" />
</map>
</option>
<option name="executionName" />
<option name="externalProjectPath" value="$PROJECT_DIR$" />
<option name="externalSystemIdString" value="GRADLE" />
<option name="scriptParameters" value="" />
<option name="taskDescriptions">
<list />
</option>
<option name="taskNames">
<list>
<option value=":importOfficeData" />
<option value="--tests" />
<option value="&quot;net.hostsharing.hsadminng.hs.office.migration.ImportOfficeData&quot;" />
</list>
</option>
<option name="vmOptions" />
</ExternalSystemSettings>
<ExternalSystemDebugServerProcess>false</ExternalSystemDebugServerProcess>
<ExternalSystemReattachDebugProcess>true</ExternalSystemReattachDebugProcess>
<EXTENSION ID="com.intellij.execution.ExternalSystemRunConfigurationJavaExtension">
<extension name="coverage" sample_coverage="false" />
</EXTENSION>
<DebugAllEnabled>false</DebugAllEnabled>
<RunAsTest>true</RunAsTest>
<method v="2" />
</configuration>
<configuration default="false" name="ImportOfficeData" type="GradleRunConfiguration" factoryName="Gradle">
<ExternalSystemSettings>
<option name="env">
<map>
<entry key="HSADMINNG_POSTGRES_ADMIN_USERNAME" value="admin" />
<entry key="HSADMINNG_POSTGRES_RESTRICTED_USERNAME" value="restricted" />
</map>
</option>
<option name="executionName" />
<option name="externalProjectPath" value="$PROJECT_DIR$" />
<option name="externalSystemIdString" value="GRADLE" />
<option name="scriptParameters" value="" />
<option name="taskDescriptions">
<list />
</option>
<option name="taskNames">
<list>
<option value=":importOfficeData" />
<option value="--tests" />
<option value="&quot;net.hostsharing.hsadminng.hs.migration.ImportOfficeData&quot;" />
</list>
</option>
<option name="vmOptions" />
</ExternalSystemSettings>
<ExternalSystemDebugServerProcess>false</ExternalSystemDebugServerProcess>
<ExternalSystemReattachDebugProcess>true</ExternalSystemReattachDebugProcess>
<EXTENSION ID="com.intellij.execution.ExternalSystemRunConfigurationJavaExtension">
<extension name="coverage" sample_coverage="false" />
</EXTENSION>
<DebugAllEnabled>false</DebugAllEnabled>
<RunAsTest>true</RunAsTest>
<method v="2" />
</configuration>
</component>

4
Jenkinsfile vendored
View File

@ -55,9 +55,9 @@ pipeline {
sh './gradlew bookingIntegrationTest hostingIntegrationTest --no-daemon' sh './gradlew bookingIntegrationTest hostingIntegrationTest --no-daemon'
} }
} }
stage('Import-Tests') { stage('Test-Imports') {
steps { steps {
sh './gradlew importOfficeData importHostingAssets --no-daemon' sh './gradlew importHostingAssets --no-daemon'
} }
} }
stage ('Scenario-Tests') { stage ('Scenario-Tests') {

17
build.gradle
View File

@ -263,7 +263,7 @@ test {
'net.hostsharing.hsadminng.**.generated.**', 'net.hostsharing.hsadminng.**.generated.**',
] ]
useJUnitPlatform { useJUnitPlatform {
excludeTags 'importOfficeData', 'importHostingAssets', 'scenarioTest' excludeTags 'importHostingAssets', 'scenarioTest'
} }
} }
@ -338,7 +338,7 @@ jacocoTestCoverageVerification {
// HOWTO: run all unit-tests which don't need a database: gw-test unitTest // HOWTO: run all unit-tests which don't need a database: gw-test unitTest
tasks.register('unitTest', Test) { tasks.register('unitTest', Test) {
useJUnitPlatform { useJUnitPlatform {
excludeTags 'importOfficeData', 'importHostingAssets', 'scenarioTest', 'generalIntegrationTest', excludeTags 'importHostingAssets', 'scenarioTest', 'generalIntegrationTest',
'officeIntegrationTest', 'bookingIntegrationTest', 'hostingIntegrationTest' 'officeIntegrationTest', 'bookingIntegrationTest', 'hostingIntegrationTest'
} }
@ -396,17 +396,6 @@ tasks.register('hostingIntegrationTest', Test) {
mustRunAfter spotlessJava mustRunAfter spotlessJava
} }
tasks.register('importOfficeData', Test) {
useJUnitPlatform {
includeTags 'importOfficeData'
}
group 'verification'
description 'run the import jobs as tests'
mustRunAfter spotlessJava
}
tasks.register('importHostingAssets', Test) { tasks.register('importHostingAssets', Test) {
useJUnitPlatform { useJUnitPlatform {
includeTags 'importHostingAssets' includeTags 'importHostingAssets'
@ -439,7 +428,7 @@ pitest {
] ]
targetTests = ['net.hostsharing.hsadminng.**.*UnitTest', 'net.hostsharing.hsadminng.**.*RestTest'] targetTests = ['net.hostsharing.hsadminng.**.*UnitTest', 'net.hostsharing.hsadminng.**.*RestTest']
excludedTestClasses = ['**AcceptanceTest*', '**IntegrationTest*', '**ImportOfficeData', '**ImportHostingAssets'] excludedTestClasses = ['**AcceptanceTest*', '**IntegrationTest*', '**ImportHostingAssets']
pitestVersion = '1.17.0' pitestVersion = '1.17.0'
junit5PluginVersion = '1.1.0' junit5PluginVersion = '1.1.0'

1313
src/test/java/net/hostsharing/hsadminng/hs/migration/BaseOfficeDataImport.java
View File

File diff suppressed because it is too large Load Diff

74
src/test/java/net/hostsharing/hsadminng/hs/migration/ImportOfficeData.java
View File

@ -1,74 +0,0 @@
package net.hostsharing.hsadminng.hs.migration;
import net.hostsharing.hsadminng.context.Context;
import net.hostsharing.hsadminng.rbac.test.JpaAttempt;
import org.junit.jupiter.api.*;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest;
import org.springframework.context.annotation.Import;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ActiveProfiles;
import java.io.File;
/*
* This 'test' includes the complete legacy 'office' data import.
*
* There is no code in 'main' because the import is not needed a normal runtime.
* There is some test data in Java resources to verify the data conversion.
* For a real import a main method will be added later
* which reads CSV files from the file system.
*
* When run on a Hostsharing database, it needs the following settings (hsh99_... just examples).
*
* In a real Hostsharing environment, these are created via (the old) hsadmin:
CREATE USER hsh99_admin WITH PASSWORD 'password';
CREATE DATABASE hsh99_hsadminng ENCODING 'UTF8' TEMPLATE template0;
REVOKE ALL ON DATABASE hsh99_hsadminng FROM public; -- why does hsadmin do that?
ALTER DATABASE hsh99_hsadminng OWNER TO hsh99_admin;
CREATE USER hsh99_restricted WITH PASSWORD 'password';
\c hsh99_hsadminng
GRANT ALL PRIVILEGES ON SCHEMA public to hsh99_admin;
* Additionally, we need these settings (because the Hostsharing DB-Admin has no CREATE right):
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
-- maybe something like that is needed for the 2nd user
-- GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public to hsh99_restricted;
* Then copy the file .tc-environment to a file named .environment (excluded from git) and fill in your specific values.
* To finally import the office data, run:
*
* gw-importOfficeTables # comes from .aliases file and uses .environment
*/
@Tag("importOfficeData")
@DataJpaTest(properties = {
"spring.datasource.url=${HSADMINNG_POSTGRES_JDBC_URL:jdbc:tc:postgresql:15.5-bookworm:///importOfficeDataTC}",
"spring.datasource.username=${HSADMINNG_POSTGRES_ADMIN_USERNAME:ADMIN}",
"spring.datasource.password=${HSADMINNG_POSTGRES_ADMIN_PASSWORD:password}",
"hsadminng.superuser=${HSADMINNG_SUPERUSER:import-superuser@hostsharing.net}",
"spring.liquibase.contexts=only-office,without-test-data"
})
@ActiveProfiles("without-test-data")
@DirtiesContext
@Import({ Context.class, JpaAttempt.class })
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
@ExtendWith(OrderedDependedTestsExtension.class)
public class ImportOfficeData extends BaseOfficeDataImport {
@Value("${spring.datasource.url}")
private String jdbcUrl;
@Test
@Order(9999)
public void dumpOfficeData() {
PostgresTestcontainer.dump(jdbcUrl, new File("build/db/released-only-office-schema-with-import-test-data.sql"));
}
}