Skip to content

Commit

Permalink
Merge pull request #5 from pagopa/config-for-nexi
Browse files Browse the repository at this point in the history
Config for Nexi Postgres
  • Loading branch information
aomegax authored Dec 12, 2023
2 parents ac7584d + e6047d6 commit 5d22a08
Show file tree
Hide file tree
Showing 90 changed files with 1,097 additions and 82 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/04h_deploy_with_github_runner.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ jobs:
cluster_name: ${{ vars.CLUSTER_NAME }}
resource_group: ${{ vars.CLUSTER_RESOURCE_GROUP }}
app_name: ${{ env.APP_NAME }}
helm_upgrade_options: "--debug"
helm_upgrade_options: "--debug --set microservice-chart.forceRedeploy=true"

cleanup_runner:
name: Cleanup Runner
Expand Down
18 changes: 11 additions & 7 deletions docker/README.md
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
# Docker Environment 🐳

## Oracle configuration
- Clone `https://github.com/oracle/docker-images` project
- Download "OracleDB" zip v19.3.0, the downloaded file will be `LINUX.ARM64_1919000_db_home`.
- Move this zip in folder `/docker-images/OracleDatabase/SingleInstance/dockerfiles/19.3.0`.
- As described [here](https://github.com/oracle/docker-images/blob/main/OracleDatabase/SingleInstance/README.md), run `./buildContainerImage.sh -e -v 19.3.0 -o '--build-arg SLIMMING=false'
- As described [here](https://github.com/oracle/docker-images/blob/main/OracleDatabase/SingleInstance/README.md), run `./buildContainerImage.sh -e -v 19.3.0 -o '--build-arg SLIMMING=false'`
- From this repository in `docker` folder, execute the commands:
```
docker run --name db-oracle \
Expand All @@ -14,14 +16,16 @@
oracle/database:19.3.0-ee
```
- Connect with your favourite client on local container for Oracle, using:
- JDBC Connection: jdbc:oracle:thin:@//localhost:1521/ORCLPDB1
- Username: system
- Password: oracle
- JDBC Connection: `jdbc:oracle:thin:@//localhost:1521/ORCLPDB1`
- Username: `system`
- Password: `oracle`
- Execute `init_oracle.sql` script in the client
- Move in `db/liquibase/` folder
- Run `sh runLiquibase.sh` script

## PostgreSQL Configuration
- From the repository `pagopa-infra`, in folder `/src/psql/local` execute the script `sh postgresql_local.sh 3.23.0 0`
- Connect with your favourite client on local container for PostgreSQL, using:
- JDBC Connection: jdbc:postgresql://localhost:5432/nodo
- Username: cfg
- Password: cfg
- JDBC Connection: `jdbc:postgresql://localhost:5432/nodo`
- Username: `cfg`
- Password: `cfg`
Binary file not shown.
3 changes: 2 additions & 1 deletion docker/db/liquibase/properties/cfg.properties
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,5 @@ parameter.schema: NODO4_CFG
parameter.tablespaceIdx: NODO4_CFG_IDX
parameter.tablespaceData: NODO4_CFG_DATA
parameter.tablespaceLob: NODO4_CFG_LOB
liquibase.hub.mode=off
liquibase.hub.mode: off
driver: oracle.jdbc.OracleDriver
4 changes: 2 additions & 2 deletions docker/db/liquibase/runLiquibase.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ set -e
db='cfg'

echo "Running liquibase in [$2/$1]"
liquibase --defaultsFile=properties/${db}.properties validate
liquibase --defaultsFile=properties/${db}.properties update
liquibase --defaultsFile=properties/${db}.properties validate --log-level debug
liquibase --defaultsFile=properties/${db}.properties update --log-level info

echo "End!!!!!!"
4 changes: 2 additions & 2 deletions helm/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ apiVersion: v2
name: pagopa-nodo-cfg-data-migration
description: A microservice that permits the migration from Nexi's Oracle database to PagoPA's PostgreSQL database
type: application
version: 0.59.0
appVersion: 1.0.0
version: 0.70.0
appVersion: 1.0.0-11-config-for-nexi
dependencies:
- name: microservice-chart
version: 2.8.0
Expand Down
11 changes: 10 additions & 1 deletion helm/values-dev.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ microservice-chart:
fullnameOverride: ""
image:
repository: ghcr.io/pagopa/pagopa-nodo-cfg-data-migration
tag: "1.0.0"
tag: "1.0.0-11-config-for-nexi"
pullPolicy: Always
livenessProbe:
httpGet:
Expand Down Expand Up @@ -67,8 +67,11 @@ microservice-chart:
ORACLEDB_CONNECTION_STRING: "jdbc:oracle:thin:@(DESCRIPTION=(ENABLE=BROKEN)(ADDRESS=(PROTOCOL=tcp)(PORT=1522)(HOST=db-nodo-pagamenti.d.db-nodo-pagamenti.com))(CONNECT_DATA=(SERVICE_NAME=NDPSPCT_PP_NODO4_CFG))(SO_KEEPALIVE=true)(TCP_KEEPIDLE=60)(TCP_KEEPINTVL=30)(TCP_KEEPCNT=15))"
ORACLEDB_USERNAME: "NODO4_CFG"
ORACLEDB_DEFAULT_SCHEMA: "NODO4_CFG"
# NEXI POSTGRESQL_CONNECTION_STRING: "jdbc:postgresql://db-postgres-ndp.d.db-nodo-pagamenti.com:5444/ndpspct?prepareThreshold=0&currentSchema=NODO4_CFG"
POSTGRESQL_CONNECTION_STRING: "jdbc:postgresql://pagopa-d-weu-nodo-flexible-postgresql.postgres.database.azure.com:6432/nodo?sslmode=require&prepareThreshold=0&currentSchema=cfg"
# NEXI POSTGRESQL_USERNAME: "pp_nodo4_cfg"
POSTGRESQL_USERNAME: "cfg"
# NEXI POSTGRESQL_DEFAULT_SCHEMA: "NODO4_CFG"
POSTGRESQL_DEFAULT_SCHEMA: "cfg"
POSTGRESQL_JDBC_BATCH_SIZE: '30'
DEFAULT_LOGGING_LEVEL: "INFO"
Expand Down Expand Up @@ -114,8 +117,14 @@ microservice-chart:
STEP_STAZIONI_BATCH_SIZE: '1000'
STEP_TIPIVERSAMENTO_BATCH_SIZE: '1000'
STEP_WFESPPLUGINCONF_BATCH_SIZE: '1000'
STEP_IBANATTRIBUTESMASTER_BATCH_SIZE: '1000'
STEP_IBANATTRIBUTES_BATCH_SIZE: '1000'
STEP_IBANMASTER_BATCH_SIZE: '1000'
STEP_IBAN_BATCH_SIZE: '1000'
STEP_ICABINARYFILE_BATCH_SIZE: '1000'
envSecret:
ORACLEDB_PASSWORD: 'oracle-db-cfg-password'
# NEXI POSTGRESQL_PASSWORD: 'db-nexi-cfg-password'
POSTGRESQL_PASSWORD: 'db-cfg-password'
OTEL_AUTH_BEARER: "otel-auth-bearer"
keyvault:
Expand Down
19 changes: 14 additions & 5 deletions helm/values-prod.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ microservice-chart:
fullnameOverride: ""
image:
repository: ghcr.io/pagopa/pagopa-nodo-cfg-data-migration
tag: "1.0.0"
tag: "1.0.0-11-config-for-nexi"
pullPolicy: Always
livenessProbe:
httpGet:
Expand Down Expand Up @@ -67,9 +67,12 @@ microservice-chart:
ORACLEDB_CONNECTION_STRING: "jdbc:oracle:thin:@(DESCRIPTION=(ENABLE=BROKEN)(ADDRESS=(PROTOCOL=tcp)(PORT=1521)(HOST=db-nodo-pagamenti.p.db-nodo-pagamenti.com))(CONNECT_DATA=(SERVICE_NAME=NDPSPCP_NODO4_CFG))(SO_KEEPALIVE=true)(TCP_KEEPIDLE=60)(TCP_KEEPINTVL=30)(TCP_KEEPCNT=15))"
ORACLEDB_USERNAME: "NODO4_CFG"
ORACLEDB_DEFAULT_SCHEMA: "NODO4_CFG"
POSTGRESQL_CONNECTION_STRING: "jdbc:postgresql://pagopa-p-weu-nodo-flexible-postgresql.postgres.database.azure.com:6432/nodo?sslmode=require&prepareThreshold=0&currentSchema=cfg"
POSTGRESQL_CONNECTION_STRING: "jdbc:postgresql://nodo-db.p.internal.postgresql.pagopa.it:6432/nodo?sslmode=require&prepareThreshold=0&currentSchema=cfg"
POSTGRESQL_USERNAME: "cfg"
POSTGRESQL_DEFAULT_SCHEMA: "cfg"
# POSTGRESQL_CONNECTION_STRING: "jdbc:postgresql://db-postgres-ndp.p.db-nodo-pagamenti.com:5444/ndpspcp?prepareThreshold=0&currentSchema=NODO4_CFG" # NEXI
# POSTGRESQL_USERNAME: "pp_nodo4_cfg" # NEXI
# POSTGRESQL_DEFAULT_SCHEMA: "NODO4_CFG"
POSTGRESQL_JDBC_BATCH_SIZE: '500'
DEFAULT_LOGGING_LEVEL: "INFO"
APP_LOGGING_LEVEL: "INFO"
Expand All @@ -78,7 +81,7 @@ microservice-chart:
CORS_CONFIGURATION: '{"origins": ["*"], "methods": ["*"]}'
OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector.elastic-system.svc:4317"
# Variables for define batch size for all steps
STEP_BINARYFILE_BATCH_SIZE: '500'
STEP_BINARYFILE_BATCH_SIZE: '100'
STEP_CANALETIPOVERSAMENTO_BATCH_SIZE: '1000'
STEP_CANALINODO_BATCH_SIZE: '1000'
STEP_CANALI_BATCH_SIZE: '1000'
Expand All @@ -91,7 +94,7 @@ microservice-chart:
STEP_CDSSERVIZIO_BATCH_SIZE: '1000'
STEP_CDSSOGGETTOSERVIZIO_BATCH_SIZE: '1000'
STEP_CDSSOGGETTO_BATCH_SIZE: '1000'
STEP_CODIFICHEPA_BATCH_SIZE: '2000'
STEP_CODIFICHEPA_BATCH_SIZE: '1000'
STEP_CODIFICHE_BATCH_SIZE: '1000'
STEP_CONFIGURATIONKEYS_BATCH_SIZE: '1000'
STEP_DIZIONARIOMETADATI_BATCH_SIZE: '1000'
Expand All @@ -105,7 +108,7 @@ microservice-chart:
STEP_INFORMATIVEPAMASTER_BATCH_SIZE: '1000'
STEP_INTERMEDIARIPA_BATCH_SIZE: '1000'
STEP_INTERMEDIARIPSP_BATCH_SIZE: '1000'
STEP_PASTAZIONEPA_BATCH_SIZE: '2000'
STEP_PASTAZIONEPA_BATCH_SIZE: '1000'
STEP_PA_BATCH_SIZE: '1000'
STEP_PDD_BATCH_SIZE: '1000'
STEP_PSPCANALETIPOVERSAMENTO_BATCH_SIZE: '1000'
Expand All @@ -114,9 +117,15 @@ microservice-chart:
STEP_STAZIONI_BATCH_SIZE: '1000'
STEP_TIPIVERSAMENTO_BATCH_SIZE: '1000'
STEP_WFESPPLUGINCONF_BATCH_SIZE: '1000'
STEP_IBANATTRIBUTESMASTER_BATCH_SIZE: '1000'
STEP_IBANATTRIBUTES_BATCH_SIZE: '1000'
STEP_IBANMASTER_BATCH_SIZE: '1000'
STEP_IBAN_BATCH_SIZE: '1000'
STEP_ICABINARYFILE_BATCH_SIZE: '1000'
envSecret:
ORACLEDB_PASSWORD: 'oracle-db-cfg-password'
POSTGRESQL_PASSWORD: 'db-cfg-password'
# POSTGRESQL_PASSWORD: 'db-nexi-cfg-password' # NEXI
OTEL_AUTH_BEARER: "otel-auth-bearer"
keyvault:
name: "pagopa-p-nodo-kv"
Expand Down
8 changes: 7 additions & 1 deletion helm/values-uat.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ microservice-chart:
fullnameOverride: ""
image:
repository: ghcr.io/pagopa/pagopa-nodo-cfg-data-migration
tag: "1.0.0"
tag: "1.0.0-11-config-for-nexi"
pullPolicy: Always
livenessProbe:
httpGet:
Expand Down Expand Up @@ -67,8 +67,12 @@ microservice-chart:
ORACLEDB_CONNECTION_STRING: "jdbc:oracle:thin:@(DESCRIPTION=(ENABLE=BROKEN)(ADDRESS=(PROTOCOL=tcp)(PORT=1524)(HOST=db-nodo-pagamenti.u.db-nodo-pagamenti.com))(CONNECT_DATA=(SERVICE_NAME=NDPSPCA_NODO4_CFG))(SO_KEEPALIVE=true)(TCP_KEEPIDLE=60)(TCP_KEEPINTVL=30)(TCP_KEEPCNT=15))"
ORACLEDB_USERNAME: "NODO4_CFG"
ORACLEDB_DEFAULT_SCHEMA: "NODO4_CFG"
# POSTGRESQL_CONNECTION_STRING: "jdbc:postgresql://db-postgres-ndp.u.db-nodo-pagamenti.com:5445/ndpspca?prepareThreshold=0&currentSchema=NODO4_CFG" # NEXI UAT
# POSTGRESQL_CONNECTION_STRING: "jdbc:postgresql://db-postgres-ndp-prf.u.db-nodo-pagamenti.com:5444/ndpspcq?prepareThreshold=0&currentSchema=NODO4_CFG" # NEXI PRF
POSTGRESQL_CONNECTION_STRING: "jdbc:postgresql://pagopa-u-weu-nodo-flexible-postgresql.postgres.database.azure.com:6432/nodo?sslmode=require&prepareThreshold=0&currentSchema=cfg"
# POSTGRESQL_USERNAME: "pp_nodo4_cfg" # NEXI
POSTGRESQL_USERNAME: "cfg"
# POSTGRESQL_DEFAULT_SCHEMA: "NODO4_CFG" # NEXI
POSTGRESQL_DEFAULT_SCHEMA: "cfg"
POSTGRESQL_JDBC_BATCH_SIZE: '30'
DEFAULT_LOGGING_LEVEL: "INFO"
Expand Down Expand Up @@ -116,6 +120,8 @@ microservice-chart:
STEP_WFESPPLUGINCONF_BATCH_SIZE: '1000'
envSecret:
ORACLEDB_PASSWORD: 'oracle-db-cfg-password'
# POSTGRESQL_PASSWORD: 'db-nexi-cfg-password' # NEXI UAT
# POSTGRESQL_PASSWORD: 'db-nexi-cfg-password-prf' # NEXI PRF
POSTGRESQL_PASSWORD: 'db-cfg-password'
OTEL_AUTH_BEARER: "otel-auth-bearer"
keyvault:
Expand Down
2 changes: 1 addition & 1 deletion openapi/openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"description": "A microservice that permits the migration from Nexi's Oracle database to PagoPA's PostgreSQL database",
"termsOfService": "https://www.pagopa.gov.it/",
"title": "data-migration",
"version": "1.0.0"
"version": "1.0.0-11-config-for-nexi"
},
"servers": [
{
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
</parent>
<groupId>it.gov.pagopa.nodo</groupId>
<artifactId>data-migration</artifactId>
<version>1.0.0</version>
<version>1.0.0-11-config-for-nexi</version>
<description>A microservice that permits the migration from Nexi's Oracle database to PagoPA's PostgreSQL database</description>
<properties>
<java.version>17</java.version>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import org.springframework.scheduling.annotation.EnableAsync;

@SpringBootApplication
@EnableAutoConfiguration
//@EnableAutoConfiguration
@EnableAsync
public class Application {

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
package it.gov.pagopa.nodo.datamigration.config.datasource;

import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import jakarta.persistence.EntityManagerFactory;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.annotation.EnableTransactionManagement;

import javax.sql.DataSource;
import java.util.Properties;

@Configuration
@EnableTransactionManagement
@EnableJpaRepositories(
entityManagerFactoryRef = "h2EntityManagerFactory",
transactionManagerRef = "h2TransactionManager",
basePackages = { "it.gov.pagopa.nodo.datamigration.repository.h2" }
)
public class H2DatasourceConfiguration {

@Value("${persistence.h2.jdbc-url}")
private String jdbcUrl;

@Value("${persistence.h2.username}")
private String username;

@Value("${persistence.h2.password}")
private String password;

// @Value("${persistence.h2.default_schema}")
// private String defaultSchema;

@Value("${persistence.h2.driver-class-name}")
private String driverClassName;

@Value("${persistence.h2.hikari.connectionTimeout}")
private String connectionTimeout;

@Value("${persistence.h2.hikari.maxLifetime}")
private String maxLifetime;

@Value("${persistence.h2.hikari.keepaliveTime}")
private String keepaliveTime;

@Value("${persistence.h2.jdbc.batch.size}")
private Integer jdbcBatchSize;

@Bean(name = "h2DataSource")
public DataSource dataSource() {
HikariConfig hikariConfig = new HikariConfig();
hikariConfig.setUsername(username);
hikariConfig.setPassword(password);
hikariConfig.setJdbcUrl(jdbcUrl);
hikariConfig.setDriverClassName(driverClassName);
hikariConfig.setConnectionTimeout(Long.parseLong(connectionTimeout));
hikariConfig.setMaxLifetime(Long.parseLong(maxLifetime));
hikariConfig.setKeepaliveTime(Long.parseLong(keepaliveTime));
return new HikariDataSource(hikariConfig);
}

@Bean(name = "h2EntityManagerFactory")
public LocalContainerEntityManagerFactoryBean entityManagerFactoryBean (
EntityManagerFactoryBuilder builder,
@Qualifier("h2DataSource") DataSource dataSource
) {
LocalContainerEntityManagerFactoryBean entityManager = builder
.dataSource(dataSource)
.packages("it.gov.pagopa.nodo.datamigration.entity")
.persistenceUnit("h2Unit")
.build();
HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter();
entityManager.setJpaVendorAdapter(vendorAdapter);


Properties props = new Properties();
props.put("hibernate.dialect", "org.hibernate.dialect.H2Dialect");
props.put("hibernate.database-platform", "org.hibernate.dialect.H2Dialect");
props.put("hibernate.jdbc.batch_size", jdbcBatchSize);
props.put("hibernate.ddl-auto", "none");
props.put("hibernate.hbm2ddl.auto", "create-drop");
props.put("hibernate.jdbc.lob.non_contextual_creation", "true");
entityManager.setJpaProperties(props);

return entityManager;
}

@Bean(name = "h2TransactionManager")
public PlatformTransactionManager transactionManager (
@Qualifier("h2EntityManagerFactory") EntityManagerFactory entityManagerFactory) {
return new JpaTransactionManager(entityManagerFactory);
}
}

Loading

0 comments on commit 5d22a08

Please sign in to comment.