Skip to content

Commit

Permalink
Merge branch 'main' into PAGOPA-1678-authorizer-aks
Browse files Browse the repository at this point in the history
  • Loading branch information
@cap-ang
cap-ang authored May 24, 2024
2 parents 8d15282 + 122c406 commit 5e0ea0f
Show file tree
Hide file tree
Showing 119 changed files with 3,598 additions and 3,509 deletions.
58 changes: 56 additions & 2 deletions .devops/nodo-db-migration-pipelines.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@ variables:
RE_SCHEMA: re
WFESP_USERNAME: wfesp
WFESP_SCHEMA: wfesp
PARTITION_USERNAME: partition
PARTITION_SCHEMA: partition

stages:
- stage: liquibase_status
Expand Down Expand Up @@ -109,7 +111,7 @@ stages:
inputs:
azureSubscription: $(AZURE_SERVICE_CONNECTION)
keyVaultName: $(KEY_VAULT_NAME)
secretsFilter: 'db-cfg-password,db-online-password,db-offline-password,db-re-password,db-wfesp-password'
secretsFilter: 'db-cfg-password,db-online-password,db-offline-password,db-re-password,db-wfesp-password,db-partition-password'
- script: |
liquibase \
--url="$(JDBC_URL)" \
Expand Down Expand Up @@ -175,6 +177,19 @@ stages:
--log-level=${{ parameters.lbLogLevel }} \
status -Dschema=$(WFESP_SCHEMA)
displayName: "wfeps db status"
- script: |
liquibase \
--url="$(JDBC_URL)" \
--username="$(PARTITION_USERNAME)" \
--password="$(db-partition-password)" \
--classpath=src/psql/nodo/liquibase/changelog/partition \
--changeLogFile=db.changelog-master-${{ parameters.dbVersion }}.xml \
--liquibaseSchemaName="$(PARTITION_SCHEMA)" \
--defaultSchemaName="$(PARTITION_SCHEMA)" \
--contexts="${{ parameters.lbContexts }}" \
--log-level=${{ parameters.lbLogLevel }} \
status -Dschema=$(PARTITION_SCHEMA)
displayName: "partition db status"
- script: |
liquibase \
--url="$(JDBC_URL)" \
Expand Down Expand Up @@ -240,6 +255,19 @@ stages:
--log-level=${{ parameters.lbLogLevel }} \
validate -Dschema=$(WFESP_SCHEMA)
displayName: "wfeps db validate"
- script: |
liquibase \
--url="$(JDBC_URL)" \
--username="$(PARTITION_USERNAME)" \
--password="$(db-partition-password)" \
--classpath=src/psql/nodo/liquibase/changelog/partition \
--changeLogFile=db.changelog-master-${{ parameters.dbVersion }}.xml \
--liquibaseSchemaName="$(PARTITION_SCHEMA)" \
--defaultSchemaName="$(PARTITION_SCHEMA)" \
--contexts="${{ parameters.lbContexts }}" \
--log-level=${{ parameters.lbLogLevel }} \
validate -Dschema=$(PARTITION_SCHEMA)
displayName: "partition db validate"
- job: approval
dependsOn: liquibase_status_job
condition: eq('${{ parameters.k8sEnv }}', 'prd')
Expand Down Expand Up @@ -288,7 +316,7 @@ stages:
inputs:
azureSubscription: $(AZURE_SERVICE_CONNECTION)
keyVaultName: $(KEY_VAULT_NAME)
secretsFilter: 'db-cfg-password,db-online-password,db-offline-password,db-re-password,db-wfesp-password'
secretsFilter: 'db-cfg-password,db-online-password,db-offline-password,db-re-password,db-wfesp-password,db-partition-password'
- script: |
liquibase \
--url="$(JDBC_URL)" \
Expand Down Expand Up @@ -419,3 +447,29 @@ stages:
--log-level=${{ parameters.lbLogLevel }} \
tag ${{ parameters.dbVersion }}
displayName: "Tag wfesp"
- script: |
liquibase \
--url="$(JDBC_URL)" \
--username="$(PARTITION_USERNAME)" \
--password="$(db-partition-password)" \
--classpath=src/psql/nodo/liquibase/changelog/partition \
--changeLogFile=db.changelog-master-${{ parameters.dbVersion }}.xml \
--liquibaseSchemaName="$(PARTITION_SCHEMA)" \
--defaultSchemaName="$(PARTITION_SCHEMA)" \
--contexts="${{ parameters.lbContexts }}" \
--log-level=${{ parameters.lbLogLevel }} \
update -Dschema=$(PARTITION_SCHEMA)
displayName: "Run Liquibase partition"
- script: |
liquibase \
--url="$(JDBC_URL)" \
--username="$(PARTITION_USERNAME)" \
--password="$(db-partition-password)" \
--classpath=src/psql/nodo/liquibase/changelog/partition \
--changeLogFile=db.changelog-master-${{ parameters.dbVersion }}.xml \
--liquibaseSchemaName="$(PARTITION_SCHEMA)" \
--defaultSchemaName="$(PARTITION_SCHEMA)" \
--contexts="${{ parameters.lbContexts }}" \
--log-level=${{ parameters.lbLogLevel }} \
tag ${{ parameters.dbVersion }}
displayName: "Tag partition"
228 changes: 227 additions & 1 deletion .devops/nodo-db-schema-pipelines.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ variables:
RE_SCHEMA: re
WFESP_USERNAME: wfesp
WFESP_SCHEMA: wfesp
PARTITION_SCHEMA: partition
PARTITION_USERNAME: partition

stages:
- stage: initialization_jobs
Expand All @@ -74,7 +76,8 @@ stages:
inputs:
azureSubscription: $(AZURE_SERVICE_CONNECTION)
keyVaultName: $(KEY_VAULT_NAME)
secretsFilter: 'db-administrator-login-password,db-cfg-password,db-online-password,db-offline-password,db-re-password,db-wfesp-password'
secretsFilter: 'db-administrator-login-password,db-cfg-password,db-online-password,db-offline-password,db-re-password,db-wfesp-password, db-partition-password'

- task: AzureCLI@2
displayName: "create all schemas"
inputs:
Expand Down Expand Up @@ -132,3 +135,226 @@ stages:
END
\$\$;
"
- task: AzureCLI@2
displayName: "create partition"
inputs:
azureSubscription: $(AZURE_SERVICE_CONNECTION)
scriptLocation: inlineScript
scriptType: bash
inlineScript: |
az config set extension.use_dynamic_install=yes_without_prompt
az postgres flexible-server execute --name $(DB_HOST) --admin-user $(ADMIN_USERNAME) \
--admin-password '$(db-administrator-login-password)' --database-name "$(DATABASE_NAME)" \
--querytext "
do \$\$
BEGIN
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '$PARTITION_USERNAME') THEN
CREATE ROLE $PARTITION_USERNAME WITH
NOSUPERUSER
CREATEDB
CREATEROLE
INHERIT
LOGIN
NOREPLICATION
NOBYPASSRLS
CONNECTION LIMIT -1;
END IF;
ALTER ROLE $PARTITION_USERNAME PASSWORD '$(db-partition-password)';
CREATE SCHEMA IF NOT EXISTS $PARTITION_SCHEMA;
GRANT azure_pg_admin TO $PARTITION_USERNAME;
GRANT $CFG_SCHEMA TO $PARTITION_USERNAME;
GRANT $OFFLINE_SCHEMA TO $PARTITION_USERNAME;
GRANT $ONLINE_SCHEMA TO $PARTITION_USERNAME;
GRANT $RE_SCHEMA TO $PARTITION_USERNAME;
GRANT $WFESP_SCHEMA TO $PARTITION_USERNAME;
GRANT pg_read_all_settings TO $PARTITION_USERNAME;
GRANT pg_read_all_stats TO $PARTITION_USERNAME;
GRANT pg_stat_scan_tables TO $PARTITION_USERNAME;
GRANT CONNECT ON DATABASE \"$(DATABASE_NAME)\" TO $PARTITION_USERNAME;
GRANT ALL ON SCHEMA $PARTITION_SCHEMA TO $PARTITION_USERNAME;
ALTER DEFAULT PRIVILEGES FOR ROLE \"$(ADMIN_USERNAME)\" IN SCHEMA $PARTITION_SCHEMA
GRANT ALL ON TABLES TO $PARTITION_USERNAME;
ALTER DEFAULT PRIVILEGES FOR ROLE \"$(ADMIN_USERNAME)\" IN SCHEMA $PARTITION_SCHEMA
GRANT ALL ON SEQUENCES TO $PARTITION_USERNAME;
ALTER DEFAULT PRIVILEGES FOR ROLE \"$(ADMIN_USERNAME)\" IN SCHEMA $PARTITION_SCHEMA
GRANT EXECUTE ON FUNCTIONS TO $PARTITION_USERNAME;
ALTER DEFAULT PRIVILEGES FOR ROLE \"$(ADMIN_USERNAME)\" IN SCHEMA $PARTITION_SCHEMA
GRANT USAGE ON TYPES TO $PARTITION_USERNAME;
END;
\$\$;
"
- task: AzureCLI@2
displayName: "configure partition"
inputs:
azureSubscription: $(AZURE_SERVICE_CONNECTION)
scriptLocation: inlineScript
scriptType: bash
inlineScript: |
az config set extension.use_dynamic_install=yes_without_prompt
az postgres flexible-server execute --name $(DB_HOST) --admin-user $(PARTITION_USERNAME) \
--admin-password '$(db-partition-password)' --database-name "$(DATABASE_NAME)" \
--querytext "
do \$\$
BEGIN
CREATE TABLE IF NOT EXISTS $(PARTITION_SCHEMA).tab_part
(
tabella character varying(100) COLLATE pg_catalog."default",
schema character varying(100) COLLATE pg_catalog."default",
prefisso_nome_indice character varying(100) COLLATE pg_catalog."default",
campi_indice character varying(100) COLLATE pg_catalog."default"
);
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO "$PARTITION_USERNAME";
ALTER DEFAULT PRIVILEGES IN SCHEMA $ONLINE_SCHEMA GRANT ALL ON TABLES TO "$PARTITION_USERNAME";
ALTER DEFAULT PRIVILEGES IN SCHEMA $RE_SCHEMA GRANT ALL ON TABLES TO "$PARTITION_USERNAME";
ALTER DEFAULT PRIVILEGES IN SCHEMA $WFESP_SCHEMA GRANT ALL ON TABLES TO "$PARTITION_USERNAME";
ALTER DEFAULT PRIVILEGES IN SCHEMA $OFFLINE_SCHEMA GRANT ALL ON TABLES TO "$PARTITION_USERNAME";
ALTER DEFAULT PRIVILEGES FOR ROLE $PARTITION_USERNAME IN SCHEMA $PARTITION_SCHEMA
GRANT ALL ON SEQUENCES TO $PARTITION_USERNAME;
ALTER DEFAULT PRIVILEGES FOR ROLE $PARTITION_USERNAME IN SCHEMA $PARTITION_SCHEMA
GRANT EXECUTE ON FUNCTIONS TO $PARTITION_USERNAME;
ALTER DEFAULT PRIVILEGES FOR ROLE $PARTITION_USERNAME IN SCHEMA $PARTITION_SCHEMA
GRANT USAGE ON TYPES TO $PARTITION_USERNAME;
GRANT ALL ON SCHEMA public TO $PARTITION_USERNAME;
ALTER DEFAULT PRIVILEGES FOR ROLE $PARTITION_USERNAME IN SCHEMA public GRANT ALL ON SEQUENCES TO $PARTITION_USERNAME;
ALTER DEFAULT PRIVILEGES FOR ROLE $PARTITION_USERNAME IN SCHEMA public GRANT EXECUTE ON FUNCTIONS TO $PARTITION_USERNAME;
ALTER DEFAULT PRIVILEGES FOR ROLE $PARTITION_USERNAME IN SCHEMA public GRANT USAGE ON TYPES TO $PARTITION_USERNAME;
ALTER DEFAULT PRIVILEGES FOR ROLE $PARTITION_USERNAME IN SCHEMA $PARTITION_SCHEMA GRANT ALL ON TABLES TO $ADMIN_USERNAME;
ALTER DEFAULT PRIVILEGES FOR ROLE $PARTITION_USERNAME IN SCHEMA $PARTITION_SCHEMA GRANT ALL ON SEQUENCES TO $ADMIN_USERNAME;
ALTER DEFAULT PRIVILEGES FOR ROLE $PARTITION_USERNAME IN SCHEMA $PARTITION_SCHEMA GRANT EXECUTE ON FUNCTIONS TO $ADMIN_USERNAME;
ALTER DEFAULT PRIVILEGES FOR ROLE $PARTITION_USERNAME IN SCHEMA $PARTITION_SCHEMA GRANT USAGE ON TYPES TO $ADMIN_USERNAME;
GRANT ALL ON TABLE $PARTITION_SCHEMA.tab_part TO $ADMIN_USERNAME;
CREATE OR REPLACE PROCEDURE $PARTITION_SCHEMA.modify_partition(a integer) LANGUAGE 'plpgsql' AS \$BODY\$
DECLARE
l_partname TEXT;
l_part_list date;
l_part_listb date;
l_partab TEXT;
L_INDEX_NAME TEXT;
l_campi_indice TEXT;
l_prefisso_indice TEXT;
l_sql text;
l_exist text;
l_var_exist integer;
loop_counter INTEGER;
tab_cursor CURSOR FOR
SELECT lower(tabella) as tabella, lower(schema) as schema, lower(prefisso_nome_indice) as prefisso_indice, lower(campi_indice) as campi_indice
FROM $PARTITION_SCHEMA.TAB_PART;
tab_record TEXT;
tab_schema TEXT;
BEGIN
IF A is null
THEN A=1;
END IF;
FOR loop_counter in 0..a
LOOP
OPEN tab_cursor;
LOOP
FETCH NEXT FROM tab_cursor INTO tab_record, tab_schema,l_prefisso_indice, l_campi_indice;
EXIT WHEN NOT FOUND;
------------------------------------------------------------------------------------------------------
IF loop_counter=0 THEN
L_PART_LIST = DATE_TRUNC('DAY', CURRENT_DATE+loop_counter-1500)::DATE;
L_PART_LISTB = DATE_TRUNC('DAY', CURRENT_DATE+loop_counter+1)::DATE;
L_PARTNAME := TAB_RECORD||'_PMINVALUE';
ELSE
L_PART_LIST = date_trunc('day', CURRENT_DATE+loop_counter)::date;
L_PART_LISTB = date_trunc('day', CURRENT_DATE+loop_counter+1)::date;
L_PARTNAME := tab_record||'_P'||to_char(CURRENT_DATE+loop_counter, 'yyyymmdd');
END IF;
L_INDEX_NAME := l_prefisso_indice||'_'||L_PARTNAME;
IF NOT EXISTS
( SELECT 1
FROM information_schema.tables
WHERE table_name=l_partname and table_schema=tab_schema
union all
SELECT 1
FROM pg_inherits
JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
JOIN pg_class child ON pg_inherits.inhrelid = child.oid
JOIN pg_namespace nmsp_parent ON nmsp_parent.oid = parent.relnamespace
JOIN pg_namespace nmsp_child ON nmsp_child.oid = child.relnamespace
WHERE child.relname=l_partname and nmsp_child.nspname =tab_schema
)
THEN
l_sql := format('CREATE TABLE %I.%I PARTITION OF %I.%I FOR VALUES FROM (%L) TO (%L)', tab_schema, l_partname, tab_schema, tab_record, l_part_list, l_part_listb);
execute l_sql;
END IF;
IF NOT EXISTS
(SELECT *
FROM pg_stat_all_indexes
WHERE indexrelname=L_INDEX_NAME and schemaname=tab_schema)
THEN
if l_campi_indice is not null then
l_sql := format('CREATE UNIQUE INDEX %I ON %I.%I USING BTREE(%s)', L_INDEX_NAME, tab_schema, l_partname, l_campi_indice);
execute l_sql;
END IF;
END IF;
------------------------------------------------------------------------------------------------------
END LOOP;
CLOSE tab_cursor;
END LOOP;
END;
\$BODY\$;
ALTER PROCEDURE $PARTITION_SCHEMA.modify_partition(integer) OWNER TO $PARTITION_USERNAME;
END;
\$\$;
"
- task: AzureCLI@2
displayName: "partition scheduling"
inputs:
azureSubscription: $(AZURE_SERVICE_CONNECTION)
scriptLocation: inlineScript
scriptType: bash
inlineScript: |
az config set extension.use_dynamic_install=yes_without_prompt
az postgres flexible-server execute --name $(DB_HOST) --admin-user $(PARTITION_USERNAME) \
--admin-password '$(db-partition-password)' --database-name "postgres" \
--querytext "
do \$\$
BEGIN
DECLARE
job_id INT;
BEGIN
SELECT j.jobid INTO job_id FROM cron.job j WHERE j.jobname = '$DATABASE_NAME';
-- use job id to cancel the schedule
PERFORM cron.unschedule(job_id);
END;
PERFORM cron.schedule_in_database('$DATABASE_NAME', '00 23 * * *', 'call partition.modify_partition(7);','$DATABASE_NAME');
END;
\$\$;
"
Loading

0 comments on commit 5e0ea0f

Please sign in to comment.