Feature/postgres oracle new merged

.azuredevops/pipeline-integration-test.yml

@@ -11,7 +11,7 @@ parameters:
     values:
       - dev
       - sit
-    default: dev
+    default: sit
 
   - name: folder
     displayName: Integration test folder
@@ -22,14 +22,19 @@ parameters:
     default: runnable,test
     displayName: Integration test tags
 
+
 variables:
   archiveName: nodo-dei-pagamenti-NRT.zip
   destinationFolderExtractArtifact: nodo/extracted
   deploy-pool: nodopagamenti-agent-pool-sit
   ${{ if eq(parameters['env'], 'dev') }}:
-    NODO_URL: "https://api.dev.platform.pagopa.it/nodo-dev"    
+    NODO_URL: "https://api.dev.platform.pagopa.it/nodo-dev"
+    file_config: $(Build.SourcesDirectory)/src/integ-test/bdd-test/resources/config_dev.json
+    file_behave: src/integ-test/bdd-test/resources/config_dev.json    
   ${{ if eq(parameters['env'], 'sit') }}:
     NODO_URL: "https://api.dev.platform.pagopa.it/nodo"
+    file_config: $(Build.SourcesDirectory)/src/integ-test/bdd-test/resources/config_sit_oracle.json
+    file_behave: src/integ-test/bdd-test/resources/config_sit_oracle.json
 
 stages:
   - stage: prepare_env
@@ -73,7 +78,8 @@ stages:
                   contents=$(jq "$key = \"$value\"" $file) && echo "${contents}" > $file
               }
 
-              file=$(Build.SourcesDirectory)/src/integ-test/bdd-test/resources/config.json
+              file=$(file_config)
+              echo ">>>>>>>>>>>>> name is $file"
 
               # replace $file ".services.\"nodo-dei-pagamenti\".url"           "https://10.230.8.5:443/nodo-pagamenti/api/v1"
               # replace $file ".services.\"nodo-dei-pagamenti\".healthcheck"   "/monitor/health"
@@ -82,35 +88,46 @@ stages:
               # replace $file ".services.\"nodo-dei-pagamenti\".refresh_config_service"  "/config/refresh/ALL"
 
               replace $file ".services.\"nodo-dei-pagamenti\".url"           "$(NODO_URL)"
-              replace $file ".services.\"nodo-dei-pagamenti\".healthcheck"   "/monitoring/v1/monitor/health"
-              replace $file ".services.\"nodo-dei-pagamenti\".soap_service"  " "
-              replace $file ".services.\"nodo-dei-pagamenti\".rest_service"  " "
-              replace $file ".services.\"nodo-dei-pagamenti\".refresh_config_service"  "/monitoring/v1/config/refresh/ALL"
+              replace $file ".services.\"nodo-dei-pagamenti\".healthcheck"   "-monitoring/monitoring/v1/monitor/health"
+              replace $file ".services.\"nodo-dei-pagamenti\".soap_service"  ""
+              replace $file ".services.\"nodo-dei-pagamenti\".rest_service"  ""
+              replace $file ".services.\"nodo-dei-pagamenti\".refresh_config_service"  "https://api.dev.platform.pagopa.it/nodo-monitoring/monitoring/v1/config/refresh/ALL"
+              replace $file ".services.\"nodo-dei-pagamenti\".subscription_key_name"  "N"
 
               replace $file ".services.\"mock-ec\".url"           "https://api.dev.platform.pagopa.it/mock-ec/api/v1"
               replace $file ".services.\"mock-ec\".healthcheck"   "/info"
               replace $file ".services.\"mock-ec\".soap_service"  ""
               replace $file ".services.\"mock-ec\".rest_service"  ""
+              replace $file ".services.\"mock-ec\".subscription_key_name"  "Y"
 
               replace $file ".services.\"secondary-mock-ec\".url"           "https://api.dev.platform.pagopa.it/secondary-mock-ec/api/v1"
               replace $file ".services.\"secondary-mock-ec\".healthcheck"   "/info"
               replace $file ".services.\"secondary-mock-ec\".soap_service"  ""
               replace $file ".services.\"secondary-mock-ec\".rest_service"  ""
+              replace $file ".services.\"secondary-mock-ec\".subscription_key_name"  "Y"
 
               replace $file ".services.\"mock-psp\".url"           "https://api.dev.platform.pagopa.it/mock-psp-service/api/v1"
               replace $file ".services.\"mock-psp\".healthcheck"   "/info"
               replace $file ".services.\"mock-psp\".soap_service"  ""
               replace $file ".services.\"mock-psp\".rest_service"  ""
+              replace $file ".services.\"mock-psp\".subscription_key_name"  "Y"
 
               replace $file ".services.\"secondary-mock-psp\".url"           "https://api.dev.platform.pagopa.it/secondary-mock-psp-service/api/v1"
               replace $file ".services.\"secondary-mock-psp\".healthcheck"   "/info"
               replace $file ".services.\"secondary-mock-psp\".soap_service"  ""
               replace $file ".services.\"secondary-mock-psp\".rest_service"  ""
+              replace $file ".services.\"secondary-mock-psp\".subscription_key_name"  "Y"
+
+              replace $file ".services.\"apicfg-testing-support\".url"           "https://api.dev.platform.pagopa.it/apiconfig/testing-support/p/v1"
+              replace $file ".services.\"apicfg-testing-support\".healthcheck"   "/info"
+              replace $file ".services.\"apicfg-testing-support\".soap_service"  ""
+              replace $file ".services.\"apicfg-testing-support\".rest_service"  ""
+              replace $file ".services.\"apicfg-testing-support\".subscription_key_name"  "Y"
 
-              replace $file ".services.\"mock-pm\".url"           "https://api.dev.platform.pagopa.it/mock-pm-sit/api/v1"
-              replace $file ".services.\"mock-pm\".healthcheck"   "/info"
-              replace $file ".services.\"mock-pm\".soap_service"  ""
-              replace $file ".services.\"mock-pm\".rest_service"  ""
+              # replace $file ".services.\"mock-pm\".url"           "https://api.dev.platform.pagopa.it/mock-pm-sit/api/v1"
+              # replace $file ".services.\"mock-pm\".healthcheck"   "/info"
+              # replace $file ".services.\"mock-pm\".soap_service"  ""
+              # replace $file ".services.\"mock-pm\".rest_service"  ""
 
 
               replace $file ".db_configuration.nodo_cfg.host"     $(db-cfg-host-${{ parameters.env }})
@@ -143,6 +160,8 @@ stages:
               replace $file ".db_configuration.wfesp.password"  $(db-wfesp-password-${{ parameters.env }})     
               replace $file ".db_configuration.wfesp.port"      $(db-wfesp-port-${{ parameters.env }})
 
+              replace $file ".env.SUBKEY"      $(apim-subscription-key-${{ parameters.env }})
+
               jq . $file
             displayName: "Configure Global Properties"
 
@@ -208,13 +227,18 @@ stages:
               echo 'Install Behave'
               sudo pip3 install behave 
               sudo pip3 install -U -r $(Pipeline.Workspace)/$(destinationFolderExtractArtifact)/requirements.txt #Install Requirements
+              sudo pip3 install cx-Oracle
+              sudo pip3 install xmltodict
+              sudo pip3 install python-dotenv
               echo python version
               python3 -V
               whereis behave
               which behave
               export PATH=/usr/local/bin/behave:/usr/bin:/usr/local/bin:$PATH #Add environment variables
 
-              behave --junit-directory=$(Pipeline.Workspace)/$(destinationFolderExtractArtifact)/report --junit $(Pipeline.Workspace)/$(destinationFolderExtractArtifact)/${{ parameters.folder }} --tags=${{ parameters.tags }}
+              fileBehave=$(Pipeline.Workspace)/$(destinationFolderExtractArtifact)/$(file_behave)
+
+              behave --junit-directory=$(Pipeline.Workspace)/$(destinationFolderExtractArtifact)/report --junit $(Pipeline.Workspace)/$(destinationFolderExtractArtifact)/${{ parameters.folder }} --tags=${{ parameters.tags }} -D conffile=$fileBehave
 
             displayName: "Run integration test"
 

.gitignore

@@ -133,3 +133,6 @@ report-nodo
 report/*
 reports
 /src/perf-test/k6/scenarios/*/test/output
+src/integ-test/bdd-test/features/steps/jupyter_notebook.ipynb
+get-pip.py
+src/integ-test/bdd-test/features/steps/jupyter_notebook.ipynb

.jenkins/Jenkinsfile

@@ -0,0 +1,44 @@
+pipeline {
+    agent { label 'NodoPagoPA' }
+    environment {
+        DOCKER_REGISTRY = "https://toolbox.sia.eu/docker_nodo_pagopa" 
+    } 
+    stages {
+
+        stage('Checkout Repo Test') {
+            steps {
+                withCredentials([gitUsernamePassword(credentialsId: 'openshift-server-sia', gitToolName: 'git-tool')]) {
+                        sh 'git config --global http.sslVerify "false" '
+                        sh 'git config --global http.postBuffer 1048576000'
+                        sh 'git config --global http.proxy http://csproxy:8080'
+                        sh 'git config --global https.proxy http://csproxy:8080'
+                        sh 'git config --global -l'
+                        sh 'git clone -b $BRANCH https://github.com/pagopa/pagopa-nodo-dei-pagamenti-test.git .'
+                }
+            }
+        }
+
+        stage('Build Integration Test image') {
+            steps {
+                script {
+                    tag = "toolbox.sia.eu/docker_nodo_pagopa/integration-test:latest"
+
+                     docker.withRegistry("${env.DOCKER_REGISTRY}", 'accenture_user' ) {
+                        dockerImage = docker.build("${tag}", '--build-arg="tags=$TAGS" --build-arg="folder=$FOLDER" .')
+                     }
+                }
+            }
+        }
+
+        stage('Push Integration Test image') {
+            steps{
+                script {
+                     docker.withRegistry("${env.DOCKER_REGISTRY}", 'accenture_user' ) {
+                         dockerImage.push()
+                     }
+                    sh("docker rmi ${tag}")
+                }
+            }
+        }
+    }    
+}

Dockerfile

@@ -0,0 +1,65 @@
+FROM toolbox.sia.eu/docker-pagopa/integration-test-base-image:1.0.0
+
+ENV http_proxy=http://csproxy:8080
+ENV https_proxy=http://csproxy:8080
+ENV no_proxy=toolbox.sia.eu
+
+
+#FROM python:3.9
+#
+##install jq and openjdk-17                                          
+#RUN apt-get update && \                                             
+#	apt-get install -y jq && \                                     
+#	apt-get install openjdk-17-jdk -y && \                             
+#	apt-get install libpq-dev python3-dev build-essential wget unzip -y
+##install allure
+#RUN wget https://github.com/allure-framework/allure2/releases/download/2.24.1/allure-2.24.1.zip && \
+#	unzip allure-2.24.1.zip && \
+#	rm allure-2.24.1.zip
+
+#copy test script	
+ADD src/integ-test test/src/integ-test	
+ADD startIntTest.sh test/startIntTest.sh
+ADD stopIntTest.sh test/stopIntTest.sh
+ADD requirements.txt test/requirements.txt
+ADD manualtrigger.py test/manualtrigger.py
+ADD entrypoint.sh test/entrypoint.sh
+
+#install requirements
+RUN pip3 install -U -r test/requirements.txt
+
+#install python-dotenv
+RUN pip3 install python-dotenv
+
+#install ps 
+RUN apt-get update && \
+	apt-get install -y procps
+
+#setting env varialbes
+ARG ARG_TAGS
+ARG ARG_FOLDER
+ENV tags=$ARG_TAGS
+ENV folder=$ARG_FOLDER
+ENV file_config=src/integ-test/bdd-test/resources/config_sit.json
+ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64
+ENV PATH=$PATH:$JAVA_HOME/bin:/allure-2.24.1/bin/
+
+#set working directory
+WORKDIR /test
+
+RUN chmod +x startIntTest.sh
+RUN chmod +x stopIntTest.sh
+RUN chmod +x entrypoint.sh
+RUN chmod -R 777 src/integ-test
+RUN mkdir /test/allure
+RUN chmod 777 /test/allure
+RUN mkdir /test/allure/allure-result
+RUN chmod 777 /test/allure/allure-result
+
+RUN echo $tags
+RUN echo $folder
+
+ENV http_proxy=http://10.79.20.33:80
+ENV https_proxy=http://10.79.20.33:80
+
+ENTRYPOINT ["./entrypoint.sh"]

DockerfilePRF

@@ -0,0 +1,50 @@
+FROM toolbox.sia.eu/docker-pagopa/performance-test-base-image:1.0.0
+
+ENV http_proxy=http://csproxy:8080
+ENV https_proxy=http://csproxy:8080
+ENV no_proxy=toolbox.sia.eu
+
+#FROM alpine:latest
+#
+##Add bash
+#RUN apk add --no-cache bash
+##INSTALL k6
+#RUN wget https://github.com/grafana/k6/releases/download/v0.47.0/k6-v0.47.0-linux-amd64.tar.gz
+#RUN	tar xzf k6-v0.47.0-linux-amd64.tar.gz
+#RUN	mv k6-v0.47.0-linux-amd64/k6 /usr/local/bin
+#RUN	rm -rf k6-v0.47.0-linux-amd64*
+#RUN apk add python3
+#RUN apk add py3-pip
+
+#add arguments
+ARG debugEnabled
+ARG rampa
+ARG blacklist
+ENV debugEn=$debugEnabled
+ENV ram=$rampa
+ENV bl=$blacklist
+
+# add k6 script and other sh
+ADD entrypointPRF.sh test/entrypointPRF.sh
+ADD startPerfTest.sh test/startPerfTest.sh
+ADD manualtriggerPRF.py test/manualtriggerPRF.py
+ADD src/perf-test test/src/perf-test
+ADD requirementsPRF.txt test/requirementsPRF.txt
+ADD stopPerfTest.sh test/stopPerfTest.sh
+
+RUN pip3 install -U -r test/requirementsPRF.txt
+
+#set working directory
+WORKDIR /test
+
+RUN chmod +x entrypointPRF.sh
+RUN chmod +x startPerfTest.sh
+RUN chmod +x stopPerfTest.sh
+RUN chmod -R 777 src/perf-test
+
+ENV http_proxy=http://10.79.20.33:80
+ENV https_proxy=http://10.79.20.33:80
+ENV K6_INFLUXDB_HTTP_PROXY=http://10.79.20.33:80 
+ENTRYPOINT ["./entrypointPRF.sh"]
+
+#CMD ["/bin/sh"]

entrypoint.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+./startIntTest.sh false & \
+python manualtrigger.py

entrypointPRF.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+#./startPerfTest.sh $debugEn $ram $bl
+python manualtriggerPRF.py

helm-prf-test/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: nodo_dei_pagamenti
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 1.0.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.0.0"

helm-prf-test/templates/deployments/prf-test-deployment.yaml

@@ -0,0 +1,58 @@
+{{- if .Values.deployments.performance_test }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Values.deployments.performance_test.name }}
+  namespace: {{ .Values.namespace_performance_tests }}
+#  annotations:
+#    helm.sh/hook: post-install,post-upgrade
+#    helm.sh/hook-delete-policy: before-hook-creation
+spec:
+  replicas: {{ .Values.deployments.performance_test.replicas }}
+  selector:
+    matchLabels:
+      app: {{ .Values.deployments.performance_test.name }}
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.deployments.performance_test.name }}
+      annotations:
+        rollme: {{ randAlphaNum 10 | quote }}
+    spec:
+      containers:
+      - name: {{ .Values.deployments.performance_test.name }}
+        image: {{ .Values.deployments.performance_test.registry_url }}/{{ .Values.deployments.performance_test.image_name }}:{{ .Values.deployments.performance_test.image_version }}
+        imagePullPolicy: {{ .Values.deployments.performance_test.imagePullPolicy }}
+        env:
+            {{- tpl (.Values.deployments.performance_test.environment | toYaml) . | nindent 10 }}
+        ports:
+            {{- toYaml .Values.deployments.performance_test.ports | nindent 10 }} 
+        resources:
+            {{- toYaml .Values.deployments.performance_test.resources | nindent 12 }}
+          {{- if .Values.deployments.performance_test.volumeMounts }}
+        volumeMounts:
+            {{- tpl (.Values.deployments.performance_test.volumeMounts | toYaml) . | nindent 10 }}
+          {{- end }}
+      {{- if .Values.deployments.performance_test.image_pull_secret }}
+      imagePullSecrets: 
+      {{- toYaml  .Values.deployments.performance_test.image_pull_secret | nindent 6 }}
+      {{- end }}
+      {{- if .Values.deployments.performance_test.hostAliases }}
+      hostAliases:
+      {{- toYaml .Values.deployments.performance_test.hostAliases | nindent 6 }}
+      {{- end }}
+      {{- if .Values.deployments.performance_test.nodeSelector }}
+      nodeSelector:
+        {{- toYaml .Values.deployments.performance_test.nodeSelector | nindent 8 }}
+      {{- end }}
+      {{- if .Values.deployments.performance_test.tolerations }}
+      tolerations:
+        {{- toYaml .Values.deployments.performance_test.tolerations | nindent 8 }}
+      {{- end }}
+      {{- if .Values.deployments.performance_test.volumes }}
+      volumes:
+        {{- tpl (.Values.deployments.performance_test.volumes | toYaml) . | nindent 8 }}
+      {{- end }}
+{{- end }}