Merge pull request #275 from TykTechnologies/v5.5.0

Tyk v5.5.0

deployments/analytics-datadog/docker-compose.yml

@@ -16,7 +16,7 @@ services:
       - .env
 
   tyk-pump-datadog:
-    image: tykio/tyk-pump-docker-pub:${PUMP_VERSION:-v1.10.0}
+    image: tykio/tyk-pump-docker-pub:${PUMP_VERSION:-v1.11.0}
     networks:
       - tyk
     volumes:

deployments/analytics-kibana/docker-compose.yml

@@ -17,7 +17,7 @@ services:
     volumes:
       - elasticsearch-data:/usr/share/elasticsearch/data
   tyk-pump-elasticsearch:
-    image: tykio/tyk-pump-docker-pub:${PUMP_VERSION:-v1.10.0}
+    image: tykio/tyk-pump-docker-pub:${PUMP_VERSION:-v1.11.0}
     networks:
       - tyk
     volumes:

deployments/analytics-splunk/docker-compose.yml

@@ -15,7 +15,7 @@ services:
       - splunk-data:/opt/splunk/var
       - splunk-data:/opt/splunk/etc
   tyk-splunk-pump:
-    image: tykio/tyk-pump-docker-pub:${PUMP_VERSION:-v1.10.0}
+    image: tykio/tyk-pump-docker-pub:${PUMP_VERSION:-v1.11.0}
     networks:
       - tyk
     volumes:

deployments/cicd/Dockerfile

@@ -0,0 +1,11 @@
+FROM jenkins/jenkins:2.319.2
+
+USER root
+
+RUN apt-get update && \
+    curl -O https://packagecloud.io/install/repositories/tyk/tyk-sync/script.deb.sh && \
+    chmod +x script.deb.sh && \
+    ./script.deb.sh && \
+    apt-get install -y tyk-sync=1.5.1
+
+USER jenkins

deployments/cicd/data/jenkins/Jenkinsfile

@@ -14,7 +14,7 @@ pipeline {
                 script {
                     if (fileExists('.tyk.json')) {
                         echo "Deploying from ${env.BRANCH_NAME}"
-                        sh "./tyk-sync sync -d http://tyk2-dashboard:3000 -s ${env.tyk2_dashboard_credentials} -p ."
+                        sh "/opt/tyk-sync/tyk-sync sync -d http://tyk2-dashboard:3000 -s ${env.tyk2_dashboard_credentials} -p ."
                     } else {
                         echo 'No files to deploy'
                     }

deployments/cicd/docker-compose.yml

@@ -1,7 +1,8 @@
 ---
 services:
   jenkins:
-    image: jenkins/jenkins:2.319.2-lts-alpine
+    build: ./deployments/cicd
+    image: tyk-demo-jenkins-tyk-sync:v1.5.1
     ports:
       - 8070:8080
     networks:

deployments/cicd/scripts/dump-tyk.sh

@@ -7,5 +7,5 @@ gitea_tyk_data_repo_path=$(cat .context-data/gitea-tyk-data-repo-path)
 docker run --rm \
   --network tyk-demo_tyk \
   -v $gitea_tyk_data_repo_path:/opt/tyk-sync/data \
-  tykio/tyk-sync:v1.4.3 \
+  tykio/tyk-sync:v1.5.1 \
   dump -d http://tyk-dashboard:3000 -s $dashboard_user_api_credentials -t data

deployments/load-balancer-nginx/bootstrap.sh

@@ -6,13 +6,13 @@ deployment="Load Balancer"
 log_start_deployment
 bootstrap_progress
 
-# log_message "Restart Gateways to load latest certificates"
-# docker restart tyk-demo-tyk-gateway-3-1 tyk-demo-tyk-gateway-4-1 1>/dev/null 2>>logs/bootstrap.log
-# if [ "$?" != 0 ]; then
-#   echo "Error when restart Gateways to load latest certificates"
-#   exit 1
-# fi
-# log_ok
+log_message "Restart Gateways to load latest certificates"
+eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-gateway-3 tyk-gateway-4 1>/dev/null 2>>logs/bootstrap.log
+if [ "$?" != 0 ]; then
+  echo "Error when restart Gateways to load latest certificates"
+  exit 1
+fi
+log_ok
 
 log_message "Restart nginx to reset load balancer"
 docker restart tyk-demo-nginx-1 1>/dev/null 2>>logs/bootstrap.log

deployments/load-balancer-nginx/docker-compose.yml

@@ -1,7 +1,7 @@
 ---
 services:
   tyk-gateway-3:
-    image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.4.0}
+    image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.5.0}
     networks:
       - tyk
     environment:
@@ -19,7 +19,7 @@ services:
       - .env
     volumes:
       - ./deployments/tyk/volumes/tyk-gateway/tyk.conf:/opt/tyk-gateway/tyk.conf
-      - ./deployments/tyk/volumes/tyk-gateway/certs:/opt/tyk-gateway/certs
+      - tyk-gateway-certs:/opt/tyk-gateway/certs
       - ./deployments/tyk/volumes/tyk-gateway/middleware:/opt/tyk-gateway/middleware
       - ./deployments/tyk/volumes/tyk-gateway/plugins:/opt/tyk-gateway/plugins
       - ./deployments/tyk/volumes/tyk-gateway/templates/error_401.json:/opt/tyk-gateway/templates/error_401.json
@@ -28,7 +28,7 @@ services:
       - tyk-redis
       - tyk-dashboard
   tyk-gateway-4:
-    image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.4.0}
+    image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.5.0}
     networks:
       - tyk
     environment:
@@ -46,7 +46,7 @@ services:
       - .env
     volumes:
       - ./deployments/tyk/volumes/tyk-gateway/tyk.conf:/opt/tyk-gateway/tyk.conf
-      - ./deployments/tyk/volumes/tyk-gateway/certs:/opt/tyk-gateway/certs
+      - tyk-gateway-certs:/opt/tyk-gateway/certs
       - ./deployments/tyk/volumes/tyk-gateway/middleware:/opt/tyk-gateway/middleware
       - ./deployments/tyk/volumes/tyk-gateway/plugins:/opt/tyk-gateway/plugins
       - ./deployments/tyk/volumes/tyk-gateway/templates/error_401.json:/opt/tyk-gateway/templates/error_401.json

deployments/mdcb/docker-compose.yml

@@ -1,7 +1,7 @@
 ---
 services:
   tyk-mdcb:
-    image: tykio/tyk-mdcb-docker:${MDCB_VERSION:-v2.6.0}
+    image: tykio/tyk-mdcb-docker:${MDCB_VERSION:-v2.7.0}
     ports:
       - 9091:9091
     networks:
@@ -15,7 +15,7 @@ services:
       - tyk-redis
       - tyk-mongo
   tyk-worker-gateway:
-    image: tykio/tyk-gateway:${GATEWAY_WORKER_VERSION:-v5.4.0}
+    image: tykio/tyk-gateway:${GATEWAY_WORKER_VERSION:-v5.5.0}
     ports:
       - 8090:8080
     networks:
@@ -28,7 +28,7 @@ services:
       - TYK_GW_OPENTELEMETRY_ENDPOINT=${OPENTELEMETRY_ENDPOINT:-false}
     volumes:
       - ./deployments/mdcb/volumes/tyk-worker-gateway/tyk.conf:/opt/tyk-gateway/tyk.conf
-      - ./deployments/tyk/volumes/tyk-gateway/certs:/opt/tyk-gateway/certs
+      - tyk-gateway-certs:/opt/tyk-gateway/certs
       - ./deployments/tyk/volumes/tyk-gateway/middleware:/opt/tyk-gateway/middleware
       - ./deployments/tyk/volumes/tyk-gateway/plugins:/opt/tyk-gateway/plugins
     depends_on:

deployments/portal/docker-compose.yml

@@ -1,7 +1,7 @@
 ---
 services:
   tyk-portal:
-    image: tykio/portal:${PORTAL_VERSION:-v1.9.0}
+    image: tykio/portal:${PORTAL_VERSION:-v1.10.0}
     networks:
       - tyk
     env_file:

deployments/slo-prometheus-grafana/docker-compose.yml

@@ -1,7 +1,7 @@
 ---
 services:
   tyk-slo-pump:
-    image: tykio/tyk-pump-docker-pub:${PUMP_VERSION:-v1.10.0}
+    image: tykio/tyk-pump-docker-pub:${PUMP_VERSION:-v1.11.0}
     ports:
       - 8091:8083
       - 8092:8092

deployments/sso/docker-compose.yml

@@ -1,14 +1,14 @@
 ---
 services:
   tyk-dashboard-sso:
-    image: tykio/tyk-dashboard:${DASHBOARD_SSO_VERSION:-v5.4.0}
+    image: tykio/tyk-dashboard:${DASHBOARD_SSO_VERSION:-v5.5.0}
     ports:
       - 3001:3000
     networks:
       - tyk
     volumes:
       - ./deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf:/opt/tyk-dashboard/tyk_analytics.conf
-      - ./deployments/tyk/volumes/tyk-dashboard/private-key.pem:/opt/tyk-dashboard/private-key.pem
+      - tyk-dashboard-certs:/opt/tyk-dashboard/certs
     environment:
       - TYK_DB_LICENSEKEY=${DASHBOARD_LICENCE:?Please set DASHBOARD_LICENCE in .env}
       - TYK_DB_SSOCUSTOMLOGINURL=http://localhost:3010/auth/tyk-dashboard/openid-connect

deployments/tyk/bootstrap.sh

@@ -53,9 +53,6 @@ bootstrap_progress
 
 # Certificates
 
-log_message "Wait for services to be ready before beginning to bootstrap"
-wait_for_liveness
-
 log_message "Checking for existing OpenSSL container"
 OPENSSL_CONTAINER_NAME="tyk-demo-openssl"
 if [ "$(docker ps -a --format '{{.Names}}' | grep -w "$OPENSSL_CONTAINER_NAME" | wc -l)" -gt 0 ]; then
@@ -67,16 +64,19 @@ fi
 bootstrap_progress
 
 log_message "Creating temporary container $OPENSSL_CONTAINER_NAME for OpenSSL usage"
-docker run -d --name $OPENSSL_CONTAINER_NAME alpine:3.20.1 tail -f /dev/null > /dev/null 2>&1
+docker run -d --name $OPENSSL_CONTAINER_NAME \
+  -v tyk-demo_tyk-gateway-certs:/tyk-gateway-certs \
+  -v tyk-demo_tyk-dashboard-certs:/tyk-dashboard-certs \
+  alpine:3.20.1 tail -f /dev/null >/dev/null 2>&1
 log_ok
 bootstrap_progress
 
 log_message "Install OpenSSL into container $OPENSSL_CONTAINER_NAME"
-docker exec -d $OPENSSL_CONTAINER_NAME apk add --no-cache openssl
+docker exec $OPENSSL_CONTAINER_NAME apk add --no-cache openssl >/dev/null 2>>logs/bootstrap.log
 # Wait for the installation to complete
 while true; do
     # Check if OpenSSL is installed by trying to get its version
-    if docker exec $OPENSSL_CONTAINER_NAME openssl version > /dev/null 2>&1; then
+    if docker exec $OPENSSL_CONTAINER_NAME openssl version >/dev/null 2>&1; then
         log_message "  OpenSSL has been successfully installed"
         break
     else
@@ -87,125 +87,64 @@ done
 
 log_message "OpenSSL version used for generating certs: $(docker exec $OPENSSL_CONTAINER_NAME openssl version)"
 
-log_message "Removing any pre-existing certs"
-rm deployments/tyk/volumes/tyk-dashboard/certs/*.pem 1> /dev/null 2>> logs/bootstrap.log
-rm deployments/tyk/volumes/tyk-gateway/certs/*.pem 1> /dev/null 2>> logs/bootstrap.log
-log_ok
-bootstrap_progress
-
 log_message "Generating self-signed certificate for TLS connections to tyk-gateway-2.localhost"
-docker exec -d $OPENSSL_CONTAINER_NAME sh -c "openssl req -x509 -newkey rsa:4096 -subj \"/CN=tyk-gateway-2.localhost\" -keyout /tmp/tls-private-key.pem -out /tmp/tls-certificate.pem -days 365 -nodes" >>logs/bootstrap.log
+docker exec $OPENSSL_CONTAINER_NAME sh -c "openssl req -x509 -newkey rsa:4096 -subj \"/CN=tyk-gateway-2.localhost\" -keyout /tyk-gateway-certs/tls-private-key.pem -out /tyk-gateway-certs/tls-certificate.pem -days 365 -nodes" >/dev/null 2>&1
 if [ "$?" -ne "0" ]; then
   echo "ERROR: Could not generate self-signed certificate"
   exit 1
 fi
-while true; do
-  docker exec $OPENSSL_CONTAINER_NAME sh -c "[ -s /tmp/tls-certificate.pem ]"
-  if [ $? -eq 0 ]; then
-    log_ok
-    bootstrap_progress
-    break;
-  else
-    log_message "  Waiting for /tmp/tls-certificate.pem to be ready"
-    bootstrap_progress
-    sleep 2
-  fi
-done
+log_ok
+bootstrap_progress
+wait_for_file "/tyk-gateway-certs/tls-certificate.pem" "$OPENSSL_CONTAINER_NAME"
+wait_for_file "/tyk-gateway-certs/tls-private-key.pem" "$OPENSSL_CONTAINER_NAME"
 
 log_message "Generating private key for secure messaging and signing"
-docker exec -d $OPENSSL_CONTAINER_NAME sh -c "openssl genrsa -out /tmp/private-key.pem 2048" >>logs/bootstrap.log
+docker exec $OPENSSL_CONTAINER_NAME sh -c "openssl genrsa -out /tyk-dashboard-certs/private-key.pem 2048" >/dev/null 2>>logs/bootstrap.log
 if [ "$?" -ne "0" ]; then
   echo "ERROR: Could not generate private key"
   exit 1
 fi
-while true; do
-  docker exec $OPENSSL_CONTAINER_NAME sh -c "[ -s /tmp/private-key.pem ]"
-  if [ $? -eq 0 ]; then
-    log_ok
-    bootstrap_progress
-    break;
-  else
-    log_message "  Waiting for /tmp/private-key.pem to be ready"
-    bootstrap_progress
-    sleep 2
-  fi
-done
+log_ok
+bootstrap_progress
+wait_for_file "/tyk-dashboard-certs/private-key.pem" "$OPENSSL_CONTAINER_NAME"
 
 log_message "Generating public key for secure messaging and signing"
-docker exec -d $OPENSSL_CONTAINER_NAME sh -c "openssl rsa -in /tmp/private-key.pem -pubout -out /tmp/public-key.pem" >>logs/bootstrap.log
+docker exec $OPENSSL_CONTAINER_NAME sh -c "openssl rsa -in /tyk-dashboard-certs/private-key.pem -pubout -out /tyk-gateway-certs/public-key.pem" >/dev/null 2>>logs/bootstrap.log
 if [ "$?" -ne "0" ]; then
   echo "ERROR: Could not generate public key"
   exit 1
 fi
-while true; do
-  docker exec $OPENSSL_CONTAINER_NAME sh -c "[ -s /tmp/public-key.pem ]"
-  if [ $? -eq 0 ]; then
-    log_ok
-    bootstrap_progress
-    break;
-  else
-    log_message "  Waiting for /tmp/public-key.pem to be ready"
-    bootstrap_progress
-    sleep 2
-  fi
-done
-
-log_message "Copying private-key.pem to dashboard volume mount"
-docker cp $OPENSSL_CONTAINER_NAME:/tmp/private-key.pem deployments/tyk/volumes/tyk-dashboard/certs >>logs/bootstrap.log
-if [ "$?" != "0" ]; then
-  echo "ERROR: Could not copy private-key.pem to dashboard volume mount"
-  exit 1
-fi
 log_ok
 bootstrap_progress
+wait_for_file "/tyk-gateway-certs/public-key.pem" "$OPENSSL_CONTAINER_NAME"
 
-log_message "Copying public-key.pem to gateway volume mount"
-docker cp $OPENSSL_CONTAINER_NAME:/tmp/public-key.pem deployments/tyk/volumes/tyk-gateway/certs >>logs/bootstrap.log
+log_message "Setting read permissions on certificate volumes"
+docker exec $OPENSSL_CONTAINER_NAME chmod -R a+r /tyk-gateway-certs >/dev/null 2>>logs/bootstrap.log
 if [ "$?" != "0" ]; then
-  echo "ERROR: Could not copy public-key.pem to gateway volume mount"
+  echo "ERROR: Could not set read permissions on /tyk-gateway-certs volume"
   exit 1
 fi
-log_ok
-bootstrap_progress
-
-log_message "Copying tls-certificate.pem to gateway volume mount"
-docker cp $OPENSSL_CONTAINER_NAME:/tmp/tls-certificate.pem deployments/tyk/volumes/tyk-gateway/certs >>logs/bootstrap.log
+docker exec $OPENSSL_CONTAINER_NAME chmod -R a+r /tyk-dashboard-certs >/dev/null 2>>logs/bootstrap.log
 if [ "$?" != "0" ]; then
-  echo "ERROR: Could not copy tls-certificate.pem to gateway volume mount"
-  exit 1
-fi
-log_ok
-bootstrap_progress
-
-log_message "Copying tls-private-key.pem to gateway volume mount"
-docker cp $OPENSSL_CONTAINER_NAME:/tmp/tls-private-key.pem deployments/tyk/volumes/tyk-gateway/certs >>logs/bootstrap.log
-if [ "$?" != "0" ]; then
-  echo "ERROR: Could not copy tls-private-key.pem to gateway volume mount"
+  echo "ERROR: Could not set read permissions on /tyk-dashboard-certs volume"
   exit 1
 fi
 log_ok
 bootstrap_progress
 
 log_message "Removing temporary OpenSSL container $OPENSSL_CONTAINER_NAME"
-docker rm -f $OPENSSL_CONTAINER_NAME
+docker rm -f $OPENSSL_CONTAINER_NAME >/dev/null 2>>logs/bootstrap.log
 if [ "$?" != "0" ]; then
   echo "ERROR: Could not remove temporary OpenSSL container $OPENSSL_CONTAINER_NAME"
   exit 1
 fi
 log_ok
 bootstrap_progress
 
-log_message "Recreating containers to ensure new certificates are loaded (tyk-gateway, tyk-gateway-2, tyk-dashboard)"
-eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-gateway tyk-gateway-2 tyk-dashboard
-# if there are gateways from other deployments connecting to this deployment 
-# (such as MDCB), then they must be recreated to. The MDCB deployment already 
-# handles recreation.
-if [ "$?" != "0" ]; then
-  echo "ERROR: Could not recreate containers"
-  exit 1
-fi
+log_message "Recreating containers to load new certificates"
+eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard
+eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-gateway tyk-gateway-2
 log_ok
-bootstrap_progress
 
 log_message "Wait for services to be available after restart"
 wait_for_liveness