diff --git a/.ci/yamllint.yml b/.ci/yamllint.yml index 4727920..1136330 100644 --- a/.ci/yamllint.yml +++ b/.ci/yamllint.yml @@ -10,3 +10,4 @@ rules: # 80 chars should be enough, but don't fail if a line is longer line-length: disable indentation: disable + comments: disable diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a964caa..2f2c909 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -94,3 +94,87 @@ jobs: with: name: logs-${{ github.run_number }} path: ${{ github.repository }}/logs + + azure-tanzu: + needs: azure + environment: Azure Tanzu + name: Azure Tanzu + runs-on: ubuntu-latest + env: + AZURE_CLIENT_ID: ${{ secrets.AZURE_SERVICE_PRINCIPAL }} + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_SERVICE_PRINCIPAL_SECRET }} + AZURE_TENANT: ${{ secrets.AZURE_TENANT }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + AZURE_SSH_PUBLIC_KEY_B64: ${{ secrets.AZURE_SSH_PUBLIC_KEY_B64}} + AZURE_RESOURCE_GROUP: nsm-ci + + steps: + - name: Checkout files + uses: actions/checkout@v4 + + - name: Install kind + uses: engineerd/setup-kind@v0.5.0 + with: + version: v0.20.0 + image: kindest/node:v1.27.0 + skipClusterCreation: true + + + - name: Install Tanzu + run: | + sudo mkdir -p /etc/apt/keyrings + curl -fsSL https://packages.vmware.com/tools/keys/VMWARE-PACKAGING-GPG-RSA-KEY.pub | sudo gpg --dearmor -o /etc/apt/keyrings/tanzu-archive-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/tanzu-archive-keyring.gpg] https://storage.googleapis.com/tanzu-cli-os-packages/apt tanzu-cli-jessie main" | sudo tee /etc/apt/sources.list.d/tanzu.list + sudo apt update + sudo apt install tanzu-cli=1.1.0 + tanzu version + tanzu config eula accept + tanzu ceip-participation set false + tanzu plugin install all --group vmware-tanzucli/essentials:v1.0.0 + tanzu plugin install --group vmware-tkg/default:v2.3.0 + - name: Setup envs (CI ONLY) + run: | + tanzu config set env.AZURE_TENANT_ID ${AZURE_TENANT} + tanzu config set env.AZURE_CLIENT_ID ${AZURE_CLIENT_ID} + tanzu config set env.AZURE_SUBSCRIPTION_ID ${AZURE_SUBSCRIPTION_ID} + tanzu config set env.AZURE_SSH_PUBLIC_KEY_B64 ${AZURE_SSH_PUBLIC_KEY_B64} + tanzu config set env.AZURE_RESOURCE_GROUP ${AZURE_RESOURCE_GROUP} + tanzu config set env.AZURE_CLIENT_SECRET ${AZURE_CLIENT_SECRET} + + - name: Setup AKS management cluster + run: | + tanzu management-cluster create -f management-cluster.yaml --timeout 30m --yes + kubectl config use-context nsm-tanzu-mg-admin@nsm-tanzu-mg + + - name: Setup AKS worker cluster + run: | + tanzu cluster create --file worker-cluster.yaml --timeout 30m --yes || true + SKIP_MULTI_AZ_VERIFY=true tanzu cluster create --file $HOME/.config/tanzu/tkg/clusterconfigs/nsm-tanzu-worker.yaml --timeout 30m + tanzu cluster kubeconfig get nsm-tanzu-worker --admin + kubectl config use-context nsm-tanzu-worker-admin@nsm-tanzu-worker + + - name: Setup go + uses: actions/setup-go@v5 + with: + go-version: 1.20.5 + + - name: Run integration tests + run: | + go test -run TestRunBasicSuite -count 1 -timeout 30m -race -v -parallel 4 + + - name: Delete NSM tanzu workload cluster + if: ${{ always() }} + run: | + tanzu cluster delete nsm-tanzu-worker --yes + kubectl config use-context nsm-tanzu-mg-admin@nsm-tanzu-mg + kubectl wait --for=delete clusters/nsm-tanzu-worker --timeout=20m + - name: Delete NSM tanzu management cluster + if: ${{ always() }} + run: | + tanzu management-cluster delete --yes + - name: Upload logs + uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: tanzu-logs-${{ github.run_number }} + path: logs diff --git a/management-cluster.yaml b/management-cluster.yaml new file mode 100644 index 0000000..eaac6a0 --- /dev/null +++ b/management-cluster.yaml @@ -0,0 +1,162 @@ +#! --------------------------------------------------------------------- +#! Basic cluster creation configuration +#! --------------------------------------------------------------------- +--- +CLUSTER_NAME: nsm-tanzu-mg + +CLUSTER_PLAN: dev +INFRASTRUCTURE_PROVIDER: azure +NAMESPACE: default +# CLUSTER_API_SERVER_PORT: +ENABLE_CEIP_PARTICIPATION: false +# ENABLE_AUDIT_LOGGING: true +# CLUSTER_CIDR: 100.96.0.0/11 +# SERVICE_CIDR: 100.64.0.0/13 +# CAPBK_BOOTSTRAP_TOKEN_TTL: 30m +OS_ARCH: amd64 +OS_NAME: ubuntu +OS_VERSION: "20.04" + +#! --------------------------------------------------------------------- +#! Node configuration +#! --------------------------------------------------------------------- + +# SIZE: +# CONTROLPLANE_SIZE: +# WORKER_SIZE: + +CONTROL_PLANE_MACHINE_COUNT: 1 +WORKER_MACHINE_COUNT: 1 +# OS_NAME: "" +# OS_VERSION: "" +# OS_ARCH: "" +# AZURE_CONTROL_PLANE_DATA_DISK_SIZE_GIB : "" +# AZURE_CONTROL_PLANE_OS_DISK_SIZE_GIB : "" +# AZURE_CONTROL_PLANE_MACHINE_TYPE : "" +# AZURE_CONTROL_PLANE_OS_DISK_STORAGE_ACCOUNT_TYPE : "" +# AZURE_ENABLE_NODE_DATA_DISK : "" +# AZURE_NODE_DATA_DISK_SIZE_GIB : "" +# AZURE_NODE_OS_DISK_SIZE_GIB : "" +# AZURE_NODE_MACHINE_TYPE : "" +# AZURE_NODE_OS_DISK_STORAGE_ACCOUNT_TYPE : "" + +#! --------------------------------------------------------------------- +#! Azure configuration +#! --------------------------------------------------------------------- +# AZURE_CONTROL_PLANE_MACHINE_TYPE: "Standard_D2as_v5" +# AZURE_NODE_MACHINE_TYPE: "Standard_D4as_v5" +AZURE_ENVIRONMENT: "AzurePublicCloud" +# AZURE_TENANT_ID: +# AZURE_CLIENT_ID: +# AZURE_CLIENT_SECRET: +# AZURE_SUBSCRIPTION_ID: +# AZURE_SSH_PUBLIC_KEY_B64: +# AZURE_RESOURCE_GROUP: + +AZURE_LOCATION: "eastus" + +# AZURE_VNET_RESOURCE_GROUP: "nsm-ci" + +# AZURE_VNET_NAME: "" +# AZURE_VNET_CIDR: "" +# AZURE_CONTROL_PLANE_SUBNET_NAME: "" +# AZURE_CONTROL_PLANE_SUBNET_CIDR: "" +# AZURE_NODE_SUBNET_NAME: "" +# AZURE_NODE_SUBNET_CIDR: "" +# AZURE_CUSTOM_TAGS : "" +# AZURE_ENABLE_PRIVATE_CLUSTER : "" +# AZURE_FRONTEND_PRIVATE_IP : "" +# AZURE_ENABLE_ACCELERATED_NETWORKING : "" + +#! --------------------------------------------------------------------- +#! Image repository configuration +#! --------------------------------------------------------------------- + +# TKG_CUSTOM_IMAGE_REPOSITORY: "" +# TKG_CUSTOM_IMAGE_REPOSITORY_CA_CERTIFICATE: "" + +#! --------------------------------------------------------------------- +#! Proxy configuration +#! --------------------------------------------------------------------- + +# TKG_HTTP_PROXY: "" +# TKG_HTTPS_PROXY: "" +# TKG_NO_PROXY: "" + +#! --------------------------------------------------------------------- +#! Machine Health Check configuration +#! --------------------------------------------------------------------- + +ENABLE_MHC: +ENABLE_MHC_CONTROL_PLANE: true +ENABLE_MHC_WORKER_NODE: true +MHC_UNKNOWN_STATUS_TIMEOUT: 15m +MHC_FALSE_STATUS_TIMEOUT: 30m + +#! --------------------------------------------------------------------- +#! Identity management configuration +#! --------------------------------------------------------------------- + +IDENTITY_MANAGEMENT_TYPE: none + +#! Settings for IDENTITY_MANAGEMENT_TYPE: "oidc" +# CERT_DURATION: 2160h +# CERT_RENEW_BEFORE: 360h +# OIDC_IDENTITY_PROVIDER_CLIENT_ID: +# OIDC_IDENTITY_PROVIDER_CLIENT_SECRET: +# OIDC_IDENTITY_PROVIDER_GROUPS_CLAIM: groups +# OIDC_IDENTITY_PROVIDER_ISSUER_URL: +# OIDC_IDENTITY_PROVIDER_SCOPES: "email,profile,groups,offline_access" +# OIDC_IDENTITY_PROVIDER_USERNAME_CLAIM: email + +#! The following two variables are used to configure Pinniped JWTAuthenticator for workload clusters +# SUPERVISOR_ISSUER_URL: +# SUPERVISOR_ISSUER_CA_BUNDLE_DATA: + +#! Settings for IDENTITY_MANAGEMENT_TYPE: "ldap" +# LDAP_BIND_DN: +# LDAP_BIND_PASSWORD: +# LDAP_HOST: +# LDAP_USER_SEARCH_BASE_DN: +# LDAP_USER_SEARCH_FILTER: +# LDAP_USER_SEARCH_ID_ATTRIBUTE: dn +# LDAP_USER_SEARCH_NAME_ATTRIBUTE: +# LDAP_GROUP_SEARCH_BASE_DN: +# LDAP_GROUP_SEARCH_FILTER: +# LDAP_GROUP_SEARCH_NAME_ATTRIBUTE: dn +# LDAP_GROUP_SEARCH_USER_ATTRIBUTE: dn +# LDAP_ROOT_CA_DATA_B64: + +#! --------------------------------------------------------------------- +#! Antrea CNI configuration +#! --------------------------------------------------------------------- + +# ANTREA_NO_SNAT: true +# ANTREA_NODEPORTLOCAL: true +# ANTREA_NODEPORTLOCAL_ENABLED: true +# ANTREA_NODEPORTLOCAL_PORTRANGE: 61000-62000 +# ANTREA_TRAFFIC_ENCAP_MODE: "encap" +# ANTREA_PROXY: true +# ANTREA_PROXY_ALL: false +# ANTREA_PROXY_LOAD_BALANCER_IPS: false +# ANTREA_PROXY_NODEPORT_ADDRS: +# ANTREA_PROXY_SKIP_SERVICES: "" +# ANTREA_POLICY: true +# ANTREA_TRACEFLOW: true +# ANTREA_DISABLE_UDP_TUNNEL_OFFLOAD: false +# ANTREA_ENABLE_USAGE_REPORTING: false +# ANTREA_EGRESS: true +# ANTREA_EGRESS_EXCEPT_CIDRS: "" +# ANTREA_FLOWEXPORTER: false +# ANTREA_FLOWEXPORTER_COLLECTOR_ADDRESS: "flow-aggregator.flow-aggregator.svc:4739:tls" +# ANTREA_FLOWEXPORTER_POLL_INTERVAL: "5s" +# ANTREA_FLOWEXPORTER_ACTIVE_TIMEOUT: "5s" +# ANTREA_FLOWEXPORTER_IDLE_TIMEOUT: "15s" +# ANTREA_IPAM: false +# ANTREA_KUBE_APISERVER_OVERRIDE: "" +# ANTREA_MULTICAST: false +# ANTREA_MULTICAST_INTERFACES: "" +# ANTREA_NETWORKPOLICY_STATS: true +# ANTREA_SERVICE_EXTERNALIP: true +# ANTREA_TRANSPORT_INTERFACE: "" +# ANTREA_TRANSPORT_INTERFACE_CIDRS: "" diff --git a/worker-cluster.yaml b/worker-cluster.yaml new file mode 100644 index 0000000..0cdd9c8 --- /dev/null +++ b/worker-cluster.yaml @@ -0,0 +1,153 @@ +#! --------------------------------------------------------------------- +#! Cluster creation basic configuration +#! --------------------------------------------------------------------- +--- +CLUSTER_NAME: nsm-tanzu-worker +CLUSTER_PLAN: dev +NAMESPACE: default +# CLUSTER_API_SERVER_PORT: +CNI: antrea + +#! --------------------------------------------------------------------- +#! Node configuration +#! --------------------------------------------------------------------- + +# SIZE: +# CONTROLPLANE_SIZE: +# WORKER_SIZE: +CONTROL_PLANE_MACHINE_COUNT: 1 +WORKER_MACHINE_COUNT: 2 +# WORKER_MACHINE_COUNT_0: +# WORKER_MACHINE_COUNT_1: +# WORKER_MACHINE_COUNT_2: +# AZURE_CONTROL_PLANE_OS_DISK_SIZE_GIB: 128 +# AZURE_CONTROL_PLANE_OS_DISK_STORAGE_ACCOUNT_TYPE: Premium_LRS +# AZURE_NODE_OS_DISK_SIZE_GIB: 128 +# AZURE_NODE_OS_DISK_STORAGE_ACCOUNT_TYPE: Premium_LRS +# AZURE_CONTROL_PLANE_DATA_DISK_SIZE_GIB: 256 +# AZURE_ENABLE_NODE_DATA_DISK: false +# AZURE_NODE_DATA_DISK_SIZE_GIB: 256 + +#! --------------------------------------------------------------------- +#! Azure Configuration +#! --------------------------------------------------------------------- + +# AZURE_CONTROL_PLANE_MACHINE_TYPE: "Standard_D2as_v5" +# AZURE_NODE_MACHINE_TYPE: "Standard_D4as_v5" +AZURE_ENVIRONMENT: "AzurePublicCloud" +# AZURE_TENANT_ID: +# AZURE_CLIENT_ID: +# AZURE_CLIENT_SECRET: +# AZURE_SUBSCRIPTION_ID: +# AZURE_SSH_PUBLIC_KEY_B64: +# AZURE_RESOURCE_GROUP: + +AZURE_LOCATION: "eastus" + +# AZURE_ENABLE_ACCELERATED_NETWORKING: true +# AZURE_RESOURCE_GROUP: "" +# AZURE_VNET_RESOURCE_GROUP: "" +# AZURE_VNET_NAME: "" +# AZURE_VNET_CIDR: "10.0.0.0/16" +# AZURE_CONTROL_PLANE_SUBNET_NAME: "" +# AZURE_CONTROL_PLANE_SUBNET_CIDR: "10.0.0.0/24" +# AZURE_CONTROL_PLANE_SUBNET_SECURITY_GROUP: "" +# AZURE_NODE_SUBNET_NAME: "" +# AZURE_NODE_SUBNET_CIDR: "10.0.1.0/24" +# AZURE_NODE_SUBNET_SECURITY_GROUP: "" +# AZURE_NODE_AZ: "" +# AZURE_NODE_AZ_1: "" +# AZURE_NODE_AZ_2: "" +# AZURE_CUSTOM_TAGS: +# AZURE_ENABLE_PRIVATE_CLUSTER: false +# AZURE_FRONTEND_PRIVATE_IP: "10.0.0.100" +# AZURE_ENABLE_CONTROL_PLANE_OUTBOUND_LB: false +# AZURE_ENABLE_NODE_OUTBOUND_LB: false +# AZURE_CONTROL_PLANE_OUTBOUND_LB_FRONTEND_IP_COUNT: 1 +# AZURE_NODE_OUTBOUND_LB_FRONTEND_IP_COUNT: 1 +# AZURE_NODE_OUTBOUND_LB_IDLE_TIMEOUT_IN_MINUTES: 4 +# AZURE_IMAGE_ID: +# AZURE_IMAGE_RESOURCE_GROUP: +# AZURE_IMAGE_NAME: +# AZURE_IMAGE_SUBSCRIPTION_ID: +# AZURE_IMAGE_GALLERY: +# AZURE_IMAGE_PUBLISHER: +# AZURE_IMAGE_OFFER: +# AZURE_IMAGE_SKU: +# AZURE_IMAGE_THIRD_PARTY: +# AZURE_IMAGE_VERSION: +# AZURE_IDENTITY_NAME: +# AZURE_IDENTITY_NAMESPACE: + +#! --------------------------------------------------------------------- +#! Common configuration +#! --------------------------------------------------------------------- + +# TKG_CUSTOM_IMAGE_REPOSITORY: "" +# TKG_CUSTOM_IMAGE_REPOSITORY_SKIP_TLS_VERIFY: false +# TKG_CUSTOM_IMAGE_REPOSITORY_CA_CERTIFICATE: "" + +# TKG_HTTP_PROXY: "" +# TKG_HTTPS_PROXY: "" +# TKG_NO_PROXY: "" +# TKG_PROXY_CA_CERT: "" + +ENABLE_AUDIT_LOGGING: false +ENABLE_DEFAULT_STORAGE_CLASS: true + +# CLUSTER_CIDR: 100.96.0.0/11 +# SERVICE_CIDR: 100.64.0.0/13 + +# OS_NAME: "" +# OS_VERSION: "" +# OS_ARCH: "" + +#! --------------------------------------------------------------------- +#! Autoscaler configuration +#! --------------------------------------------------------------------- + +ENABLE_AUTOSCALER: false +# AUTOSCALER_MAX_NODES_TOTAL: "0" +# AUTOSCALER_SCALE_DOWN_DELAY_AFTER_ADD: "10m" +# AUTOSCALER_SCALE_DOWN_DELAY_AFTER_DELETE: "10s" +# AUTOSCALER_SCALE_DOWN_DELAY_AFTER_FAILURE: "3m" +# AUTOSCALER_SCALE_DOWN_UNNEEDED_TIME: "10m" +# AUTOSCALER_MAX_NODE_PROVISION_TIME: "15m" +# AUTOSCALER_MIN_SIZE_0: +# AUTOSCALER_MAX_SIZE_0: +# AUTOSCALER_MIN_SIZE_1: +# AUTOSCALER_MAX_SIZE_1: +# AUTOSCALER_MIN_SIZE_2: +# AUTOSCALER_MAX_SIZE_2: + +#! --------------------------------------------------------------------- +#! Antrea CNI configuration +#! --------------------------------------------------------------------- + +# ANTREA_NO_SNAT: false +# ANTREA_DISABLE_UDP_TUNNEL_OFFLOAD: false +# ANTREA_TRAFFIC_ENCAP_MODE: "encap" +# ANTREA_EGRESS_EXCEPT_CIDRS: "" +# ANTREA_NODEPORTLOCAL_ENABLED: true +# ANTREA_NODEPORTLOCAL_PORTRANGE: 61000-62000 +# ANTREA_PROXY: true +# ANTREA_PROXY_ALL: false +# ANTREA_PROXY_NODEPORT_ADDRS: "" +# ANTREA_PROXY_SKIP_SERVICES: "" +# ANTREA_PROXY_LOAD_BALANCER_IPS: false +# ANTREA_FLOWEXPORTER_COLLECTOR_ADDRESS: "flow-aggregator.flow-aggregator.svc:4739:tls" +# ANTREA_FLOWEXPORTER_POLL_INTERVAL: "5s" +# ANTREA_FLOWEXPORTER_ACTIVE_TIMEOUT: "30s" +# ANTREA_FLOWEXPORTER_IDLE_TIMEOUT: "15s" +# ANTREA_KUBE_APISERVER_OVERRIDE: +# ANTREA_TRANSPORT_INTERFACE: +# ANTREA_TRANSPORT_INTERFACE_CIDRS: "" +# ANTREA_MULTICAST_INTERFACES: "" +# ANTREA_MULTICAST_IGMPQUERY_INTERVAL: "125s" +# ANTREA_TUNNEL_TYPE: geneve +# ANTREA_ENABLE_USAGE_REPORTING: false +# ANTREA_ENABLE_BRIDGING_MODE: false +# ANTREA_DISABLE_TXCHECKSUM_OFFLOAD: false +# ANTREA_DNS_SERVER_OVERRIDE: "" +# ANTREA_MULTICLUSTER_ENABLE: false +# ANTREA_MULTICLUSTER_NAMESPACE: ""