WIP DNM kube-proxy-test

ci-operator/config/openshift/sdn/openshift-sdn-master.yaml

@@ -64,6 +64,12 @@ tests:
       ALL_PROJECTS: "true"
       PROJECT_NAME: sdn
     workflow: openshift-ci-security
+- as: kube-proxy-test
+  steps:
+    cluster_profile: gcp-3
+    test:
+    - ref: sdn-kube-proxy
+    workflow: openshift-e2e-gcp
 zz_generated_metadata:
   branch: master
   org: openshift

ci-operator/jobs/openshift/sdn/openshift-sdn-master-presubmits.yaml

@@ -54,6 +54,88 @@ presubmits:
         secret:
           secretName: result-aggregator
     trigger: (?m)^/test( | .* )images,?($|\s.*)
+  - agent: kubernetes
+    always_run: true
+    branches:
+    - ^master$
+    - ^master-
+    cluster: build02
+    context: ci/prow/kube-proxy-test
+    decorate: true
+    labels:
+      ci-operator.openshift.io/cloud: gcp
+      ci-operator.openshift.io/cloud-cluster-profile: gcp-3
+      ci.openshift.io/generator: prowgen
+      pj-rehearse.openshift.io/can-be-rehearsed: "true"
+    name: pull-ci-openshift-sdn-master-kube-proxy-test
+    rerun_command: /test kube-proxy-test
+    spec:
+      containers:
+      - args:
+        - --gcs-upload-secret=/secrets/gcs/service-account.json
+        - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+        - --lease-server-credentials-file=/etc/boskos/credentials
+        - --report-credentials-file=/etc/report/credentials
+        - --secret-dir=/secrets/ci-pull-credentials
+        - --secret-dir=/usr/local/kube-proxy-test-cluster-profile
+        - --target=kube-proxy-test
+        command:
+        - ci-operator
+        image: ci-operator:latest
+        imagePullPolicy: Always
+        name: ""
+        resources:
+          requests:
+            cpu: 10m
+        volumeMounts:
+        - mountPath: /etc/boskos
+          name: boskos
+          readOnly: true
+        - mountPath: /secrets/ci-pull-credentials
+          name: ci-pull-credentials
+          readOnly: true
+        - mountPath: /usr/local/kube-proxy-test-cluster-profile
+          name: cluster-profile
+        - mountPath: /secrets/gcs
+          name: gcs-credentials
+          readOnly: true
+        - mountPath: /secrets/manifest-tool
+          name: manifest-tool-local-pusher
+          readOnly: true
+        - mountPath: /etc/pull-secret
+          name: pull-secret
+          readOnly: true
+        - mountPath: /etc/report
+          name: result-aggregator
+          readOnly: true
+      serviceAccountName: ci-operator
+      volumes:
+      - name: boskos
+        secret:
+          items:
+          - key: credentials
+            path: credentials
+          secretName: boskos-credentials
+      - name: ci-pull-credentials
+        secret:
+          secretName: ci-pull-credentials
+      - name: cluster-profile
+        projected:
+          sources:
+          - secret:
+              name: cluster-secrets-gcp-3
+          - configMap:
+              name: cluster-profile-gcp-3
+      - name: manifest-tool-local-pusher
+        secret:
+          secretName: manifest-tool-local-pusher
+      - name: pull-secret
+        secret:
+          secretName: registry-pull-credentials
+      - name: result-aggregator
+        secret:
+          secretName: result-aggregator
+    trigger: (?m)^/test( | .* )kube-proxy-test,?($|\s.*)
   - agent: kubernetes
     always_run: false
     branches:

ci-operator/step-registry/sdn/kube-proxy/OWNERS

@@ -0,0 +1 @@
+../OWNERS

ci-operator/step-registry/sdn/kube-proxy/sdn-kube-proxy-commands.sh

@@ -0,0 +1,234 @@
+#!/bin/sh
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+# This script tests the kube-proxy image without actually using it as
+# part of the infrastructure of a cluster. It is intended to be copied
+# to the kubernetes-tests image for use in CI and should have no
+# dependencies beyond oc and basic shell stuff.
+
+# There is no good way to "properly" test the kube-proxy image in
+# OpenShift CI, because it is only used as a dependency of third-party
+# software (e.g. Calico); no fully-RH-supported configuration uses it.
+#
+# However, since we don't apply any kube-proxy-specific patches to our
+# tree, we can assume that it *mostly* works, since we are building
+# from sources that passed upstream testing. This script is just to
+# confirm that our build is not somehow completely broken (e.g.
+# immediate segfault due to a bad build environment).
+
+if [[ -z "${KUBE_PROXY_IMAGE}" ]]; then
+    echo "KUBE_PROXY_IMAGE not set" 1>&2
+    exit 1
+fi
+
+TMPDIR=$(mktemp --tmpdir -d kube-proxy.XXXXXX)
+function cleanup() {
+    oc delete namespace kube-proxy-test || true
+    oc delete clusterrole kube-proxy-test || true
+    oc delete clusterrolebinding kube-proxy-test || true
+    rm -rf "${TMPDIR}"
+}
+trap "cleanup" EXIT
+
+function indent() {
+    sed -e 's/^/  /' "$@"
+    echo ""
+}
+
+# Decide what kube-proxy mode to use.
+# (jsonpath expression copied from types_cluster_version.go)
+OCP_VERSION=$(oc get clusterversion version -o jsonpath='{.status.history[?(@.state=="Completed")].version}')
+case "${OCP_VERSION}" in
+    4.17.*|4.18.*)
+        # 4.17 and 4.18 always use RHEL 9 (and nftables mode was still alpha in 4.17), so
+        # use iptables mode
+        PROXY_MODE="iptables"
+        ;;
+    *)
+        # 4.19 and later may use RHEL 10, so use nftables mode
+        PROXY_MODE="nftables"
+        ;;
+esac
+
+echo "Setting up Namespace and RBAC"
+oc create -f - <<EOF
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kube-proxy-test
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/warn: privileged
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kube-proxy-test
+rules:
+- apiGroups: [""]
+  resources:
+  - namespaces
+  - endpoints
+  - services
+  - pods
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups: ["discovery.k8s.io"]
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-proxy-test
+  namespace: kube-proxy-test
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-proxy-test
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-proxy-test
+subjects:
+- kind: ServiceAccount
+  name: kube-proxy-test
+  namespace: kube-proxy-test
+EOF
+echo ""
+
+# We run kube-proxy in a pod-network pod, so that it can create rules
+# in that pod's network namespace without interfering with
+# ovn-kubernetes in the host network namespace.
+#
+# We need to manually set all of the conntrack values to 0 so it won't
+# try to set the sysctls (which would fail). This is the most fragile
+# part of this script in terms of future compatibility. Likewise, we
+# need to set .iptables.localhostNodePorts=false so it won't try to
+# set the sysctl associated with that. (The nftables mode never tries
+# to set that sysctl.)
+oc create -f - <<EOF
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config
+  namespace: kube-proxy-test
+data:
+  kube-proxy-config.yaml: |-
+    apiVersion: kubeproxy.config.k8s.io/v1alpha1
+    kind: KubeProxyConfiguration
+    conntrack:
+      maxPerCore: 0
+      min: 0
+      tcpCloseWaitTimeout: 0s
+      tcpEstablishedTimeout: 0s
+      udpStreamTimeout: 0s
+      udpTimeout: 0s
+    iptables:
+      localhostNodePorts: false
+    mode: ${PROXY_MODE}
+EOF
+echo "config is:"
+oc get configmap -n kube-proxy-test config -o yaml | indent
+
+# The --hostname-override is needed to fake out the node detection,
+# since we aren't running in a host-network pod. (The fact that we're
+# cheating here means we'll end up generating incorrect NodePort rules
+# but that doesn't matter.)
+oc create -f - <<EOF
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-proxy
+  namespace: kube-proxy-test
+spec:
+  containers:
+  - name: kube-proxy
+    image: ${KUBE_PROXY_IMAGE}
+    command:
+    - /bin/sh
+    - -c
+    - exec kube-proxy --hostname-override "\${NODENAME}" --config /config/kube-proxy-config.yaml
+    env:
+    - name: NODENAME
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.nodeName
+    securityContext:
+      privileged: true
+    volumeMounts:
+    - mountPath: /config
+      name: config
+      readOnly: true
+  serviceAccountName: kube-proxy-test
+  volumes:
+  - name: config
+    configMap:
+      name: config
+EOF
+echo "pod is:"
+oc get pod -n kube-proxy-test kube-proxy -o yaml | indent
+
+echo "Waiting for kube-proxy to program initial ${PROXY_MODE} rules..."
+function kube_proxy_synced() {
+    oc exec -n kube-proxy-test kube-proxy -- curl -s http://127.0.0.1:10249/metrics |& grep -q '^kubeproxy_sync_proxy_rules_duration_seconds_count [^0]'
+}
+synced=false
+for count in $(seq 1 10); do
+    if kube_proxy_synced; then
+        synced=true
+        break
+    fi
+    sleep 5
+done
+if [[ "${synced}" != true ]]; then
+    echo "kube-proxy failed to sync to ${PROXY_MODE}:"
+    oc logs -n kube-proxy-test kube-proxy |& indent
+    exit 1
+fi
+
+# Dump the ruleset; since RHEL9 uses iptables-nft, kube-proxy's rules will show up in the
+# nft ruleset regardless of whether kube-proxy is using iptables or nftables.
+echo "Dumping rules"
+oc exec -n kube-proxy-test kube-proxy -- nft list ruleset >& "${TMPDIR}/nft.out"
+
+# We don't want to hardcode any assumptions about what kube-proxy's rules look like, but
+# it necessarily must be the case that every clusterIP and endpoint IP appears somewhere
+# in the output.
+exitcode=0
+for service in kubernetes.default dns-default.openshift-dns router-default.openshift-ingress; do
+    name="${service%.*}"
+    namespace="${service#*.}"
+    clusterIP="$(oc get service -n ${namespace} ${name} -o jsonpath='{.spec.clusterIP}')"
+    endpointIPs="$(oc get endpointslices -n ${namespace} -l kubernetes.io/service-name=${name} -o jsonpath='{range .items[*]}{range .endpoints[0].addresses[*]}{@} {end}{end}')"
+    echo "Looking for ${service} cluster IP (${clusterIP}) and endpoint IPs (${endpointIPs}) in ruleset"
+    for ip in ${clusterIP} ${endpointIPs}; do
+        if ! grep --quiet --fixed-strings " ${ip} " "${TMPDIR}/nft.out"; then
+            echo "Did not find IP ${ip} (from service ${name} in namespace ${namespace}) in ruleset" 1>&2
+            exitcode=1
+        fi
+    done
+done
+echo ""
+
+if [[ "${exitcode}" == 1 ]]; then
+    echo "Ruleset was:"
+    indent "${TMPDIR}/nft.out"
+
+    echo "kube-proxy logs:"
+    oc logs -n kube-proxy-test kube-proxy |& indent
+fi
+
+exit "${exitcode}"

ci-operator/step-registry/sdn/kube-proxy/sdn-kube-proxy-ref.metadata.json

@@ -0,0 +1,11 @@
+{
+	"path": "sdn/kube-proxy/sdn-kube-proxy-ref.yaml",
+	"owners": {
+		"approvers": [
+			"sdn-approvers"
+		],
+		"reviewers": [
+			"sdn-reviewers"
+		]
+	}
+}

ci-operator/step-registry/sdn/kube-proxy/sdn-kube-proxy-ref.yaml

@@ -0,0 +1,17 @@
+ref:
+  as: sdn-kube-proxy
+  from_image:
+    namespace: ocp
+    name: cli-yq
+    tag: latest
+  commands: sdn-kube-proxy-commands.sh
+  dependencies:
+  - name: "kube-proxy"
+    env: "KUBE_PROXY_IMAGE"
+  grace_period: 5m
+  resources:
+    requests:
+      cpu: 10m
+      memory: 100Mi
+  documentation: |-
+    Tests the standalone kube-proxy image