Skip to content

OCPBUGS-121387: wait for ready endpoints before idling #31620

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
redhat-chai-bot wants to merge 5 commits into
base: main
Choose a base branch
from
+132 −44
Open

OCPBUGS-121387: wait for ready endpoints before idling #31620

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9f639b1
OCPBUGS-121387: wait for ready endpoints before idling
redhat-chai-bot Sep 7, 2026
7cc89d8
OCPBUGS-121387: preserve idle wait diagnostics
redhat-chai-bot Sep 8, 2026
db62c36
OCPBUGS-121387: stabilize idle wake-up probe
redhat-chai-bot Sep 8, 2026
054c9ca
OCPBUGS-121387: harden idle wake-up coverage
redhat-chai-bot Sep 8, 2026
a3518d1
OCPBUGS-121387: limit idle readiness fix scope
redhat-chai-bot Sep 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 53 additions & 44 deletions test/extended/cli/idle.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
g "github.com/onsi/ginkgo/v2"
o "github.com/onsi/gomega"

discoveryv1 "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/klog/v2"
Expand All @@ -20,9 +21,25 @@ import (
const (
idledAnnotation = "idling.alpha.openshift.io/idled-at"
prevScaleAnnotation = "idling.alpha.openshift.io/previous-scale"
scaledReplicaCount = "2"
scaledReplicaCount = 2
)

func readyPodEndpointCount(endpointSlices []discoveryv1.EndpointSlice) int {
readyPods := map[string]struct{}{}
for _, endpointSlice := range endpointSlices {
for _, endpoint := range endpointSlice.Endpoints {
if endpoint.TargetRef == nil || endpoint.TargetRef.Kind != "Pod" {
continue
}
if endpoint.Conditions.Ready == nil || *endpoint.Conditions.Ready {
key := fmt.Sprintf("%s/%s/%s", endpoint.TargetRef.Namespace, endpoint.TargetRef.Name, endpoint.TargetRef.UID)
readyPods[key] = struct{}{}
}
}
}
return len(readyPods)
}

var _ = g.Describe("[sig-cli] oc idle [apigroup:apps.openshift.io][apigroup:route.openshift.io][apigroup:project.openshift.io][apigroup:image.openshift.io]", func() {
defer g.GinkgoRecover()

Expand Down Expand Up @@ -51,24 +68,13 @@ var _ = g.Describe("[sig-cli] oc idle [apigroup:apps.openshift.io][apigroup:rout
o.Expect(dcList.Items).Should(o.HaveLen(1))
deploymentConfigName = dcList.Items[0].Name

expectedOutput = fmt.Sprintf("The service will unidle DeploymentConfig \"%s/%s\" to %s replicas once it receives traffic", projectName, deploymentConfigName, scaledReplicaCount)
expectedOutput = fmt.Sprintf("The service will unidle DeploymentConfig \"%s/%s\" to %d replicas once it receives traffic", projectName, deploymentConfigName, scaledReplicaCount)

err = oc.Run("describe").Args("deploymentconfigs", deploymentConfigName).Execute()
o.Expect(err).NotTo(o.HaveOccurred())

ctx := context.Background()
g.By("wait until idling-echo endpoint is ready")
err = wait.PollUntilContextTimeout(ctx, time.Second, 60*time.Second, true, func(ctx context.Context) (done bool, err error) {
err = oc.Run("describe").Args("endpointslices", "-l", "kubernetes.io/service-name=idling-echo").Execute()
if err != nil {
return false, nil
}

return true, nil
})
o.Expect(err).NotTo(o.HaveOccurred())

g.By("wait until replicationcontroller is ready")
g.By("wait until replicationcontroller exists")
err = wait.PollUntilContextTimeout(ctx, time.Second, 60*time.Second, true, func(ctx context.Context) (done bool, err error) {
err = oc.Run("get").Args("replicationcontroller", fmt.Sprintf("%s-1", deploymentConfigName)).Execute()
if err != nil {
Expand All @@ -79,33 +85,40 @@ var _ = g.Describe("[sig-cli] oc idle [apigroup:apps.openshift.io][apigroup:rout
})
o.Expect(err).NotTo(o.HaveOccurred())

g.By(fmt.Sprintf("scale deploymentconfig to %s replicas", scaledReplicaCount))
err = oc.Run("scale").Args("replicationcontroller", fmt.Sprintf("%s-1", deploymentConfigName), fmt.Sprintf("--replicas=%s", scaledReplicaCount)).Execute()
g.By(fmt.Sprintf("scale deploymentconfig to %d replicas", scaledReplicaCount))
err = oc.Run("scale").Args("replicationcontroller", fmt.Sprintf("%s-1", deploymentConfigName), fmt.Sprintf("--replicas=%d", scaledReplicaCount)).Execute()
o.Expect(err).NotTo(o.HaveOccurred())

g.By(fmt.Sprintf("wait until pod is scaled to %s", scaledReplicaCount))
g.By(fmt.Sprintf("wait until pod is scaled to %d", scaledReplicaCount))
err = wait.PollUntilContextTimeout(ctx, time.Second, 60*time.Second, true, func(ctx context.Context) (done bool, err error) {
out, err := oc.Run("get").Args("pods", "-l", "app=idling-echo", "--template={{ len .items }}", "--output=go-template").Output()
if err != nil {
return false, err
}

if out != scaledReplicaCount {
if out != fmt.Sprint(scaledReplicaCount) {
return false, nil
}

return true, nil
})
o.Expect(err).NotTo(o.HaveOccurred())

g.By(fmt.Sprintf("wait until endpoint addresses are scaled to %s", scaledReplicaCount))
g.By(fmt.Sprintf("wait until %d ready pod-backed endpoints are available", scaledReplicaCount))
err = wait.PollUntilContextTimeout(ctx, time.Second, 5*time.Minute, true, func(ctx context.Context) (done bool, err error) {
out, err := oc.Run("get").Args("endpointslices", "-l", "kubernetes.io/service-name=idling-echo", "--template={{ len (index .items 0).endpoints }}", "--output=go-template").Output()
if err != nil || out != scaledReplicaCount {
endpointSlices, err := oc.KubeClient().DiscoveryV1().EndpointSlices(projectName).List(ctx, metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", discoveryv1.LabelServiceName, "idling-echo")})
if err != nil {
Comment thread
coderabbitai[bot] marked this conversation as resolved.
klog.Info("Failed to list EndpointSlices for the idling test; retrying...")
return false, nil
}

return true, nil
rc, err := oc.KubeClient().CoreV1().ReplicationControllers(projectName).Get(ctx, fmt.Sprintf("%s-1", deploymentConfigName), metav1.GetOptions{})
if err != nil {
klog.Info("Failed to get the ReplicationController for the idling test; retrying...")
return false, nil
}

return rc.Status.ReadyReplicas == int32(scaledReplicaCount) && readyPodEndpointCount(endpointSlices.Items) == scaledReplicaCount, nil
})
o.Expect(err).NotTo(o.HaveOccurred())
})
Expand Down Expand Up @@ -154,7 +167,7 @@ var _ = g.Describe("[sig-cli] oc idle [apigroup:apps.openshift.io][apigroup:rout
dcObj, err := oc.AdminAppsClient().AppsV1().DeploymentConfigs(projectName).Get(context.TODO(), deploymentConfigName, metav1.GetOptions{})
o.Expect(err).NotTo(o.HaveOccurred())
out = dcObj.Annotations[prevScaleAnnotation]
o.Expect(out).To(o.Equal(scaledReplicaCount))
o.Expect(out).To(o.Equal(fmt.Sprint(scaledReplicaCount)))
})
})

Expand Down Expand Up @@ -186,24 +199,13 @@ var _ = g.Describe("[sig-cli] oc idle Deployments [apigroup:route.openshift.io][
o.Expect(dcList.Items).Should(o.HaveLen(1))
deploymentName = dcList.Items[0].Name

expectedOutput = fmt.Sprintf("The service will unidle Deployment \"%s/%s\" to %s replicas once it receives traffic", projectName, deploymentName, scaledReplicaCount)
expectedOutput = fmt.Sprintf("The service will unidle Deployment \"%s/%s\" to %d replicas once it receives traffic", projectName, deploymentName, scaledReplicaCount)

err = oc.Run("describe").Args("deployments", deploymentName).Execute()
o.Expect(err).NotTo(o.HaveOccurred())

ctx := context.Background()
g.By("wait until idling-echo endpoint is ready")
err = wait.PollUntilContextTimeout(ctx, time.Second, 60*time.Second, true, func(ctx context.Context) (done bool, err error) {
err = oc.Run("describe").Args("endpointslices", "-l", "kubernetes.io/service-name=idling-echo").Execute()
if err != nil {
return false, nil
}

return true, nil
})
o.Expect(err).NotTo(o.HaveOccurred())

g.By("wait until replicaset is ready")
g.By("wait until replicaset exists")
var rsName string
err = wait.PollUntilContextTimeout(ctx, time.Second, 60*time.Second, true, func(ctx context.Context) (done bool, err error) {
rsList, err := oc.AdminKubeClient().AppsV1().ReplicaSets(projectName).List(context.TODO(), metav1.ListOptions{LabelSelector: "app=idling-echo,deployment=idling-echo"})
Expand All @@ -222,33 +224,40 @@ var _ = g.Describe("[sig-cli] oc idle Deployments [apigroup:route.openshift.io][
})
o.Expect(err).NotTo(o.HaveOccurred())

g.By(fmt.Sprintf("scale deployment to %s replicas", scaledReplicaCount))
err = oc.Run("scale").Args("replicaset", rsName, fmt.Sprintf("--replicas=%s", scaledReplicaCount)).Execute()
g.By(fmt.Sprintf("scale deployment to %d replicas", scaledReplicaCount))
err = oc.Run("scale").Args("replicaset", rsName, fmt.Sprintf("--replicas=%d", scaledReplicaCount)).Execute()
o.Expect(err).NotTo(o.HaveOccurred())

g.By(fmt.Sprintf("wait until pod is scaled to %s", scaledReplicaCount))
g.By(fmt.Sprintf("wait until pod is scaled to %d", scaledReplicaCount))
err = wait.PollUntilContextTimeout(ctx, time.Second, 60*time.Second, true, func(ctx context.Context) (done bool, err error) {
out, err := oc.Run("get").Args("pods", "-l", "app=idling-echo", "--template={{ len .items }}", "--output=go-template").Output()
if err != nil {
return false, err
}

if out != scaledReplicaCount {
if out != fmt.Sprint(scaledReplicaCount) {
return false, nil
}

return true, nil
})
o.Expect(err).NotTo(o.HaveOccurred())

g.By(fmt.Sprintf("wait until endpoint addresses are scaled to %s", scaledReplicaCount))
g.By(fmt.Sprintf("wait until %d ready pod-backed endpoints are available", scaledReplicaCount))
err = wait.PollUntilContextTimeout(ctx, time.Second, 5*time.Minute, true, func(ctx context.Context) (done bool, err error) {
out, err := oc.Run("get").Args("endpointslices", "-l", "kubernetes.io/service-name=idling-echo", "--template={{ len (index .items 0).endpoints }}", "--output=go-template").Output()
if err != nil || out != scaledReplicaCount {
endpointSlices, err := oc.KubeClient().DiscoveryV1().EndpointSlices(projectName).List(ctx, metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", discoveryv1.LabelServiceName, "idling-echo")})
if err != nil {
klog.Info("Failed to list EndpointSlices for the idling test; retrying...")
return false, nil
}

return true, nil
rs, err := oc.KubeClient().AppsV1().ReplicaSets(projectName).Get(ctx, rsName, metav1.GetOptions{})
if err != nil {
klog.Info("Failed to get the ReplicaSet for the idling test; retrying...")
return false, nil
}

return rs.Status.ReadyReplicas == int32(scaledReplicaCount) && readyPodEndpointCount(endpointSlices.Items) == scaledReplicaCount, nil
})
o.Expect(err).NotTo(o.HaveOccurred())
})
Expand Down
79 changes: 79 additions & 0 deletions test/extended/cli/idle_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
package cli

import (
"testing"

corev1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
"k8s.io/utils/ptr"
)

func TestReadyPodEndpointCount(t *testing.T) {
tests := []struct {
name string
endpointSlices []discoveryv1.EndpointSlice
expected int
}{
{
name: "entries for unready pods do not count",
endpointSlices: []discoveryv1.EndpointSlice{{
Endpoints: []discoveryv1.Endpoint{
{TargetRef: &corev1.ObjectReference{Kind: "Pod", Name: "pod-1"}, Conditions: discoveryv1.EndpointConditions{Ready: ptr.To(false)}},
{TargetRef: &corev1.ObjectReference{Kind: "Pod", Name: "pod-2"}, Conditions: discoveryv1.EndpointConditions{Ready: ptr.To(false)}},
},
}},
expected: 0,
},
{
name: "ready and unset-ready pod endpoints count",
endpointSlices: []discoveryv1.EndpointSlice{
{
Endpoints: []discoveryv1.Endpoint{
{TargetRef: &corev1.ObjectReference{Kind: "Pod", Name: "pod-1"}, Conditions: discoveryv1.EndpointConditions{Ready: ptr.To(true)}},
{TargetRef: &corev1.ObjectReference{Kind: "Pod", Name: "pod-2"}},
},
},
{
Endpoints: []discoveryv1.Endpoint{
{TargetRef: &corev1.ObjectReference{Kind: "Pod", Name: "pod-3"}, Conditions: discoveryv1.EndpointConditions{Ready: ptr.To(false)}},
},
},
},
expected: 2,
},
{
name: "the same ready pod in multiple slices counts once",
endpointSlices: []discoveryv1.EndpointSlice{
{
Endpoints: []discoveryv1.Endpoint{
{TargetRef: &corev1.ObjectReference{Kind: "Pod", Namespace: "test", Name: "pod-1", UID: "pod-uid"}, Conditions: discoveryv1.EndpointConditions{Ready: ptr.To(true)}},
},
},
{
Endpoints: []discoveryv1.Endpoint{
{TargetRef: &corev1.ObjectReference{Kind: "Pod", Namespace: "test", Name: "pod-1", UID: "pod-uid"}, Conditions: discoveryv1.EndpointConditions{Ready: ptr.To(true)}},
},
},
},
expected: 1,
},
{
name: "ready endpoints without pod references do not count",
endpointSlices: []discoveryv1.EndpointSlice{{
Endpoints: []discoveryv1.Endpoint{
{Conditions: discoveryv1.EndpointConditions{Ready: ptr.To(true)}},
{TargetRef: &corev1.ObjectReference{Kind: "Node", Name: "node-1"}, Conditions: discoveryv1.EndpointConditions{Ready: ptr.To(true)}},
},
}},
expected: 0,
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if actual := readyPodEndpointCount(test.endpointSlices); actual != test.expected {
t.Fatalf("readyPodEndpointCount() = %d, want %d", actual, test.expected)
}
})
}
}