test: cover workload health checks and assessRelease edge cases (3d42fd9c) · Commits · OSL / MDG HypO / code / org.etsi.osl.hypo.core / org.etsi.osl.hypo.core.service.monitor

cmd/sessions/session_test.go

+87 −0

Changes for cmd/sessions/session_test.go: 87 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -2,6 +2,7 @@ package sessions

		import (
		"context"
		"errors"
		"testing"
		"time"

		@@ -9,6 +10,7 @@ import (
		"k8s.io/apimachinery/pkg/runtime"
		"k8s.io/apimachinery/pkg/runtime/schema"
		dynamicfake "k8s.io/client-go/dynamic/fake"
		k8stesting "k8s.io/client-go/testing"

		"labs.etsi.org/rep/osl/hypo/code/org.etsi.osl.hypo.core/base.go/pkg/util"
		"labs.etsi.org/rep/osl/hypo/code/org.etsi.osl.hypo.core/service.monitor/cmd/datamodels/messages"
		@@ -234,3 +236,88 @@ func TestLogAssessment_TracksTransitions(t *testing.T) {
		t.Fatalf("lastStatus = %q, want Degraded", s.lastStatus)
		}
		}

		// TestAssessRelease_SpansAllNamespaces verifies workloads from every namespace of the
		// release are assessed, and a Degraded one in the second namespace sinks the verdict.
		func TestAssessRelease_SpansAllNamespaces(t *testing.T) {
		healthy := helmDeployment("api", 1, healthyDeploymentStatus(1))
		broken := helmDeployment("worker", 1, map[string]any{
		"observedGeneration": int64(1),
		"conditions": []any{map[string]any{
		"type": "Progressing", "status": "False", "reason": "ProgressDeadlineExceeded",
		}},
		})
		broken.SetNamespace("ns2")
		client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), testGVRToListKind, healthy, broken)
		s := New([]string{"ns1", "ns2"}, "svc-1", client, testLogger(t))

		got := s.assessRelease(context.Background())

		if len(got.Resources) != 2 {
		t.Fatalf("resources = %d, want 2 (one per namespace)", len(got.Resources))
		}
		if got.Status != string(health.HealthStatusDegraded) {
		t.Fatalf("status = %q (%s), want Degraded", got.Status, got.Message)
		}
		}

		// TestAssessRelease_FailedNonHookJobIsDegraded verifies that, unlike hook Jobs, a regular
		// Job that failed is assessed and drags an otherwise healthy release to Degraded.
		func TestAssessRelease_FailedNonHookJobIsDegraded(t *testing.T) {
		dep := helmDeployment("api", 1, healthyDeploymentStatus(1))
		job := &unstructured.Unstructured{Object: map[string]any{
		"apiVersion": "batch/v1",
		"kind": "Job",
		"metadata": map[string]any{"name": "seed", "namespace": "ns1"},
		"status": map[string]any{"conditions": []any{map[string]any{
		"type": "Failed", "status": "True", "message": "BackoffLimitExceeded",
		}}},
		}}
		s := newFakeSession(t, dep, job)

		got := s.assessRelease(context.Background())

		if len(got.Resources) != 2 {
		t.Fatalf("resources = %d, want 2", len(got.Resources))
		}
		if got.Status != string(health.HealthStatusDegraded) {
		t.Fatalf("status = %q (%s), want Degraded", got.Status, got.Message)
		}
		}

		// TestAssessRelease_ListErrorSkipsOnlyThatKind verifies a List failure for one resource
		// kind is logged and skipped rather than aborting the whole assessment.
		func TestAssessRelease_ListErrorSkipsOnlyThatKind(t *testing.T) {
		dep := helmDeployment("api", 1, healthyDeploymentStatus(1))
		client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), testGVRToListKind, dep)
		client.PrependReactor("list", "jobs", func(k8stesting.Action) (bool, runtime.Object, error) {
		return true, nil, errors.New("forbidden")
		})
		s := New([]string{"ns1"}, "svc-1", client, testLogger(t))

		got := s.assessRelease(context.Background())

		if got.Status != string(health.HealthStatusHealthy) {
		t.Fatalf("status = %q (%s), want Healthy", got.Status, got.Message)
		}
		if len(got.Resources) != 1 {
		t.Fatalf("resources = %d, want 1", len(got.Resources))
		}
		}

		// TestAssessRelease_UnassessableWorkloadIsUnknown verifies a workload whose health can't
		// be computed is still reported, as Unknown, instead of being silently dropped.
		func TestAssessRelease_UnassessableWorkloadIsUnknown(t *testing.T) {
		dep := helmDeployment("api", 1, healthyDeploymentStatus(1))
		unstructured.SetNestedField(dep.Object, "two", "spec", "replicas") //nolint:errcheck
		s := newFakeSession(t, dep)

		got := s.assessRelease(context.Background())

		if len(got.Resources) != 1 {
		t.Fatalf("resources = %d, want 1", len(got.Resources))
		}
		if got.Status != string(health.HealthStatusUnknown) {
		t.Fatalf("status = %q (%s), want Unknown", got.Status, got.Message)
		}
		}

internal/health/health_test.go

0 → 100644

+363 −0

Changes for internal/health/health_test.go: 363 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		package health

		import (
		"errors"
		"testing"

		metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
		"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
		"k8s.io/apimachinery/pkg/runtime/schema"
		)

		// Tests cover the dispatcher and the four workload kinds service.monitor actually watches
		// (Deployment, StatefulSet, DaemonSet, Job). The other vendored checks are unused.

		func obj(apiVersion, kind string, generation int64, spec, status map[string]any) *unstructured.Unstructured {
		o := map[string]any{
		"apiVersion": apiVersion,
		"kind": kind,
		"metadata": map[string]any{"name": "x", "namespace": "ns1", "generation": generation},
		}
		if spec != nil {
		o["spec"] = spec
		}
		if status != nil {
		o["status"] = status
		}
		return &unstructured.Unstructured{Object: o}
		}

		func assertHealth(t testing.T, u unstructured.Unstructured, want HealthStatusCode) {
		t.Helper()
		got, err := GetResourceHealth(u, nil)
		if err != nil {
		t.Fatalf("unexpected error: %v", err)
		}
		if got == nil {
		t.Fatal("health is nil")
		}
		if got.Status != want {
		t.Fatalf("status = %q (%s), want %q", got.Status, got.Message, want)
		}
		}

		func TestDeploymentHealth(t *testing.T) {
		cases := []struct {
		name string
		gen int64
		spec map[string]any
		status map[string]any
		want HealthStatusCode
		}{
		{
		name: "rolled out",
		gen: 1,
		spec: map[string]any{"replicas": int64(2)},
		status: map[string]any{"observedGeneration": int64(1), "replicas": int64(2), "updatedReplicas": int64(2), "availableReplicas": int64(2)},
		want: HealthStatusHealthy,
		},
		{
		name: "paused",
		gen: 1,
		spec: map[string]any{"replicas": int64(1), "paused": true},
		status: map[string]any{"observedGeneration": int64(1)},
		want: HealthStatusSuspended,
		},
		{
		name: "generation not yet observed",
		gen: 2,
		spec: map[string]any{"replicas": int64(1)},
		status: map[string]any{"observedGeneration": int64(1), "replicas": int64(1), "updatedReplicas": int64(1), "availableReplicas": int64(1)},
		want: HealthStatusProgressing,
		},
		{
		name: "not all replicas updated",
		gen: 1,
		spec: map[string]any{"replicas": int64(3)},
		status: map[string]any{"observedGeneration": int64(1), "replicas": int64(3), "updatedReplicas": int64(1), "availableReplicas": int64(3)},
		want: HealthStatusProgressing,
		},
		{
		name: "updated replicas not yet available",
		gen: 1,
		spec: map[string]any{"replicas": int64(2)},
		status: map[string]any{"observedGeneration": int64(1), "replicas": int64(2), "updatedReplicas": int64(2), "availableReplicas": int64(1)},
		want: HealthStatusProgressing,
		},
		{
		name: "progress deadline exceeded",
		gen: 1,
		spec: map[string]any{"replicas": int64(1)},
		status: map[string]any{
		"observedGeneration": int64(1), "replicas": int64(1), "updatedReplicas": int64(0),
		"conditions": []any{map[string]any{"type": "Progressing", "status": "False", "reason": "ProgressDeadlineExceeded"}},
		},
		want: HealthStatusDegraded,
		},
		}
		for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
		assertHealth(t, obj("apps/v1", "Deployment", tc.gen, tc.spec, tc.status), tc.want)
		})
		}
		}

		func TestStatefulSetHealth(t *testing.T) {
		cases := []struct {
		name string
		gen int64
		spec map[string]any
		status map[string]any
		want HealthStatusCode
		}{
		{
		name: "never observed",
		gen: 1,
		spec: map[string]any{"replicas": int64(1)},
		status: map[string]any{},
		want: HealthStatusProgressing,
		},
		{
		name: "pods not ready",
		gen: 1,
		spec: map[string]any{"replicas": int64(3)},
		status: map[string]any{"observedGeneration": int64(1), "readyReplicas": int64(1)},
		want: HealthStatusProgressing,
		},
		{
		name: "rolling update in progress",
		gen: 1,
		spec: map[string]any{"replicas": int64(1)},
		status: map[string]any{"observedGeneration": int64(1), "readyReplicas": int64(1), "currentRevision": "r1", "updateRevision": "r2"},
		want: HealthStatusProgressing,
		},
		{
		name: "rolled out",
		gen: 1,
		spec: map[string]any{"replicas": int64(1)},
		status: map[string]any{"observedGeneration": int64(1), "readyReplicas": int64(1), "currentRevision": "r1", "updateRevision": "r1"},
		want: HealthStatusHealthy,
		},
		{
		name: "OnDelete strategy with ready pods",
		gen: 1,
		spec: map[string]any{"replicas": int64(1), "updateStrategy": map[string]any{"type": "OnDelete"}},
		status: map[string]any{"observedGeneration": int64(1), "readyReplicas": int64(1), "currentRevision": "r1", "updateRevision": "r2"},
		want: HealthStatusHealthy,
		},
		{
		name: "partitioned rollout incomplete",
		gen: 1,
		spec: map[string]any{
		"replicas": int64(3),
		"updateStrategy": map[string]any{"type": "RollingUpdate", "rollingUpdate": map[string]any{"partition": int64(1)}},
		},
		status: map[string]any{"observedGeneration": int64(1), "readyReplicas": int64(3), "updatedReplicas": int64(1)},
		want: HealthStatusProgressing,
		},
		}
		for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
		assertHealth(t, obj("apps/v1", "StatefulSet", tc.gen, tc.spec, tc.status), tc.want)
		})
		}
		}

		func TestDaemonSetHealth(t *testing.T) {
		cases := []struct {
		name string
		gen int64
		spec map[string]any
		status map[string]any
		want HealthStatusCode
		}{
		{
		name: "rolled out",
		gen: 1,
		status: map[string]any{"observedGeneration": int64(1), "desiredNumberScheduled": int64(2), "updatedNumberScheduled": int64(2), "numberAvailable": int64(2)},
		want: HealthStatusHealthy,
		},
		{
		name: "generation not yet observed",
		gen: 2,
		status: map[string]any{"observedGeneration": int64(1), "desiredNumberScheduled": int64(2), "updatedNumberScheduled": int64(2), "numberAvailable": int64(2)},
		want: HealthStatusProgressing,
		},
		{
		name: "pods not updated",
		gen: 1,
		status: map[string]any{"observedGeneration": int64(1), "desiredNumberScheduled": int64(3), "updatedNumberScheduled": int64(1), "numberAvailable": int64(3)},
		want: HealthStatusProgressing,
		},
		{
		name: "pods not available",
		gen: 1,
		status: map[string]any{"observedGeneration": int64(1), "desiredNumberScheduled": int64(3), "updatedNumberScheduled": int64(3), "numberAvailable": int64(1)},
		want: HealthStatusProgressing,
		},
		{
		name: "OnDelete strategy is healthy regardless of update count",
		gen: 1,
		spec: map[string]any{"updateStrategy": map[string]any{"type": "OnDelete"}},
		status: map[string]any{"observedGeneration": int64(1), "desiredNumberScheduled": int64(3), "updatedNumberScheduled": int64(0)},
		want: HealthStatusHealthy,
		},
		}
		for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
		assertHealth(t, obj("apps/v1", "DaemonSet", tc.gen, tc.spec, tc.status), tc.want)
		})
		}
		}

		func TestJobHealth(t *testing.T) {
		cases := []struct {
		name string
		conditions []any
		want HealthStatusCode
		wantMsg string
		}{
		{name: "running", conditions: nil, want: HealthStatusProgressing},
		{
		name: "complete",
		conditions: []any{map[string]any{"type": "Complete", "status": "True"}},
		want: HealthStatusHealthy,
		},
		{
		name: "failed",
		conditions: []any{map[string]any{"type": "Failed", "status": "True", "message": "BackoffLimitExceeded"}},
		want: HealthStatusDegraded,
		wantMsg: "BackoffLimitExceeded",
		},
		{
		name: "failed wins over complete",
		conditions: []any{
		map[string]any{"type": "Complete", "status": "True"},
		map[string]any{"type": "Failed", "status": "True", "message": "boom"},
		},
		want: HealthStatusDegraded,
		wantMsg: "boom",
		},
		}
		for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
		u := obj("batch/v1", "Job", 1, nil, map[string]any{"conditions": tc.conditions})
		assertHealth(t, u, tc.want)
		if tc.wantMsg != "" {
		got, _ := GetResourceHealth(u, nil)
		if got.Message != tc.wantMsg {
		t.Errorf("message = %q, want %q", got.Message, tc.wantMsg)
		}
		}
		})
		}
		}

		func TestGetResourceHealth_PendingDeletionIsProgressing(t *testing.T) {
		u := obj("apps/v1", "Deployment", 1, map[string]any{"replicas": int64(1)},
		map[string]any{"observedGeneration": int64(1), "replicas": int64(1), "updatedReplicas": int64(1), "availableReplicas": int64(1)})
		now := metav1.Now()
		u.SetDeletionTimestamp(&now)

		assertHealth(t, u, HealthStatusProgressing)
		}

		func TestGetResourceHealth_UnsupportedKindReturnsNil(t *testing.T) {
		got, err := GetResourceHealth(obj("v1", "ConfigMap", 1, nil, nil), nil)
		if err != nil {
		t.Fatalf("unexpected error: %v", err)
		}
		if got != nil {
		t.Fatalf("health = %+v, want nil for an unsupported kind", got)
		}
		}

		func TestGetResourceHealth_ConversionErrorIsUnknown(t *testing.T) {
		// spec.replicas must be an integer; a string makes the typed conversion fail.
		u := obj("apps/v1", "Deployment", 1, map[string]any{"replicas": "two"}, nil)

		got, err := GetResourceHealth(u, nil)
		if err == nil {
		t.Fatal("expected a conversion error")
		}
		if got == nil \|\| got.Status != HealthStatusUnknown {
		t.Fatalf("health = %+v, want Unknown", got)
		}
		}

		func TestGetResourceHealth_UnsupportedVersionIsUnknown(t *testing.T) {
		// Group/kind match the Deployment check, but the version isn't apps/v1.
		u := obj("apps/v1beta1", "Deployment", 1, nil, nil)

		got, err := GetResourceHealth(u, nil)
		if err == nil {
		t.Fatal("expected an unsupported GVK error")
		}
		if got == nil \|\| got.Status != HealthStatusUnknown {
		t.Fatalf("health = %+v, want Unknown", got)
		}
		}

		type stubOverride struct {
		health *HealthStatus
		err error
		}

		func (s stubOverride) GetResourceHealth(unstructured.Unstructured) (HealthStatus, error) {
		return s.health, s.err
		}

		func TestGetResourceHealth_Override(t *testing.T) {
		// A Job with no conditions is Progressing by the built-in check.
		job := obj("batch/v1", "Job", 1, nil, map[string]any{})

		t.Run("override result wins", func(t *testing.T) {
		got, err := GetResourceHealth(job, stubOverride{health: &HealthStatus{Status: HealthStatusHealthy}})
		if err != nil \|\| got.Status != HealthStatusHealthy {
		t.Fatalf("got %+v, %v; want Healthy", got, err)
		}
		})
		t.Run("nil override result falls back to built-in", func(t *testing.T) {
		got, err := GetResourceHealth(job, stubOverride{})
		if err != nil \|\| got.Status != HealthStatusProgressing {
		t.Fatalf("got %+v, %v; want Progressing", got, err)
		}
		})
		t.Run("override error is Unknown", func(t *testing.T) {
		got, err := GetResourceHealth(job, stubOverride{err: errors.New("boom")})
		if err == nil \|\| got.Status != HealthStatusUnknown \|\| got.Message != "boom" {
		t.Fatalf("got %+v, %v; want Unknown with message boom", got, err)
		}
		})
		}

		func TestGetHealthCheckFunc_WatchedKinds(t *testing.T) {
		for _, gvk := range []schema.GroupVersionKind{
		{Group: "apps", Version: "v1", Kind: DeploymentKind},
		{Group: "apps", Version: "v1", Kind: StatefulSetKind},
		{Group: "apps", Version: "v1", Kind: DaemonSetKind},
		{Group: "batch", Version: "v1", Kind: JobKind},
		} {
		if GetHealthCheckFunc(gvk) == nil {
		t.Errorf("no health check for %s", gvk)
		}
		}
		}

		func TestIsWorse(t *testing.T) {
		cases := []struct {
		current, new HealthStatusCode
		want bool
		}{
		{HealthStatusHealthy, HealthStatusDegraded, true},
		{HealthStatusProgressing, HealthStatusMissing, true},
		{HealthStatusDegraded, HealthStatusHealthy, false},
		{HealthStatusProgressing, HealthStatusProgressing, false},
		{HealthStatusDegraded, HealthStatusUnknown, true},
		}
		for _, tc := range cases {
		if got := IsWorse(tc.current, tc.new); got != tc.want {
		t.Errorf("IsWorse(%s, %s) = %v, want %v", tc.current, tc.new, got, tc.want)
		}
		}
		}