Commit bf58faa3 authored by Vasilis Katopodis's avatar Vasilis Katopodis
Browse files

feat: create managed cluster kubeconfig secrets for OCM scheduler deploys

parent b4424cb2
Loading
Loading
Loading
Loading
Loading
+2 −0
Changes for README.md: 2 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -277,6 +277,8 @@ The Package Manager integrates with the **Open Cluster Management (OCM)** framew
- **`clusterManager`**: Set to `ocm` to activate the OCM path.
- **`clusterMetadata`**: A Base64-encoded YAML manifest that describes workload-to-cluster mapping rules and the target scheduler namespace.

For the OCM scheduler chart itself (installed on the hub as a plain Helm order, without `clusterManager: ocm`), at deploy each `managedClusters[]` entry in its values with a `kubeconfigSecretId` is resolved through the Registry API and created as an Opaque Secret named after the entry's `name` (key `kubeconfig`) before `helm install`. See `docs/kafka-api.md`.

During the parse phase, the Package Manager decodes `clusterMetadata`, extracts the `namespace` field, and stores it as the Helm chart annotation `sonata.engine/namespace`. This annotation is then read by the deploy, activate, and terminate handlers to determine the correct namespace of the OCM scheduler, instead of defaulting to the `serviceId`.

An example `clusterMetadata` payload (after Base64 decoding) for the **P2CODE** scheduler:
+14 −2
Changes for docs/kafka-api.md: 14 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -229,7 +229,8 @@ Sent for a private repository, or whenever `clusterMetadata` yields a scheduler
    "servicePackageValuesYml": "<base64 values.yaml>"
  },
  "kubernetesServiceID": "2e5b1f9a-6c3d-4a2e-8b0a-9f7c1d4e5a6b",
  "secretNames": ["regcred"]
  "secretNames": ["regcred"],
  "kubeconfigSecretIds": ["<id-managed-1>", "<id-managed-2>"]
}
```

@@ -238,6 +239,16 @@ Sent for a private repository, or whenever `clusterMetadata` yields a scheduler
| `serviceOperationRequest` | the inbound request plus its process ID; echoed back. Termination always sends `activateService: false` |
| `kubernetesServiceID` | the inbound `k8sServiceId` |
| `secretNames` | image-pull-secret names from the chart (deployment only; `null` otherwise) |
| `kubeconfigSecretIds` | deployment only; `managedClusters[].kubeconfigSecretId` from `servicePackageValuesYml` (entries with both `name` and `kubeconfigSecretId`). Omitted when there are none |

The OCM scheduler chart's values list the managed clusters whose kubeconfigs it needs:

```yaml
managedClusters:
  - name: managed-1            # name of the Secret created in the release namespace
    location: managed-1
    kubeconfigSecretId: <id>   # package-registry lookup key
```

## Registry replies

@@ -279,10 +290,11 @@ it fails later, at the private-repo fetch.
| `serviceOperationRequest` | echoed; `serviceId` must be at least 5 characters |
| `kubeConfigAsBase64` | required; the cluster kubeconfig |
| `imagePullSecrets` | `[{ "secretName", "url", "username", "password" }]`, created in the target namespace (non-OCM deploys) |
| `kubeconfigSecrets` | `[{ "id", "kubeConfigAsBase64" }]`, one per requested `kubeconfigSecretIds` entry. Each becomes an Opaque Secret named after the matching `managedClusters[].name`, key `kubeconfig`, in the target namespace (non-OCM deploys). A requested ID missing from the reply fails the deploy |
| `schedulerNamespace` | non-empty means OCM: used as the target namespace instead of `serviceId`, and selects `OCMStrategy` |
| `requestedCompletionDate` | (inside `serviceOperationRequest`) required when `type` is `deployment` |

`kubeConfigAsBase64` and `servicePackageValuesYml` are base64-decoded after validation
`kubeConfigAsBase64`, `servicePackageValuesYml` and each `kubeconfigSecrets[].kubeConfigAsBase64` are base64-decoded after validation
(`KubeconfigMessageDecoder`).

## SA kubeconfig store
+50 −2
Changes for kubernetes/kubernetes.go: 50 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -37,6 +37,14 @@ type Auth struct {
	Auth     string `json:"auth"`
}

// KubeconfigSecret is an Opaque Secret holding a kubeconfig under KubeconfigSecretKey.
type KubeconfigSecret struct {
	Name       string
	Kubeconfig []byte
}

const KubeconfigSecretKey = "kubeconfig"

// CreateClient creates a new Kubernetes client from kubeconfig bytes
func CreateClient(log logging.Logger, contextTag string, kubeconfigBytes []byte) (*KubernetesClient, error) {
	config, err := clientcmd.RESTConfigFromKubeConfig(kubeconfigBytes)
@@ -128,6 +136,29 @@ func (client *KubernetesClient) ProcessImagePullSecrets(namespace string, secret
	return nil
}

func (client *KubernetesClient) ProcessKubeconfigSecrets(namespace string, secrets []KubeconfigSecret) error {
	client.Log.Debug("%s msg=\"Processing kubeconfig secrets for target namespace: %s\"", client.ContextTag, namespace)

	if err := client.createNamespace(namespace); err != nil {
		return err
	}

	for _, kubeconfigSecret := range secrets {
		secret := createKubeconfigSecret(kubeconfigSecret.Name, namespace, kubeconfigSecret.Kubeconfig)
		err := client.withRetry("apply kubeconfig secret "+kubeconfigSecret.Name, func() error {
			return client.applySecret(secret)
		})
		if err != nil {
			client.Log.Error(err, "%s msg=\"Failed to apply kubeconfig secret: secretName=%q\"", client.ContextTag, kubeconfigSecret.Name)
			return fmt.Errorf("creation of kubeconfig secret %q failed: %w", kubeconfigSecret.Name, err)
		}
		client.Log.Info("%s msg=\"Successfully set up kubeconfig secret: secretName=%q\"", client.ContextTag, kubeconfigSecret.Name)
	}

	client.Log.Debug("%s msg=\"Successfully processed kubeconfig secrets.\"", client.ContextTag)
	return nil
}

func (client *KubernetesClient) DeleteNamespace(namespace string) error {
	client.Log.Info("%s msg=\"Deleting namespace: %s\"", client.ContextTag, namespace)
	err := client.CoreV1().Namespaces().Delete(context.TODO(), namespace, metav1.DeleteOptions{})
@@ -233,17 +264,34 @@ func (client *KubernetesClient) createImagePullSecret(name, namespace string, im
	}, nil
}

func createKubeconfigSecret(name, namespace string, kubeconfig []byte) *v1.Secret {
	return &v1.Secret{
		TypeMeta: metav1.TypeMeta{
			APIVersion: "v1",
			Kind:       "Secret",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      name,
			Namespace: namespace,
		},
		Type: v1.SecretTypeOpaque,
		Data: map[string][]byte{
			KubeconfigSecretKey: kubeconfig,
		},
	}
}

func (client *KubernetesClient) applySecret(secret *v1.Secret) error {
	_, err := client.CoreV1().Secrets(secret.Namespace).Get(context.TODO(), secret.Name, metav1.GetOptions{})
	if err == nil {
		client.Log.Debug("%s msg=\"Image pull secret %s already exists. Skipping creation.\"", client.ContextTag, secret.Name)
		client.Log.Debug("%s msg=\"Secret %s already exists. Skipping creation.\"", client.ContextTag, secret.Name)
		return nil
	}
	if !errors.IsNotFound(err) {
		return fmt.Errorf("error checking for existing secret: %w", err)
	}

	client.Log.Debug("%s msg=\"Creating new image pull secret: %s\"", client.ContextTag, secret.Name)
	client.Log.Debug("%s msg=\"Creating new secret: %s\"", client.ContextTag, secret.Name)
	_, err = client.CoreV1().Secrets(secret.Namespace).Create(context.TODO(), secret, metav1.CreateOptions{})
	if err != nil {
		return fmt.Errorf("error creating secret: %w", err)
+31 −0
Changes for kubernetes/kubernetes_test.go: 31 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -160,3 +160,34 @@ func TestGetServiceEndpoints(t *testing.T) {
	assert.Equal(t, "192.168.1.100", endpoints[0].IP)
	assert.Equal(t, int32(30001), endpoints[0].Port)
}

func TestProcessKubeconfigSecrets(t *testing.T) {
	clientset := fake.NewSimpleClientset()
	client := &KubernetesClient{
		Interface:  clientset,
		Log:        &mockLogger{},
		ContextTag: "test_context=ProcessKubeconfigSecrets",
	}

	nsName := "scheduler-namespace"
	secrets := []KubeconfigSecret{
		{Name: "managed-1", Kubeconfig: []byte("kubeconfig-managed-1")},
		{Name: "managed-2", Kubeconfig: []byte("kubeconfig-managed-2")},
	}

	err := client.ProcessKubeconfigSecrets(nsName, secrets)
	assert.NoError(t, err)

	_, err = clientset.CoreV1().Namespaces().Get(context.TODO(), nsName, metav1.GetOptions{})
	assert.NoError(t, err)

	for _, s := range secrets {
		created, err := clientset.CoreV1().Secrets(nsName).Get(context.TODO(), s.Name, metav1.GetOptions{})
		assert.NoError(t, err, "secret %s should exist", s.Name)
		assert.Equal(t, v1.SecretTypeOpaque, created.Type)
		assert.Equal(t, s.Kubeconfig, created.Data[KubeconfigSecretKey])
	}

	err = client.ProcessKubeconfigSecrets(nsName, secrets)
	assert.NoError(t, err, "re-processing existing secrets should succeed")
}
+2 −0
Changes for ops/deploy_strategy.go: 2 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ import (
	"labs.etsi.org/rep/osl/hypo/code/org.etsi.osl.hypo.core/pkg-manager.helm/engines"
	"labs.etsi.org/rep/osl/hypo/code/org.etsi.osl.hypo.core/pkg-manager.helm/helm"
	"labs.etsi.org/rep/osl/hypo/code/org.etsi.osl.hypo.core/pkg-manager.helm/internal/logging"
	"labs.etsi.org/rep/osl/hypo/code/org.etsi.osl.hypo.core/pkg-manager.helm/kubernetes"
	"labs.etsi.org/rep/osl/hypo/code/org.etsi.osl.hypo.core/pkg-manager.helm/registryapi"
)

@@ -24,6 +25,7 @@ type DeployInputs struct {
	Kubeconfig           []byte
	ValuesYalm           string
	ImagePullSecrets     []*registryapi.ImagePullSecret
	KubeconfigSecrets    []kubernetes.KubeconfigSecret
	RegistryClient       *engines.HelmRegistryClient
	HelmClient           *helm.HelmClient
	ReleaseName          string
Loading