Commit 4daf0c3c authored by Vasilis Katopodis's avatar Vasilis Katopodis
Browse files

feat: retry transient k8s API errors and improve logging

parent 9f727173

engines/convertEngine.go

+24 −25
Changes for engines/convertEngine.go: 24 added lines, 25 removed lines.
Original line number Diff line number Diff line
@@ -34,21 +34,23 @@ func NewConvertEngine(serviceLogger logging.Logger, gitEngine GitEngineInterface 
	}

}



func (convertEngine *ConvertEngine) CheckIfRepoIsPublic(ctx context.Context, gitRepoURL string, artifactVersion string) (bool, error) {

func (convertEngine *ConvertEngine) CheckIfRepoIsPublic(ctx context.Context, serviceID string, gitRepoURL string, artifactVersion string) (bool, error) {

	gitEngine := convertEngine.gitEngine

	return gitEngine.CheckIfRepoIsPublic(ctx, gitRepoURL, artifactVersion)

	return gitEngine.CheckIfRepoIsPublic(ctx, serviceID, gitRepoURL, artifactVersion)

}



func (convertEngine *ConvertEngine) ConvertKubernetesToHelmCharts(ctx context.Context, serviceOrder datamodel.ServiceOrder, username, password string) (*ConversionResult, error) {

	gitEngine := convertEngine.gitEngine



	err := gitEngine.Clone(ctx, serviceOrder.ServiceRepositoryURL, serviceOrder.ServiceArtifactVersion, username, password)

	serviceID := serviceOrder.ServiceID



	err := gitEngine.Clone(ctx, serviceID, serviceOrder.ServiceRepositoryURL, serviceOrder.ServiceArtifactVersion, username, password)

	if err != nil {

		return nil, err

	}



	exportedFilesPath := path.Join(defaultExportPath, serviceOrder.ServiceOrderID)

	helmChartsPath, err := convertEngine.exportKubernetesManifests(serviceOrder.ServiceArtifactIdentifier, exportedFilesPath)

	helmChartsPath, err := convertEngine.exportKubernetesManifests(serviceID, serviceOrder.ServiceArtifactIdentifier, exportedFilesPath)

	if err != nil {

		return nil, err

	}
@@ -62,36 +64,33 @@ func (convertEngine *ConvertEngine) ConvertDockerComposeToHelmCharts(ctx context 


	dockerRepo := convertServiceOrderToDockerComposeRepo(serviceOrder)



	logger.Debug(

		"Attempting to convert a docker compose named '%v' from repository '%s' to helm.",

		dockerRepo.ComposeFileName,

		dockerRepo.URL,

	)

	serviceID := serviceOrder.ServiceID

	logger.Debug("service_id=%s msg=\"Converting docker compose file %s from repository %s to helm.\"", serviceID, dockerRepo.ComposeFileName, dockerRepo.URL)



	err := dockerRepo.Validate()

	if err != nil {

		return nil, fmt.Errorf("DockerComposeRepo - Validation failed: %w", err)

	}



	err = gitEngine.Clone(ctx, dockerRepo.URL, dockerRepo.Tag, username, password)

	err = gitEngine.Clone(ctx, serviceID, dockerRepo.URL, dockerRepo.Tag, username, password)

	if err != nil {

		return nil, err

	}



	exportedFilesPath := path.Join(defaultExportPath, dockerRepo.ServiceOrderID)

	composeFilePath, err := convertEngine.exportComposeFiles(dockerRepo.ComposeFileName, dockerRepo.EnvFileName, exportedFilesPath)

	composeFilePath, err := convertEngine.exportComposeFiles(serviceID, dockerRepo.ComposeFileName, dockerRepo.EnvFileName, exportedFilesPath)

	if err != nil {

		return nil, err

	}



	return convertEngine.convertDockerComposeToHelm(composeFilePath, exportedFilesPath)

	return convertEngine.convertDockerComposeToHelm(serviceID, composeFilePath, exportedFilesPath)

}



func (convertEngine *ConvertEngine) convertDockerComposeToHelm(composeFilePath string, exportedFilesPath string) (*ConversionResult, error) {

func (convertEngine *ConvertEngine) convertDockerComposeToHelm(serviceID string, composeFilePath string, exportedFilesPath string) (*ConversionResult, error) {

	helmChartsPath := path.Join(exportedFilesPath, "charts")

	komposeEngine := convertEngine.komposeEngine



	err := komposeEngine.ConvertToHelm(composeFilePath, helmChartsPath)

	err := komposeEngine.ConvertToHelm(serviceID, composeFilePath, helmChartsPath)

	if err != nil {

		return nil, fmt.Errorf("DockerComposeRepo - Failed to convert docker-compose to helm: %w", err)

	}
@@ -126,11 +125,11 @@ func (convertEngine *ConvertEngine) convertKubernetesToHelm(kubernetesDir string 
	return result, nil

}



func (convertEngine *ConvertEngine) exportKubernetesManifests(kubernetesDir string, exportedFilesPath string) (string, error) {

func (convertEngine *ConvertEngine) exportKubernetesManifests(serviceID string, kubernetesDir string, exportedFilesPath string) (string, error) {

	composeFilePath := path.Join(exportedFilesPath, kubernetesDir)



	convertEngine.logger.Debug("Exporting kubernetes manifests directory.")

	err := convertEngine.gitEngine.ExportDirectory(kubernetesDir, composeFilePath)

	convertEngine.logger.Debug("service_id=%s msg=\"Exporting kubernetes manifests directory.\"", serviceID)

	err := convertEngine.gitEngine.ExportDirectory(serviceID, kubernetesDir, composeFilePath)

	if err != nil {

		return "", err

	}
@@ -138,17 +137,17 @@ func (convertEngine *ConvertEngine) exportKubernetesManifests(kubernetesDir stri 
	return composeFilePath, nil

}



func (convertEngine *ConvertEngine) exportComposeFiles(dockerComposeFile string, envFile string, exportedFilesPath string) (string, error) {

func (convertEngine *ConvertEngine) exportComposeFiles(serviceID string, dockerComposeFile string, envFile string, exportedFilesPath string) (string, error) {

	composeFilePath := path.Join(exportedFilesPath, dockerComposeFile)



	convertEngine.logger.Debug("Exporting docker compose file.")

	err := convertEngine.gitEngine.ExportFile(dockerComposeFile, composeFilePath)

	convertEngine.logger.Debug("service_id=%s msg=\"Exporting docker compose file.\"", serviceID)

	err := convertEngine.gitEngine.ExportFile(serviceID, dockerComposeFile, composeFilePath)

	if err != nil {

		return "", err

	}



	if envFile != "" {

		err := convertEngine.handleEnvFile(envFile, composeFilePath, exportedFilesPath)

		err := convertEngine.handleEnvFile(serviceID, envFile, composeFilePath, exportedFilesPath)

		if err != nil {

			return "", err

		}
@@ -161,11 +160,11 @@ var ReadFile = os.ReadFile 
var ReadEnv = godotenv.Read

var WriteFile = os.WriteFile



func (convertEngine *ConvertEngine) handleEnvFile(envFile string, composeFilePath string, exportedFilesPath string) error {

	convertEngine.logger.Debug("Exporting env file.")

func (convertEngine *ConvertEngine) handleEnvFile(serviceID string, envFile string, composeFilePath string, exportedFilesPath string) error {

	convertEngine.logger.Debug("service_id=%s msg=\"Exporting env file.\"", serviceID)

	envFilePath := path.Join(exportedFilesPath, ".env")



	err := convertEngine.gitEngine.ExportFile(envFile, envFilePath)

	err := convertEngine.gitEngine.ExportFile(serviceID, envFile, envFilePath)

	if err != nil {

		return fmt.Errorf("DockerComposeRepo - Failed to export the env file: %w", err)

	}
@@ -180,7 +179,7 @@ func (convertEngine *ConvertEngine) handleEnvFile(envFile string, composeFilePat 
		return fmt.Errorf("DockerComposeRepo - Failed to read exported compose file: %w", err)

	}



	convertEngine.logger.Debug("Embedding the .env variables to docker compose '%s'.", composeFilePath)

	convertEngine.logger.Debug("service_id=%s msg=\"Embedding .env variables into docker compose file: %s\"", serviceID, composeFilePath)

	composeAsString := string(composeAsBytes)

	for envName, envValue := range envs {

		composeAsString = strings.ReplaceAll(composeAsString, "${"+envName+"}", envValue)

engines/convertEngine_test.go

+5 −5
Changes for engines/convertEngine_test.go: 5 added lines, 5 removed lines.
Original line number Diff line number Diff line
@@ -17,19 +17,19 @@ var loggerConfig util.LoggerConfig = util.LoggerConfig{ 


type MockGitEngine struct{}



func (m *MockGitEngine) CheckIfRepoIsPublic(_ context.Context, _ string, _ string) (bool, error) {

func (m *MockGitEngine) CheckIfRepoIsPublic(_ context.Context, _ string, _ string, _ string) (bool, error) {

	return false, nil

}



func (m *MockGitEngine) Clone(_ context.Context, _ string, _ string, _ string, _ string) error {

func (m *MockGitEngine) Clone(_ context.Context, _ string, _ string, _ string, _ string, _ string) error {

	return nil

}



func (m *MockGitEngine) ExportFile(_ string, _ string) error {

func (m *MockGitEngine) ExportFile(_ string, _ string, _ string) error {

	return nil

}



func (m *MockGitEngine) ExportDirectory(_ string, _ string) error {

func (m *MockGitEngine) ExportDirectory(_ string, _ string, _ string) error {

	return nil

}
@@ -74,7 +74,7 @@ func TestHandleEnvFile(t *testing.T) { 
	envFile := "test.env"

	composeFilePath := "test.yml"

	exportedFilesPath := "/tmp"

	err := convertEngine.handleEnvFile(envFile, composeFilePath, exportedFilesPath)

	err := convertEngine.handleEnvFile("test-service", envFile, composeFilePath, exportedFilesPath)



	// Assert that the expected result is returned.

	assert.NoError(t, err)

engines/gitEngine.go

+14 −14
Changes for engines/gitEngine.go: 14 added lines, 14 removed lines.
Original line number Diff line number Diff line
@@ -26,10 +26,10 @@ import ( 
var gitTransportMu sync.Mutex



type GitEngineInterface interface {

	CheckIfRepoIsPublic(ctx context.Context, repositoryURL string, tag string) (bool, error)

	Clone(ctx context.Context, repositoryURL string, tag string, username string, password string) error

	ExportFile(fileName string, exportComposeFilePath string) error

	ExportDirectory(dirName string, exportDirPath string) error

	CheckIfRepoIsPublic(ctx context.Context, serviceID string, repositoryURL string, tag string) (bool, error)

	Clone(ctx context.Context, serviceID string, repositoryURL string, tag string, username string, password string) error

	ExportFile(serviceID string, fileName string, exportComposeFilePath string) error

	ExportDirectory(serviceID string, dirName string, exportDirPath string) error

}



type GitEngine struct {
@@ -54,8 +54,8 @@ func installTracedGitTransport(ctx context.Context) { 
// CheckIfRepoIsPublic probes the repository without credentials.

// Returns true if the repo is publicly accessible, false if authentication

// is required, and true with a non-nil error for unexpected failures.

func (gitEngine *GitEngine) CheckIfRepoIsPublic(ctx context.Context, repositoryURL, tag string) (bool, error) {

	gitEngine.logger.Debug("Attempting to check if the '%v' repository is public", repositoryURL)

func (gitEngine *GitEngine) CheckIfRepoIsPublic(ctx context.Context, serviceID, repositoryURL, tag string) (bool, error) {

	gitEngine.logger.Debug("service_id=%s msg=\"Checking if git repository is public: %s\"", serviceID, repositoryURL)



	cloneOptions := &git.CloneOptions{

		URL: repositoryURL,
@@ -82,8 +82,8 @@ func (gitEngine *GitEngine) CheckIfRepoIsPublic(ctx context.Context, repositoryU 
	return true, err

}



func (gitEngine *GitEngine) Clone(ctx context.Context, repositoryURL, tag, username, password string) error {

	gitEngine.logger.Debug("Attempting to clone the '%v' repository", repositoryURL)

func (gitEngine *GitEngine) Clone(ctx context.Context, serviceID, repositoryURL, tag, username, password string) error {

	gitEngine.logger.Debug("service_id=%s msg=\"Cloning git repository: %s, Tag: %s\"", serviceID, repositoryURL, tag)



	cloneOptions := &git.CloneOptions{

		URL: repositoryURL,
@@ -109,7 +109,7 @@ func (gitEngine *GitEngine) Clone(ctx context.Context, repositoryURL, tag, usern 
		return fmt.Errorf("failed to clone the repository: %w", err)

	}



	gitEngine.logger.Debug("Attempting to retrieve the repository's work tree.")

	gitEngine.logger.Debug("service_id=%s msg=\"Retrieving the repository work tree.\"", serviceID)

	worktree, err := repository.Worktree()

	if err != nil {

		return fmt.Errorf("could not retrieve the repository's work tree: %w", err)
@@ -119,8 +119,8 @@ func (gitEngine *GitEngine) Clone(ctx context.Context, repositoryURL, tag, usern 
	return nil

}



func (gitEngine *GitEngine) ExportFile(fileName string, exportComposeFilePath string) error {

	gitEngine.logger.Debug("Attempting to export compose file tmp directory '%s'.", exportComposeFilePath)

func (gitEngine *GitEngine) ExportFile(serviceID string, fileName string, exportComposeFilePath string) error {

	gitEngine.logger.Debug("service_id=%s msg=\"Exporting file %s to: %s\"", serviceID, fileName, exportComposeFilePath)



	gitMemoryFile, err := gitEngine.fileSystem.Open(fileName)

	if err != nil {
@@ -146,8 +146,8 @@ func (gitEngine *GitEngine) ExportFile(fileName string, exportComposeFilePath st 
	return nil

}



func (gitEngine *GitEngine) ExportDirectory(dirName string, exportDirPath string) error {

	gitEngine.logger.Debug("Attempting to export file to tmp directory '%s'.", exportDirPath)

func (gitEngine *GitEngine) ExportDirectory(serviceID string, dirName string, exportDirPath string) error {

	gitEngine.logger.Debug("service_id=%s msg=\"Exporting directory %s to: %s\"", serviceID, dirName, exportDirPath)



	// List the files in the directory

	files, err := gitEngine.fileSystem.ReadDir(dirName)
@@ -158,7 +158,7 @@ func (gitEngine *GitEngine) ExportDirectory(dirName string, exportDirPath string 
	// Iterate over the files and export each one

	for _, file := range files {

		fileName := file.Name()

		gitEngine.logger.Debug("Attempting to retrieve the file named '%s' from a cached fileSystem.", fileName)

		gitEngine.logger.Debug("service_id=%s msg=\"Exporting file from cloned repository: %s\"", serviceID, fileName)



		gitMemoryFile, err := gitEngine.fileSystem.Open(filepath.Join(dirName, fileName))

		if err != nil {

engines/komposeEngine.go

+4 −4
Changes for engines/komposeEngine.go: 4 added lines, 4 removed lines.
Original line number Diff line number Diff line
@@ -45,7 +45,7 @@ func validate(composeFilePath string, exportHelmChartFilePath string) error { 
	return nil

}



func (komposeEngine *KomposeEngine) ConvertToHelm(composeFilePath string, exportHelmChartFilePath string) error {

func (komposeEngine *KomposeEngine) ConvertToHelm(serviceID string, composeFilePath string, exportHelmChartFilePath string) error {



	err := validate(composeFilePath, exportHelmChartFilePath)

	if err != nil {
@@ -53,7 +53,7 @@ func (komposeEngine *KomposeEngine) ConvertToHelm(composeFilePath string, export 
	}



	logger := komposeEngine.logger

	logger.Debug("Attempting to convert docker compose file '%s' to helm chart", composeFilePath)

	logger.Debug("service_id=%s msg=\"Converting docker compose file to helm chart: %s\"", serviceID, composeFilePath)



	convertOptions := kobject.ConvertOptions{

		CreateChart:      true,
@@ -66,13 +66,13 @@ func (komposeEngine *KomposeEngine) ConvertToHelm(composeFilePath string, export 
		ToStdout:         false,

	}



	logger.Debug("Validating  the compose file.")

	logger.Debug("service_id=%s msg=\"Validating the compose file.\"", serviceID)

	err = komposeApp.ValidateComposeFile(&convertOptions)

	if err != nil {

		return fmt.Errorf("failed to validate the compose file: %w", err)

	}



	logger.Debug("Converting the compose file  to helm chart.")

	logger.Debug("service_id=%s msg=\"Running kompose conversion.\"", serviceID)

	_, err = convert(convertOptions)

	if err != nil {

		return fmt.Errorf("failed to convert the compose file: %w", err)

engines/komposeEngine_test.go

+1 −1
Changes for engines/komposeEngine_test.go: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -37,7 +37,7 @@ func TestValidateKompose(t *testing.T) { 


	for _, tt := range tests {

		t.Run(tt.name, func(t *testing.T) {

			err := komposeEngine.ConvertToHelm(tt.composeFilePath, tt.exportHelmChartFilePath)

			err := komposeEngine.ConvertToHelm("test-service", tt.composeFilePath, tt.exportHelmChartFilePath)

			if (err != nil) != tt.wantErr {

				t.Errorf("ValidateIPOrHost() error = %v, wantErr %v", err, tt.wantErr)

			}