Commit 567d2fe0 authored by Muhammad Umair Khan's avatar Muhammad Umair Khan
Browse files

Improve meepctl build caching, CLI sudo authorization, and pyinfra deployment configuration 

- Implement checksum caching (.build_checksum) across all meepctl build targets (Go microservices and frontend applications), storing cache files cleanly inside bin/<target>/.build_checksum.
- Add --no-cache CLI option to meepctl build to bypass cache and force binary recompilation when required.
- Configure /etc/sudoers.d/meepctl during installation in install.sh to allow non-interactive sudo execution for certificate trust and runtime restart operations in meepctl deploy.
- Add environment variable loading (.env) and configuration helpers in pyinfra to externalize target host addresses, user credentials, and OAuth provider configuration.
- Improve idempotency and reliability of pyinfra Kubernetes cluster initialization, Calico CNI setup, containerd runtime configuration, and development environment automation.
parent a02ad134

.gitignore

+2 −0
Original line number Diff line number Diff line
@@ -16,3 +16,5 @@ config/secrets.yaml 
.meepctl-repocfg.yaml

config/api/

charts/grafana/dashboards/mec-sandbox.json

pyinfra/pyinfra-venv/

pyinfra/.env

.gitmodules

+1 −1
Original line number Diff line number Diff line 
[submodule "etsi-mec-sandbox-frontend"]

	path = etsi-mec-sandbox-frontend

	url = https://labs.etsi.org/rep/mec/etsi-mec-sandbox-frontend.git

	branch = STF678_Task4
 
	branch = STF_685

go-apps/meepctl/cmd/build.go

+62 −34
Original line number Diff line number Diff line
@@ -20,6 +20,7 @@ import ( 
	"fmt"

	"os"

	"os/exec"

	"path/filepath"

	"sort"

	"strings"

	"time"
@@ -33,6 +34,7 @@ import ( 
type BuildData struct {

	codecov       bool

	nolint        bool

	noCache       bool

	coreGoApps    []string

	coreJsApps    []string

	sandboxGoApps []string
@@ -83,6 +85,7 @@ func init() { 
	// Set build-specific flags

	buildCmd.Flags().BoolVar(&buildData.codecov, "codecov", false, "Build a code coverage binary (dev. option)")

	buildCmd.Flags().BoolVar(&buildData.nolint, "nolint", false, "Disable linting")

	buildCmd.Flags().BoolVar(&buildData.noCache, "no-cache", false, "Build binaries without using checksum cache")



	// Add command

	rootCmd.AddCommand(buildCmd)
@@ -184,35 +187,11 @@ func buildFrontend(targetName string, repo string, cobraCmd *cobra.Command) { 
	locDeps := utils.RepoCfg.GetStringMapString(repo + targetName + ".local-deps")



	// Checksum optimization

	binDirExists := false

	if _, err := os.Stat(binDir); !os.IsNotExist(err) {

		binDirExists = true

	}



	checksumFile := srcDir + "/.build_checksum"

	findArgs := srcDir

	for _, depDir := range locDeps {

		findArgs += " " + gitDir + "/" + depDir

	}



	cmdStr := fmt.Sprintf("find %s -type f -not -name '.build_checksum' -not -path '*/node_modules/*' -not -path '*/dist/*' -not -path '*/bin/*' -not -path '*/.git/*' 2>/dev/null | sort | xargs md5sum 2>/dev/null | md5sum | cut -d' ' -f1", findArgs)

	checksumCmd := exec.Command("sh", "-c", cmdStr)

	outBytes, err := checksumCmd.Output()

	currentChecksum := ""

	if err == nil {

		currentChecksum = strings.TrimSpace(string(outBytes))



		if binDirExists && currentChecksum != "" {

			savedChecksumBytes, err := os.ReadFile(checksumFile)

			if err == nil {

				savedChecksum := strings.TrimSpace(string(savedChecksumBytes))

				if currentChecksum == savedChecksum {

	checksumFile, currentChecksum, skipped := checkBuildCache(srcDir, binDir, binDir, gitDir, locDeps)

	if skipped {

		fmt.Println(utils.FormatStep("   + skipping build (no changes detected)"))

		return

	}

			}

		}

	}



	// dependencies

	fmt.Println(utils.FormatStep("   + checking external dependencies"))
@@ -281,12 +260,7 @@ func buildFrontend(targetName string, repo string, cobraCmd *cobra.Command) { 
		fmt.Println(utils.FormatError("Error: " + err.Error()))

		fmt.Println(out)

	} else {

		if currentChecksum != "" {

			err = os.WriteFile(checksumFile, []byte(currentChecksum), 0644)

			if err != nil {

				fmt.Println(utils.FormatError("Error saving checksum: " + err.Error()))

			}

		}

		saveBuildCache(checksumFile, currentChecksum)

	}

}
@@ -299,6 +273,13 @@ func buildGoApp(targetName string, repo string, cobraCmd *cobra.Command) { 
	codecovCapable := utils.RepoCfg.GetBool(repo + targetName + ".codecov")

	lintEnabled := utils.RepoCfg.GetBool(repo + targetName + ".lint")



	locDeps := utils.RepoCfg.GetStringMapString(repo + targetName + ".local-deps")

	checksumFile, currentChecksum, skipped := checkBuildCache(srcDir, binDir+"/"+targetName, binDir, gitDir, locDeps)

	if skipped {

		fmt.Println(utils.FormatStep("   + skipping build (no changes detected)"))

		return

	}



	// dependencies

	fmt.Println(utils.FormatStep("   + checking external dependencies"))

	cmd := exec.Command("go", "mod", "vendor")
@@ -349,6 +330,8 @@ func buildGoApp(targetName string, repo string, cobraCmd *cobra.Command) { 
	if err != nil {

		fmt.Println(utils.FormatError("Error: " + err.Error()))

		fmt.Println(out)

	} else {

		saveBuildCache(checksumFile, currentChecksum)

	}

}
@@ -381,3 +364,48 @@ func fixDeps(targetName string, repo string, cobraCmd *cobra.Command) { 
		}

	}

}



func checkBuildCache(srcDir string, binFile string, binDir string, gitDir string, locDeps map[string]string) (string, string, bool) {

	binExists := false

	if _, err := os.Stat(binFile); !os.IsNotExist(err) {

		binExists = true

	}



	checksumFile := binDir + "/.build_checksum"

	findArgs := srcDir

	for _, depDir := range locDeps {

		findArgs += " " + gitDir + "/" + depDir

	}



	cmdStr := fmt.Sprintf("find %s -type f -not -name '.build_checksum' -not -path '*/node_modules/*' -not -path '*/vendor/*' -not -path '*/dist/*' -not -path '*/bin/*' -not -path '*/.git/*' 2>/dev/null | sort | xargs md5sum 2>/dev/null | md5sum | cut -d' ' -f1", findArgs)

	checksumCmd := exec.Command("sh", "-c", cmdStr)

	outBytes, err := checksumCmd.Output()

	currentChecksum := ""

	if err == nil {

		currentChecksum = strings.TrimSpace(string(outBytes))

		if !buildData.noCache && binExists && currentChecksum != "" {

			savedChecksumBytes, err := os.ReadFile(checksumFile)

			if err == nil {

				savedChecksum := strings.TrimSpace(string(savedChecksumBytes))

				if currentChecksum == savedChecksum {

					return checksumFile, currentChecksum, true

				}

			}

		}

	}

	return checksumFile, currentChecksum, false

}



func saveBuildCache(checksumFile string, currentChecksum string) {

	if currentChecksum != "" {

		err := os.MkdirAll(filepath.Dir(checksumFile), 0755)

		if err != nil {

			fmt.Println(utils.FormatError("Error creating directory: " + err.Error()))

			return

		}

		err = os.WriteFile(checksumFile, []byte(currentChecksum), 0644)

		if err != nil {

			fmt.Println(utils.FormatError("Error saving checksum: " + err.Error()))

		}

	}

}

go-apps/meepctl/cmd/deploy.go

+24 −0
Original line number Diff line number Diff line
@@ -19,9 +19,11 @@ package cmd 
import (

	"errors"

	"fmt"

	"net"

	"os"

	"os/exec"

	"sort"



	"strconv"

	"strings"

	"sync"
@@ -901,6 +903,28 @@ func deployCreateRegistryCerts(chart string, cobraCmd *cobra.Command) { 
	certdir := deployData.workdir + "/certs"

	cmd := exec.Command("sh", "-c", chart+"/create-k8s-ca-signed-cert.sh --certdir "+certdir)

	_, _ = utils.ExecuteCmd(cmd, cobraCmd)

	ca := utils.RepoCfg.GetString("repo.deployment.ingress.ca")

	host := utils.RepoCfg.GetString("repo.deployment.ingress.host")

	isIP := false

	if ip := net.ParseIP(host); ip != nil {

		isIP = true

	}

	if ca == "self-signed" || isIP {

		trustRegistryCerts(certdir, cobraCmd)

	}

}



func trustRegistryCerts(certdir string, cobraCmd *cobra.Command) {

	cmd := exec.Command("sudo", "cp", "-f", "/etc/kubernetes/pki/ca.crt", "/usr/local/share/ca-certificates/kubernetes-ca.crt")

	_, _ = utils.ExecuteCmd(cmd, cobraCmd)

	cmd = exec.Command("sh", "-c", "for f in "+certdir+"/*.pem; do [ -f \"$f\" ] && sudo cp -f \"$f\" \"/usr/local/share/ca-certificates/$(basename \"$f\" .pem).crt\"; done")

	_, _ = utils.ExecuteCmd(cmd, cobraCmd)

	cmd = exec.Command("sudo", "update-ca-certificates")

	_, _ = utils.ExecuteCmd(cmd, cobraCmd)

	cmd = exec.Command("sudo", "systemctl", "restart", "containerd")

	_, _ = utils.ExecuteCmd(cmd, cobraCmd)

	cmd = exec.Command("sudo", "systemctl", "restart", "docker")

	_, _ = utils.ExecuteCmd(cmd, cobraCmd)

}



func deployCreateIngressCerts(chart string, cobraCmd *cobra.Command) {

go-apps/meepctl/install.sh

+12 −0
Original line number Diff line number Diff line
@@ -57,6 +57,18 @@ echo "" 
printf "%b\n" "${BLUE}${BOLD}➤ Running go install${NC}"

go install



echo ""

# Configure sudoers for meepctl certificate trust operations

printf "%b\n" "${BLUE}${BOLD}➤ Configuring sudoers NOPASSWD for meepctl certificate operations${NC}"

SUDOERS_RULE="${USER} ALL=(ALL:ALL) NOPASSWD: /usr/bin/cp *, /bin/cp *, /usr/sbin/update-ca-certificates, /usr/bin/update-ca-certificates, /usr/bin/systemctl restart containerd, /usr/bin/systemctl restart docker, /bin/systemctl restart containerd, /bin/systemctl restart docker"

if [ -n "$SUDO_PASSWORD" ]; then

    echo "$SUDO_PASSWORD" | sudo -S sh -c "echo '$SUDOERS_RULE' > /etc/sudoers.d/meepctl && chmod 0440 /etc/sudoers.d/meepctl" 2>/dev/null || true

elif sudo -n true 2>/dev/null; then

    sudo sh -c "echo '$SUDOERS_RULE' > /etc/sudoers.d/meepctl && chmod 0440 /etc/sudoers.d/meepctl" 2>/dev/null || true

else

    sudo sh -c "echo '$SUDOERS_RULE' > /etc/sudoers.d/meepctl && chmod 0440 /etc/sudoers.d/meepctl" || true

fi



echo ""

printf "%b\n" "${GREEN}${BOLD}✨ ${IMAGE_NAME} installation completed successfully!${NC}"

echo ""