Commit ba49c1f5 authored by Muhammad Umair Khan's avatar Muhammad Umair Khan
Browse files

meepctl: show build and image download progress; fix pod egress and DNS setup 

meepctl
- dockerize: replace raw docker output with one live status line per
  build/push (current step, step output, layer downloads, elapsed time);
  full output with -v. On failure, print the last lines of docker output
  and skip the push so a stale image is never published.
- deploy dep/core: after installing, follow the image pulls of the new
  pods and report size, time, speed and source registry per image, plus
  failed pulls; skipped when nothing was installed, or with --no-progress.
- deploy: set prometheus-operator admission webhook failurePolicy=Ignore
  so charts creating PrometheusRules (meep-thanos) no longer fail while
  the operator is still starting.
- helm: print helm's error output when an install fails.

pyinfra
- docker: set ip-forward-no-drop in daemon.json so Docker restarts no
  longer set the FORWARD policy to DROP, which cut pod internet access
  and broke CoreDNS upstream lookups (GitHub OAuth sign-in failures).
- CoreDNS: forward upstreams are configurable via CLUSTER_DNS_SERVERS
  (default Cloudflare then Google, tried in order with policy
  sequential); servers not answering from the node are skipped.
parent 260e3c6a

deploy/pyinfra/.env.example

+8 −0
Changes for deploy/pyinfra/.env.example: 8 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -23,6 +23,14 @@ MEC_HOST_ADDRESS="" 
# The protocol to use for the ingress (http or https)

MEC_PROTOCOL="https"



# Upstream DNS servers CoreDNS forwards external lookups to, in priority order

# (space or comma separated): the first one is used, the next only if it fails.

# Each is tested from the node during deployment and unreachable ones are

# skipped. Leave empty for the default: Cloudflare, then Google

# ("1.1.1.1 1.0.0.1 8.8.8.8 8.8.4.4").

# Set this on networks that only allow an internal resolver (e.g. "10.0.0.53").

CLUSTER_DNS_SERVERS=""



# ----------------------------------------------------

# GitHub OAuth Secrets

# ----------------------------------------------------

deploy/pyinfra/group_data/all.py

+7 −0
Changes for deploy/pyinfra/group_data/all.py: 7 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -23,6 +23,13 @@ mec_host_address = __get_mec_host_address() 
mec_sandbox_dir = __os.environ.get("MEC_SANDBOX_DIR", f"{target_home}/etsi-mec-sandbox")

mec_frontend_dir = __os.environ.get("MEC_FRONTEND_DIR", f"{mec_sandbox_dir}/etsi-mec-sandbox-frontend")



# -----------------------------------------------------------------------------

# Cluster DNS

# -----------------------------------------------------------------------------

# Upstream resolvers for CoreDNS, in priority order: Cloudflare first, Google

# as fallback. Unreachable ones are skipped (some routers/ISPs block 1.1.1.1).

cluster_dns_servers = (__os.environ.get("CLUSTER_DNS_SERVERS") or "1.1.1.1 1.0.0.1 8.8.8.8 8.8.4.4").replace(",", " ").split()



# -----------------------------------------------------------------------------

# OAuth Configuration (GitHub & GitLab)

# -----------------------------------------------------------------------------

deploy/pyinfra/lib/operations/kubernetes.py

+81 −1
Changes for deploy/pyinfra/lib/operations/kubernetes.py: 81 added lines, 1 removed line.
Original line number Diff line number Diff line 
import shlex



from pyinfra import host

from pyinfra.api import FactBase, operation, StringCommand

from pyinfra.api import FactBase, operation, OperationError, StringCommand

from pyinfra.facts.files import File



def _get_kubeconfig_env(kubeconfig):
@@ -22,6 +24,41 @@ class ConfigMap(FactBase): 
        return f"{env_str}kubectl get configmap {configmap_name} -n {namespace} -o yaml || true"





class ReachableDnsServers(FactBase):

    """

    Returns the subset of the given DNS servers that answer a plain DNS query

    (UDP/53) from this host. Pod traffic leaves through the node, so this is

    what CoreDNS will be able to reach; some networks block specific public

    resolvers (e.g. 1.1.1.1 is often hijacked by routers/ISPs).

    """

    requires_command = "python3"



    def default(self):

        return []



    def command(self, servers):

        return (

            "python3 - " + " ".join(servers) + " <<'EOF'\n"

            "import socket, struct, sys\n"

            "query = struct.pack('>HHHHHH', 0x4d45, 0x0100, 1, 0, 0, 0)\n"

            "query += b''.join(bytes([len(p)]) + p.encode() for p in 'github.com'.split('.'))\n"

            "query += b'\\0' + struct.pack('>HH', 1, 1)\n"

            "for server in sys.argv[1:]:\n"

            "    try:\n"

            "        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n"

            "        s.settimeout(2)\n"

            "        s.sendto(query, (server, 53))\n"

            "        if s.recv(512)[:2] == query[:2]:\n"

            "            print(server)\n"

            "    except OSError:\n"

            "        pass\n"

            "EOF"

        )



    def process(self, output):

        return [line.strip() for line in output if line.strip()]





@operation()

def apply(manifest_path, kubeconfig=None, server_side=False, wait_resource=None, wait_condition="Available", wait_namespace=None, wait_timeout="300s"):

    """
@@ -102,6 +139,49 @@ def patch_configmap(configmap_name, namespace, search_string, replace_string, ro 
        yield StringCommand(f"{env_str}kubectl rollout restart {rollout_restart} -n {namespace}")

        yield StringCommand(f"{env_str}kubectl rollout status {rollout_restart} -n {namespace} --timeout=60s")



@operation()

def configure_coredns_upstreams(servers, kubeconfig=None):

    """

    Points CoreDNS at the given upstream resolvers, keeping only those that

    actually answer from this node. Servers are tried strictly in the given

    order (`policy sequential`), so later ones are fallbacks, not load-shared.

    Idempotent: rewrites whatever `forward .` block exists (kubeadm's

    /etc/resolv.conf or a previous list) and leaves the ConfigMap alone when

    it already matches.

    """

    reachable = host.get_fact(ReachableDnsServers, servers=servers)

    unreachable = [s for s in servers if s not in reachable]

    if unreachable:

        host.noop(f"Skipping unreachable DNS servers: {', '.join(unreachable)}")

    if not reachable:

        raise OperationError(

            f"None of the DNS servers {servers} answer from this host; "

            "set CLUSTER_DNS_SERVERS in .env to resolvers reachable on this network"

        )



    forward = "forward . " + " ".join(reachable) + " {"

    block = forward + "\n       max_concurrent 1000\n       policy sequential\n    }"

    current_yaml = host.get_fact(ConfigMap, configmap_name="coredns", namespace="kube-system", kubeconfig=kubeconfig)

    if current_yaml and forward in current_yaml and "policy sequential" in current_yaml:

        return  # Already configured



    # Replace the whole forward block (header and options) on the node, so

    # re-runs with a different server list never duplicate options

    rewrite = (

        "import json, re, sys; "

        "cm = json.load(sys.stdin); "

        "cm['data']['Corefile'] = re.sub(r'forward \\. [^{]*\\{[^}]*\\}', lambda m: sys.argv[1], cm['data']['Corefile'], count=1); "

        "json.dump(cm, sys.stdout)"

    )

    env_str = _get_kubeconfig_env(kubeconfig)

    yield StringCommand(

        f"{env_str}kubectl get configmap coredns -n kube-system -o json | "

        f"python3 -c {shlex.quote(rewrite)} {shlex.quote(block)} | "

        f"{env_str}kubectl apply -f -"

    )

    yield StringCommand(f"{env_str}kubectl rollout restart deployment/coredns -n kube-system")

    yield StringCommand(f"{env_str}kubectl rollout status deployment/coredns -n kube-system --timeout=60s")



@operation()

def init_control_plane(pod_network_cidr):

    """

deploy/pyinfra/tasks/container_runtime/docker.py

+25 −0
Changes for deploy/pyinfra/tasks/container_runtime/docker.py: 25 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -99,6 +99,31 @@ if socket_stat != "missing" and socket_stat != "docker:660": 
        _sudo=True

    )



# Stop Docker from setting the iptables FORWARD policy to DROP on every start.

# That policy also drops Kubernetes pod egress (Calico's own accept rules live

# in a separate nftables table, and a drop in any table wins), so after any

# Docker restart (e.g. meepctl's registry cert trust step) pods lose internet

# access and CoreDNS upstream lookups time out ("server misbehaving").

# Merges the key into an existing daemon.json and restarts Docker only when it changed.

server.shell(

    name="Configure Docker daemon to not drop forwarded (pod) traffic",

    commands=[

        "python3 - <<'EOF'\n"

        "import json, os, subprocess\n"

        "path = '/etc/docker/daemon.json'\n"

        "cfg = json.load(open(path)) if os.path.exists(path) and os.path.getsize(path) else {}\n"

        "if cfg.get('ip-forward-no-drop') is not True:\n"

        "    cfg['ip-forward-no-drop'] = True\n"

        "    os.makedirs('/etc/docker', exist_ok=True)\n"

        "    json.dump(cfg, open(path, 'w'), indent=2)\n"

        "    subprocess.run(['systemctl', 'restart', 'docker'], check=True)\n"

        "EOF",

        # Undo a DROP policy already set by an earlier Docker start

        "iptables -P FORWARD ACCEPT",

    ],

    _sudo=True,

)



# Configure Docker MTU to 1450 to match Calico overlay and prevent DNS timeouts

# server.shell(

#     name="Configure Docker daemon.json with MTU 1450",

deploy/pyinfra/tasks/k8s_cluster/cni_calico.py

+5 −9
Changes for deploy/pyinfra/tasks/k8s_cluster/cni_calico.py: 5 added lines, 9 removed lines.
Original line number Diff line number Diff line
@@ -50,15 +50,11 @@ kubernetes.wait_for_condition( 
    kubeconfig=kubeconfig_path

)



# Patch CoreDNS ConfigMap to use public DNS resolvers and restart it



kubernetes.patch_configmap(

    name="Patch CoreDNS ConfigMap to use public DNS resolvers and restart it",

    configmap_name="coredns",

    namespace="kube-system",

    search_string=r"forward \. /etc/resolv\.conf",

    replace_string="forward . 8.8.8.8 1.1.1.1",

    rollout_restart="deployment/coredns",

# Point CoreDNS at public resolvers reachable from this network (unreachable

# ones are dropped, so a blocked resolver can't cause slow/failed lookups)

kubernetes.configure_coredns_upstreams(

    name="Configure CoreDNS upstream DNS servers",

    servers=host.data.get('cluster_dns_servers'),

    kubeconfig=kubeconfig_path

)