Initial movement of code from nginx to helper (e06c17b5) · Commits · OCF / capif

services/docker-compose-capif.yml

+1 −1

Changes for services/docker-compose-capif.yml: 1 added line, 1 removed line.
Original line number	Diff line number	Diff line
		@@ -22,7 +22,7 @@ services:
		volumes:
		- ${SERVICES_DIR}/helper/config.yaml:/usr/src/app/config.yaml
		# - ${SERVICES_DIR}/nginx/certs:/usr/src/app/helper_service/server_certs:ro
		# - ${SERVICES_DIR}/helper/helper_service/certs:/usr/src/app/helper_service/certs
		- ${SERVICES_DIR}/helper/helper_service/certs:/usr/src/app/helper_service/certs

		extra_hosts:
		- host.docker.internal:host-gateway

services/helper/helper_service/app.py

+184 −38

Changes for services/helper/helper_service/app.py: 184 added lines, 38 removed lines.
Original line number	Diff line number	Diff line
		@@ -15,10 +15,16 @@ from cryptography import x509
		from cryptography.hazmat.primitives import hashes, serialization
		from cryptography.hazmat.primitives.asymmetric import rsa
		from cryptography.x509.oid import NameOID
		from datetime import datetime, timedelta, timezone


		CCF_CERTIFICATE_NAME = "server"

		# --- Paths setup: make 'services' discoverable so "import api..." works ---
		BASE_DIR = Path(__file__).resolve().parent
		SERVICES_DIR = BASE_DIR / "services"
		# Save superadmin private key
		CERTS_DIR = Path(__file__).resolve().parent / "certs"

		# Insert services directory at front of sys.path
		if SERVICES_DIR.is_dir():
		@@ -73,25 +79,119 @@ def configure_logging(app):
		l.propagate = False
		l.setLevel(numeric_level)

		def certificate_needs_renewal(
		certificate_path: Path,
		renewal_window_days: int = 7,
		) -> bool:
		"""
		Return True if the certificate is expired or will expire
		within the specified renewal window.
		"""
		certificate = x509.load_pem_x509_certificate(
		certificate_path.read_bytes()
		)

		# Log level setup
		log_level = os.getenv('LOG_LEVEL', 'INFO').upper()
		numeric_level = getattr(logging, log_level, logging.INFO)
		configure_logging(app.app)
		# For newer versions of the cryptography package.
		expiration_date = certificate.not_valid_after_utc

		# Logger for this module
		logger = logging.getLogger(__name__)
		current_time = datetime.now(timezone.utc)
		renewal_deadline = current_time + timedelta(
		days=renewal_window_days
		)

		logger.debug(f"Certificate expires on: {expiration_date.isoformat()}")
		logger.debug(
		f"Time remaining: {expiration_date - current_time}"
		)

		return expiration_date <= renewal_deadline

		def load_private_key(
		private_key_path: Path,
		password: bytes \| None = None):
		try:
		return serialization.load_pem_private_key(
		private_key_path.read_bytes(),
		password=password,
		)
		except ValueError as error:
		logger.error(
		f"Could not load private key from {private_key_path}. Check whether the key is encrypted and whether the password is correct."
		)
		return None

		def obtain_ccf_certificates(ccf_id,certificate_name, ca_name="ca.crt"):
		logger.info(f"Checking if vault has information of {ccf_id}")
		url = 'http://{}:{}/v1/secret/data/capif/{}/nginx'.format(config["ca_factory"]["url"], config["ca_factory"]["port"], ccf_id)
		headers = {'X-Vault-Token': f"{config["ca_factory"]["token"]}"}
		response = requests.request("GET", url, headers=headers, verify = config["ca_factory"].get("verify", False))
		if response.status_code != 200:
		logger.error(f"Failed to obtain certificates for {ccf_id}: {response.text}")
		return None
		else:
		key_path = CERTS_DIR / (certificate_name+".key")
		pub_path = CERTS_DIR / (certificate_name+".pem")
		crt_path = CERTS_DIR / (certificate_name+".crt")
		ca_path = CERTS_DIR / (ca_name)
		data = response.json().get("data", {}).get("data", {})
		if not data:
		logger.error(f"No certificate data found for {ccf_id}")
		return None

		key = data.get("server_key")
		crt = data.get("server_crt")
		pub = data.get("server_pub")
		ca = data.get("ca")

		if not all([key, crt, pub, ca]):
		logger.error(f"Incomplete certificate data for {ccf_id}")
		return None

		with open(key_path, "w") as f:
		f.write(key)
		with open(crt_path, "w") as f:
		f.write(crt)
		with open(pub_path, "w") as f:
		f.write(pub)
		with open(ca_path, "w") as f:
		f.write(ca)

		if certificate_needs_renewal(
		crt_path,
		renewal_window_days=30,
		):
		logger.debug("The certificate is expired or will expire soon. remove local file")
		# Request and store a new certificate.
		os.remove(crt_path)
		os.remove(pub_path)

		else:
		logger.debug("The certificate is still valid.")

		# Create a superadmin CSR and keys
		try:
		os.chmod(key_path, 0o600)
		except Exception as e:
		logger.warning(f"Could not chmod {key_path} to 600: {e}")

		def create_certificate(certificate_name, common_name, ttl="43000h", upload_to_vault=False, ca_name="ca.crt"):
		key_path = CERTS_DIR / (certificate_name+".key")
		pub_path = CERTS_DIR / (certificate_name+".pem")
		crt_path = CERTS_DIR / (certificate_name+".crt")
		ca_path = CERTS_DIR / (ca_name)
		if not key_path.exists() or not crt_path.exists():
		key = None
		if key_path.exists():
		key = load_private_key(key_path)
		if key is None:
		# Create a superadmin CSR and keys if there is no crt
		key = rsa.generate_private_key(public_exponent=65537, key_size=2048)

		subject = x509.Name([
		x509.NameAttribute(NameOID.ORGANIZATION_NAME, 'OCF helper'),
		x509.NameAttribute(NameOID.ORGANIZATIONAL_UNIT_NAME, 'helper'),
		x509.NameAttribute(NameOID.LOCALITY_NAME, 'Madrid'),
		x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, 'Madrid'),
		x509.NameAttribute(NameOID.COUNTRY_NAME, 'ES'),
		# x509.NameAttribute(NameOID.COMMON_NAME, "superadmin{}".format(os.getenv("CAPIF_HOSTNAME"))),
		x509.NameAttribute(NameOID.EMAIL_ADDRESS, 'helper@tid.es')
		])
		req = x509.CertificateSigningRequestBuilder().subject_name(subject).sign(key, hashes.SHA256())
		@@ -103,21 +203,6 @@ private_key = key.private_bytes(
		encryption_algorithm=serialization.NoEncryption()
		)

		# Save superadmin private key
		CERTS_DIR = Path(__file__).resolve().parent / "certs"

		try:
		# If it exists but it's not a directory -> fail early with a clear error
		if CERTS_DIR.exists() and not CERTS_DIR.is_dir():
		raise RuntimeError(f"'certs' exists but is not a directory: {CERTS_DIR}")

		CERTS_DIR.mkdir(parents=True, exist_ok=True)

		# Quick sanity check: can we write there?
		if not os.access(CERTS_DIR, os.W_OK):
		raise PermissionError(f"No write permission on certs dir: {CERTS_DIR}")

		key_path = CERTS_DIR / "superadmin.key"
		with open(key_path, "wb") as f:
		f.write(private_key)

		@@ -127,33 +212,55 @@ try:
		except Exception as e:
		logger.warning(f"Could not chmod {key_path} to 600: {e}")

		logger.info(f"Superadmin key written to {key_path}")

		except Exception:
		logger.exception(f"Failed to write superadmin key under {CERTS_DIR}")
		raise

		logger.info(f"{certificate_name} key written to {key_path}")

		# Request superadmin certificate
		# Request certificate
		url = 'http://{}:{}/v1/pki_int/sign/my-ca'.format(config["ca_factory"]["url"], config["ca_factory"]["port"])
		headers = {'X-Vault-Token': f"{config["ca_factory"]["token"]}"}
		data = {
		'format':'pem_bundle',
		'ttl': ttl_superadmin_cert,
		'ttl': ttl,
		'csr': csr_request,
		# 'common_name': "superadmin{}".format(os.getenv("CAPIF_HOSTNAME")),
		'common_name': "superadmin",
		'common_name': common_name,
		'alt_names': "{}".format(os.getenv("CAPIF_HOSTNAME"))
		}

		response = requests.request("POST", url, headers=headers, data=data, verify = config["ca_factory"].get("verify", False))
		superadmin_cert = json.loads(response.text)['data']['certificate']
		logger.info(f"Superadmin Cert:\n{superadmin_cert}")
		cert = json.loads(response.text)['data']['certificate']
		logger.info(f"{certificate_name} Cert:\n{cert}")

		# Save the superadmin certificate
		with open(CERTS_DIR / "superadmin.crt", "wb") as cert_file:
		cert_file.write(superadmin_cert.encode("utf-8"))
		with open(crt_path, "wb") as cert_file:
		cert_file.write(cert.encode("utf-8"))

		certificate = x509.load_pem_x509_certificate(cert.encode("utf-8"))
		public_key_pem = certificate.public_key().public_bytes(
		encoding=serialization.Encoding.PEM,
		format=serialization.PublicFormat.SubjectPublicKeyInfo
		)
		with open(pub_path, "wb") as pub_file:
		pub_file.write(public_key_pem)

		if upload_to_vault:
		if ca_path.exists():
		url = 'http://{}:{}/v1/secret/data/capif/{}/nginx'.format(config["ca_factory"]["url"], config["ca_factory"]["port"], db.get_ccf_id())
		headers = {'X-Vault-Token': f"{config["ca_factory"]["token"]}"}
		data = {
		'server_key': open(key_path, "rb").read(),
		'server_crt': open(crt_path, "rb").read(),
		'server_pub': open(pub_path, "rb").read(),
		'ca': open(ca_path, "rb").read()
		}

		response = requests.request("POST", url, headers=headers, data=data, verify = config["ca_factory"].get("verify", False))
		else:
		logger.warning(f"CA path does not exist: {ca_path}")
		else:
		logger.info(f"{certifcate_name} already exists at {crt_path}")

		def obtain_ca(ca_name="ca.crt"):
		ca_file_path = CERTS_DIR / ca_name
		if not ca_file_path.exists():
		url = f"http://{config['ca_factory']['url']}:{config['ca_factory']['port']}/v1/secret/data/ca"
		headers = {

		@@ -163,9 +270,48 @@ response = requests.request("GET", url, headers=headers, verify = config["ca_fac

		ca_root = json.loads(response.text)['data']['data']['ca']
		logger.info(f"CA root:\n{ca_root}")
		with open(CERTS_DIR / "ca_root.crt", "wb") as cert_file:
		with open(ca_file_path, "wb") as cert_file:
		cert_file.write(ca_root.encode("utf-8"))
		else:
		logger.info(f"CA root already exists at {ca_file_path}")

		def create_cert_folder():
		try:
		if CERTS_DIR.exists() and not CERTS_DIR.is_dir():
		raise RuntimeError(f"'certs' exists but is not a directory: {CERTS_DIR}")

		CERTS_DIR.mkdir(parents=True, exist_ok=True)

		# Quick sanity check: can we write there?
		if not os.access(CERTS_DIR, os.W_OK):
		raise PermissionError(f"No write permission on certs dir: {CERTS_DIR}")
		except Exception:
		logger.exception(f"Failed to create {CERTS_DIR} folder")
		raise


		# Log level setup
		log_level = os.getenv('LOG_LEVEL', 'INFO').upper()
		numeric_level = getattr(logging, log_level, logging.INFO)
		configure_logging(app.app)

		# Logger for this module
		logger = logging.getLogger(__name__)

		# Try to get information from Vault
		obtain_ccf_certificates(ccf_id=db.get_ccf_id(),certificate_name=CCF_CERTIFICATE_NAME)

		# Check and create CERT_DIR
		create_cert_folder()

		# Get CA Root
		obtain_ca()

		# Create superadmin Certificate
		create_certificate(certificate_name="superadmin", common_name="superadmin", ttl=ttl_superadmin_cert)

		# Create Server Certificate
		create_certificate(certificate_name=CCF_CERTIFICATE_NAME, common_name=db.get_ccf_id(), ttl="43000h", upload_to_vault=True)

		package_paths = config.get("package_paths", {})

services/helper/helper_service/db/db.py

+5 −0

Changes for services/helper/helper_service/db/db.py: 5 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -93,6 +93,11 @@ class MongoDatabse():
		else:
		print("Capif_configuration already contains data with a unique ccf_id. No default values inserted.")

		def get_ccf_id(self):
		capif_col = self.get_col_by_name(self.capif_configuration)
		existing_config = capif_col.find_one({}, {"_id": 0})
		return existing_config.get("ccf_id") if existing_config else None


		_singleton = None

services/helper/helper_service/services/api/core/helper_operations.py

+2 −2

Changes for services/helper/helper_service/services/api/core/helper_operations.py: 2 added lines, 2 removed lines.
Original line number	Diff line number	Diff line
		@@ -185,14 +185,14 @@ class HelperOperations:
		current_app.logger.debug(f"Removing Invoker: {invoker["api_invoker_id"]}")
		url = 'https://{}/api-invoker-management/v1/onboardedInvokers/{}'.format(os.getenv('CAPIF_HOSTNAME'), invoker["api_invoker_id"])
		requests.request("DELETE", url, cert=(
		'/usr/src/app/helper_service/certs/superadmin.crt', '/usr/src/app/helper_service/certs/superadmin.key'), verify='/usr/src/app/helper_service/certs/ca_root.crt')
		'/usr/src/app/helper_service/certs/superadmin.crt', '/usr/src/app/helper_service/certs/superadmin.key'), verify='/usr/src/app/helper_service/certs/ca.crt')

		for provider in provider_col.find({'uuid':uuid}, {"_id":0}):
		current_app.logger.debug(f"Removing Provider: {provider["api_prov_dom_id"]}")
		url = 'https://{}/api-provider-management/v1/registrations/{}'.format(os.getenv('CAPIF_HOSTNAME'), provider["api_prov_dom_id"])

		requests.request("DELETE", url, cert=(
		'/usr/src/app/helper_service/certs/superadmin.crt', '/usr/src/app/helper_service/certs/superadmin.key'), verify='/usr/src/app/helper_service/certs/ca_root.crt')
		'/usr/src/app/helper_service/certs/superadmin.crt', '/usr/src/app/helper_service/certs/superadmin.key'), verify='/usr/src/app/helper_service/certs/ca.crt')
		except Exception as e:
		current_app.logger.debug(f"Error deleting user entities: {e}")
		jsonify(message=f"Error deleting user entities: {e}"), 500

services/helper/prepare_helper.sh

+39 −39

Changes for services/helper/prepare_helper.sh: 39 added lines, 39 removed lines.
Original line number	Diff line number	Diff line
		@@ -2,45 +2,45 @@

		VAULT_ADDR="http://$VAULT_HOSTNAME:$VAULT_PORT"
		VAULT_TOKEN=$VAULT_ACCESS_TOKEN
		CERTS_DIR=/usr/src/app/helper_service/certs
		mkdir -p "$CERTS_DIR"
		MAX_RETRIES=60; RETRY_DELAY=10
		# CERTS_DIR=/usr/src/app/helper_service/certs
		# mkdir -p "$CERTS_DIR"
		# MAX_RETRIES=60; RETRY_DELAY=10

		fetch_certs() {
		local attempt=0 ccf_id="" resp crt key ca
		while [ -z "$ccf_id" ] && [ $attempt -lt $MAX_RETRIES ]; do
		attempt=$((attempt+1))
		ccf_id=$(curl -sS --max-time 10 "http://helper:8080/helper/api/getCcfId" \| jq -r '.ccf_id // empty')
		[ -z "$ccf_id" ] && sleep $RETRY_DELAY
		done
		[ -z "$ccf_id" ] && echo "[prepare_helper] no ccf_id; certs not fetched" && return 1
		attempt=0
		while [ $attempt -lt $MAX_RETRIES ]; do
		attempt=$((attempt+1))
		resp=$(curl -s -k --header "X-Vault-Token: $VAULT_TOKEN" \
		--request GET "$VAULT_ADDR/v1/secret/data/capif/${ccf_id}/nginx")
		# skip this attempt if Vault didn't return valid JSON yet
		if ! printf '%s' "$resp" \| jq -e . >/dev/null 2>&1; then
		echo "[prepare_helper] invalid/empty JSON from Vault, retrying"
		sleep $RETRY_DELAY
		continue
		fi
		crt=$(printf '%s' "$resp" \| jq -r '.data.data.server_crt // empty')
		key=$(printf '%s' "$resp" \| jq -r '.data.data.server_key // empty')
		ca=$(printf '%s' "$resp" \| jq -r '.data.data.ca // empty')
		if [ -n "$crt" ] && [ -n "$key" ] && [ -n "$ca" ]; then
		printf '%s\n' "$crt" > "$CERTS_DIR/server.crt"
		printf '%s\n' "$key" > "$CERTS_DIR/server.key"
		printf '%s\n' "$ca" > "$CERTS_DIR/ca.crt"
		chmod 600 "$CERTS_DIR/server.key"
		echo "[prepare_helper] certs fetched"
		return 0
		fi
		sleep $RETRY_DELAY
		done
		echo "[prepare_helper] timed out waiting for certs in Vault"
		return 1
		}
		# fetch_certs() {
		# local attempt=0 ccf_id="" resp crt key ca
		# while [ -z "$ccf_id" ] && [ $attempt -lt $MAX_RETRIES ]; do
		# attempt=$((attempt+1))
		# ccf_id=$(curl -sS --max-time 10 "http://helper:8080/helper/api/getCcfId" \| jq -r '.ccf_id // empty')
		# [ -z "$ccf_id" ] && sleep $RETRY_DELAY
		# done
		# [ -z "$ccf_id" ] && echo "[prepare_helper] no ccf_id; certs not fetched" && return 1
		# attempt=0
		# while [ $attempt -lt $MAX_RETRIES ]; do
		# attempt=$((attempt+1))
		# resp=$(curl -s -k --header "X-Vault-Token: $VAULT_TOKEN" \
		# --request GET "$VAULT_ADDR/v1/secret/data/capif/${ccf_id}/nginx")
		# # skip this attempt if Vault didn't return valid JSON yet
		# if ! printf '%s' "$resp" \| jq -e . >/dev/null 2>&1; then
		# echo "[prepare_helper] invalid/empty JSON from Vault, retrying"
		# sleep $RETRY_DELAY
		# continue
		# fi
		# crt=$(printf '%s' "$resp" \| jq -r '.data.data.server_crt // empty')
		# key=$(printf '%s' "$resp" \| jq -r '.data.data.server_key // empty')
		# ca=$(printf '%s' "$resp" \| jq -r '.data.data.ca // empty')
		# if [ -n "$crt" ] && [ -n "$key" ] && [ -n "$ca" ]; then
		# printf '%s\n' "$crt" > "$CERTS_DIR/server.crt"
		# printf '%s\n' "$key" > "$CERTS_DIR/server.key"
		# printf '%s\n' "$ca" > "$CERTS_DIR/ca.crt"
		# chmod 600 "$CERTS_DIR/server.key"
		# echo "[prepare_helper] certs fetched"
		# return 0
		# fi
		# sleep $RETRY_DELAY
		# done
		# echo "[prepare_helper] timed out waiting for certs in Vault"
		# return 1
		# }

		# helper calling itself. But helper's gunicorn only starts after this loop.
		# So nothing is listening on :8080 during the loop
		@@ -49,7 +49,7 @@ fetch_certs() {
		# So helper blocking on the certs waits for something that can't happen until helper is up
		# So helper must start gunicorn first and fetch the certs in the background.

		fetch_certs & # background — keep :8080 free so nginx can call getCcfId
		# fetch_certs & # background — keep :8080 free so nginx can call getCcfId

		exec gunicorn -k uvicorn.workers.UvicornWorker --bind 0.0.0.0:8080 \
		--chdir /usr/src/app/helper_service wsgi:asgi_app
		No newline at end of file