Commit 35768f96 authored by George Papathanail's avatar George Papathanail
Browse files

fix: retry JWKS warm-up in /readyz while keycloak is unhealthy

parent 6705eeb5

src/open_exposure_gateway/api/platform/health.py

+4 −0
Changes for src/open_exposure_gateway/api/platform/health.py: 4 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -33,6 +33,10 @@ async def readyz( 
    token_verifier: Annotated[TokenVerifierPort, Depends(get_token_verifier)],

) -> HealthResponse:

    settings = get_settings()

    if not token_verifier.is_healthy:

        # Retry here: an unready pod gets no token-bearing traffic, so nothing

        # else would re-fetch the JWKS once Keycloak comes back.

        await token_verifier.warm()

    if not publisher.is_connected or not database_is_healthy or not token_verifier.is_healthy:

        response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE

        return HealthResponse(

src/open_exposure_gateway/ports/token_verifier_port.py

+2 −0
Changes for src/open_exposure_gateway/ports/token_verifier_port.py: 2 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -6,5 +6,7 @@ from open_exposure_gateway.core.auth.tokens import DecodedClaims 
class TokenVerifierPort(Protocol):

    async def verify(self, token: str) -> DecodedClaims: ...



    async def warm(self) -> None: ...



    @property

    def is_healthy(self) -> bool: ...

tests/unit/fakes.py

+9 −2
Changes for tests/unit/fakes.py: 9 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -88,16 +88,23 @@ Handler = Callable[[dict[str, Any]], Awaitable[None]] 




class FakeTokenVerifier:

    """Only is_healthy is exercised: get_caller_context is always overridden

    with fake_caller_context in these tests, so verify() never gets called."""

    """Only is_healthy and warm() are exercised: get_caller_context is always

    overridden with fake_caller_context in these tests, so verify() never gets

    called."""



    def __init__(self) -> None:

        self.healthy = True

        self.keycloak_reachable = True

        self.warm_calls = 0



    @property

    def is_healthy(self) -> bool:

        return self.healthy



    async def warm(self) -> None:

        self.warm_calls += 1

        self.healthy = self.keycloak_reachable





class FakeDataBus:

    def __init__(self) -> None:

tests/unit/test_platform_flows.py

+21 −0
Changes for tests/unit/test_platform_flows.py: 21 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -25,12 +25,33 @@ class TestReadyzFlow: 
        """architecture.md E3: /readyz fails if Keycloak's JWKS is unreachable."""

        fake_verifier = FakeTokenVerifier()

        fake_verifier.healthy = False

        fake_verifier.keycloak_reachable = False

        app.dependency_overrides[get_token_verifier] = lambda: fake_verifier



        response = api_client.get("/platform/readyz")



        assert response.status_code == 503



    def test_recovers_once_keycloak_is_reachable(self, api_client: TestClient) -> None:

        """Keycloak down at startup must not latch the pod unready forever."""

        fake_verifier = FakeTokenVerifier()

        fake_verifier.healthy = False

        fake_verifier.keycloak_reachable = False

        app.dependency_overrides[get_token_verifier] = lambda: fake_verifier



        assert api_client.get("/platform/readyz").status_code == 503



        fake_verifier.keycloak_reachable = True

        assert api_client.get("/platform/readyz").status_code == 200

        assert fake_verifier.warm_calls == 2



    def test_does_not_refetch_jwks_when_healthy(self, api_client: TestClient) -> None:

        fake_verifier = FakeTokenVerifier()

        app.dependency_overrides[get_token_verifier] = lambda: fake_verifier



        assert api_client.get("/platform/readyz").status_code == 200

        assert fake_verifier.warm_calls == 0





class TestHealthzFlow:

    def test_liveness_does_not_depend_on_downstreams(