[sunbeam-clusterd] implement tls certificates interface

Optionally allow clusterd to be integrated with tls certificates
interface. When integrated, get a certificates managed by the leader
replacing the cluster certificates auto-generated by microcluster.

Change-Id: Ia019bd533962976ddc68e2b93bcdcbe28a5cff9c
Signed-off-by: Guillaume Boutry <guillaume.boutry@canonical.com>

charms/sunbeam-clusterd/charmcraft.yaml

@@ -52,3 +52,8 @@ config:
     debug:
       default: False
       type: boolean
+
+requires:
+  certificates:
+    interface: tls-certificates
+    optional: True

charms/sunbeam-clusterd/src/charm.py

@@ -21,7 +21,9 @@ This charm manages a clusterd deployment. Clusterd is a service storing
 every metadata about a sunbeam deployment.
 """
 
+import hashlib
 import logging
+import socket
 from pathlib import (
     Path,
 )
@@ -30,11 +32,18 @@ import clusterd
 import ops.framework
 import ops_sunbeam.charm as sunbeam_charm
 import ops_sunbeam.guard as sunbeam_guard
+import ops_sunbeam.relation_handlers as sunbeam_rhandlers
 import requests
 import tenacity
 from charms.operator_libs_linux.v2 import (
     snap,
 )
+from charms.tls_certificates_interface.v3.tls_certificates import (
+    generate_csr,
+)
+from cryptography import (
+    x509,
+)
 from ops.main import (
     main,
 )
@@ -55,6 +64,87 @@ def _identity(x: bool) -> bool:
     return x
 
 
+class ClusterCertificatesHandler(sunbeam_rhandlers.TlsCertificatesHandler):
+    """Handler for certificates interface."""
+
+    def get_entity(self) -> ops.Unit | ops.Application:
+        """Return the entity for the key store."""
+        return self.charm.model.app
+
+    def key_names(self) -> list[str]:
+        """Return the key names managed by this relation.
+
+        First key is considered as default key.
+        """
+        return ["main", "client"]
+
+    def csrs(self) -> dict[str, bytes]:
+        """Return a dict of generated csrs for self.key_names().
+
+        The method calling this method will ensure that all keys have a matching
+        csr.
+        """
+        main_key = self._private_keys.get("main")
+        client_key = self._private_keys.get("client")
+        if not main_key or not client_key:
+            return {}
+        return {
+            "main": generate_csr(
+                private_key=main_key.encode(),
+                subject=self.charm.app.name,
+                sans_ip=self.sans_ips,
+                sans_dns=self.sans_dns,
+                additional_critical_extensions=[
+                    x509.KeyUsage(
+                        digital_signature=True,
+                        content_commitment=False,
+                        key_encipherment=True,
+                        data_encipherment=False,
+                        key_agreement=False,
+                        key_cert_sign=False,
+                        crl_sign=False,
+                        encipher_only=False,
+                        decipher_only=False,
+                    ),
+                    x509.ExtendedKeyUsage({x509.OID_SERVER_AUTH}),
+                ],
+            ),
+            "client": generate_csr(
+                private_key=client_key.encode(),
+                subject=self.charm.app.name + "-client",
+                additional_critical_extensions=[
+                    x509.KeyUsage(
+                        digital_signature=True,
+                        content_commitment=False,
+                        key_encipherment=True,
+                        data_encipherment=False,
+                        key_agreement=False,
+                        key_cert_sign=False,
+                        crl_sign=False,
+                        encipher_only=False,
+                        decipher_only=False,
+                    ),
+                    x509.ExtendedKeyUsage({x509.OID_CLIENT_AUTH}),
+                ],
+            ),
+        }
+
+    def get_client_keypair(self) -> dict[str, str]:
+        """Return client keypair with the CA."""
+        client_key = self.store.get_private_key("client")
+        client_csr = self.store.get_csr("client")
+        if client_key is None or client_csr is None:
+            return {}
+        for cert in self.get_certs():
+            if cert.csr == client_csr:
+                return {
+                    "certificate-authority": cert.ca,
+                    "certificate": cert.certificate,
+                    "private-key-secret": client_key,
+                }
+        return {}
+
+
 class SunbeamClusterdCharm(sunbeam_charm.OSBaseOperatorCharm):
     """Charm the service."""
 
@@ -65,7 +155,9 @@ class SunbeamClusterdCharm(sunbeam_charm.OSBaseOperatorCharm):
     def __init__(self, framework: ops.Framework) -> None:
         """Run constructor."""
         super().__init__(framework)
-        self._state.set_default(channel="config", departed=False)
+        self._state.set_default(
+            channel="config", departed=False, certs_hash=""
+        )
         self.framework.observe(self.on.install, self._on_install)
         self.framework.observe(self.on.stop, self._on_stop)
         self.framework.observe(
@@ -88,8 +180,26 @@ class SunbeamClusterdCharm(sunbeam_charm.OSBaseOperatorCharm):
                 "peers" in self.mandatory_relations,
             )
             handlers.append(self.peers)
+        if self.can_add_handler("certificates", handlers):
+            self.certs = ClusterCertificatesHandler(
+                self,
+                "certificates",
+                self.configure_charm,
+                self.get_domain_name_sans(),
+                self.get_sans_ips(),
+                mandatory="certificates" in self.mandatory_relations,
+            )
+            handlers.append(self.certs)
         return super().get_relation_handlers(handlers)
 
+    def get_domain_name_sans(self) -> list[str]:
+        """Return domain name sans."""
+        return [socket.gethostname()]
+
+    def get_sans_ips(self) -> list[str]:
+        """Return Subject Alternate Names to use in cert for service."""
+        return ["127.0.0.1"]
+
     def _on_install(self, event: ops.InstallEvent) -> None:
         """Handle install event."""
         try:
@@ -115,13 +225,23 @@ class SunbeamClusterdCharm(sunbeam_charm.OSBaseOperatorCharm):
         """Handle get-credentials action."""
         if not self.peers.interface.state.joined:
             event.fail("Clusterd not joined yet")
+            return
+
+        credentials = {}
+        if relation := self.model.get_relation(self.certs.relation_name):
+            if relation.active:
+                credentials = self.certs.get_client_keypair()
+            if not credentials:
+                event.fail("No credentials found yet")
+                return
 
         event.set_results(
             {
                 "url": "https://"
                 + self._binding_address()
                 + ":"
-                + str(self.clusterd_port)
+                + str(self.clusterd_port),
+                **credentials,
             }
         )
 
@@ -174,10 +294,11 @@ class SunbeamClusterdCharm(sunbeam_charm.OSBaseOperatorCharm):
         if not self.clusterd_ready():
             logger.debug("Clusterd not ready yet.")
             event.defer()
-            return
+            raise sunbeam_guard.WaitingExceptionError("Clusterd not ready yet")
         if not self.is_leader_ready():
             self.bootstrap_cluster()
             self.peers.interface.state.joined = True
+        self.configure_certificates()
         super().configure_app_leader(event)
         if isinstance(event, ClusterdNewNodeEvent):
             self.add_node_to_cluster(event)
@@ -200,6 +321,26 @@ class SunbeamClusterdCharm(sunbeam_charm.OSBaseOperatorCharm):
         }
         self.set_snap_data(snap_data)
 
+    def configure_certificates(self):
+        """Configure certificates."""
+        if not self.unit.is_leader():
+            logger.debug("Not leader, skipping certificate configuration.")
+            return
+        if not self.certs.ready:
+            logger.debug("Certificates not ready yet.")
+            return
+        certs = self.certs.context()
+        certs_hash = hashlib.sha256(bytes(str(certs), "utf-8")).hexdigest()
+        if certs_hash == self._state.certs_hash:
+            logger.debug("Certificates have not changed.")
+            return
+        self._clusterd.set_certs(
+            ca=certs["ca_cert_main"],
+            key=certs["key_main"],
+            cert=certs["cert_main"],
+        )
+        self._state.certs_hash = certs_hash
+
     def set_snap_data(self, snap_data: dict):
         """Set snap data on local snap."""
         cache = snap.SnapCache()
@@ -242,7 +383,6 @@ class SunbeamClusterdCharm(sunbeam_charm.OSBaseOperatorCharm):
             self.unit.name.replace("/", "-"),
             self._binding_address() + ":" + str(self.clusterd_port),
         )
-        self.status.set(ops.ActiveStatus())
 
     def add_node_to_cluster(self, event: ClusterdNewNodeEvent) -> None:
         """Generate token for node joining."""
@@ -277,7 +417,6 @@ class SunbeamClusterdCharm(sunbeam_charm.OSBaseOperatorCharm):
         already_left = self._wait_for_roles_to_settle_before_removal(
             event, self_departing
         )
-        self.status.set(ops.ActiveStatus())
         if already_left:
             return
 
@@ -309,7 +448,6 @@ class SunbeamClusterdCharm(sunbeam_charm.OSBaseOperatorCharm):
             raise sunbeam_guard.WaitingExceptionError(
                 "Waiting for roles to settle"
             )
-        self.status.set(ops.ActiveStatus())
 
     def _wait_for_roles_to_settle_before_removal(
         self, event: ops.EventBase, self_departing: bool
@@ -399,7 +537,6 @@ class SunbeamClusterdCharm(sunbeam_charm.OSBaseOperatorCharm):
             logger.debug("Member %s is still pending", member)
             event.defer()
             return
-        self.status.set(ops.ActiveStatus())
 
     def _wait_until_role_set(self, name: str) -> bool:
         @tenacity.retry(

charms/sunbeam-clusterd/src/clusterd.py

@@ -82,6 +82,9 @@ class ClusterdClient:
     def _post(self, path, data=None, json=None, **kwargs):
         return self._request("post", path, data=data, json=json, **kwargs)
 
+    def _put(self, path, data=None, json=None, **kwargs):
+        return self._request("put", path, data=data, json=json, **kwargs)
+
     def _delete(self, path, **kwargs):
         return self._request("delete", path, **kwargs)
 
@@ -177,3 +180,15 @@ class ClusterdClient:
         data = {"name": name}
         result = self._post("/cluster/1.0/tokens", data=json.dumps(data))
         return str(result["metadata"])
+
+    def set_certs(self, ca: str, cert: str, key: str):
+        """Configure cluster certificates.
+
+        The CA is not set in the cluster certificates, but in the config endpoint.
+        This is because we don't want microcluster to go full CA-mode.
+        """
+        self._put("/1.0/config/cluster-ca", data=ca)
+        data = {"public_key": cert, "private_key": key}
+        self._put(
+            "/cluster/internal/cluster/certificates", data=json.dumps(data)
+        )

common.sh

@@ -165,6 +165,7 @@ EXTERNAL_OPENSTACK_IMAGES_SYNC_LIBS=(
 
 EXTERNAL_SUNBEAM_CLUSTERD_LIBS=(
 	"operator_libs_linux"
+	"tls_certificates_interface"
 )
 
 EXTERNAL_SUNBEAM_MACHINE_LIBS=(

tests/local/zaza/sunbeam/charm_tests/clusterd/tests.py

@@ -13,17 +13,34 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import base64
+import contextlib
 import json
 import logging
 import subprocess
+import tempfile
 import unittest
 from random import shuffle
 from typing import Tuple
 
 import requests
+import requests.adapters
 import tenacity
+import zaza
 import zaza.model as model
 import zaza.openstack.charm_tests.test_utils as test_utils
+from juju.client import client
+from juju.model import Model
+
+
+@contextlib.contextmanager
+def keypair(certificate: bytes, private_key: bytes):
+    with tempfile.NamedTemporaryFile() as cert_file, tempfile.NamedTemporaryFile() as key_file:
+        cert_file.write(certificate)
+        cert_file.flush()
+        key_file.write(private_key)
+        key_file.flush()
+        yield (cert_file.name, key_file.name)
 
 
 class ClusterdTest(test_utils.BaseCharmTest):
@@ -69,16 +86,46 @@ class ClusterdTest(test_utils.BaseCharmTest):
         for unit in units:
             model.block_until_unit_wl_status(unit, "active", timeout=60 * 5)
 
+    async def _read_secret(
+        self, model: Model, secret_id: str
+    ) -> dict[str, str]:
+        facade = client.SecretsFacade.from_connection(model.connection())
+        secrets = await facade.ListSecrets(
+            filter_={"uri": secret_id}, show_secrets=True
+        )
+        if len(secrets.results) != 1:
+            self.fail("Secret not found")
+        return secrets["results"][0].value.data
+
     def test_100_connect_to_clusterd(self):
         """Try sending data to an endpoint."""
         action = model.run_action_on_leader(
             self.application_name, "get-credentials"
         )
         url = action.data["results"]["url"] + "/1.0/config/100_connect"
-        response = requests.put(url, json={"data": "test"}, verify=False)
+        private_key_secret = action.data["results"].get("private-key-secret")
-        response.raise_for_status()
+        certificate = action.data["results"].get("certificate")
-        response = requests.get(url, verify=False)
+        if private_key_secret is None or certificate is None:
-        response.raise_for_status()
+            context = contextlib.nullcontext()
+            logging.debug("Request made without mTLS")
+        else:
+            model_impl = zaza.sync_wrapper(model.get_model)()
+            private_key = base64.b64decode(
+                zaza.sync_wrapper(self._read_secret)(
+                    model_impl, private_key_secret
+                )["private-key"]
+            )
+            context = keypair(certificate.encode(), private_key)
+            logging.debug("Request made with mTLS")
+
+        with context as cert:
+            response = requests.put(
+                url, json={"data": "test"}, verify=False, cert=cert
+            )
+            response.raise_for_status()
+            response = requests.get(url, verify=False, cert=cert)
+            response.raise_for_status()
+
         self.assertEqual(
             json.loads(response.json()["metadata"])["data"], "test"
         )
@@ -106,14 +153,11 @@ class ClusterdTest(test_utils.BaseCharmTest):
         """Scale back to 3."""
         self._add_2_units()
 
-
     @unittest.skip("Skip until scale down stable")
     def test_203_scale_down_to_2_units(self):
         """Scale down to 2 units for voter/spare test."""
         leader = model.get_lead_unit_name(self.application_name)
-        model.destroy_unit(
+        model.destroy_unit(self.application_name, leader, wait_disappear=True)
-            self.application_name, leader, wait_disappear=True
-        )
         model.block_until_all_units_idle()
 
         units = self._get_units()