From c81a45e5f992df87a4cb27c1e3b635046aa55aa2 Mon Sep 17 00:00:00 2001 From: Guillaume Boutry Date: Thu, 27 Jun 2024 15:52:07 +0200 Subject: [PATCH] [sunbeam-clusterd] implement tls certificates interface Optionally allow clusterd to be integrated with tls certificates interface. When integrated, get a certificates managed by the leader replacing the cluster certificates auto-generated by microcluster. Change-Id: Ia019bd533962976ddc68e2b93bcdcbe28a5cff9c Signed-off-by: Guillaume Boutry --- charms/sunbeam-clusterd/charmcraft.yaml | 5 + charms/sunbeam-clusterd/src/charm.py | 151 +++++++++++++++++- charms/sunbeam-clusterd/src/clusterd.py | 15 ++ common.sh | 1 + .../sunbeam/charm_tests/clusterd/tests.py | 60 ++++++- 5 files changed, 217 insertions(+), 15 deletions(-) diff --git a/charms/sunbeam-clusterd/charmcraft.yaml b/charms/sunbeam-clusterd/charmcraft.yaml index 18d687bd..3f59e880 100644 --- a/charms/sunbeam-clusterd/charmcraft.yaml +++ b/charms/sunbeam-clusterd/charmcraft.yaml @@ -52,3 +52,8 @@ config: debug: default: False type: boolean + +requires: + certificates: + interface: tls-certificates + optional: True diff --git a/charms/sunbeam-clusterd/src/charm.py b/charms/sunbeam-clusterd/src/charm.py index 0aa4a234..cbb1d36c 100755 --- a/charms/sunbeam-clusterd/src/charm.py +++ b/charms/sunbeam-clusterd/src/charm.py @@ -21,7 +21,9 @@ This charm manages a clusterd deployment. Clusterd is a service storing every metadata about a sunbeam deployment. """ +import hashlib import logging +import socket from pathlib import ( Path, ) @@ -30,11 +32,18 @@ import clusterd import ops.framework import ops_sunbeam.charm as sunbeam_charm import ops_sunbeam.guard as sunbeam_guard +import ops_sunbeam.relation_handlers as sunbeam_rhandlers import requests import tenacity from charms.operator_libs_linux.v2 import ( snap, ) +from charms.tls_certificates_interface.v3.tls_certificates import ( + generate_csr, +) +from cryptography import ( + x509, +) from ops.main import ( main, ) @@ -55,6 +64,87 @@ def _identity(x: bool) -> bool: return x +class ClusterCertificatesHandler(sunbeam_rhandlers.TlsCertificatesHandler): + """Handler for certificates interface.""" + + def get_entity(self) -> ops.Unit | ops.Application: + """Return the entity for the key store.""" + return self.charm.model.app + + def key_names(self) -> list[str]: + """Return the key names managed by this relation. + + First key is considered as default key. + """ + return ["main", "client"] + + def csrs(self) -> dict[str, bytes]: + """Return a dict of generated csrs for self.key_names(). + + The method calling this method will ensure that all keys have a matching + csr. + """ + main_key = self._private_keys.get("main") + client_key = self._private_keys.get("client") + if not main_key or not client_key: + return {} + return { + "main": generate_csr( + private_key=main_key.encode(), + subject=self.charm.app.name, + sans_ip=self.sans_ips, + sans_dns=self.sans_dns, + additional_critical_extensions=[ + x509.KeyUsage( + digital_signature=True, + content_commitment=False, + key_encipherment=True, + data_encipherment=False, + key_agreement=False, + key_cert_sign=False, + crl_sign=False, + encipher_only=False, + decipher_only=False, + ), + x509.ExtendedKeyUsage({x509.OID_SERVER_AUTH}), + ], + ), + "client": generate_csr( + private_key=client_key.encode(), + subject=self.charm.app.name + "-client", + additional_critical_extensions=[ + x509.KeyUsage( + digital_signature=True, + content_commitment=False, + key_encipherment=True, + data_encipherment=False, + key_agreement=False, + key_cert_sign=False, + crl_sign=False, + encipher_only=False, + decipher_only=False, + ), + x509.ExtendedKeyUsage({x509.OID_CLIENT_AUTH}), + ], + ), + } + + def get_client_keypair(self) -> dict[str, str]: + """Return client keypair with the CA.""" + client_key = self.store.get_private_key("client") + client_csr = self.store.get_csr("client") + if client_key is None or client_csr is None: + return {} + for cert in self.get_certs(): + if cert.csr == client_csr: + return { + "certificate-authority": cert.ca, + "certificate": cert.certificate, + "private-key-secret": client_key, + } + return {} + + class SunbeamClusterdCharm(sunbeam_charm.OSBaseOperatorCharm): """Charm the service.""" @@ -65,7 +155,9 @@ class SunbeamClusterdCharm(sunbeam_charm.OSBaseOperatorCharm): def __init__(self, framework: ops.Framework) -> None: """Run constructor.""" super().__init__(framework) - self._state.set_default(channel="config", departed=False) + self._state.set_default( + channel="config", departed=False, certs_hash="" + ) self.framework.observe(self.on.install, self._on_install) self.framework.observe(self.on.stop, self._on_stop) self.framework.observe( @@ -88,8 +180,26 @@ class SunbeamClusterdCharm(sunbeam_charm.OSBaseOperatorCharm): "peers" in self.mandatory_relations, ) handlers.append(self.peers) + if self.can_add_handler("certificates", handlers): + self.certs = ClusterCertificatesHandler( + self, + "certificates", + self.configure_charm, + self.get_domain_name_sans(), + self.get_sans_ips(), + mandatory="certificates" in self.mandatory_relations, + ) + handlers.append(self.certs) return super().get_relation_handlers(handlers) + def get_domain_name_sans(self) -> list[str]: + """Return domain name sans.""" + return [socket.gethostname()] + + def get_sans_ips(self) -> list[str]: + """Return Subject Alternate Names to use in cert for service.""" + return ["127.0.0.1"] + def _on_install(self, event: ops.InstallEvent) -> None: """Handle install event.""" try: @@ -115,13 +225,23 @@ class SunbeamClusterdCharm(sunbeam_charm.OSBaseOperatorCharm): """Handle get-credentials action.""" if not self.peers.interface.state.joined: event.fail("Clusterd not joined yet") + return + + credentials = {} + if relation := self.model.get_relation(self.certs.relation_name): + if relation.active: + credentials = self.certs.get_client_keypair() + if not credentials: + event.fail("No credentials found yet") + return event.set_results( { "url": "https://" + self._binding_address() + ":" - + str(self.clusterd_port) + + str(self.clusterd_port), + **credentials, } ) @@ -174,10 +294,11 @@ class SunbeamClusterdCharm(sunbeam_charm.OSBaseOperatorCharm): if not self.clusterd_ready(): logger.debug("Clusterd not ready yet.") event.defer() - return + raise sunbeam_guard.WaitingExceptionError("Clusterd not ready yet") if not self.is_leader_ready(): self.bootstrap_cluster() self.peers.interface.state.joined = True + self.configure_certificates() super().configure_app_leader(event) if isinstance(event, ClusterdNewNodeEvent): self.add_node_to_cluster(event) @@ -200,6 +321,26 @@ class SunbeamClusterdCharm(sunbeam_charm.OSBaseOperatorCharm): } self.set_snap_data(snap_data) + def configure_certificates(self): + """Configure certificates.""" + if not self.unit.is_leader(): + logger.debug("Not leader, skipping certificate configuration.") + return + if not self.certs.ready: + logger.debug("Certificates not ready yet.") + return + certs = self.certs.context() + certs_hash = hashlib.sha256(bytes(str(certs), "utf-8")).hexdigest() + if certs_hash == self._state.certs_hash: + logger.debug("Certificates have not changed.") + return + self._clusterd.set_certs( + ca=certs["ca_cert_main"], + key=certs["key_main"], + cert=certs["cert_main"], + ) + self._state.certs_hash = certs_hash + def set_snap_data(self, snap_data: dict): """Set snap data on local snap.""" cache = snap.SnapCache() @@ -242,7 +383,6 @@ class SunbeamClusterdCharm(sunbeam_charm.OSBaseOperatorCharm): self.unit.name.replace("/", "-"), self._binding_address() + ":" + str(self.clusterd_port), ) - self.status.set(ops.ActiveStatus()) def add_node_to_cluster(self, event: ClusterdNewNodeEvent) -> None: """Generate token for node joining.""" @@ -277,7 +417,6 @@ class SunbeamClusterdCharm(sunbeam_charm.OSBaseOperatorCharm): already_left = self._wait_for_roles_to_settle_before_removal( event, self_departing ) - self.status.set(ops.ActiveStatus()) if already_left: return @@ -309,7 +448,6 @@ class SunbeamClusterdCharm(sunbeam_charm.OSBaseOperatorCharm): raise sunbeam_guard.WaitingExceptionError( "Waiting for roles to settle" ) - self.status.set(ops.ActiveStatus()) def _wait_for_roles_to_settle_before_removal( self, event: ops.EventBase, self_departing: bool @@ -399,7 +537,6 @@ class SunbeamClusterdCharm(sunbeam_charm.OSBaseOperatorCharm): logger.debug("Member %s is still pending", member) event.defer() return - self.status.set(ops.ActiveStatus()) def _wait_until_role_set(self, name: str) -> bool: @tenacity.retry( diff --git a/charms/sunbeam-clusterd/src/clusterd.py b/charms/sunbeam-clusterd/src/clusterd.py index f0f7a292..ebaed97d 100644 --- a/charms/sunbeam-clusterd/src/clusterd.py +++ b/charms/sunbeam-clusterd/src/clusterd.py @@ -82,6 +82,9 @@ class ClusterdClient: def _post(self, path, data=None, json=None, **kwargs): return self._request("post", path, data=data, json=json, **kwargs) + def _put(self, path, data=None, json=None, **kwargs): + return self._request("put", path, data=data, json=json, **kwargs) + def _delete(self, path, **kwargs): return self._request("delete", path, **kwargs) @@ -177,3 +180,15 @@ class ClusterdClient: data = {"name": name} result = self._post("/cluster/1.0/tokens", data=json.dumps(data)) return str(result["metadata"]) + + def set_certs(self, ca: str, cert: str, key: str): + """Configure cluster certificates. + + The CA is not set in the cluster certificates, but in the config endpoint. + This is because we don't want microcluster to go full CA-mode. + """ + self._put("/1.0/config/cluster-ca", data=ca) + data = {"public_key": cert, "private_key": key} + self._put( + "/cluster/internal/cluster/certificates", data=json.dumps(data) + ) diff --git a/common.sh b/common.sh index ecd7134f..75578b73 100644 --- a/common.sh +++ b/common.sh @@ -165,6 +165,7 @@ EXTERNAL_OPENSTACK_IMAGES_SYNC_LIBS=( EXTERNAL_SUNBEAM_CLUSTERD_LIBS=( "operator_libs_linux" + "tls_certificates_interface" ) EXTERNAL_SUNBEAM_MACHINE_LIBS=( diff --git a/tests/local/zaza/sunbeam/charm_tests/clusterd/tests.py b/tests/local/zaza/sunbeam/charm_tests/clusterd/tests.py index ac63c8bf..4bcf5d05 100644 --- a/tests/local/zaza/sunbeam/charm_tests/clusterd/tests.py +++ b/tests/local/zaza/sunbeam/charm_tests/clusterd/tests.py @@ -13,17 +13,34 @@ # See the License for the specific language governing permissions and # limitations under the License. +import base64 +import contextlib import json import logging import subprocess +import tempfile import unittest from random import shuffle from typing import Tuple import requests +import requests.adapters import tenacity +import zaza import zaza.model as model import zaza.openstack.charm_tests.test_utils as test_utils +from juju.client import client +from juju.model import Model + + +@contextlib.contextmanager +def keypair(certificate: bytes, private_key: bytes): + with tempfile.NamedTemporaryFile() as cert_file, tempfile.NamedTemporaryFile() as key_file: + cert_file.write(certificate) + cert_file.flush() + key_file.write(private_key) + key_file.flush() + yield (cert_file.name, key_file.name) class ClusterdTest(test_utils.BaseCharmTest): @@ -69,16 +86,46 @@ class ClusterdTest(test_utils.BaseCharmTest): for unit in units: model.block_until_unit_wl_status(unit, "active", timeout=60 * 5) + async def _read_secret( + self, model: Model, secret_id: str + ) -> dict[str, str]: + facade = client.SecretsFacade.from_connection(model.connection()) + secrets = await facade.ListSecrets( + filter_={"uri": secret_id}, show_secrets=True + ) + if len(secrets.results) != 1: + self.fail("Secret not found") + return secrets["results"][0].value.data + def test_100_connect_to_clusterd(self): """Try sending data to an endpoint.""" action = model.run_action_on_leader( self.application_name, "get-credentials" ) url = action.data["results"]["url"] + "/1.0/config/100_connect" - response = requests.put(url, json={"data": "test"}, verify=False) - response.raise_for_status() - response = requests.get(url, verify=False) - response.raise_for_status() + private_key_secret = action.data["results"].get("private-key-secret") + certificate = action.data["results"].get("certificate") + if private_key_secret is None or certificate is None: + context = contextlib.nullcontext() + logging.debug("Request made without mTLS") + else: + model_impl = zaza.sync_wrapper(model.get_model)() + private_key = base64.b64decode( + zaza.sync_wrapper(self._read_secret)( + model_impl, private_key_secret + )["private-key"] + ) + context = keypair(certificate.encode(), private_key) + logging.debug("Request made with mTLS") + + with context as cert: + response = requests.put( + url, json={"data": "test"}, verify=False, cert=cert + ) + response.raise_for_status() + response = requests.get(url, verify=False, cert=cert) + response.raise_for_status() + self.assertEqual( json.loads(response.json()["metadata"])["data"], "test" ) @@ -106,14 +153,11 @@ class ClusterdTest(test_utils.BaseCharmTest): """Scale back to 3.""" self._add_2_units() - @unittest.skip("Skip until scale down stable") def test_203_scale_down_to_2_units(self): """Scale down to 2 units for voter/spare test.""" leader = model.get_lead_unit_name(self.application_name) - model.destroy_unit( - self.application_name, leader, wait_disappear=True - ) + model.destroy_unit(self.application_name, leader, wait_disappear=True) model.block_until_all_units_idle() units = self._get_units()