diff --git a/ops-sunbeam/ops_sunbeam/charm.py b/ops-sunbeam/ops_sunbeam/charm.py index fa4264eb..81b96c1b 100644 --- a/ops-sunbeam/ops_sunbeam/charm.py +++ b/ops-sunbeam/ops_sunbeam/charm.py @@ -58,6 +58,8 @@ import ops_sunbeam.compound_status as compound_status import ops_sunbeam.config_contexts as sunbeam_config_contexts import ops_sunbeam.container_handlers as sunbeam_chandlers import ops_sunbeam.core as sunbeam_core +import ops_sunbeam.guard as sunbeam_guard +import ops_sunbeam.job_ctrl as sunbeam_job_ctrl import ops_sunbeam.relation_handlers as sunbeam_rhandlers logger = logging.getLogger(__name__) @@ -81,10 +83,14 @@ class OSBaseOperatorCharm(ops.charm.CharmBase): "by ops_sunbeam" ) ) + # unit_bootstrapped is stored in the local unit storage which is lost + # when the pod is replaced, so this will revert to False on charm + # upgrade or upgrade of the payload container. + self._state.set_default(unit_bootstrapped=False) self.status = compound_status.Status("workload", priority=100) self.status_pool = compound_status.StatusPool(self) self.status_pool.add(self.status) - self._state.set_default(bootstrapped=False) + self.relation_handlers = self.get_relation_handlers() self.bootstrap_status = compound_status.Status( "bootstrap", priority=90 ) @@ -93,7 +99,6 @@ class OSBaseOperatorCharm(ops.charm.CharmBase): self.bootstrap_status.set( MaintenanceStatus("Service not bootstrapped") ) - self.relation_handlers = self.get_relation_handlers() self.pebble_handlers = self.get_pebble_handlers() self.framework.observe(self.on.config_changed, self._on_config_changed) @@ -242,19 +247,22 @@ class OSBaseOperatorCharm(ops.charm.CharmBase): if h.container_name in container_names ] - # flake8: noqa: C901 - def configure_charm(self, event: ops.framework.EventBase) -> None: - """Catchall handler to configure charm services.""" + def check_leader_ready(self): + """Check the leader is reporting as ready.""" if self.supports_peer_relation and not ( self.unit.is_leader() or self.is_leader_ready() ): - logging.debug("Leader not ready") - return + raise sunbeam_guard.WaitingExceptionError("Leader not ready") + def check_relation_handlers_ready(self): + """Check all relation handlers are ready.""" if not self.relation_handlers_ready(): - logging.debug("Aborting charm relations not ready") - return + raise sunbeam_guard.WaitingExceptionError( + "Not all relations are ready" + ) + def init_container_services(self): + """Run init on pebble handlers that are ready.""" for ph in self.pebble_handlers: if ph.pebble_ready: logging.debug(f"Running init for {ph.service_name}") @@ -264,32 +272,64 @@ class OSBaseOperatorCharm(ops.charm.CharmBase): f"Not running init for {ph.service_name}," " container not ready" ) + raise sunbeam_guard.WaitingExceptionError( + "Payload container not ready" + ) + def check_pebble_handlers_ready(self): + """Check pebble handlers are ready.""" for ph in self.pebble_handlers: if not ph.service_ready: logging.debug( f"Aborting container {ph.service_name} service not ready" ) - return - - if not self.bootstrapped(): - if not self._do_bootstrap(): - self._state.bootstrapped = False - logging.warning( - "Failed to bootstrap the service, event deferred" + raise sunbeam_guard.WaitingExceptionError( + "Container service not ready" ) - # Defer the event to re-trigger the bootstrap process - event.defer() - return - if self.unit.is_leader() and self.supports_peer_relation: - self.set_leader_ready() - # Add healthchecks to the plan + def configure_unit(self, event: ops.framework.EventBase) -> None: + """Run configuration on this unit.""" + self.check_leader_ready() + self.check_relation_handlers_ready() + self.init_container_services() + self.check_pebble_handlers_ready() + self.run_db_sync() + self._state.unit_bootstrapped = True + + def configure_app_leader(self, event): + """Run global app setup. + + These are tasks that should only be run once per application and only + the leader runs them. + """ + self.set_leader_ready() + + def configure_app_non_leader(self, event): + """Setup steps for a non-leader after leader has bootstrapped.""" + if not self.bootstrapped: + raise sunbeam_guard.WaitingExceptionError("Leader not ready") + + def configure_app(self, event): + """Check on (and run if leader) app wide tasks.""" + if self.unit.is_leader(): + self.configure_app_leader(event) + else: + self.configure_app_non_leader(event) + + def add_pebble_health_checks(self): + """Add health checks for services in payload containers.""" for ph in self.pebble_handlers: ph.add_healthchecks() - self.status.set(ActiveStatus("")) - self._state.bootstrapped = True + def configure_charm(self, event: ops.framework.EventBase) -> None: + """Catchall handler to configure charm services.""" + with sunbeam_guard.guard(self, "Bootstrapping"): + self.configure_unit(event) + self.configure_app(event) + self.bootstrap_status.set(ActiveStatus()) + self.add_pebble_health_checks() + logger.info("Setting active status") + self.status.set(ActiveStatus("")) @property def supports_peer_relation(self) -> bool: @@ -387,7 +427,7 @@ class OSBaseOperatorCharm(ops.charm.CharmBase): def bootstrapped(self) -> bool: """Determine whether the service has been bootstrapped.""" - return self._state.bootstrapped + return self._state.unit_bootstrapped and self.is_leader_ready() def leader_set(self, settings: dict = None, **kwargs) -> None: """Juju set data in peer data bag.""" @@ -426,8 +466,9 @@ class OSBaseOperatorCharm(ops.charm.CharmBase): if warnings: for line in warnings.splitlines(): logger.warning("DB Sync Out: %s", line.strip()) - logging.debug("Output from database sync: \n%s", out) + logging.debug("Output from database sync: \n%s", out) + @sunbeam_job_ctrl.run_once_per_unit("db-sync") def run_db_sync(self) -> None: """Run DB sync to init DB. @@ -451,23 +492,6 @@ class OSBaseOperatorCharm(ops.charm.CharmBase): "Not DB sync ran. Charm does not specify self.db_sync_cmds" ) - def _do_bootstrap(self) -> bool: - """Perform bootstrap. - - :return: Return True if bootstrap is success - :rtype: bool - """ - try: - self.run_db_sync() - self.bootstrap_status.set(ActiveStatus()) - return True - except ops.pebble.ExecError as e: - logger.exception("Failed to bootstrap") - logger.error("Exited with code %d. Stderr:", e.exit_code) - for line in e.stderr.splitlines(): - logger.error(" %s", line) - return False - class OSBaseOperatorAPICharm(OSBaseOperatorCharm): """Base class for OpenStack API operators.""" diff --git a/ops-sunbeam/ops_sunbeam/job_ctrl.py b/ops-sunbeam/ops_sunbeam/job_ctrl.py index f5acc8b2..61f843ca 100644 --- a/ops-sunbeam/ops_sunbeam/job_ctrl.py +++ b/ops-sunbeam/ops_sunbeam/job_ctrl.py @@ -23,10 +23,14 @@ import time from functools import ( wraps, ) +from typing import ( + TYPE_CHECKING, +) import ops.framework -import ops_sunbeam +if TYPE_CHECKING: + import ops_sunbeam.charm logger = logging.getLogger(__name__) @@ -52,7 +56,7 @@ def run_once_per_unit(label): def wrap(f): @wraps(f) def wrapped_f( - charm: ops_sunbeam.charm.OSBaseOperatorCharm, *args, **kwargs + charm: "ops_sunbeam.charm.OSBaseOperatorCharm", *args, **kwargs ): """Run once decorator.