starlingx/config
Code Issues Proposed changes

Merge "Ensure configuration is applied before updating control plane status"

Browse Source
This commit is contained in:
Zuul 2025-04-08 16:12:05 +00:00 committed by Gerrit Code Review
commit c21198ec4e
2 changed files with 82 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
sysinv/sysinv/sysinv/sysinv/conductor/manager.py
View File

@ -10702,19 +10702,43 @@ class ConductorManager(service.PeriodicService):
wait_fixed=1000)
def handle_k8s_upgrade_control_plane_success(self, context, kube_upgrade_obj, host_uuid,
new_state, fail_state):
def check_alarm_status(host_name, kube_host_upgrade_obj, kube_upgrade_obj):
# Monitor the alarm status in a separate thread and update the
# control plane status if no alarm is detected.
check_alarm_count = 5
while check_alarm_count > 0:
try:
alarms = self.fm_api.get_faults_by_id(
fm_constants.FM_ALARM_ID_SYSCONFIG_OUT_OF_DATE)
time.sleep(5)
check_alarm_count -= 1
if not alarms:
# The control plane update was successful
LOG.info("Control plane was updated for host %s" % host_name)
kube_host_upgrade_obj.status = None
kube_host_upgrade_obj.save()
kube_upgrade_obj.state = new_state
kube_upgrade_obj.save()
return
except Exception:
LOG.error("Error checking alarm status for host %s:" % host_name)
kube_host_upgrade_obj.status = \
kubernetes.KUBE_HOST_UPGRADING_CONTROL_PLANE_FAILED
kube_host_upgrade_obj.save()
kube_upgrade_obj.state = fail_state
kube_upgrade_obj.save()
return
host_obj = objects.host.get_by_uuid(context, host_uuid)
host_name = host_obj.hostname
kube_host_upgrade_obj = objects.kube_host_upgrade.get_by_host_id(
context, host_obj.id)
target_version = kube_host_upgrade_obj.target_version
kube_operator = kubernetes.KubeOperator()
cp_versions = kube_operator.kube_get_control_plane_versions()
LOG.info("Checking control plane update on host %s, "
"cp_versions = %s, target_version = %s" %
(host_name, cp_versions, target_version))
if cp_versions.get(host_name, None) != target_version:
LOG.warning("Control plane upgrade failed for host %s" %
host_name)
@ -10725,12 +10749,7 @@ class ConductorManager(service.PeriodicService):
kube_upgrade_obj.save()
return False
# The control plane update was successful
LOG.info("Control plane was updated for host %s" % host_name)
kube_host_upgrade_obj.status = None
kube_host_upgrade_obj.save()
kube_upgrade_obj.state = new_state
kube_upgrade_obj.save()
greenthread.spawn(check_alarm_status, host_name, kube_host_upgrade_obj, kube_upgrade_obj)
return True
def report_config_status(self, context, iconfig, status, error=None):

56
sysinv/sysinv/sysinv/sysinv/tests/conductor/test_manager.py
View File

@ -1860,7 +1860,8 @@ class ManagerTestCase(base.DbTestCase):
mock_config_apply_runtime_manifest.assert_called_with(mock.ANY, '273cfafd-886d-43ec-9478-8328727b34cc',
config_dict)
def test_handle_k8s_upgrade_control_plane_success_first_master(self):
@mock.patch('eventlet.greenthread.spawn')
def test_handle_k8s_upgrade_control_plane_success_first_master(self, mock_spawn):
# Create controller-0
config_uuid = str(uuid.uuid4())
c0 = self._create_test_ihost(
@ -1891,11 +1892,56 @@ class ManagerTestCase(base.DbTestCase):
fail_state = kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED
kube_upgrade_obj = objects.kube_upgrade.get_one(context)
self.service.fm_api.get_faults_by_id.return_value = ['250.001']
p = mock.patch('time.sleep')
p.start().return_value = None
self.addCleanup(p.stop)
mock_spawn.side_effect = lambda func, *args, **kwargs: func(*args, **kwargs)
self.service.handle_k8s_upgrade_control_plane_success(self.context,
kube_upgrade_obj, c0.uuid, new_state, fail_state)
self.assertEqual(kube_upgrade_obj.state,
new_state)
@mock.patch('eventlet.greenthread.spawn')
def test_handle_k8s_upgrade_control_plane_success_alarm_exception(self, mock_spawn):
# Create controller-0
config_uuid = str(uuid.uuid4())
c0 = self._create_test_ihost(
personality=constants.CONTROLLER,
hostname='controller-0',
uuid=str(uuid.uuid4()),
config_status=None,
config_applied=config_uuid,
config_target=config_uuid,
invprovision=constants.PROVISIONED,
administrative=constants.ADMIN_UNLOCKED,
operational=constants.OPERATIONAL_ENABLED,
availability=constants.AVAILABILITY_ONLINE,
)
# Set the target version for controller-0
self.dbapi.kube_host_upgrade_update(1, {'target_version': 'v1.42.2'})
utils.create_test_kube_upgrade(
from_version='v1.42.1',
to_version='v1.42.2',
state=kubernetes.KUBE_UPGRADED_FIRST_MASTER,
)
self.kube_get_control_plane_versions_result = {
'controller-0': 'v1.42.2'}
new_state = kubernetes.KUBE_UPGRADED_FIRST_MASTER
fail_state = kubernetes.KUBE_UPGRADING_FIRST_MASTER_FAILED
# Speed up the test
kubernetes.MANIFEST_APPLY_INTERVAL = 1
self.service.fm_api.get_faults_by_id.side_effect = Exception("API failure")
mock_spawn.side_effect = lambda func, *args, **kwargs: func(*args, **kwargs)
kube_upgrade_obj = objects.kube_upgrade.get_one(context)
self.service.handle_k8s_upgrade_control_plane_success(self.context, kube_upgrade_obj,
c0.uuid, new_state, fail_state)
self.assertEqual(kube_upgrade_obj.state,
fail_state)
@mock.patch("time.sleep", return_value=None) # Skip actual wait
@mock.patch('sysinv.objects.host.get_by_uuid')
@mock.patch('sysinv.objects.kube_host_upgrade.get_by_host_id')
@ -2006,7 +2052,8 @@ class ManagerTestCase(base.DbTestCase):
mock_config_apply_runtime_manifest.assert_called_with(mock.ANY, '273cfafd-886d-43ec-9478-8328727b34cc',
config_dict)
def test_handle_k8s_upgrade_control_plane_success_second_master(self):
@mock.patch('eventlet.greenthread.spawn')
def test_handle_k8s_upgrade_control_plane_success_second_master(self, mock_spawn):
# Create controller-0
config_uuid = str(uuid.uuid4())
self._create_test_ihost(
@ -2056,6 +2103,11 @@ class ManagerTestCase(base.DbTestCase):
fail_state = kubernetes.KUBE_UPGRADING_SECOND_MASTER_FAILED
kube_upgrade_obj = objects.kube_upgrade.get_one(context)
self.service.fm_api.get_faults_by_id.return_value = ['250.001']
p = mock.patch('time.sleep')
p.start().return_value = None
self.addCleanup(p.stop)
mock_spawn.side_effect = lambda func, *args, **kwargs: func(*args, **kwargs)
self.service.handle_k8s_upgrade_control_plane_success(self.context,
kube_upgrade_obj, c1.uuid, new_state, fail_state)
self.assertEqual(kube_upgrade_obj.state,