diff --git a/neutron_tempest_plugin/common/ssh.py b/neutron_tempest_plugin/common/ssh.py index ea30a282..96f0ef9c 100644 --- a/neutron_tempest_plugin/common/ssh.py +++ b/neutron_tempest_plugin/common/ssh.py @@ -14,12 +14,15 @@ import locale import os +import socket import time from oslo_log import log import paramiko +import six from tempest.lib.common import ssh from tempest.lib import exceptions +import tenacity from neutron_tempest_plugin import config from neutron_tempest_plugin import exceptions as exc @@ -29,6 +32,16 @@ CONF = config.CONF LOG = log.getLogger(__name__) +RETRY_EXCEPTIONS = (exceptions.TimeoutException, paramiko.SSHException, + socket.error) +if six.PY2: + # NOTE(ralonsoh): TimeoutError was added in 3.3 and corresponds to + # OSError(errno.ETIMEDOUT) + RETRY_EXCEPTIONS += (OSError, ) +else: + RETRY_EXCEPTIONS += (TimeoutError, ) + + class Client(ssh.Client): default_ssh_lang = 'en_US.UTF-8' @@ -179,6 +192,11 @@ class Client(ssh.Client): user=self.username, password=self.password) + @tenacity.retry( + stop=tenacity.stop_after_attempt(10), + wait=tenacity.wait_fixed(1), + retry=tenacity.retry_if_exception_type(RETRY_EXCEPTIONS), + reraise=True) def exec_command(self, cmd, encoding="utf-8", timeout=None): if timeout: original_timeout = self.timeout diff --git a/neutron_tempest_plugin/scenario/test_qos.py b/neutron_tempest_plugin/scenario/test_qos.py index 82a5391c..ba8cc884 100644 --- a/neutron_tempest_plugin/scenario/test_qos.py +++ b/neutron_tempest_plugin/scenario/test_qos.py @@ -85,9 +85,9 @@ class QoSTestMixin(object): cmd = ("(dd if=/dev/zero bs=%(bs)d count=%(count)d of=%(file_path)s) " % {'bs': self.BUFFER_SIZE, 'count': self.COUNT, 'file_path': self.FILE_PATH}) - ssh_client.exec_command(cmd) + ssh_client.exec_command(cmd, timeout=5) cmd = "stat -c %%s %s" % self.FILE_PATH - filesize = ssh_client.exec_command(cmd) + filesize = ssh_client.exec_command(cmd, timeout=5) if int(filesize.strip()) != self.FILE_SIZE: raise sc_exceptions.FileCreationFailedException( file=self.FILE_PATH) @@ -96,7 +96,7 @@ class QoSTestMixin(object): def _kill_nc_process(ssh_client): cmd = "killall -q nc" try: - ssh_client.exec_command(cmd) + ssh_client.exec_command(cmd, timeout=5) except exceptions.SSHExecCommandFailed: pass @@ -104,7 +104,7 @@ class QoSTestMixin(object): self._kill_nc_process(ssh_client) cmd = ("(nc -ll -p %(port)d < %(file_path)s > /dev/null &)" % { 'port': port, 'file_path': self.FILE_PATH}) - ssh_client.exec_command(cmd) + ssh_client.exec_command(cmd, timeout=5) # Open TCP socket to remote VM and download big file start_time = time.time() diff --git a/requirements.txt b/requirements.txt index bb836d17..2febb7e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ oslo.utils>=3.33.0 # Apache-2.0 paramiko>=2.0.0 # LGPLv2.1+ six>=1.10.0 # MIT tempest>=17.1.0 # Apache-2.0 +tenacity>=3.2.1 # Apache-2.0 ddt>=1.0.1 # MIT testtools>=2.2.0 # MIT testscenarios>=0.4 # Apache-2.0/BSD