Merge "Add retry decorator to SSH "execute" method"

neutron_tempest_plugin/common/ssh.py

@@ -14,12 +14,15 @@
 
 import locale
 import os
+import socket
 import time
 
 from oslo_log import log
 import paramiko
+import six
 from tempest.lib.common import ssh
 from tempest.lib import exceptions
+import tenacity
 
 from neutron_tempest_plugin import config
 from neutron_tempest_plugin import exceptions as exc
@@ -29,6 +32,16 @@ CONF = config.CONF
 LOG = log.getLogger(__name__)
 
 
+RETRY_EXCEPTIONS = (exceptions.TimeoutException, paramiko.SSHException,
+                    socket.error)
+if six.PY2:
+    # NOTE(ralonsoh): TimeoutError was added in 3.3 and corresponds to
+    # OSError(errno.ETIMEDOUT)
+    RETRY_EXCEPTIONS += (OSError, )
+else:
+    RETRY_EXCEPTIONS += (TimeoutError, )
+
+
 class Client(ssh.Client):
 
     default_ssh_lang = 'en_US.UTF-8'
@@ -179,6 +192,11 @@ class Client(ssh.Client):
                                         user=self.username,
                                         password=self.password)
 
+    @tenacity.retry(
+        stop=tenacity.stop_after_attempt(10),
+        wait=tenacity.wait_fixed(1),
+        retry=tenacity.retry_if_exception_type(RETRY_EXCEPTIONS),
+        reraise=True)
     def exec_command(self, cmd, encoding="utf-8", timeout=None):
         if timeout:
             original_timeout = self.timeout

neutron_tempest_plugin/scenario/test_qos.py

@@ -85,9 +85,9 @@ class QoSTestMixin(object):
         cmd = ("(dd if=/dev/zero bs=%(bs)d count=%(count)d of=%(file_path)s) "
                % {'bs': self.BUFFER_SIZE, 'count': self.COUNT,
                'file_path': self.FILE_PATH})
-        ssh_client.exec_command(cmd)
+        ssh_client.exec_command(cmd, timeout=5)
         cmd = "stat -c %%s %s" % self.FILE_PATH
-        filesize = ssh_client.exec_command(cmd)
+        filesize = ssh_client.exec_command(cmd, timeout=5)
         if int(filesize.strip()) != self.FILE_SIZE:
             raise sc_exceptions.FileCreationFailedException(
                 file=self.FILE_PATH)
@@ -96,7 +96,7 @@ class QoSTestMixin(object):
     def _kill_nc_process(ssh_client):
         cmd = "killall -q nc"
         try:
-            ssh_client.exec_command(cmd)
+            ssh_client.exec_command(cmd, timeout=5)
         except exceptions.SSHExecCommandFailed:
             pass
 
@@ -104,7 +104,7 @@ class QoSTestMixin(object):
         self._kill_nc_process(ssh_client)
         cmd = ("(nc -ll -p %(port)d < %(file_path)s > /dev/null &)" % {
                 'port': port, 'file_path': self.FILE_PATH})
-        ssh_client.exec_command(cmd)
+        ssh_client.exec_command(cmd, timeout=5)
 
         # Open TCP socket to remote VM and download big file
         start_time = time.time()

requirements.txt

@@ -13,6 +13,7 @@ oslo.utils>=3.33.0 # Apache-2.0
 paramiko>=2.0.0 # LGPLv2.1+
 six>=1.10.0 # MIT
 tempest>=17.1.0 # Apache-2.0
+tenacity>=3.2.1 # Apache-2.0
 ddt>=1.0.1 # MIT
 testtools>=2.2.0 # MIT
 testscenarios>=0.4 # Apache-2.0/BSD