Add retry decorator to SSH "execute" method

In case of SSH timeout (TimeoutException, TimeoutError), the
tenacity.retry decorator retries the execution of the SSH
"execute" method up to 10 times.

Some SSH execute calls, related to QoS scenario tests, have been
enhanced by setting a relatively small timeout value. The commands
executed should be quick enough to be executed in this amount of time.
In case of timeout (due to communication problems), the retry decorator
will send again the command to be executed.

Change-Id: Idc0d55b776f499a4bc5d8c9d9a549f0af8f3fac0
Closes-Bug: #1844516

neutron_tempest_plugin/common/ssh.py

@@ -14,12 +14,15 @@
 
 import locale
 import os
+import socket
 import time
 
 from oslo_log import log
 import paramiko
+import six
 from tempest.lib.common import ssh
 from tempest.lib import exceptions
+import tenacity
 
 from neutron_tempest_plugin import config
 from neutron_tempest_plugin import exceptions as exc
@@ -29,6 +32,16 @@ CONF = config.CONF
 LOG = log.getLogger(__name__)
 
 
+RETRY_EXCEPTIONS = (exceptions.TimeoutException, paramiko.SSHException,
+                    socket.error)
+if six.PY2:
+    # NOTE(ralonsoh): TimeoutError was added in 3.3 and corresponds to
+    # OSError(errno.ETIMEDOUT)
+    RETRY_EXCEPTIONS += (OSError, )
+else:
+    RETRY_EXCEPTIONS += (TimeoutError, )
+
+
 class Client(ssh.Client):
 
     default_ssh_lang = 'en_US.UTF-8'
@@ -179,6 +192,11 @@ class Client(ssh.Client):
                                         user=self.username,
                                         password=self.password)
 
+    @tenacity.retry(
+        stop=tenacity.stop_after_attempt(10),
+        wait=tenacity.wait_fixed(1),
+        retry=tenacity.retry_if_exception_type(RETRY_EXCEPTIONS),
+        reraise=True)
     def exec_command(self, cmd, encoding="utf-8", timeout=None):
         if timeout:
             original_timeout = self.timeout

neutron_tempest_plugin/scenario/test_qos.py

@@ -85,9 +85,9 @@ class QoSTestMixin(object):
         cmd = ("(dd if=/dev/zero bs=%(bs)d count=%(count)d of=%(file_path)s) "
                % {'bs': self.BUFFER_SIZE, 'count': self.COUNT,
                'file_path': self.FILE_PATH})
-        ssh_client.exec_command(cmd)
+        ssh_client.exec_command(cmd, timeout=5)
         cmd = "stat -c %%s %s" % self.FILE_PATH
-        filesize = ssh_client.exec_command(cmd)
+        filesize = ssh_client.exec_command(cmd, timeout=5)
         if int(filesize.strip()) != self.FILE_SIZE:
             raise sc_exceptions.FileCreationFailedException(
                 file=self.FILE_PATH)
@@ -96,7 +96,7 @@ class QoSTestMixin(object):
     def _kill_nc_process(ssh_client):
         cmd = "killall -q nc"
         try:
-            ssh_client.exec_command(cmd)
+            ssh_client.exec_command(cmd, timeout=5)
         except exceptions.SSHExecCommandFailed:
             pass
 
@@ -104,7 +104,7 @@ class QoSTestMixin(object):
         self._kill_nc_process(ssh_client)
         cmd = ("(nc -ll -p %(port)d < %(file_path)s > /dev/null &)" % {
                 'port': port, 'file_path': self.FILE_PATH})
-        ssh_client.exec_command(cmd)
+        ssh_client.exec_command(cmd, timeout=5)
 
         # Open TCP socket to remote VM and download big file
         start_time = time.time()

requirements.txt

@@ -13,6 +13,7 @@ oslo.utils>=3.33.0 # Apache-2.0
 paramiko>=2.0.0 # LGPLv2.1+
 six>=1.10.0 # MIT
 tempest>=17.1.0 # Apache-2.0
+tenacity>=3.2.1 # Apache-2.0
 ddt>=1.0.1 # MIT
 testtools>=2.2.0 # MIT
 testscenarios>=0.4 # Apache-2.0/BSD