Linting: Make Shipyard mostly pep8 compliant

This PS makes shipyard mostly pep8 compliant, with the exeption of:
./shipyard_airflow/plugins/rest_api_plugin.py which will require some
rework to make compliant.

Change-Id: I913f7ac696eb5558c1fed7c4f4104211bf4b5876

alembic/versions/51b92375e5c4_initial_shipyard_base.py

@@ -73,6 +73,7 @@ def upgrade():
                   server_default=func.now()),
     )
 
+
 def downgrade():
     """
     Remove the database objects created by this revision

shipyard_airflow/conf/config.py

@@ -214,6 +214,7 @@ SECTIONS = [
     ),
 ]
 
+
 def register_opts(conf):
     """
     Registers all the sections in this module.

shipyard_airflow/conf/opts.py

@@ -26,6 +26,7 @@ class ConfigSection(object):
     """
     Defines a configuration section
     """
+
     def __init__(self, name, title, options, help=None):
         self.name = name
         self.title = title

shipyard_airflow/control/action_helper.py

@@ -30,6 +30,7 @@ DAG_STATE_MAPPING = {
     'PAUSED': 'Paused'
 }
 
+
 def determine_lifecycle(dag_status=None):
     """
     Convert a dag_status to an action_lifecycle value
@@ -38,6 +39,7 @@ def determine_lifecycle(dag_status=None):
         dag_status = 'NONE'
     return DAG_STATE_MAPPING.get(dag_status.upper())
 
+
 def format_action_steps(action_id, steps):
     """
     Converts a list of action step database records to desired format
@@ -51,6 +53,7 @@ def format_action_steps(action_id, steps):
                                           index=idx + 1))
     return steps_response
 
+
 def format_step(action_id, step, index):
     """
     reformat a step (dictionary) into a common response format

shipyard_airflow/control/api.py

@@ -77,10 +77,12 @@ def start_api():
 
     return control_api
 
+
 class VersionsResource(BaseResource):
     """
     Lists the versions supported by this API
     """
+
     def on_get(self, req, resp):
         resp.body = self.to_json({
             'v1.0': {

shipyard_airflow/control/base.py

@@ -128,10 +128,12 @@ class BaseResource(object):
         """
         self.log_message(ctx, logging.ERROR, msg)
 
+
 class ShipyardRequestContext(object):
     """
     Context object for shipyard resource requests
     """
+
     def __init__(self):
         self.log_level = 'error'
         self.user = None

shipyard_airflow/dags/deploy_site.py

@@ -54,6 +54,8 @@ Define push function to store the content of 'action' that is
 defined via 'dag_run' in XCOM so that it can be used by the
 Operators
 """
+
+
 def xcom_push(**kwargs):
     # Pushes action XCom
     kwargs['ti'].xcom_push(key='action',

shipyard_airflow/dags/drydock_deploy_site.py

@@ -35,6 +35,7 @@ promenade_conf = config.get('drydock', 'prom_yaml')
 parent_dag = 'deploy_site'
 child_dag = 'deploy_site.drydock_build'
 
+
 def create_drydock_client(parent_dag_name, child_dag_name, args):
     '''
     Create Drydock Client
@@ -56,6 +57,7 @@ def create_drydock_client(parent_dag_name, child_dag_name, args):
 
     return dag
 
+
 def drydock_get_design_id(parent_dag_name, child_dag_name, args):
     '''
     Get Design ID
@@ -73,6 +75,7 @@ def drydock_get_design_id(parent_dag_name, child_dag_name, args):
 
     return dag
 
+
 def drydock_load_parts(parent_dag_name, child_dag_name, args):
     '''
     Load DryDock Yaml
@@ -91,6 +94,7 @@ def drydock_load_parts(parent_dag_name, child_dag_name, args):
 
     return dag
 
+
 def promenade_load_parts(parent_dag_name, child_dag_name, args):
     '''
     Load Promenade Yaml
@@ -109,6 +113,7 @@ def promenade_load_parts(parent_dag_name, child_dag_name, args):
 
     return dag
 
+
 def drydock_verify_site(parent_dag_name, child_dag_name, args):
     '''
     Verify connectivity between DryDock and MAAS
@@ -126,6 +131,7 @@ def drydock_verify_site(parent_dag_name, child_dag_name, args):
 
     return dag
 
+
 def drydock_prepare_site(parent_dag_name, child_dag_name, args):
     '''
     Prepare site for deployment
@@ -143,6 +149,7 @@ def drydock_prepare_site(parent_dag_name, child_dag_name, args):
 
     return dag
 
+
 def drydock_prepare_node(parent_dag_name, child_dag_name, args):
     '''
     Prepare nodes for deployment
@@ -160,6 +167,7 @@ def drydock_prepare_node(parent_dag_name, child_dag_name, args):
 
     return dag
 
+
 def drydock_deploy_node(parent_dag_name, child_dag_name, args):
     '''
     Deploy Nodes
@@ -188,6 +196,7 @@ DRYDOCK_PREPARE_SITE_DAG_NAME = 'drydock_prepare_site'
 DRYDOCK_PREPARE_NODE_DAG_NAME = 'drydock_prepare_node'
 DRYDOCK_DEPLOY_NODE_DAG_NAME = 'drydock_deploy_node'
 
+
 def deploy_site_drydock(parent_dag_name, child_dag_name, args):
     '''
     Puts all of the drydock deploy site into atomic unit

shipyard_airflow/dags/drydock_operator_child.py

@@ -18,6 +18,7 @@ import configparser
 from airflow import DAG
 from airflow.operators import DryDockOperator
 
+
 def sub_dag(parent_dag_name, child_dag_name, args, schedule_interval):
     dag = DAG(
         '%s.%s' % (parent_dag_name, child_dag_name),

shipyard_airflow/dags/drydock_operator_parent.py

@@ -44,6 +44,8 @@ main_dag = DAG(
 # Define push function to store the content of 'action' that is
 # defined via 'dag_run' in XCOM so that it can be used by the
 # DryDock Operators
+
+
 def push(**kwargs):
     # Pushes action XCom
     kwargs['ti'].xcom_push(key='action',

shipyard_airflow/dags/preflight_checks.py

@@ -24,6 +24,7 @@ config_path = '/usr/local/airflow/plugins/shipyard.conf'
 # Note that the shipyard.conf file needs to be placed on a volume
 # that can be accessed by the containers
 
+
 def k8s_preflight_check(parent_dag_name, child_dag_name, args):
     '''
     The k8s_preflight_check checks that k8s is in a good state for

shipyard_airflow/dags/redeploy_server.py

@@ -52,6 +52,8 @@ Define push function to store the content of 'action' that is
 defined via 'dag_run' in XCOM so that it can be used by the
 Operators
 """
+
+
 def xcom_push(**kwargs):
     # Pushes action XCom
     kwargs['ti'].xcom_push(key='action',

shipyard_airflow/dags/update_site.py

@@ -54,6 +54,8 @@ Define push function to store the content of 'action' that is
 defined via 'dag_run' in XCOM so that it can be used by the
 Operators
 """
+
+
 def xcom_push(**kwargs):
     # Pushes action XCom
     kwargs['ti'].xcom_push(key='action',

shipyard_airflow/db/common_db.py

@@ -19,6 +19,7 @@ from shipyard_airflow.errors import DatabaseError
 
 LOG = logging.getLogger(__name__)
 
+
 class DbAccess:
     """
     Base class for simple database access

shipyard_airflow/db/errors.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 class AirflowStateError(Exception):
     def __init__(self, message=""):
         """

shipyard_airflow/db/shipyard_db.py

@@ -28,6 +28,7 @@ from shipyard_airflow.db.common_db import DbAccess
 LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
 
+
 class ShipyardDbAccess(DbAccess):
     """
     Shipyard database access

shipyard_airflow/errors.py

@@ -70,6 +70,7 @@ def format_resp(req,
     resp.content_type = 'application/json'
     resp.status = status_code
 
+
 def default_error_serializer(req, resp, exception):
     """
     Writes the default error message body, when we don't handle it otherwise
@@ -84,6 +85,7 @@ def default_error_serializer(req, resp, exception):
         error_list=[{'message': exception.description}]
     )
 
+
 def default_exception_handler(ex, req, resp, params):
     """
     Catch-all execption handler for standardized output.
@@ -110,6 +112,7 @@ class AppError(Exception):
     """
     Base error containing enough information to make a shipyard formatted error
     """
+
     def __init__(self,
                  title='Internal Server Error',
                  description=None,
@@ -146,6 +149,7 @@ class AirflowError(AppError):
     """
     An error to handle errors returned by the Airflow API
     """
+
     def __init__(self, description=None, error_list=None):
         super().__init__(
             title='Error response from Airflow',
@@ -155,10 +159,12 @@ class AirflowError(AppError):
             retry=False
         )
 
+
 class DatabaseError(AppError):
     """
     An error to handle general api errors.
     """
+
     def __init__(self,
                  description=None,
                  error_list=None,
@@ -178,6 +184,7 @@ class ApiError(AppError):
     """
     An error to handle general api errors.
     """
+
     def __init__(self,
                  description="",
                  error_list=None,
@@ -197,6 +204,7 @@ class InvalidFormatError(AppError):
     """
     An exception to cover invalid input formatting
     """
+
     def __init__(self, title, description="Not Specified", error_list=None):
 
         super().__init__(

shipyard_airflow/plugins/drydock_operators.py

@@ -403,32 +403,33 @@ class DryDockOperator(BaseOperator):
     def drydock_query_task(self, drydock_client, interval, time_out,
                            task_id, desired_state):
 
-            # Calculate number of times to execute the 'for' loop
-            end_range = int(time_out / interval)
+        # Calculate number of times to execute the 'for' loop
+        end_range = int(time_out / interval)
 
-            # Query task state
-            for i in range(0, end_range + 1):
+        # Query task state
+        for i in range(0, end_range + 1):
 
-                # Retrieve current task state
-                task_state = drydock_client.get_task(self.task_id)
-                logging.info(task_state)
+            # Retrieve current task state
+            task_state = drydock_client.get_task(self.task_id)
+            logging.info(task_state)
 
-                # Return Time Out Exception
-                if task_state['status'] == 'running' and i == end_range:
-                    logging.info('Timed Out!')
-                    return 'timed_out'
+            # Return Time Out Exception
+            if task_state['status'] == 'running' and i == end_range:
+                logging.info('Timed Out!')
+                return 'timed_out'
 
-                # Exit 'for' loop if task is in 'complete' state
-                if task_state['status'] == 'complete':
-                    break
-                else:
-                    time.sleep(interval)
-
-            # Get final task state
-            if task_state['result'] == desired_state:
-                return drydock_client.get_task(self.task_id)
+            # Exit 'for' loop if task is in 'complete' state
+            if task_state['status'] == 'complete':
+                break
             else:
-                return 'task_failed'
+                time.sleep(interval)
+
+        # Get final task state
+        if task_state['result'] == desired_state:
+            return drydock_client.get_task(self.task_id)
+        else:
+            return 'task_failed'
+
 
 class DryDockClientPlugin(AirflowPlugin):
     name = "drydock_client_plugin"

shipyard_airflow/plugins/rest_api_plugin.py

@@ -1,4 +1,4 @@
-# Original Source Code: https://github.com/teamclairvoyant/airflow-rest-api-plugin 
+# Original Source Code: https://github.com/teamclairvoyant/airflow-rest-api-plugin
 # Date when airflow-rest-api-plugin source codes were downloaded: 2017-06-27
 # Kept majority of the codes and removed those that are not needed for our purpose
 
@@ -780,4 +780,3 @@ class REST_API_Plugin(AirflowPlugin):
     executors = []
     admin_views = [rest_api_view]
     menu_links = []
-

tests/unit/control/__init__.py

@@ -15,6 +15,7 @@ import pytest
 
 from shipyard_airflow.conf import config
 
+
 @pytest.fixture
 def setup_config():
     """

tests/unit/control/test_actions_api.py

@@ -142,6 +142,7 @@ def tasks_db():
         },
     ]
 
+
 def airflow_stub(**kwargs):
     """
     asserts that the airflow invocation method was called with the right
@@ -152,12 +153,14 @@ def airflow_stub(**kwargs):
     print(kwargs)
     return '2017-09-06 14:10:08.528402'
 
+
 def insert_action_stub(**kwargs):
     """
     asserts that the insert action was called with the right parameters
     """
     assert kwargs['action']
 
+
 def audit_control_command_db(action_audit):
     """
     Stub for inserting the invoke record
@@ -167,6 +170,7 @@ def audit_control_command_db(action_audit):
 
 context = ShipyardRequestContext()
 
+
 def test_get_all_actions():
     """
     Tests the main response from get all actions
@@ -188,6 +192,7 @@ def test_get_all_actions():
             assert len(action['steps']) == 3
             assert action['dag_status'] == 'SUCCESS'
 
+
 def test_create_action():
     action_resource = ActionsResource()
     action_resource.get_all_actions_db = actions_db

tests/unit/control/test_actions_control_api.py

@@ -47,6 +47,7 @@ def actions_db(action_id):
             'context_marker': '8-4-4-4-12a'
         }
 
+
 def control_dag_run(dag_id,
                     execution_date,
                     expected_state,
@@ -56,9 +57,11 @@ def control_dag_run(dag_id,
     else:
         pass
 
+
 def audit_control_command_db(action_audit):
     pass
 
+
 def test_get_action():
     """
     Tests the main response from get all actions

tests/unit/control/test_actions_id_api.py

@@ -20,6 +20,7 @@ DATE_TWO = datetime(2017, 9, 13, 11, 13, 5, 57000)
 DATE_ONE_STR = DATE_ONE.strftime('%Y-%m-%dT%H:%M:%S')
 DATE_TWO_STR = DATE_TWO.strftime('%Y-%m-%dT%H:%M:%S')
 
+
 def actions_db(action_id):
     """
     replaces the actual db call
@@ -35,6 +36,7 @@ def actions_db(action_id):
         'context_marker': '8-4-4-4-12a'
     }
 
+
 def dag_runs_db(dag_id, execution_date):
     """
     replaces the actual db call
@@ -49,6 +51,7 @@ def dag_runs_db(dag_id, execution_date):
         'end_date': DATE_ONE
     }]
 
+
 def tasks_db(dag_id, execution_date):
     """
     replaces the actual db call
@@ -98,6 +101,7 @@ def tasks_db(dag_id, execution_date):
         }
     ]
 
+
 def get_validations(action_id):
     """
     Stub to return validations
@@ -111,6 +115,7 @@ def get_validations(action_id):
         }
     ]
 
+
 def get_ac_audit(action_id):
     """
     Stub to return command audit response
@@ -132,6 +137,7 @@ def get_ac_audit(action_id):
         }
     ]
 
+
 def test_get_action():
     """
     Tests the main response from get all actions

tests/unit/control/test_actions_steps_id_api.py

@@ -38,6 +38,7 @@ def actions_db(action_id):
         'context_marker': '8-4-4-4-12a'
     }
 
+
 def tasks_db(dag_id, execution_date):
     """
     replaces the actual db call
@@ -87,6 +88,7 @@ def tasks_db(dag_id, execution_date):
         }
     ]
 
+
 def test_get_action_steps():
     """
     Tests the main response from get all actions

tests/unit/control/test_actions_validations_id_api.py

@@ -17,6 +17,7 @@ from shipyard_airflow.control.actions_validations_id_api import (
 )
 from shipyard_airflow.errors import ApiError
 
+
 def actions_db(action_id):
     """
     replaces the actual db call
@@ -35,6 +36,7 @@ def actions_db(action_id):
             'context_marker': '8-4-4-4-12a'
         }
 
+
 def get_validations(validation_id):
     """
     Stub to return validations
@@ -49,6 +51,7 @@ def get_validations(validation_id):
     else:
         return None
 
+
 def test_get_action_validation():
     """
     Tests the main response from get all actions

tests/unit/plugins/test_concurrency_check_operator.py

@@ -98,6 +98,7 @@ def test_find_conflicting_dag():
     assert cco.find_conflicting_dag('buffalo') != 'monkey'
     assert cco.find_conflicting_dag('buffalo') == 'chicken'
 
+
 def test_execute_exception():
     """
     Run the whole execute function for testing
@@ -114,6 +115,7 @@ def test_execute_exception():
     except AirflowException as airflow_exception:
         assert 'Aborting run' in airflow_exception.args[0]
 
+
 def test_execute_success():
     """
     Run the whole execute function for testing - successfully!

tox.ini

@@ -28,7 +28,7 @@ commands = oslopolicy-sample-generator --config-file=generator/policy-generator.
 # NOTE(Bryan Strassner) ignoring F841 because of the airflow example pattern
 #     of naming variables even if they aren't used for DAGs and Operators.
 #     Doing so adds readability and context in this case.
-ignore = E302,H306,D100,D101,D102,F841
+ignore = F841
 # NOTE(Bryan Strassner) excluding 3rd party and generated code that is brought into the
 #     codebase.
 exclude = *plugins/rest_api_plugin.py,*lib/python*,*egg,.git*,*.md,.tox*,alembic/env.py,build/*