Update to UCP layout standard

- Create Makefile for image build
- Move Dockerfile into images/pegleg
- Move pegleg module src to src/bin/pegleg

Change-Id: I8fd728888ecfd75fe857da253d6c8cd4fd83f89c

.dockerignore

@@ -0,0 +1,4 @@
+**/__pycache__
+**/.tox
+**/.eggs
+**/pegleg.egg-info

Makefile

@@ -0,0 +1,61 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+PEGLEG_BUILD_CTX           ?= src/bin/pegleg
+IMAGE_NAME                 ?= pegleg
+DOCKER_REGISTRY            ?= attcomdev
+IMAGE_TAG                  ?= latest
+HELM                       ?= helm
+PROXY                      ?= http://proxy_url
+USE_PROXY                  ?= false
+PUSH_IMAGE                 ?= false
+LABEL                      ?= commit-id
+IMAGE                      ?= $(IMAGE_PREFIX)/$(PEGLEG_IMAGE_NAME):$(IMAGE_TAG)
+export
+
+# Build all docker images for this project
+.PHONY: images
+images: build_pegleg
+
+# Run an image locally and exercise simple tests
+.PHONY: run_images
+run_images: run_pegleg
+
+# Run the drydock container and exercise simple tests
+.PHONY: run_pegleg
+run_pegleg: build_pegleg
+	tools/pegleg.sh --help
+
+# Perform Linting
+.PHONY: lint
+lint: py_lint
+
+.PHONY: build_pegleg
+build_pegleg:
+ifeq ($(USE_PROXY), true)
+	docker build -t $(IMAGE) --network=host --label $(LABEL) -f images/pegleg/Dockerfile --build-arg ctx_base=$(PEGLEG_BUILD_CTX) --build-arg http_proxy=$(PROXY) --build-arg https_proxy=$(PROXY) .
+else
+	docker build -t $(IMAGE) --network=host --label $(LABEL) -f images/pegleg/Dockerfile --build-arg ctx_base=$(PEGLEG_BUILD_CTX) .
+endif
+ifeq ($(PUSH_IMAGE), true)
+	docker push $(IMAGE)
+endif
+
+.PHONY: clean
+clean:
+	rm -rf build
+
+.PHONY: py_lint
+py_lint:
+	cd src/bin/pegleg;tox -e lint

images/pegleg/Dockerfile

@@ -3,8 +3,10 @@ FROM python:3.6
 VOLUME /var/pegleg
 WORKDIR /var/pegleg
 
-COPY requirements.txt /opt/pegleg/requirements.txt
+ARG ctx_base=src/bin/pegleg
+
+COPY ${ctx_base}/requirements.txt /opt/pegleg/requirements.txt
 RUN pip3 install --no-cache-dir -r /opt/pegleg/requirements.txt
 
-COPY . /opt/pegleg
+COPY ${ctx_base} /opt/pegleg
 RUN pip3 install -e /opt/pegleg

src/bin/pegleg/pegleg/cli.py

@@ -32,6 +32,7 @@ def main(ctx, *, verbose):
 def site():
     pass
 
+
 @site.command(help='Output complete config for one site')
 @click.option(
     '-o',
@@ -44,6 +45,7 @@ def site():
 def collect(*, output_stream, site_name):
     engine.site.collect(site_name, output_stream)
 
+
 @site.command(help='Find sites impacted by changed files')
 @click.option(
     '-i',
@@ -99,6 +101,7 @@ def show(*, output_stream, site_name):
 def render(*, output_stream, site_name):
     engine.site.render(site_name, output_stream)
 
+
 def _validate_revision_callback(_ctx, _param, value):
     if value is not None and value.startswith('v'):
         return value
@@ -130,7 +133,9 @@ LINT_OPTION = click.option(
     required=False,
     type=click.BOOL,
     default=True,
-    help="Raise deckhand exception on missing substition sources. Defaults to True.")
+    help=
+    "Raise deckhand exception on missing substition sources. Defaults to True."
+)
 
 
 @stub.command('global', help='Add global structure for a new revision')

src/bin/pegleg/pegleg/engine/lint.py

@@ -113,16 +113,17 @@ def _verify_document(document, schemas, filename):
     if document.get('schema') in MANDATORY_ENCRYPTED_TYPES:
         storage_policy = document.get('metadata', {}).get('storagePolicy')
         if storage_policy != 'encrypted':
-            errors.append(
+            errors.append('%s (document %s) is a secret, but has unexpected '
-                    '%s (document %s) is a secret, but has unexpected storagePolicy: "%s"'
+                          'storagePolicy: "%s"' % (filename, name,
-                    % (filename, name, storage_policy))
+                                                   storage_policy))
 
         if not _filename_in_section(filename, 'secrets/'):
             errors.append(
-                    '%s (document %s) is a secret, is not stored in a secrets path'
+                '%s (document %s) is a secret, is not stored in a secrets path'
-                    % (filename, name))
+                % (filename, name))
     return errors
 
+
 def _verify_deckhand_render(fail_on_missing_sub_src=False):
 
     documents = []
@@ -138,6 +139,7 @@ def _verify_deckhand_render(fail_on_missing_sub_src=False):
     )
     return errors
 
+
 def _layer(data):
     if hasattr(data, 'get'):
         return data.get('metadata', {}).get('layeringDefinition',

src/bin/pegleg/pegleg/engine/site.py

@@ -26,6 +26,7 @@ def impacted(input_stream, output_stream):
     for site_name in sorted(impacted_sites):
         output_stream.write(site_name + '\n')
 
+
 def render(site_name, output_stream):
     documents = []
     for filename in util.definition.site_files(site_name):
@@ -33,11 +34,11 @@ def render(site_name, output_stream):
             documents.extend(list(yaml.safe_load_all(f)))
 
     rendered_documents, errors = util.deckhand.deckhand_render(
-        documents=documents
+        documents=documents)
-    )
     for d in documents:
         output_stream.writelines(yaml.dump(d))
 
+
 def list_(output_stream):
     fieldnames = ['site_name', 'site_type', 'revision']
     writer = csv.DictWriter(

src/bin/pegleg/pegleg/engine/util/deckhand.py

@@ -1,6 +1,7 @@
 from deckhand.engine import layering
 from deckhand import errors as dh_errors
 
+
 def load_schemas_from_docs(documents):
     '''
     Fills the cache of known schemas from the document set
@@ -20,7 +21,10 @@ def load_schemas_from_docs(documents):
 
     return schema_set, errors
 
-def deckhand_render(documents=[], fail_on_missing_sub_src=False, validate=False):
+
+def deckhand_render(documents=[],
+                    fail_on_missing_sub_src=False,
+                    validate=False):
 
     errors = []
     rendered_documents = []

src/bin/pegleg/tox.ini

@@ -1,6 +1,9 @@
 [tox]
 envlist = lint
 
+[testenv]
+basepython=python3
+
 [testenv:fmt]
 deps = yapf==0.20.0
 commands =
@@ -13,3 +16,6 @@ deps =
 commands =
     yapf -rd {toxinidir}/pegleg
     flake8 {toxinidir}/pegleg
+
+[flake8]
+ignore = E251

tools/pegleg.sh

@@ -14,16 +14,10 @@ else
   WORKSPACE=$(realpath "${SCRIPT_DIR}/..")
 fi
 
-IMAGE_PEGLEG=${IMAGE_PEGLEG:-quay.io/attcomdev/pegleg:latest}
+IMAGE=${IMAGE:-quay.io/attcomdev/pegleg:latest}
 
-if [[ -z ${http_proxy} && -z ${https_proxy} ]]
+docker run --rm -t \
-then
+    --net=none \
-    docker build --network=host -q --rm -t "${IMAGE_PEGLEG}" "${SOURCE_DIR}" > /dev/null
+    -v "${WORKSPACE}:/workspace" \
-else
+    "${IMAGE}" \
-    docker build --network=host -q --rm -t "${IMAGE_PEGLEG}" --build-arg http_proxy=${http_proxy} --build-arg https_proxy=${https_proxy}  "${SOURCE_DIR}" > /dev/null
-fi
-
-docker run --net=none --rm -t \
-    -v "${WORKSPACE}:/var/pegleg" \
-    "${IMAGE_PEGLEG}" \
     pegleg "${@}"

tools/pegleg/.dockerignore

@@ -1,4 +0,0 @@
-__pycache__
-.tox
-.eggs
-pegleg.egg-info