Add CSAR processing support

Add support for processing CSAR files and include necessary unit tests.

Note: Since CSAR files used do not already exist on github a temporary
URL has been used. Once the patch merges a follow-on patch will be
submitted to update the URL to a github URL.

Change-Id: Ibf44be1368ed0144ecf415068b0ca5aea5aa6134
Partially-Implements: blueprint tosca-csar-translation

toscaparser/prereq/csar.py

@@ -11,18 +11,26 @@
 #    under the License.
 
 import os.path
+import requests
 import tempfile
 import yaml
 import zipfile
 
 from toscaparser.common.exception import ValidationError
 from toscaparser.utils.gettextutils import _
+from toscaparser.utils.urlutils import UrlUtils
+
+try:  # Python 2.x
+    from BytesIO import BytesIO
+except ImportError:  # Python 3.x
+    from io import BytesIO
 
 
 class CSAR(object):
 
-    def __init__(self, csar_file):
+    def __init__(self, csar_file, a_file=True):
         self.csar_file = csar_file
+        self.a_file = a_file
         self.is_validated = False
 
     def validate(self):
@@ -31,10 +39,18 @@ class CSAR(object):
         self.is_validated = True
 
         # validate that the file exists
-        if not os.path.isfile(self.csar_file):
+        if self.a_file and not os.path.isfile(self.csar_file):
             err_msg = (_('The file %s does not exist.') % self.csar_file)
             raise ValidationError(message=err_msg)
 
+        if not self.a_file:  # a URL
+            if not UrlUtils.validate_url(self.csar_file):
+                err_msg = (_('The URL %s does not exist.') % self.csar_file)
+                raise ValidationError(message=err_msg)
+            else:
+                response = requests.get(self.csar_file)
+                self.csar_file = BytesIO(response.content)
+
         # validate that it is a valid zip file
         if not zipfile.is_zipfile(self.csar_file):
             err_msg = (_('The file %s is not a valid zip file.')

toscaparser/tests/data/CSAR/tosca_single_instance_wordpress/Definitions/tosca_single_instance_wordpress.yaml

@@ -58,10 +58,10 @@ topology_template:
            configure:
               implementation: Scripts/MYSQLDatabase/configure.sh
               inputs:
-                db_name: { get_property: [ SELF, db_name ] }
-                db_user: { get_property: [ SELF, db_user ] }
-                db_password: { get_property: [ SELF, db_password ] }
-                db_root_password: { get_property: [ mysql_dbms, dbms_root_password ] }
+                db_name: { get_property: [ SELF, name ] }
+                db_user: { get_property: [ SELF, user ] }
+                db_password: { get_property: [ SELF, password ] }
+                db_root_password: { get_property: [ mysql_dbms, root_password ] }
 
     mysql_dbms:
       type: tosca.nodes.DBMS
@@ -87,20 +87,21 @@ topology_template:
         Standard:
           create: Scripts/WebServer/install.sh
           start: Scripts/WebServer/start.sh
-  server:
-    type: tosca.nodes.Compute
-    capabilities:
-      host:
-        properties:
-          disk_size: 10 GB
-          num_cpus: { get_input: cpus }
-          mem_size: 4096 MB
-      os:
-        properties:
-          architecture: x86_64
-          type: Linux
-          distribution: Fedora
-          version: 18.0
+
+    server:
+      type: tosca.nodes.Compute
+      capabilities:
+        host:
+          properties:
+            disk_size: 10 GB
+            num_cpus: { get_input: cpus }
+            mem_size: 4096 MB
+        os:
+          properties:
+            architecture: x86_64
+            type: Linux
+            distribution: Fedora
+            version: 18.0
 
   outputs:
     website_url:

toscaparser/tests/test_toscatpl.py

@@ -423,3 +423,14 @@ class ToscaTemplateTest(TestCase):
                      'tosca_single_instance_wordpress_with_url_import.yaml')
         tosca = ToscaTemplate(tosca_tpl, False)
         self.assertTrue(tosca.topology_template.custom_defs)
+
+    def test_csar_parsing_wordpress(self):
+        csar_archive = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)),
+            'data/CSAR/csar_wordpress.zip')
+        self.assertTrue(ToscaTemplate(csar_archive))
+
+    def test_csar_parsing_elk_url_based(self):
+        csar_archive = ('https://ibm.box.com/shared/static/'
+                        'k9vtus4jes1epl7vfojbcscgsd80inzv.zip')
+        self.assertTrue(ToscaTemplate(csar_archive, False))

toscaparser/tosca_template.py

@@ -12,15 +12,15 @@
 
 
 import logging
+import os
 
 from toscaparser.common.exception import InvalidTemplateVersion
 from toscaparser.common.exception import MissingRequiredFieldError
 from toscaparser.common.exception import UnknownFieldError
+import toscaparser.imports
+from toscaparser.prereq.csar import CSAR
 from toscaparser.topology_template import TopologyTemplate
 from toscaparser.tpl_relationship_graph import ToscaGraph
-
-import toscaparser.imports
-import toscaparser.utils.urlutils
 import toscaparser.utils.yamlparser
 
 
@@ -49,9 +49,9 @@ class ToscaTemplate(object):
 
     '''Load the template data.'''
     def __init__(self, path, a_file=True, parsed_params=None):
-        self.tpl = YAML_LOADER(path, a_file)
-        self.path = path
         self.a_file = a_file
+        self.path = self._get_path(path)
+        self.tpl = YAML_LOADER(self.path, self.a_file)
         self.parsed_params = parsed_params
         self._validate_field()
         self.version = self._tpl_version()
@@ -151,3 +151,17 @@ class ToscaTemplate(object):
             raise InvalidTemplateVersion(
                 what=version,
                 valid_versions=', '. join(self.VALID_TEMPLATE_VERSIONS))
+
+    def _get_path(self, path):
+        if path.lower().endswith('.yaml'):
+            return path
+        elif path.lower().endswith('.zip'):
+            # a CSAR archive
+            csar = CSAR(path, self.a_file)
+            csar.validate()
+            folder = csar.decompress()
+            self.a_file = True  # the file has been decompressed locally
+            return os.path.join(folder, csar.get_main_template())
+        else:
+            raise ValueError(_("%(path)s is not a valid file.")
+                             % {'path': path})