diff --git a/toscaparser/tests/data/tosca_single_instance_wordpress_with_local_abspath_import.yaml b/toscaparser/tests/data/tosca_single_instance_wordpress_with_local_abspath_import.yaml new file mode 100644 index 00000000..45aec7f3 --- /dev/null +++ b/toscaparser/tests/data/tosca_single_instance_wordpress_with_local_abspath_import.yaml @@ -0,0 +1,120 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: > + TOSCA simple profile with wordpress, web server and mysql on the same server. + +imports: + - /toscaparser/tests/data/custom_types/wordpress.yaml + +topology_template: + inputs: + cpus: + type: integer + description: Number of CPUs for the server. + constraints: + - valid_values: [ 1, 2, 4, 8 ] + default: 1 + db_name: + type: string + description: The name of the database. + default: wordpress + db_user: + type: string + description: The user name of the DB user. + default: wp_user + db_pwd: + type: string + description: The WordPress database admin account password. + default: wp_pass + db_root_pwd: + type: string + description: Root password for MySQL. + db_port: + type: PortDef + description: Port for the MySQL database. + default: 3306 + + node_templates: + wordpress: + type: tosca.nodes.WebApplication.WordPress + requirements: + - host: webserver + - database_endpoint: mysql_database + interfaces: + Standard: + create: wordpress/wordpress_install.sh + configure: + implementation: wordpress/wordpress_configure.sh + inputs: + wp_db_name: wordpress + wp_db_user: wp_user + wp_db_password: wp_pass + + mysql_database: + type: tosca.nodes.Database + properties: + name: { get_input: db_name } + user: { get_input: db_user } + password: { get_input: db_pwd } + capabilities: + database_endpoint: + properties: + port: { get_input: db_port } + requirements: + - host: + node: mysql_dbms + interfaces: + Standard: + configure: + implementation: mysql/mysql_database_configure.sh + inputs: + db_name: wordpress + db_user: wp_user + db_password: wp_pass + db_root_password: passw0rd + mysql_dbms: + type: tosca.nodes.DBMS + properties: + root_password: { get_input: db_root_pwd } + port: { get_input: db_port } + requirements: + - host: server + interfaces: + Standard: + create: + implementation: mysql/mysql_dbms_install.sh + inputs: + db_root_password: passw0rd + start: mysql/mysql_dbms_start.sh + configure: + implementation: mysql/mysql_dbms_configure.sh + inputs: + db_port: 3366 + + webserver: + type: tosca.nodes.WebServer + requirements: + - host: server + interfaces: + Standard: + create: webserver/webserver_install.sh + start: webserver/webserver_start.sh + server: + type: tosca.nodes.Compute + capabilities: + host: + properties: + disk_size: 10 GB + num_cpus: { get_input: cpus } + mem_size: 4096 MB + os: + properties: + architecture: x86_64 + type: Linux + distribution: Ubuntu + version: 14.04 + + outputs: + website_url: + description: URL for Wordpress wiki. + value: { get_attribute: [server, private_address] } diff --git a/toscaparser/tests/data/tosca_single_instance_wordpress_with_url_import.yaml b/toscaparser/tests/data/tosca_single_instance_wordpress_with_url_import.yaml new file mode 100644 index 00000000..e5f1580b --- /dev/null +++ b/toscaparser/tests/data/tosca_single_instance_wordpress_with_url_import.yaml @@ -0,0 +1,120 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: > + TOSCA simple profile with wordpress, web server and mysql on the same server. + +imports: + - https://raw.githubusercontent.com/openstack/heat-translator/master/translator/tests/data/custom_types/wordpress.yaml + +topology_template: + inputs: + cpus: + type: integer + description: Number of CPUs for the server. + constraints: + - valid_values: [ 1, 2, 4, 8 ] + default: 1 + db_name: + type: string + description: The name of the database. + default: wordpress + db_user: + type: string + description: The user name of the DB user. + default: wp_user + db_pwd: + type: string + description: The WordPress database admin account password. + default: wp_pass + db_root_pwd: + type: string + description: Root password for MySQL. + db_port: + type: PortDef + description: Port for the MySQL database. + default: 3306 + + node_templates: + wordpress: + type: tosca.nodes.WebApplication.WordPress + requirements: + - host: webserver + - database_endpoint: mysql_database + interfaces: + Standard: + create: wordpress/wordpress_install.sh + configure: + implementation: wordpress/wordpress_configure.sh + inputs: + wp_db_name: wordpress + wp_db_user: wp_user + wp_db_password: wp_pass + + mysql_database: + type: tosca.nodes.Database + properties: + name: { get_input: db_name } + user: { get_input: db_user } + password: { get_input: db_pwd } + capabilities: + database_endpoint: + properties: + port: { get_input: db_port } + requirements: + - host: + node: mysql_dbms + interfaces: + Standard: + configure: + implementation: mysql/mysql_database_configure.sh + inputs: + db_name: wordpress + db_user: wp_user + db_password: wp_pass + db_root_password: passw0rd + mysql_dbms: + type: tosca.nodes.DBMS + properties: + root_password: { get_input: db_root_pwd } + port: { get_input: db_port } + requirements: + - host: server + interfaces: + Standard: + create: + implementation: mysql/mysql_dbms_install.sh + inputs: + db_root_password: passw0rd + start: mysql/mysql_dbms_start.sh + configure: + implementation: mysql/mysql_dbms_configure.sh + inputs: + db_port: 3366 + + webserver: + type: tosca.nodes.WebServer + requirements: + - host: server + interfaces: + Standard: + create: webserver/webserver_install.sh + start: webserver/webserver_start.sh + server: + type: tosca.nodes.Compute + capabilities: + host: + properties: + disk_size: 10 GB + num_cpus: { get_input: cpus } + mem_size: 4096 MB + os: + properties: + architecture: x86_64 + type: Linux + distribution: Ubuntu + version: 14.04 + + outputs: + website_url: + description: URL for Wordpress wiki. + value: { get_attribute: [server, private_address] } diff --git a/toscaparser/tests/test_toscatpl.py b/toscaparser/tests/test_toscatpl.py index b85939de..6f121677 100644 --- a/toscaparser/tests/test_toscatpl.py +++ b/toscaparser/tests/test_toscatpl.py @@ -382,3 +382,35 @@ class ToscaTemplateTest(TestCase): custom_def).get_capabilities_objects()) self.assertEqual('Type "tosca.capabilities.TestCapability" is not ' 'a valid type.', six.text_type(err)) + + def test_local_template_with_local_relpath_import(self): + tosca_tpl = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "data/tosca_single_instance_wordpress.yaml") + tosca = ToscaTemplate(tosca_tpl) + self.assertIsNotNone(tosca.topology_template.custom_defs) + + def test_local_template_with_url_import(self): + tosca_tpl = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "data/tosca_single_instance_wordpress_with_url_import.yaml") + tosca = ToscaTemplate(tosca_tpl) + self.assertIsNotNone(tosca.topology_template.custom_defs) + + def test_url_template_with_local_relpath_import(self): + tosca_tpl = ('https://raw.githubusercontent.com/openstack/' + 'tosca-parser/master/toscaparser/tests/data/' + 'tosca_single_instance_wordpress.yaml') + tosca = ToscaTemplate(tosca_tpl, False) + self.assertIsNotNone(tosca.topology_template.custom_defs) + + def test_url_template_with_local_abspath_import(self): + tosca_tpl = ('http://tinyurl.com/nfbwjwd') + err = self.assertRaises(ImportError, ToscaTemplate, tosca_tpl, False) + self.assertEqual('Absolute file name cannot be used for a URL-based ' + 'input template.', err.__str__()) + + def test_url_template_with_url_import(self): + tosca_tpl = ('http://tinyurl.com/ow76273') + tosca = ToscaTemplate(tosca_tpl, False) + self.assertIsNotNone(tosca.topology_template.custom_defs) diff --git a/toscaparser/tests/test_utils.py b/toscaparser/tests/test_utils.py new file mode 100644 index 00000000..52ab5f63 --- /dev/null +++ b/toscaparser/tests/test_utils.py @@ -0,0 +1,44 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from toscaparser.tests.base import TestCase +import toscaparser.utils.urlutils + + +class UrlUtilsTest(TestCase): + + url_utils = toscaparser.utils.urlutils.UrlUtils + + def test_urlutils_validate_url(self): + self.assertTrue(self.url_utils.validate_url("http://www.github.com/")) + self.assertTrue( + self.url_utils.validate_url("https://github.com:81/a/2/a.b")) + self.assertTrue(self.url_utils.validate_url("ftp://github.com")) + self.assertFalse(self.url_utils.validate_url("github.com")) + self.assertFalse(self.url_utils.validate_url("123")) + self.assertFalse(self.url_utils.validate_url("a/b/c")) + + def test_urlutils_join_url(self): + self.assertEqual( + self.url_utils.join_url("http://github.com/proj1", "proj2"), + "http://github.com/proj2") + self.assertEqual( + self.url_utils.join_url("http://github.com/proj1/scripts/a.js", + "b.js"), + "http://github.com/proj1/scripts/b.js") + self.assertEqual( + self.url_utils.join_url("http://github.com/proj1/scripts", "b.js"), + "http://github.com/proj1/b.js") + self.assertEqual( + self.url_utils.join_url("http://github.com/proj1/scripts", + "scripts/b.js"), + "http://github.com/proj1/scripts/b.js") diff --git a/toscaparser/tosca_template.py b/toscaparser/tosca_template.py index 2b35a63e..ec05688a 100644 --- a/toscaparser/tosca_template.py +++ b/toscaparser/tosca_template.py @@ -19,6 +19,8 @@ from toscaparser.common.exception import MissingRequiredFieldError from toscaparser.common.exception import UnknownFieldError from toscaparser.topology_template import TopologyTemplate from toscaparser.tpl_relationship_graph import ToscaGraph +from toscaparser.utils.gettextutils import _ +import toscaparser.utils.urlutils import toscaparser.utils.yamlparser @@ -44,9 +46,10 @@ class ToscaTemplate(object): VALID_TEMPLATE_VERSIONS = ['tosca_simple_yaml_1_0'] '''Load the template data.''' - def __init__(self, path, parsed_params=None): - self.tpl = YAML_LOADER(path) + def __init__(self, path, a_file=True, parsed_params=None): + self.tpl = YAML_LOADER(path, a_file) self.path = path + self.a_file = a_file self.parsed_params = parsed_params self._validate_field() self.version = self._tpl_version() @@ -111,17 +114,54 @@ class ToscaTemplate(object): return custom_defs def _get_custom_types(self, type_definition): - # Handle custom types defined in outer template file + """Handle custom types defined in imported template files + + This method loads the custom type definitions referenced in "imports" + section of the TOSCA YAML template by determining whether each import + is specified via a file reference (by relative or absolute path) or a + URL reference. It then assigns the correct value to "def_file" variable + so the YAML content of those imports can be loaded. + + Possibilities: + +----------+--------+------------------------------+ + | template | import | comment | + +----------+--------+------------------------------+ + | file | file | OK | + | file | URL | OK | + | URL | file | file must be a relative path | + | URL | URL | OK | + +----------+--------+------------------------------+ + """ + custom_defs = {} imports = self._tpl_imports() if imports: + main_a_file = os.path.isfile(self.path) for definition in imports: - if os.path.isabs(definition): - def_file = definition - else: - tpl_dir = os.path.dirname(os.path.abspath(self.path)) - def_file = os.path.join(tpl_dir, definition) - custom_type = YAML_LOADER(def_file) + def_file = definition + a_file = False + if main_a_file: + if os.path.isfile(definition): + a_file = True + else: + full_path = os.path.join( + os.path.dirname(os.path.abspath(self.path)), + definition) + if os.path.isfile(full_path): + a_file = True + def_file = full_path + else: # main_a_url + a_url = toscaparser.utils.urlutils.UrlUtils.\ + validate_url(definition) + if not a_url: + if os.path.isabs(definition): + raise ImportError(_("Absolute file name cannot be " + "used for a URL-based input " + "template.")) + def_file = toscaparser.utils.urlutils.UrlUtils.\ + join_url(self.path, definition) + + custom_type = YAML_LOADER(def_file, a_file) outer_custom_types = custom_type.get(type_definition) if outer_custom_types: custom_defs.update(outer_custom_types) diff --git a/toscaparser/utils/urlutils.py b/toscaparser/utils/urlutils.py new file mode 100644 index 00000000..e1b40560 --- /dev/null +++ b/toscaparser/utils/urlutils.py @@ -0,0 +1,43 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from six.moves.urllib.parse import urljoin +from six.moves.urllib.parse import urlparse +from toscaparser.utils.gettextutils import _ + + +class UrlUtils(object): + + @staticmethod + def validate_url(path): + """Validates whether the given path is a URL or not. + + If the given path includes a scheme (http, https, ftp, ...) and a net + location (a domain name such as www.github.com) it is validated as a + URL. + """ + parsed = urlparse(path) + return bool(parsed.scheme) and bool(parsed.netloc) + + @staticmethod + def join_url(url, relative_path): + """Builds a new URL from the given URL and the relative path. + + Example: + url: http://www.githib.com/openstack/heat + relative_path: heat-translator + - joined: http://www.githib.com/openstack/heat-translator + """ + if not UrlUtils.validate_url(url): + raise ValueError(_("Provided URL is invalid.")) + return urljoin(url, relative_path) diff --git a/toscaparser/utils/yamlparser.py b/toscaparser/utils/yamlparser.py index d2f406a5..a7131512 100644 --- a/toscaparser/utils/yamlparser.py +++ b/toscaparser/utils/yamlparser.py @@ -14,15 +14,23 @@ import codecs from collections import OrderedDict import yaml +try: + # Python 3.x + import urllib.request as urllib2 +except ImportError: + # Python 2.x + import urllib2 + if hasattr(yaml, 'CSafeLoader'): yaml_loader = yaml.CSafeLoader else: yaml_loader = yaml.SafeLoader -def load_yaml(path): - with codecs.open(path, encoding='utf-8', errors='strict') as f: - return yaml.load(f.read(), Loader=yaml_loader) +def load_yaml(path, a_file=True): + f = codecs.open(path, encoding='utf-8', errors='strict') if a_file \ + else urllib2.urlopen(path) + return yaml.load(f.read(), Loader=yaml_loader) def simple_parse(tmpl_str):