openstack/tosca-parser
Code Issues Proposed changes

Handle URLs for input templates and imports

Browse Source 
Allow input templates and imported custom types to be provided as URLs
with TOSCA-Parser auto-detecting the type (file vs URL), and add
necessary unit tests.

Note, since for some test cases currently the required file does not
exist on github, we temporarily use files hosted somewhere else. Once
the patch is merged a follow-on patch will be submitted to fix that
issue and use URLs of the new files that are submitted with this patch.

Partial-Bug: #1340748
Partially Implements: blueprint tosca-namespaces

Change-Id: Idec0318fa456ebccd552f67726eee8905714aa91
This commit is contained in:
Vahid Hashemian 2015-09-08 12:51:46 -07:00
parent 4941a93f85
commit 457a80379f
7 changed files with 419 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
toscaparser/tests/data/tosca_single_instance_wordpress_with_local_abspath_import.yaml Normal file
View File

@ -0,0 +1,120 @@
tosca_definitions_version: tosca_simple_yaml_1_0
description: >
TOSCA simple profile with wordpress, web server and mysql on the same server.
imports:
- /toscaparser/tests/data/custom_types/wordpress.yaml
topology_template:
inputs:
cpus:
type: integer
description: Number of CPUs for the server.
constraints:
- valid_values: [ 1, 2, 4, 8 ]
default: 1
db_name:
type: string
description: The name of the database.
default: wordpress
db_user:
type: string
description: The user name of the DB user.
default: wp_user
db_pwd:
type: string
description: The WordPress database admin account password.
default: wp_pass
db_root_pwd:
type: string
description: Root password for MySQL.
db_port:
type: PortDef
description: Port for the MySQL database.
default: 3306
node_templates:
wordpress:
type: tosca.nodes.WebApplication.WordPress
requirements:
- host: webserver
- database_endpoint: mysql_database
interfaces:
Standard:
create: wordpress/wordpress_install.sh
configure:
implementation: wordpress/wordpress_configure.sh
inputs:
wp_db_name: wordpress
wp_db_user: wp_user
wp_db_password: wp_pass
mysql_database:
type: tosca.nodes.Database
properties:
name: { get_input: db_name }
user: { get_input: db_user }
password: { get_input: db_pwd }
capabilities:
database_endpoint:
properties:
port: { get_input: db_port }
requirements:
- host:
node: mysql_dbms
interfaces:
Standard:
configure:
implementation: mysql/mysql_database_configure.sh
inputs:
db_name: wordpress
db_user: wp_user
db_password: wp_pass
db_root_password: passw0rd
mysql_dbms:
type: tosca.nodes.DBMS
properties:
root_password: { get_input: db_root_pwd }
port: { get_input: db_port }
requirements:
- host: server
interfaces:
Standard:
create:
implementation: mysql/mysql_dbms_install.sh
inputs:
db_root_password: passw0rd
start: mysql/mysql_dbms_start.sh
configure:
implementation: mysql/mysql_dbms_configure.sh
inputs:
db_port: 3366
webserver:
type: tosca.nodes.WebServer
requirements:
- host: server
interfaces:
Standard:
create: webserver/webserver_install.sh
start: webserver/webserver_start.sh
server:
type: tosca.nodes.Compute
capabilities:
host:
properties:
disk_size: 10 GB
num_cpus: { get_input: cpus }
mem_size: 4096 MB
os:
properties:
architecture: x86_64
type: Linux
distribution: Ubuntu
version: 14.04
outputs:
website_url:
description: URL for Wordpress wiki.
value: { get_attribute: [server, private_address] }

120
toscaparser/tests/data/tosca_single_instance_wordpress_with_url_import.yaml Normal file
View File

@ -0,0 +1,120 @@
tosca_definitions_version: tosca_simple_yaml_1_0
description: >
TOSCA simple profile with wordpress, web server and mysql on the same server.
imports:
- https://raw.githubusercontent.com/openstack/heat-translator/master/translator/tests/data/custom_types/wordpress.yaml
topology_template:
inputs:
cpus:
type: integer
description: Number of CPUs for the server.
constraints:
- valid_values: [ 1, 2, 4, 8 ]
default: 1
db_name:
type: string
description: The name of the database.
default: wordpress
db_user:
type: string
description: The user name of the DB user.
default: wp_user
db_pwd:
type: string
description: The WordPress database admin account password.
default: wp_pass
db_root_pwd:
type: string
description: Root password for MySQL.
db_port:
type: PortDef
description: Port for the MySQL database.
default: 3306
node_templates:
wordpress:
type: tosca.nodes.WebApplication.WordPress
requirements:
- host: webserver
- database_endpoint: mysql_database
interfaces:
Standard:
create: wordpress/wordpress_install.sh
configure:
implementation: wordpress/wordpress_configure.sh
inputs:
wp_db_name: wordpress
wp_db_user: wp_user
wp_db_password: wp_pass
mysql_database:
type: tosca.nodes.Database
properties:
name: { get_input: db_name }
user: { get_input: db_user }
password: { get_input: db_pwd }
capabilities:
database_endpoint:
properties:
port: { get_input: db_port }
requirements:
- host:
node: mysql_dbms
interfaces:
Standard:
configure:
implementation: mysql/mysql_database_configure.sh
inputs:
db_name: wordpress
db_user: wp_user
db_password: wp_pass
db_root_password: passw0rd
mysql_dbms:
type: tosca.nodes.DBMS
properties:
root_password: { get_input: db_root_pwd }
port: { get_input: db_port }
requirements:
- host: server
interfaces:
Standard:
create:
implementation: mysql/mysql_dbms_install.sh
inputs:
db_root_password: passw0rd
start: mysql/mysql_dbms_start.sh
configure:
implementation: mysql/mysql_dbms_configure.sh
inputs:
db_port: 3366
webserver:
type: tosca.nodes.WebServer
requirements:
- host: server
interfaces:
Standard:
create: webserver/webserver_install.sh
start: webserver/webserver_start.sh
server:
type: tosca.nodes.Compute
capabilities:
host:
properties:
disk_size: 10 GB
num_cpus: { get_input: cpus }
mem_size: 4096 MB
os:
properties:
architecture: x86_64
type: Linux
distribution: Ubuntu
version: 14.04
outputs:
website_url:
description: URL for Wordpress wiki.
value: { get_attribute: [server, private_address] }

32
toscaparser/tests/test_toscatpl.py
View File

@ -382,3 +382,35 @@ class ToscaTemplateTest(TestCase):
custom_def).get_capabilities_objects())
self.assertEqual('Type "tosca.capabilities.TestCapability" is not '
'a valid type.', six.text_type(err))
def test_local_template_with_local_relpath_import(self):
tosca_tpl = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"data/tosca_single_instance_wordpress.yaml")
tosca = ToscaTemplate(tosca_tpl)
self.assertIsNotNone(tosca.topology_template.custom_defs)
def test_local_template_with_url_import(self):
tosca_tpl = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"data/tosca_single_instance_wordpress_with_url_import.yaml")
tosca = ToscaTemplate(tosca_tpl)
self.assertIsNotNone(tosca.topology_template.custom_defs)
def test_url_template_with_local_relpath_import(self):
tosca_tpl = ('https://raw.githubusercontent.com/openstack/'
'tosca-parser/master/toscaparser/tests/data/'
'tosca_single_instance_wordpress.yaml')
tosca = ToscaTemplate(tosca_tpl, False)
self.assertIsNotNone(tosca.topology_template.custom_defs)
def test_url_template_with_local_abspath_import(self):
tosca_tpl = ('http://tinyurl.com/nfbwjwd')
err = self.assertRaises(ImportError, ToscaTemplate, tosca_tpl, False)
self.assertEqual('Absolute file name cannot be used for a URL-based '
'input template.', err.__str__())
def test_url_template_with_url_import(self):
tosca_tpl = ('http://tinyurl.com/ow76273')
tosca = ToscaTemplate(tosca_tpl, False)
self.assertIsNotNone(tosca.topology_template.custom_defs)

44
toscaparser/tests/test_utils.py Normal file
View File

@ -0,0 +1,44 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from toscaparser.tests.base import TestCase
import toscaparser.utils.urlutils
class UrlUtilsTest(TestCase):
url_utils = toscaparser.utils.urlutils.UrlUtils
def test_urlutils_validate_url(self):
self.assertTrue(self.url_utils.validate_url("http://www.github.com/"))
self.assertTrue(
self.url_utils.validate_url("https://github.com:81/a/2/a.b"))
self.assertTrue(self.url_utils.validate_url("ftp://github.com"))
self.assertFalse(self.url_utils.validate_url("github.com"))
self.assertFalse(self.url_utils.validate_url("123"))
self.assertFalse(self.url_utils.validate_url("a/b/c"))
def test_urlutils_join_url(self):
self.assertEqual(
self.url_utils.join_url("http://github.com/proj1", "proj2"),
"http://github.com/proj2")
self.assertEqual(
self.url_utils.join_url("http://github.com/proj1/scripts/a.js",
"b.js"),
"http://github.com/proj1/scripts/b.js")
self.assertEqual(
self.url_utils.join_url("http://github.com/proj1/scripts", "b.js"),
"http://github.com/proj1/b.js")
self.assertEqual(
self.url_utils.join_url("http://github.com/proj1/scripts",
"scripts/b.js"),
"http://github.com/proj1/scripts/b.js")

58
toscaparser/tosca_template.py
View File

@ -19,6 +19,8 @@ from toscaparser.common.exception import MissingRequiredFieldError
from toscaparser.common.exception import UnknownFieldError
from toscaparser.topology_template import TopologyTemplate
from toscaparser.tpl_relationship_graph import ToscaGraph
from toscaparser.utils.gettextutils import _
import toscaparser.utils.urlutils
import toscaparser.utils.yamlparser
@ -44,9 +46,10 @@ class ToscaTemplate(object):
VALID_TEMPLATE_VERSIONS = ['tosca_simple_yaml_1_0']
'''Load the template data.'''
def __init__(self, path, parsed_params=None):
self.tpl = YAML_LOADER(path)
def __init__(self, path, a_file=True, parsed_params=None):
self.tpl = YAML_LOADER(path, a_file)
self.path = path
self.a_file = a_file
self.parsed_params = parsed_params
self._validate_field()
self.version = self._tpl_version()
@ -111,17 +114,54 @@ class ToscaTemplate(object):
return custom_defs
def _get_custom_types(self, type_definition):
# Handle custom types defined in outer template file
"""Handle custom types defined in imported template files
This method loads the custom type definitions referenced in "imports"
section of the TOSCA YAML template by determining whether each import
is specified via a file reference (by relative or absolute path) or a
URL reference. It then assigns the correct value to "def_file" variable
so the YAML content of those imports can be loaded.
Possibilities:
+----------+--------+------------------------------+
| template | import | comment |
+----------+--------+------------------------------+
| file | file | OK |
| file | URL | OK |
| URL | file | file must be a relative path |
| URL | URL | OK |
+----------+--------+------------------------------+
"""
custom_defs = {}
imports = self._tpl_imports()
if imports:
main_a_file = os.path.isfile(self.path)
for definition in imports:
if os.path.isabs(definition):
def_file = definition
else:
tpl_dir = os.path.dirname(os.path.abspath(self.path))
def_file = os.path.join(tpl_dir, definition)
custom_type = YAML_LOADER(def_file)
def_file = definition
a_file = False
if main_a_file:
if os.path.isfile(definition):
a_file = True
else:
full_path = os.path.join(
os.path.dirname(os.path.abspath(self.path)),
definition)
if os.path.isfile(full_path):
a_file = True
def_file = full_path
else: # main_a_url
a_url = toscaparser.utils.urlutils.UrlUtils.\
validate_url(definition)
if not a_url:
if os.path.isabs(definition):
raise ImportError(_("Absolute file name cannot be "
"used for a URL-based input "
"template."))
def_file = toscaparser.utils.urlutils.UrlUtils.\
join_url(self.path, definition)
custom_type = YAML_LOADER(def_file, a_file)
outer_custom_types = custom_type.get(type_definition)
if outer_custom_types:
custom_defs.update(outer_custom_types)

43
toscaparser/utils/urlutils.py Normal file
View File

@ -0,0 +1,43 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from six.moves.urllib.parse import urljoin
from six.moves.urllib.parse import urlparse
from toscaparser.utils.gettextutils import _
class UrlUtils(object):
@staticmethod
def validate_url(path):
"""Validates whether the given path is a URL or not.
If the given path includes a scheme (http, https, ftp, ...) and a net
location (a domain name such as www.github.com) it is validated as a
URL.
"""
parsed = urlparse(path)
return bool(parsed.scheme) and bool(parsed.netloc)
@staticmethod
def join_url(url, relative_path):
"""Builds a new URL from the given URL and the relative path.
Example:
url: http://www.githib.com/openstack/heat
relative_path: heat-translator
- joined: http://www.githib.com/openstack/heat-translator
"""
if not UrlUtils.validate_url(url):
raise ValueError(_("Provided URL is invalid."))
return urljoin(url, relative_path)

14
toscaparser/utils/yamlparser.py
View File

@ -14,15 +14,23 @@ import codecs
from collections import OrderedDict
import yaml
try:
# Python 3.x
import urllib.request as urllib2
except ImportError:
# Python 2.x
import urllib2
if hasattr(yaml, 'CSafeLoader'):
yaml_loader = yaml.CSafeLoader
else:
yaml_loader = yaml.SafeLoader
def load_yaml(path):
with codecs.open(path, encoding='utf-8', errors='strict') as f:
return yaml.load(f.read(), Loader=yaml_loader)
def load_yaml(path, a_file=True):
f = codecs.open(path, encoding='utf-8', errors='strict') if a_file \
else urllib2.urlopen(path)
return yaml.load(f.read(), Loader=yaml_loader)
def simple_parse(tmpl_str):