diff --git a/toscaparser/tests/base.py b/toscaparser/tests/base.py index 26198897..b7856250 100644 --- a/toscaparser/tests/base.py +++ b/toscaparser/tests/base.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +import codecs import os import fixtures @@ -63,3 +64,23 @@ class TestCase(testscenarios.TestWithScenarios, testtools.TestCase): os.path.dirname(os.path.abspath(__file__)), 'data', filename)) + + +class MockTestClass(): + comp_urldict = {} + + def mock_urlopen_method(self, path: str): + """Open local file instead of opening file at external URL. + + When using this method, you need to set the URL to be replaced and the + file path to open instead in "comp_urldict". + This method checks whether the URL passed as an argument exists in the + key of "comp_urldict", and if it exists, opens the file with the path + set in the value of "comp_urldict". + """ + if self.comp_urldict.get(path): + file_path = self.comp_urldict.get(path) + else: + file_path = path + + return codecs.open(file_path, encoding='utf-8', errors='strict') diff --git a/toscaparser/tests/data/CSAR/csar_wordpress_with_url_import_and_script.zip b/toscaparser/tests/data/CSAR/csar_wordpress_with_url_import_and_script.zip index 5dedfcdf..8d98d0a5 100644 Binary files a/toscaparser/tests/data/CSAR/csar_wordpress_with_url_import_and_script.zip and b/toscaparser/tests/data/CSAR/csar_wordpress_with_url_import_and_script.zip differ diff --git a/toscaparser/tests/data/repositories/test_repositories_definition.yaml b/toscaparser/tests/data/repositories/test_repositories_definition.yaml index c2856c8f..1b82ddb3 100644 --- a/toscaparser/tests/data/repositories/test_repositories_definition.yaml +++ b/toscaparser/tests/data/repositories/test_repositories_definition.yaml @@ -3,7 +3,7 @@ tosca_definitions_version: tosca_simple_yaml_1_0 repositories: some_repository: description: Some repo - url: https://raw.githubusercontent.com/openstack/tosca-parser/master/toscaparser/tests/data/custom_types/ + url: https://example.com/custom_types/ credential: #type: Credential token_type: basic_auth token: myusername:mypassword diff --git a/toscaparser/tests/data/tosca_single_instance_wordpress_with_url_import.yaml b/toscaparser/tests/data/tosca_single_instance_wordpress_with_url_import.yaml index e5f1580b..63a691eb 100644 --- a/toscaparser/tests/data/tosca_single_instance_wordpress_with_url_import.yaml +++ b/toscaparser/tests/data/tosca_single_instance_wordpress_with_url_import.yaml @@ -4,7 +4,7 @@ description: > TOSCA simple profile with wordpress, web server and mysql on the same server. imports: - - https://raw.githubusercontent.com/openstack/heat-translator/master/translator/tests/data/custom_types/wordpress.yaml + - https://example.com/custom_types/wordpress.yaml topology_template: inputs: diff --git a/toscaparser/tests/test_prereq.py b/toscaparser/tests/test_prereq.py index 94ccbb39..cefcdb45 100644 --- a/toscaparser/tests/test_prereq.py +++ b/toscaparser/tests/test_prereq.py @@ -11,16 +11,21 @@ # under the License. import os +import requests import shutil +from unittest import mock +import urllib import zipfile from toscaparser.common.exception import ExceptionCollector from toscaparser.common.exception import URLException from toscaparser.common.exception import ValidationError from toscaparser.prereq.csar import CSAR +from toscaparser.tests.base import MockTestClass from toscaparser.tests.base import TestCase import toscaparser.utils from toscaparser.utils.gettextutils import _ +from toscaparser.utils.urlutils import UrlUtils class CSARPrereqTest(TestCase): @@ -46,9 +51,20 @@ class CSARPrereqTest(TestCase): error = self.assertRaises(ValidationError, csar.validate) self.assertEqual(_('"%s" is not a valid zip file.') % path, str(error)) - def test_url_is_zip(self): - path = "https://github.com/openstack/tosca-parser/raw/master/" \ - "toscaparser/tests/data/CSAR/csar_not_zip.zip" + @mock.patch.object(requests, 'get') + def test_url_is_zip(self, mock_requests_get): + path = "https://example.com/csar_not_zip.zip" + + class TestResponse: + content: bytes + + response = TestResponse() + file_path = os.path.join(self.base_path, "data/CSAR/csar_not_zip.zip") + + with open(file_path, 'br') as f: + response.content = f.read() + + mock_requests_get.return_value = response csar = CSAR(path, False) error = self.assertRaises(ValidationError, csar.validate) self.assertEqual(_('"%s" is not a valid zip file.') % path, str(error)) @@ -167,9 +183,20 @@ class CSARPrereqTest(TestCase): self.assertTrue(csar.temp_dir is None or not os.path.exists(csar.temp_dir)) - def test_valid_csar_with_url_import_and_script(self): + @mock.patch.object(urllib.request, 'urlopen') + @mock.patch.object(UrlUtils, 'url_accessible') + def test_valid_csar_with_url_import_and_script( + self, mock_url_accessible, mock_urlopen): path = os.path.join(self.base_path, "data/CSAR/csar_wordpress_with_url" "_import_and_script.zip") + mock_path = "https://example.com/wordpress.yaml" + + mockclass = MockTestClass() + mockclass.comp_urldict = { + mock_path: os.path.join(self.base_path, + "data/custom_types/wordpress.yaml")} + mock_urlopen.side_effect = mockclass.mock_urlopen_method + mock_url_accessible.return_value = True csar = CSAR(path) self.assertTrue(csar.validate()) self.assertTrue(csar.temp_dir is None or diff --git a/toscaparser/tests/test_toscatpl.py b/toscaparser/tests/test_toscatpl.py index 203305e4..02b5b7db 100644 --- a/toscaparser/tests/test_toscatpl.py +++ b/toscaparser/tests/test_toscatpl.py @@ -11,6 +11,9 @@ # under the License. import os +import requests +from unittest import mock +import urllib from toscaparser.common import exception import toscaparser.elements.interfaces as ifaces @@ -19,9 +22,11 @@ from toscaparser.elements.portspectype import PortSpec from toscaparser.functions import GetInput from toscaparser.functions import GetProperty from toscaparser.nodetemplate import NodeTemplate +from toscaparser.tests.base import MockTestClass from toscaparser.tests.base import TestCase from toscaparser.tosca_template import ToscaTemplate from toscaparser.utils.gettextutils import _ +from toscaparser.utils.urlutils import UrlUtils import toscaparser.utils.yamlparser @@ -513,18 +518,34 @@ class ToscaTemplateTest(TestCase): tosca = ToscaTemplate(tosca_tpl, parsed_params=params) self.assertTrue(tosca.topology_template.custom_defs) - def test_local_template_with_url_import(self): + @mock.patch.object(ToscaTemplate, '_tpl_imports') + def test_local_template_with_url_import(self, mock_tpl_imports): tosca_tpl = os.path.join( os.path.dirname(os.path.abspath(__file__)), "data/tosca_single_instance_wordpress_with_url_import.yaml") + import_file_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "data/custom_types/wordpress.yaml") + mock_tpl_imports.return_value = [import_file_path] tosca = ToscaTemplate(tosca_tpl, parsed_params={'db_root_pwd': '123456'}) self.assertTrue(tosca.topology_template.custom_defs) - def test_url_template_with_local_relpath_import(self): - tosca_tpl = ('https://raw.githubusercontent.com/openstack/' - 'tosca-parser/master/toscaparser/tests/data/' - 'tosca_single_instance_wordpress.yaml') + @mock.patch.object(urllib.request, 'urlopen') + def test_url_template_with_local_relpath_import(self, mock_urlopen): + filenames = ['tosca_single_instance_wordpress.yaml', + 'wordpress.yaml'] + tosca_tpl = f'https://example.com/{filenames[0]}' + mock_path = f'https://example.com/custom_types/{filenames[1]}' + + mockclass = MockTestClass() + mockclass.comp_urldict = { + tosca_tpl: os.path.join(os.path.dirname(os.path.abspath(__file__)), + f'data/{filenames[0]}'), + mock_path: os.path.join(os.path.dirname(os.path.abspath(__file__)), + f'data/custom_types/{filenames[1]}')} + mock_urlopen.side_effect = mockclass.mock_urlopen_method + tosca = ToscaTemplate(tosca_tpl, a_file=False, parsed_params={"db_name": "mysql", "db_user": "mysql", @@ -534,11 +555,18 @@ class ToscaTemplateTest(TestCase): "cpus": 4}) self.assertTrue(tosca.topology_template.custom_defs) - def test_url_template_with_local_abspath_import(self): - tosca_tpl = ('https://raw.githubusercontent.com/openstack/' - 'tosca-parser/master/toscaparser/tests/data/' - 'tosca_single_instance_wordpress_with_local_abspath_' - 'import.yaml') + @mock.patch.object(urllib.request, 'urlopen') + def test_url_template_with_local_abspath_import(self, mock_urlopen): + filename = ('tosca_single_instance_wordpress_with_local_abspath_' + 'import.yaml') + tosca_tpl = f'https://example.com/{filename}' + + mockclass = MockTestClass() + mockclass.comp_urldict = { + tosca_tpl: os.path.join(os.path.dirname(os.path.abspath(__file__)), + f'data/{filename}')} + + mock_urlopen.side_effect = mockclass.mock_urlopen_method self.assertRaises(exception.ValidationError, ToscaTemplate, tosca_tpl, None, False) err_msg = (_('Absolute file name "/tmp/tosca-parser/toscaparser/tests' @@ -548,10 +576,30 @@ class ToscaTemplateTest(TestCase): exception.ExceptionCollector.assertExceptionMessage(ImportError, err_msg) - def test_url_template_with_url_import(self): - tosca_tpl = ('https://raw.githubusercontent.com/openstack/' - 'tosca-parser/master/toscaparser/tests/data/' - 'tosca_single_instance_wordpress_with_url_import.yaml') + @mock.patch.object(UrlUtils, 'join_url') + @mock.patch.object(os.path, 'isabs') + @mock.patch.object(ToscaTemplate, '_tpl_imports') + @mock.patch.object(urllib.request, 'urlopen') + def test_url_template_with_url_import( + self, mock_urlopen, mock_tpl_imports, mock_isabs, mock_join_url): + filename = 'tosca_single_instance_wordpress_with_url_import.yaml' + tosca_tpl = f'https://example.com/{filename}' + + import_file_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "data/custom_types/wordpress.yaml") + + mockclass = MockTestClass() + mockclass.comp_urldict = { + tosca_tpl: os.path.join(os.path.dirname(os.path.abspath(__file__)), + f'data/{filename}') + } + + mock_urlopen.side_effect = mockclass.mock_urlopen_method + mock_tpl_imports.return_value = [import_file_path] + mock_isabs.return_value = False + mock_join_url.return_value = import_file_path + tosca = ToscaTemplate(tosca_tpl, a_file=False, parsed_params={"db_root_pwd": "1234"}) self.assertTrue(tosca.topology_template.custom_defs) @@ -568,9 +616,23 @@ class ToscaTemplateTest(TestCase): "db_port": 3306, "cpus": 4})) - def test_csar_parsing_elk_url_based(self): - csar_archive = ('https://github.com/openstack/tosca-parser/raw/master/' - 'toscaparser/tests/data/CSAR/csar_elk.zip') + @mock.patch.object(requests, 'get') + def test_csar_parsing_elk_url_based(self, mock_requests_get): + csar_archive = 'https://example.com/csar_elk.zip' + + class TestResponse: + content: bytes + + response = TestResponse() + file_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "data/CSAR/csar_elk.zip") + + with open(file_path, 'br') as f: + response.content = f.read() + f.close() + + mock_requests_get.return_value = response self.assertTrue(ToscaTemplate(csar_archive, a_file=False, parsed_params={"my_cpus": 4})) @@ -731,7 +793,8 @@ class ToscaTemplateTest(TestCase): self.assertEqual(tosca.version, "tosca_simple_yaml_1_0") - def test_yaml_dict_tpl_with_params_and_url_import(self): + @mock.patch.object(ToscaTemplate, '_tpl_imports') + def test_yaml_dict_tpl_with_params_and_url_import(self, mock_tpl_imports): test_tpl = os.path.join( os.path.dirname(os.path.abspath(__file__)), "data/tosca_single_instance_wordpress_with_url_import.yaml") @@ -740,6 +803,10 @@ class ToscaTemplateTest(TestCase): params = {'db_name': 'my_wordpress', 'db_user': 'my_db_user', 'db_root_pwd': 'mypasswd'} + import_file_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "data/custom_types/wordpress.yaml") + mock_tpl_imports.return_value = [import_file_path] tosca = ToscaTemplate(parsed_params=params, yaml_dict_tpl=yaml_dict_tpl) @@ -878,10 +945,19 @@ class ToscaTemplateTest(TestCase): "data/test_attributes_inheritance.yaml") ToscaTemplate(tosca_tpl) - def test_repositories_definition(self): + @mock.patch.object(urllib.request, 'urlopen') + def test_repositories_definition(self, mock_urlopen): tosca_tpl = os.path.join( os.path.dirname(os.path.abspath(__file__)), "data/repositories/test_repositories_definition.yaml") + filename = 'compute_with_prop.yaml' + mock_path = f'https://example.com/custom_types/{filename}' + + mockclass = MockTestClass() + mockclass.comp_urldict = { + mock_path: os.path.join(os.path.dirname(os.path.abspath(__file__)), + f'data/custom_types/{filename}')} + mock_urlopen.side_effect = mockclass.mock_urlopen_method ToscaTemplate(tosca_tpl) def test_custom_caps_def(self): @@ -1014,7 +1090,8 @@ class ToscaTemplateTest(TestCase): self.assertRaises(exception.ValidationError, ToscaTemplate, tosca_tpl, None) - def test_local_custom_defs(self): + @mock.patch.object(ToscaTemplate, '_tpl_imports') + def test_local_custom_defs(self, mock_tpl_imports): """Compare if custom defs on local and remote the same.""" tosca_tpl = os.path.join( @@ -1025,13 +1102,15 @@ class ToscaTemplateTest(TestCase): os.path.dirname(os.path.abspath(__file__)), "data/custom_types/wordpress.yaml") remote_def = ( - "https://raw.githubusercontent.com/openstack/" - "tosca-parser/master/toscaparser/tests/data/custom_types/" - "wordpress.yaml") + "https://example.com/custom_types/wordpress.yaml") local_defs = {remote_def: local_def} params = {'db_name': 'my_wordpress', 'db_user': 'my_db_user', 'db_root_pwd': '12345678'} + import_file_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "data/custom_types/wordpress.yaml") + mock_tpl_imports.return_value = [import_file_path] tosca = ToscaTemplate(tosca_tpl, parsed_params=params) tosca_local = ToscaTemplate(tosca_tpl, parsed_params=params, local_defs=local_defs) diff --git a/toscaparser/tests/test_toscatplvalidation.py b/toscaparser/tests/test_toscatplvalidation.py index dc5c0d76..7c5bfef8 100644 --- a/toscaparser/tests/test_toscatplvalidation.py +++ b/toscaparser/tests/test_toscatplvalidation.py @@ -11,6 +11,8 @@ # under the License. import os +from unittest import mock +import urllib from toscaparser.common import exception from toscaparser.imports import ImportsLoader @@ -21,6 +23,7 @@ from toscaparser.policy import Policy from toscaparser.relationship_template import RelationshipTemplate from toscaparser.repositories import Repository from toscaparser.reservation import Reservation +from toscaparser.tests.base import MockTestClass from toscaparser.tests.base import TestCase from toscaparser.topology_template import TopologyTemplate from toscaparser.tosca_template import ToscaTemplate @@ -329,29 +332,47 @@ class ToscaTemplateValidationTest(TestCase): tpl_snippet, path, None) self.assertEqual(errormsg, err.__str__()) - def test_imports_without_import_name(self): + @mock.patch.object(urllib.request, 'urlopen') + def test_imports_without_import_name(self, mock_urlopen): tpl_snippet = ''' imports: - custom_types/paypalpizzastore_nodejs_app.yaml - - https://raw.githubusercontent.com/openstack/\ -tosca-parser/master/toscaparser/tests/data/custom_types/wordpress.yaml + - https://example.com/custom_types/wordpress.yaml ''' - path = 'toscaparser/tests/data/tosca_elk.yaml' + mock_path = "https://example.com/custom_types/wordpress.yaml" + + mockclass = MockTestClass() + mockclass.comp_urldict = { + mock_path: os.path.join(os.path.dirname(os.path.abspath(__file__)), + "data/custom_types/wordpress.yaml")} + mock_urlopen.side_effect = mockclass.mock_urlopen_method + path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "data/tosca_elk.yaml") custom_defs = self._imports_content_test(tpl_snippet, path, "node_types") self.assertTrue(custom_defs) - def test_imports_wth_import_name(self): + @mock.patch.object(urllib.request, 'urlopen') + def test_imports_wth_import_name(self, mock_urlopen): tpl_snippet = ''' imports: - some_definitions: custom_types/paypalpizzastore_nodejs_app.yaml - more_definitions: - file: 'https://raw.githubusercontent.com/openstack/tosca-parser\ -/master/toscaparser/tests/data/custom_types/wordpress.yaml' + file: 'https://example.com/custom_types/wordpress.yaml' namespace_prefix: single_instance_wordpress ''' - path = 'toscaparser/tests/data/tosca_elk.yaml' + mock_path = "https://example.com/custom_types/wordpress.yaml" + + mockclass = MockTestClass() + mockclass.comp_urldict = { + mock_path: os.path.join(os.path.dirname(os.path.abspath(__file__)), + "data/custom_types/wordpress.yaml")} + mock_urlopen.side_effect = mockclass.mock_urlopen_method + path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "data/tosca_elk.yaml") custom_defs = self._imports_content_test(tpl_snippet, path, "node_types") @@ -385,16 +406,25 @@ tosca-parser/master/toscaparser/tests/data/custom_types/wordpress.yaml tpl_snippet, None, None) self.assertEqual(errormsg, err.__str__()) - def test_imports_duplicate_name(self): + @mock.patch.object(urllib.request, 'urlopen') + def test_imports_duplicate_name(self, mock_urlopen): tpl_snippet = ''' imports: - - some_definitions: https://raw.githubusercontent.com/openstack/\ -tosca-parser/master/toscaparser/tests/data/custom_types/wordpress.yaml + - some_definitions: https://example.com/custom_types/wordpress.yaml - some_definitions: file: my_defns/my_typesdefs_n.yaml ''' + mock_path = "https://example.com/custom_types/wordpress.yaml" + + mockclass = MockTestClass() + mockclass.comp_urldict = { + mock_path: os.path.join(os.path.dirname(os.path.abspath(__file__)), + "data/custom_types/wordpress.yaml")} + mock_urlopen.side_effect = mockclass.mock_urlopen_method errormsg = _('Duplicate import name "some_definitions" was found.') - path = 'toscaparser/tests/data/tosca_elk.yaml' + path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "data/tosca_elk.yaml") err = self.assertRaises(exception.ValidationError, self._imports_content_test, tpl_snippet, path, None) @@ -417,46 +447,72 @@ tosca-parser/master/toscaparser/tests/data/custom_types/wordpress.yaml tpl_snippet, path, None) self.assertEqual(errormsg, err.__str__()) - def test_imports_file_with_uri(self): + @mock.patch.object(urllib.request, 'urlopen') + @mock.patch.object(ToscaTemplate, '_tpl_imports') + def test_imports_file_with_uri(self, mock_tpl_imports, mock_urlopen): tpl_snippet = ''' imports: - more_definitions: - file: https://raw.githubusercontent.com/openstack/\ -tosca-parser/master/toscaparser/tests/data/custom_types/wordpress.yaml + file: https://example.com/custom_types/wordpress.yaml ''' - path = 'https://raw.githubusercontent.com/openstack/\ -tosca-parser/master/toscaparser/tests/data/\ -tosca_single_instance_wordpress_with_url_import.yaml' + path = ('https://example.com/tosca_single_instance_wordpress_' + 'with_url_import.yaml') + mock_path = "https://example.com/custom_types/wordpress.yaml" + + mockclass = MockTestClass() + mockclass.comp_urldict = { + mock_path: os.path.join(os.path.dirname(os.path.abspath(__file__)), + "data/custom_types/wordpress.yaml")} + mock_urlopen.side_effect = mockclass.mock_urlopen_method + import_file_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "data/custom_types/wordpress.yaml") + mock_tpl_imports.return_value = [import_file_path] custom_defs = self._imports_content_test(tpl_snippet, path, "node_types") self.assertTrue(custom_defs.get("tosca.nodes." "WebApplication.WordPress")) - def test_imports_file_namespace_fields(self): + @mock.patch.object(urllib.request, 'urlopen') + def test_imports_file_namespace_fields(self, mock_urlopen): tpl_snippet = ''' imports: - more_definitions: - file: https://raw.githubusercontent.com/openstack/\ -heat-translator/master/translator/tests/data/custom_types/wordpress.yaml + file: https://example.com/custom_types/wordpress.yaml namespace_prefix: mycompany namespace_uri: http://docs.oasis-open.org/tosca/ns/simple/yaml/1.0 ''' - path = 'toscaparser/tests/data/tosca_elk.yaml' + mock_path = "https://example.com/custom_types/wordpress.yaml" + + mockclass = MockTestClass() + mockclass.comp_urldict = { + mock_path: os.path.join(os.path.dirname(os.path.abspath(__file__)), + "data/custom_types/wordpress.yaml")} + mock_urlopen.side_effect = mockclass.mock_urlopen_method + path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "data/tosca_elk.yaml") custom_defs = self._imports_content_test(tpl_snippet, path, "node_types") self.assertTrue(custom_defs.get("mycompany.tosca.nodes." "WebApplication.WordPress")) - def test_imports_with_local_defs(self): + @mock.patch.object(urllib.request, 'urlopen') + def test_imports_with_local_defs(self, mock_urlopen): """Compare custom types on local and remote.""" ctypes = { - "remote": ("https://raw.githubusercontent.com/openstack/" - "heat-translator/master/translator/tests/data/" - "custom_types/wordpress.yaml"), + "remote": "https://example.com/custom_types/wordpress.yaml", "local": "../data/wordpress.yaml"} + mock_path = "https://example.com/custom_types/wordpress.yaml" + + mockclass = MockTestClass() + mockclass.comp_urldict = { + mock_path: os.path.join(os.path.dirname(os.path.abspath(__file__)), + "data/custom_types/wordpress.yaml")} + mock_urlopen.side_effect = mockclass.mock_urlopen_method tpl_snippet = ''' imports: @@ -464,7 +520,9 @@ heat-translator/master/translator/tests/data/custom_types/wordpress.yaml '''.format(ctypes["remote"]) local_defs = {ctypes["remote"]: ctypes["local"]} - path = 'toscaparser/tests/data/tosca_elk.yaml' + path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "data/tosca_elk.yaml") imports = (toscaparser.utils.yamlparser. simple_parse(tpl_snippet)['imports']) ld1 = ImportsLoader(imports, path, "node_types")