diff --git a/Authors b/Authors index 9aad104a7998..1679d2dee150 100644 --- a/Authors +++ b/Authors @@ -33,6 +33,7 @@ Jonathan Bryce Jordan Rinke Josh Durgin Josh Kearney +Josh Kleinpeter Joshua McKenty Justin Santa Barbara Kei Masumoto diff --git a/bin/nova-ajax-console-proxy b/bin/nova-ajax-console-proxy index bbd60badeb1c..b4ba157e16b1 100755 --- a/bin/nova-ajax-console-proxy +++ b/bin/nova-ajax-console-proxy @@ -1,5 +1,5 @@ #!/usr/bin/env python -# pylint: disable-msg=C0103 +# pylint: disable=C0103 # vim: tabstop=4 shiftwidth=4 softtabstop=4 # Copyright 2010 United States Government as represented by the diff --git a/bin/nova-api b/bin/nova-api index 06bb855cb9ff..a1088c23d276 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -1,5 +1,5 @@ #!/usr/bin/env python -# pylint: disable-msg=C0103 +# pylint: disable=C0103 # vim: tabstop=4 shiftwidth=4 softtabstop=4 # Copyright 2010 United States Government as represented by the diff --git a/bin/nova-direct-api b/bin/nova-direct-api index bf29d9a5e71e..a2c9f1557625 100755 --- a/bin/nova-direct-api +++ b/bin/nova-direct-api @@ -1,5 +1,5 @@ #!/usr/bin/env python -# pylint: disable-msg=C0103 +# pylint: disable=C0103 # vim: tabstop=4 shiftwidth=4 softtabstop=4 # Copyright 2010 United States Government as represented by the diff --git a/bin/nova-instancemonitor b/bin/nova-instancemonitor index 24cc9fd2345c..b9d4e49d7d74 100755 --- a/bin/nova-instancemonitor +++ b/bin/nova-instancemonitor @@ -50,7 +50,7 @@ if __name__ == '__main__': if __name__ == '__builtin__': LOG.warn(_('Starting instance monitor')) - # pylint: disable-msg=C0103 + # pylint: disable=C0103 monitor = monitor.InstanceMonitor() # This is the parent service that twistd will be looking for when it diff --git a/bin/nova-manage b/bin/nova-manage index a4d820209a6c..69cbf6f95529 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -518,11 +518,12 @@ class NetworkCommands(object): network_size=None, vlan_start=None, vpn_start=None, fixed_range_v6=None, label='public'): """Creates fixed ips for host by range - arguments: [fixed_range=FLAG], [num_networks=FLAG], + arguments: fixed_range=FLAG, [num_networks=FLAG], [network_size=FLAG], [vlan_start=FLAG], [vpn_start=FLAG], [fixed_range_v6=FLAG]""" if not fixed_range: - fixed_range = FLAGS.fixed_range + raise TypeError(_('Fixed range in the form of 10.0.0.0/8 is ' + 'required to create networks.')) if not num_networks: num_networks = FLAGS.num_networks if not network_size: @@ -579,8 +580,10 @@ class VmCommands(object): ctxt = context.get_admin_context() instance_id = ec2utils.ec2_id_to_id(ec2_id) - if FLAGS.connection_type != 'libvirt': - msg = _('Only KVM is supported for now. Sorry!') + if (FLAGS.connection_type != 'libvirt' or + (FLAGS.connection_type == 'libvirt' and + FLAGS.libvirt_type not in ['kvm', 'qemu'])): + msg = _('Only KVM and QEmu are supported for now. Sorry!') raise exception.Error(msg) if (FLAGS.volume_driver != 'nova.volume.driver.AOEDriver' and \ @@ -872,7 +875,7 @@ class InstanceTypeCommands(object): if name == None: inst_types = instance_types.get_all_types() elif name == "--all": - inst_types = instance_types.get_all_types(1) + inst_types = instance_types.get_all_types(True) else: inst_types = instance_types.get_instance_type(name) except exception.DBError, e: diff --git a/bin/nova-objectstore b/bin/nova-objectstore index 9fbe228a2c64..94ef2a8d5019 100755 --- a/bin/nova-objectstore +++ b/bin/nova-objectstore @@ -49,4 +49,4 @@ if __name__ == '__main__': twistd.serve(__file__) if __name__ == '__builtin__': - application = handler.get_application() # pylint: disable-msg=C0103 + application = handler.get_application() # pylint: disable=C0103 diff --git a/contrib/boto_v6/ec2/connection.py b/contrib/boto_v6/ec2/connection.py index 23466e5d70c8..868c93c11c50 100644 --- a/contrib/boto_v6/ec2/connection.py +++ b/contrib/boto_v6/ec2/connection.py @@ -4,8 +4,10 @@ Created on 2010/12/20 @author: Nachi Ueno ''' import boto +import base64 import boto.ec2 from boto_v6.ec2.instance import ReservationV6 +from boto.ec2.securitygroup import SecurityGroup class EC2ConnectionV6(boto.ec2.EC2Connection): @@ -39,3 +41,101 @@ class EC2ConnectionV6(boto.ec2.EC2Connection): self.build_filter_params(params, filters) return self.get_list('DescribeInstancesV6', params, [('item', ReservationV6)]) + + def run_instances(self, image_id, min_count=1, max_count=1, + key_name=None, security_groups=None, + user_data=None, addressing_type=None, + instance_type='m1.small', placement=None, + kernel_id=None, ramdisk_id=None, + monitoring_enabled=False, subnet_id=None, + block_device_map=None): + """ + Runs an image on EC2. + + :type image_id: string + :param image_id: The ID of the image to run + + :type min_count: int + :param min_count: The minimum number of instances to launch + + :type max_count: int + :param max_count: The maximum number of instances to launch + + :type key_name: string + :param key_name: The name of the key pair with which to + launch instances + + :type security_groups: list of strings + :param security_groups: The names of the security groups with + which to associate instances + + :type user_data: string + :param user_data: The user data passed to the launched instances + + :type instance_type: string + :param instance_type: The type of instance to run + (m1.small, m1.large, m1.xlarge) + + :type placement: string + :param placement: The availability zone in which to launch + the instances + + :type kernel_id: string + :param kernel_id: The ID of the kernel with which to + launch the instances + + :type ramdisk_id: string + :param ramdisk_id: The ID of the RAM disk with which to + launch the instances + + :type monitoring_enabled: bool + :param monitoring_enabled: Enable CloudWatch monitoring + on the instance. + + :type subnet_id: string + :param subnet_id: The subnet ID within which to launch + the instances for VPC. + + :type block_device_map: + :class:`boto.ec2.blockdevicemapping.BlockDeviceMapping` + :param block_device_map: A BlockDeviceMapping data structure + describing the EBS volumes associated + with the Image. + + :rtype: Reservation + :return: The :class:`boto.ec2.instance.ReservationV6` + associated with the request for machines + """ + params = {'ImageId': image_id, + 'MinCount': min_count, + 'MaxCount': max_count} + if key_name: + params['KeyName'] = key_name + if security_groups: + l = [] + for group in security_groups: + if isinstance(group, SecurityGroup): + l.append(group.name) + else: + l.append(group) + self.build_list_params(params, l, 'SecurityGroup') + if user_data: + params['UserData'] = base64.b64encode(user_data) + if addressing_type: + params['AddressingType'] = addressing_type + if instance_type: + params['InstanceType'] = instance_type + if placement: + params['Placement.AvailabilityZone'] = placement + if kernel_id: + params['KernelId'] = kernel_id + if ramdisk_id: + params['RamdiskId'] = ramdisk_id + if monitoring_enabled: + params['Monitoring.Enabled'] = 'true' + if subnet_id: + params['SubnetId'] = subnet_id + if block_device_map: + block_device_map.build_list_params(params) + return self.get_object('RunInstances', params, + ReservationV6, verb='POST') diff --git a/etc/api-paste.ini b/etc/api-paste.ini index 9f7e93d4c33f..d95350fc76a8 100644 --- a/etc/api-paste.ini +++ b/etc/api-paste.ini @@ -68,6 +68,7 @@ paste.app_factory = nova.api.ec2.metadatarequesthandler:MetadataRequestHandler.f use = egg:Paste#urlmap /: osversions /v1.0: openstackapi +/v1.1: openstackapi [pipeline:openstackapi] pipeline = faultwrap auth ratelimit osapiapp @@ -79,7 +80,7 @@ paste.filter_factory = nova.api.openstack:FaultWrapper.factory paste.filter_factory = nova.api.openstack.auth:AuthMiddleware.factory [filter:ratelimit] -paste.filter_factory = nova.api.openstack.ratelimiting:RateLimitingMiddleware.factory +paste.filter_factory = nova.api.openstack.limits:RateLimitingMiddleware.factory [app:osapiapp] paste.app_factory = nova.api.openstack:APIRouter.factory diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index fccebca5d904..20701cfa8732 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -31,7 +31,7 @@ from nova import log as logging from nova import utils from nova import wsgi from nova.api.ec2 import apirequest -from nova.api.ec2 import cloud +from nova.api.ec2 import ec2utils from nova.auth import manager @@ -319,13 +319,13 @@ class Executor(wsgi.Application): except exception.InstanceNotFound as ex: LOG.info(_('InstanceNotFound raised: %s'), unicode(ex), context=context) - ec2_id = cloud.id_to_ec2_id(ex.instance_id) + ec2_id = ec2utils.id_to_ec2_id(ex.instance_id) message = _('Instance %s not found') % ec2_id return self._error(req, context, type(ex).__name__, message) except exception.VolumeNotFound as ex: LOG.info(_('VolumeNotFound raised: %s'), unicode(ex), context=context) - ec2_id = cloud.id_to_ec2_id(ex.volume_id, 'vol-%08x') + ec2_id = ec2utils.id_to_ec2_id(ex.volume_id, 'vol-%08x') message = _('Volume %s not found') % ec2_id return self._error(req, context, type(ex).__name__, message) except exception.NotFound as ex: diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index ce3cff337a01..b4c352b08a80 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -33,6 +33,7 @@ from nova.api.openstack import backup_schedules from nova.api.openstack import consoles from nova.api.openstack import flavors from nova.api.openstack import images +from nova.api.openstack import limits from nova.api.openstack import servers from nova.api.openstack import shared_ip_groups from nova.api.openstack import users @@ -114,12 +115,17 @@ class APIRouter(wsgi.Router): mapper.resource("image", "images", controller=images.Controller(), collection={'detail': 'GET'}) + mapper.resource("flavor", "flavors", controller=flavors.Controller(), collection={'detail': 'GET'}) + mapper.resource("shared_ip_group", "shared_ip_groups", collection={'detail': 'GET'}, controller=shared_ip_groups.Controller()) + _limits = limits.LimitsController() + mapper.resource("limit", "limits", controller=_limits) + super(APIRouter, self).__init__(mapper) @@ -128,8 +134,11 @@ class Versions(wsgi.Application): def __call__(self, req): """Respond to a request for all OpenStack API versions.""" response = { - "versions": [ - dict(status="CURRENT", id="v1.0")]} + "versions": [ + dict(status="DEPRECATED", id="v1.0"), + dict(status="CURRENT", id="v1.1"), + ], + } metadata = { "application/xml": { "attributes": dict(version=["status", "id"])}} diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index f3a9bdeca5ee..5aa5e099b923 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -69,6 +69,8 @@ class AuthMiddleware(wsgi.Middleware): return faults.Fault(webob.exc.HTTPUnauthorized()) req.environ['nova.context'] = context.RequestContext(user, account) + version = req.path.split('/')[1].replace('v', '') + req.environ['api.version'] = version return self.application def has_authentication(self, req): diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index 74ac21024589..d6679de010a3 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -74,3 +74,7 @@ def get_image_id_from_image_hash(image_service, context, image_hash): if abs(hash(image_id)) == int(image_hash): return image_id raise exception.NotFound(image_hash) + + +def get_api_version(req): + return req.environ.get('api.version') diff --git a/nova/api/openstack/faults.py b/nova/api/openstack/faults.py index 2fd733299f77..0e9c4b26f4c8 100644 --- a/nova/api/openstack/faults.py +++ b/nova/api/openstack/faults.py @@ -61,3 +61,42 @@ class Fault(webob.exc.HTTPException): content_type = req.best_match_content_type() self.wrapped_exc.body = serializer.serialize(fault_data, content_type) return self.wrapped_exc + + +class OverLimitFault(webob.exc.HTTPException): + """ + Rate-limited request response. + """ + + _serialization_metadata = { + "application/xml": { + "attributes": { + "overLimitFault": "code", + }, + }, + } + + def __init__(self, message, details, retry_time): + """ + Initialize new `OverLimitFault` with relevant information. + """ + self.wrapped_exc = webob.exc.HTTPForbidden() + self.content = { + "overLimitFault": { + "code": self.wrapped_exc.status_int, + "message": message, + "details": details, + }, + } + + @webob.dec.wsgify(RequestClass=wsgi.Request) + def __call__(self, request): + """ + Return the wrapped exception with a serialized body conforming to our + error format. + """ + serializer = wsgi.Serializer(self._serialization_metadata) + content_type = request.best_match_content_type() + content = serializer.serialize(self.content, content_type) + self.wrapped_exc.body = content + return self.wrapped_exc diff --git a/nova/api/openstack/flavors.py b/nova/api/openstack/flavors.py index 1c440b3a990c..c99b945fb4f6 100644 --- a/nova/api/openstack/flavors.py +++ b/nova/api/openstack/flavors.py @@ -22,6 +22,7 @@ from nova import context from nova.api.openstack import faults from nova.api.openstack import common from nova.compute import instance_types +from nova.api.openstack.views import flavors as flavors_views from nova import wsgi import nova.api.openstack @@ -36,7 +37,7 @@ class Controller(wsgi.Controller): def index(self, req): """Return all flavors in brief.""" - return dict(flavors=[dict(id=flavor['flavorid'], name=flavor['name']) + return dict(flavors=[dict(id=flavor['id'], name=flavor['name']) for flavor in self.detail(req)['flavors']]) def detail(self, req): @@ -47,14 +48,18 @@ class Controller(wsgi.Controller): def show(self, req, id): """Return data about the given flavor id.""" ctxt = req.environ['nova.context'] - values = db.instance_type_get_by_flavor_id(ctxt, id) - values['id'] = values['flavorid'] + flavor = db.api.instance_type_get_by_flavor_id(ctxt, id) + values = { + "id": flavor["flavorid"], + "name": flavor["name"], + "ram": flavor["memory_mb"], + "disk": flavor["local_gb"], + } return dict(flavor=values) - raise faults.Fault(exc.HTTPNotFound()) def _all_ids(self, req): """Return the list of all flavorids.""" ctxt = req.environ['nova.context'] - inst_types = db.instance_type_get_all(ctxt) + inst_types = db.api.instance_type_get_all(ctxt) flavor_ids = [inst_types[i]['flavorid'] for i in inst_types.keys()] return sorted(flavor_ids) diff --git a/nova/api/openstack/limits.py b/nova/api/openstack/limits.py new file mode 100644 index 000000000000..efc7d193d56e --- /dev/null +++ b/nova/api/openstack/limits.py @@ -0,0 +1,358 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License.import datetime + +""" +Module dedicated functions/classes dealing with rate limiting requests. +""" + +import copy +import httplib +import json +import math +import re +import time +import urllib +import webob.exc + +from collections import defaultdict + +from webob.dec import wsgify + +from nova import wsgi +from nova.api.openstack import faults +from nova.wsgi import Controller +from nova.wsgi import Middleware + + +# Convenience constants for the limits dictionary passed to Limiter(). +PER_SECOND = 1 +PER_MINUTE = 60 +PER_HOUR = 60 * 60 +PER_DAY = 60 * 60 * 24 + + +class LimitsController(Controller): + """ + Controller for accessing limits in the OpenStack API. + """ + + _serialization_metadata = { + "application/xml": { + "attributes": { + "limit": ["verb", "URI", "regex", "value", "unit", + "resetTime", "remaining", "name"], + }, + "plurals": { + "rate": "limit", + }, + }, + } + + def index(self, req): + """ + Return all global and rate limit information. + """ + abs_limits = {} + rate_limits = req.environ.get("nova.limits", []) + + return { + "limits": { + "rate": rate_limits, + "absolute": abs_limits, + }, + } + + +class Limit(object): + """ + Stores information about a limit for HTTP requets. + """ + + UNITS = { + 1: "SECOND", + 60: "MINUTE", + 60 * 60: "HOUR", + 60 * 60 * 24: "DAY", + } + + def __init__(self, verb, uri, regex, value, unit): + """ + Initialize a new `Limit`. + + @param verb: HTTP verb (POST, PUT, etc.) + @param uri: Human-readable URI + @param regex: Regular expression format for this limit + @param value: Integer number of requests which can be made + @param unit: Unit of measure for the value parameter + """ + self.verb = verb + self.uri = uri + self.regex = regex + self.value = int(value) + self.unit = unit + self.unit_string = self.display_unit().lower() + self.remaining = int(value) + + if value <= 0: + raise ValueError("Limit value must be > 0") + + self.last_request = None + self.next_request = None + + self.water_level = 0 + self.capacity = self.unit + self.request_value = float(self.capacity) / float(self.value) + self.error_message = _("Only %(value)s %(verb)s request(s) can be "\ + "made to %(uri)s every %(unit_string)s." % self.__dict__) + + def __call__(self, verb, url): + """ + Represents a call to this limit from a relevant request. + + @param verb: string http verb (POST, GET, etc.) + @param url: string URL + """ + if self.verb != verb or not re.match(self.regex, url): + return + + now = self._get_time() + + if self.last_request is None: + self.last_request = now + + leak_value = now - self.last_request + + self.water_level -= leak_value + self.water_level = max(self.water_level, 0) + self.water_level += self.request_value + + difference = self.water_level - self.capacity + + self.last_request = now + + if difference > 0: + self.water_level -= self.request_value + self.next_request = now + difference + return difference + + cap = self.capacity + water = self.water_level + val = self.value + + self.remaining = math.floor(((cap - water) / cap) * val) + self.next_request = now + + def _get_time(self): + """Retrieve the current time. Broken out for testability.""" + return time.time() + + def display_unit(self): + """Display the string name of the unit.""" + return self.UNITS.get(self.unit, "UNKNOWN") + + def display(self): + """Return a useful representation of this class.""" + return { + "verb": self.verb, + "URI": self.uri, + "regex": self.regex, + "value": self.value, + "remaining": int(self.remaining), + "unit": self.display_unit(), + "resetTime": int(self.next_request or self._get_time()), + } + +# "Limit" format is a dictionary with the HTTP verb, human-readable URI, +# a regular-expression to match, value and unit of measure (PER_DAY, etc.) + +DEFAULT_LIMITS = [ + Limit("POST", "*", ".*", 10, PER_MINUTE), + Limit("POST", "*/servers", "^/servers", 50, PER_DAY), + Limit("PUT", "*", ".*", 10, PER_MINUTE), + Limit("GET", "*changes-since*", ".*changes-since.*", 3, PER_MINUTE), + Limit("DELETE", "*", ".*", 100, PER_MINUTE), +] + + +class RateLimitingMiddleware(Middleware): + """ + Rate-limits requests passing through this middleware. All limit information + is stored in memory for this implementation. + """ + + def __init__(self, application, limits=None): + """ + Initialize new `RateLimitingMiddleware`, which wraps the given WSGI + application and sets up the given limits. + + @param application: WSGI application to wrap + @param limits: List of dictionaries describing limits + """ + Middleware.__init__(self, application) + self._limiter = Limiter(limits or DEFAULT_LIMITS) + + @wsgify(RequestClass=wsgi.Request) + def __call__(self, req): + """ + Represents a single call through this middleware. We should record the + request if we have a limit relevant to it. If no limit is relevant to + the request, ignore it. + + If the request should be rate limited, return a fault telling the user + they are over the limit and need to retry later. + """ + verb = req.method + url = req.url + context = req.environ.get("nova.context") + + if context: + username = context.user_id + else: + username = None + + delay, error = self._limiter.check_for_delay(verb, url, username) + + if delay: + msg = _("This request was rate-limited.") + retry = time.time() + delay + return faults.OverLimitFault(msg, error, retry) + + req.environ["nova.limits"] = self._limiter.get_limits(username) + + return self.application + + +class Limiter(object): + """ + Rate-limit checking class which handles limits in memory. + """ + + def __init__(self, limits): + """ + Initialize the new `Limiter`. + + @param limits: List of `Limit` objects + """ + self.limits = copy.deepcopy(limits) + self.levels = defaultdict(lambda: copy.deepcopy(limits)) + + def get_limits(self, username=None): + """ + Return the limits for a given user. + """ + return [limit.display() for limit in self.levels[username]] + + def check_for_delay(self, verb, url, username=None): + """ + Check the given verb/user/user triplet for limit. + + @return: Tuple of delay (in seconds) and error message (or None, None) + """ + delays = [] + + for limit in self.levels[username]: + delay = limit(verb, url) + if delay: + delays.append((delay, limit.error_message)) + + if delays: + delays.sort() + return delays[0] + + return None, None + + +class WsgiLimiter(object): + """ + Rate-limit checking from a WSGI application. Uses an in-memory `Limiter`. + + To use: + POST / with JSON data such as: + { + "verb" : GET, + "path" : "/servers" + } + + and receive a 204 No Content, or a 403 Forbidden with an X-Wait-Seconds + header containing the number of seconds to wait before the action would + succeed. + """ + + def __init__(self, limits=None): + """ + Initialize the new `WsgiLimiter`. + + @param limits: List of `Limit` objects + """ + self._limiter = Limiter(limits or DEFAULT_LIMITS) + + @wsgify(RequestClass=wsgi.Request) + def __call__(self, request): + """ + Handles a call to this application. Returns 204 if the request is + acceptable to the limiter, else a 403 is returned with a relevant + header indicating when the request *will* succeed. + """ + if request.method != "POST": + raise webob.exc.HTTPMethodNotAllowed() + + try: + info = dict(json.loads(request.body)) + except ValueError: + raise webob.exc.HTTPBadRequest() + + username = request.path_info_pop() + verb = info.get("verb") + path = info.get("path") + + delay, error = self._limiter.check_for_delay(verb, path, username) + + if delay: + headers = {"X-Wait-Seconds": "%.2f" % delay} + return webob.exc.HTTPForbidden(headers=headers, explanation=error) + else: + return webob.exc.HTTPNoContent() + + +class WsgiLimiterProxy(object): + """ + Rate-limit requests based on answers from a remote source. + """ + + def __init__(self, limiter_address): + """ + Initialize the new `WsgiLimiterProxy`. + + @param limiter_address: IP/port combination of where to request limit + """ + self.limiter_address = limiter_address + + def check_for_delay(self, verb, path, username=None): + body = json.dumps({"verb": verb, "path": path}) + headers = {"Content-Type": "application/json"} + + conn = httplib.HTTPConnection(self.limiter_address) + + if username: + conn.request("POST", "/%s" % (username), body, headers) + else: + conn.request("POST", "/", body, headers) + + resp = conn.getresponse() + + if 200 >= resp.status < 300: + return None, None + + return resp.getheader("X-Wait-Seconds"), resp.read() or None diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 05045045752b..db5942e92bfb 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -31,114 +31,69 @@ from nova import wsgi from nova import utils from nova.api.openstack import common from nova.api.openstack import faults +from nova.api.openstack.views import servers as servers_views +from nova.api.openstack.views import addresses as addresses_views from nova.auth import manager as auth_manager from nova.compute import instance_types from nova.compute import power_state -from nova.quota import QuotaError +prom nova.quota import QuotaError import nova.api.openstack LOG = logging.getLogger('server') - - FLAGS = flags.FLAGS -def _translate_detail_keys(inst): - """ Coerces into dictionary format, mapping everything to Rackspace-like - attributes for return""" - power_mapping = { - None: 'build', - power_state.NOSTATE: 'build', - power_state.RUNNING: 'active', - power_state.BLOCKED: 'active', - power_state.SUSPENDED: 'suspended', - power_state.PAUSED: 'paused', - power_state.SHUTDOWN: 'active', - power_state.SHUTOFF: 'active', - power_state.CRASHED: 'error', - power_state.FAILED: 'error'} - inst_dict = {} - - mapped_keys = dict(status='state', imageId='image_id', - flavorId='instance_type', name='display_name', id='id') - - for k, v in mapped_keys.iteritems(): - inst_dict[k] = inst[v] - - ctxt = context.get_admin_context() - try: - migration = db.migration_get_by_instance_and_status(ctxt, - inst['id'], 'finished') - inst_dict['status'] = 'resize-confirm' - except Exception, e: - inst_dict['status'] = power_mapping[inst_dict['status']] - inst_dict['addresses'] = dict(public=[], private=[]) - - # grab single private fixed ip - private_ips = utils.get_from_path(inst, 'fixed_ip/address') - inst_dict['addresses']['private'] = private_ips - - # grab all public floating ips - public_ips = utils.get_from_path(inst, 'fixed_ip/floating_ips/address') - inst_dict['addresses']['public'] = public_ips - - # Return the metadata as a dictionary - metadata = {} - for item in inst['metadata']: - metadata[item['key']] = item['value'] - inst_dict['metadata'] = metadata - - inst_dict['hostId'] = '' - if inst['host']: - inst_dict['hostId'] = hashlib.sha224(inst['host']).hexdigest() - - return dict(server=inst_dict) - - -def _translate_keys(inst): - """ Coerces into dictionary format, excluding all model attributes - save for id and name """ - return dict(server=dict(id=inst['id'], name=inst['display_name'])) - - -class Controller(wsgi.Controller): +plass Controller(wsgi.Controller): """ The Server API controller for the OpenStack API """ _serialization_metadata = { 'application/xml': { "attributes": { "server": ["id", "imageId", "name", "flavorId", "hostId", - "status", "progress", "adminPass"]}}} + "status", "progress", "adminPass", "flavorRef", + "imageRef"]}}} def __init__(self): self.compute_api = compute.API() self._image_service = utils.import_object(FLAGS.image_service) super(Controller, self).__init__() + def ips(self, req, id): + try: + instance = self.compute_api.get(req.environ['nova.context'], id) + except exception.NotFound: + return faults.Fault(exc.HTTPNotFound()) + + builder = addresses_views.get_view_builder(req) + return builder.build(instance) + def index(self, req): """ Returns a list of server names and ids for a given user """ - return self._items(req, entity_maker=_translate_keys) + return self._items(req, is_detail=False) def detail(self, req): """ Returns a list of server details for a given user """ - return self._items(req, entity_maker=_translate_detail_keys) + return self._items(req, is_detail=True) - def _items(self, req, entity_maker): + def _items(self, req, is_detail): """Returns a list of servers for a given user. - entity_maker - either _translate_detail_keys or _translate_keys + builder - the response model builder """ instance_list = self.compute_api.get_all(req.environ['nova.context']) limited_list = common.limited(instance_list, req) - res = [entity_maker(inst)['server'] for inst in limited_list] - return dict(servers=res) + builder = servers_views.get_view_builder(req) + servers = [builder.build(inst, is_detail)['server'] + for inst in limited_list] + return dict(servers=servers) def show(self, req, id): """ Returns server details by server id """ try: instance = self.compute_api.get(req.environ['nova.context'], id) - return _translate_detail_keys(instance) + builder = servers_views.get_view_builder(req) + return builder.build(instance, is_detail=True) except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) @@ -181,8 +136,10 @@ class Controller(wsgi.Controller): for k, v in env['server']['metadata'].items(): metadata.append({'key': k, 'value': v}) - personality = env['server'].get('personality', []) - injected_files = self._get_injected_files(personality) + personality = env['server'].get('personality') + injected_files = [] + if personality: + injected_files = self._get_injected_files(personality) try: instances = self.compute_api.create( @@ -200,7 +157,8 @@ class Controller(wsgi.Controller): except QuotaError as error: self._handle_quota_errors(error) - server = _translate_keys(instances[0]) + builder = servers_views.get_view_builder(req) + server = builder.build(instances[0], is_detail=False) password = "%s%s" % (server['server']['name'][:4], utils.generate_password(12)) server['server']['adminPass'] = password @@ -229,6 +187,7 @@ class Controller(wsgi.Controller): underlying compute service. """ injected_files = [] + for item in personality: try: path = item['path'] diff --git a/nova/api/openstack/users.py b/nova/api/openstack/users.py index ebd0f451253c..d3ab3d553afa 100644 --- a/nova/api/openstack/users.py +++ b/nova/api/openstack/users.py @@ -13,13 +13,14 @@ # License for the specific language governing permissions and limitations # under the License. -import common +from webob import exc from nova import exception from nova import flags from nova import log as logging from nova import wsgi - +from nova.api.openstack import common +from nova.api.openstack import faults from nova.auth import manager FLAGS = flags.FLAGS @@ -63,7 +64,17 @@ class Controller(wsgi.Controller): def show(self, req, id): """Return data about the given user id""" - user = self.manager.get_user(id) + + #NOTE(justinsb): The drivers are a little inconsistent in how they + # deal with "NotFound" - some throw, some return None. + try: + user = self.manager.get_user(id) + except exception.NotFound: + user = None + + if user is None: + raise faults.Fault(exc.HTTPNotFound()) + return dict(user=_translate_keys(user)) def delete(self, req, id): diff --git a/nova/api/openstack/views/__init__.py b/nova/api/openstack/views/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/nova/api/openstack/views/addresses.py b/nova/api/openstack/views/addresses.py new file mode 100644 index 000000000000..9d392aacea7e --- /dev/null +++ b/nova/api/openstack/views/addresses.py @@ -0,0 +1,54 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010-2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova import utils +from nova.api.openstack import common + + +def get_view_builder(req): + ''' + A factory method that returns the correct builder based on the version of + the api requested. + ''' + version = common.get_api_version(req) + if version == '1.1': + return ViewBuilder_1_1() + else: + return ViewBuilder_1_0() + + +class ViewBuilder(object): + ''' Models a server addresses response as a python dictionary.''' + + def build(self, inst): + raise NotImplementedError() + + +class ViewBuilder_1_0(ViewBuilder): + def build(self, inst): + private_ips = utils.get_from_path(inst, 'fixed_ip/address') + public_ips = utils.get_from_path(inst, 'fixed_ip/floating_ips/address') + return dict(public=public_ips, private=private_ips) + + +class ViewBuilder_1_1(ViewBuilder): + def build(self, inst): + private_ips = utils.get_from_path(inst, 'fixed_ip/address') + private_ips = [dict(version=4, addr=a) for a in private_ips] + public_ips = utils.get_from_path(inst, 'fixed_ip/floating_ips/address') + public_ips = [dict(version=4, addr=a) for a in public_ips] + return dict(public=public_ips, private=private_ips) diff --git a/nova/api/openstack/views/flavors.py b/nova/api/openstack/views/flavors.py new file mode 100644 index 000000000000..dd2e75a7acd9 --- /dev/null +++ b/nova/api/openstack/views/flavors.py @@ -0,0 +1,51 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010-2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova.api.openstack import common + + +def get_view_builder(req): + ''' + A factory method that returns the correct builder based on the version of + the api requested. + ''' + version = common.get_api_version(req) + base_url = req.application_url + if version == '1.1': + return ViewBuilder_1_1(base_url) + else: + return ViewBuilder_1_0() + + +class ViewBuilder(object): + def __init__(self): + pass + + def build(self, flavor_obj): + raise NotImplementedError() + + +class ViewBuilder_1_1(ViewBuilder): + def __init__(self, base_url): + self.base_url = base_url + + def generate_href(self, flavor_id): + return "%s/flavors/%s" % (self.base_url, flavor_id) + + +class ViewBuilder_1_0(ViewBuilder): + pass diff --git a/nova/api/openstack/views/images.py b/nova/api/openstack/views/images.py new file mode 100644 index 000000000000..2369a8f9d1a2 --- /dev/null +++ b/nova/api/openstack/views/images.py @@ -0,0 +1,51 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010-2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova.api.openstack import common + + +def get_view_builder(req): + ''' + A factory method that returns the correct builder based on the version of + the api requested. + ''' + version = common.get_api_version(req) + base_url = req.application_url + if version == '1.1': + return ViewBuilder_1_1(base_url) + else: + return ViewBuilder_1_0() + + +class ViewBuilder(object): + def __init__(self): + pass + + def build(self, image_obj): + raise NotImplementedError() + + +class ViewBuilder_1_1(ViewBuilder): + def __init__(self, base_url): + self.base_url = base_url + + def generate_href(self, image_id): + return "%s/images/%s" % (self.base_url, image_id) + + +class ViewBuilder_1_0(ViewBuilder): + pass diff --git a/nova/api/openstack/views/servers.py b/nova/api/openstack/views/servers.py new file mode 100644 index 000000000000..6d54a7a7eccb --- /dev/null +++ b/nova/api/openstack/views/servers.py @@ -0,0 +1,138 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010-2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import hashlib +from nova.compute import power_state +from nova.api.openstack import common +from nova.api.openstack.views import addresses as addresses_view +from nova.api.openstack.views import flavors as flavors_view +from nova.api.openstack.views import images as images_view +from nova import utils + + +def get_view_builder(req): + ''' + A factory method that returns the correct builder based on the version of + the api requested. + ''' + version = common.get_api_version(req) + addresses_builder = addresses_view.get_view_builder(req) + if version == '1.1': + flavor_builder = flavors_view.get_view_builder(req) + image_builder = images_view.get_view_builder(req) + return ViewBuilder_1_1(addresses_builder, flavor_builder, + image_builder) + else: + return ViewBuilder_1_0(addresses_builder) + + +class ViewBuilder(object): + ''' + Models a server response as a python dictionary. + Abstract methods: _build_image, _build_flavor + ''' + + def __init__(self, addresses_builder): + self.addresses_builder = addresses_builder + + def build(self, inst, is_detail): + """ + Coerces into dictionary format, mapping everything to + Rackspace-like attributes for return + """ + if is_detail: + return self._build_detail(inst) + else: + return self._build_simple(inst) + + def _build_simple(self, inst): + return dict(server=dict(id=inst['id'], name=inst['display_name'])) + + def _build_detail(self, inst): + power_mapping = { + None: 'build', + power_state.NOSTATE: 'build', + power_state.RUNNING: 'active', + power_state.BLOCKED: 'active', + power_state.SUSPENDED: 'suspended', + power_state.PAUSED: 'paused', + power_state.SHUTDOWN: 'active', + power_state.SHUTOFF: 'active', + power_state.CRASHED: 'error', + power_state.FAILED: 'error'} + inst_dict = {} + + #mapped_keys = dict(status='state', imageId='image_id', + # flavorId='instance_type', name='display_name', id='id') + + mapped_keys = dict(status='state', name='display_name', id='id') + + for k, v in mapped_keys.iteritems(): + inst_dict[k] = inst[v] + + inst_dict['status'] = power_mapping[inst_dict['status']] + try: + migration = db.migration_get_by_instance_and_status(ctxt, + inst['id'], 'finished') + inst_dict['status'] = 'resize-confirm' + except Exception, e: + inst_dict['status'] = power_mapping[inst_dict['status']] + inst_dict['addresses'] = self.addresses_builder.build(inst) + + # Return the metadata as a dictionary + metadata = {} + for item in inst['metadata']: + metadata[item['key']] = item['value'] + inst_dict['metadata'] = metadata + + inst_dict['hostId'] = '' + if inst['host']: + inst_dict['hostId'] = hashlib.sha224(inst['host']).hexdigest() + + self._build_image(inst_dict, inst) + self._build_flavor(inst_dict, inst) + + return dict(server=inst_dict) + + def _build_image(self, response, inst): + raise NotImplementedError() + + def _build_flavor(self, response, inst): + raise NotImplementedError() + + +class ViewBuilder_1_0(ViewBuilder): + def _build_image(self, response, inst): + response["imageId"] = inst["image_id"] + + def _build_flavor(self, response, inst): + response["flavorId"] = inst["instance_type"] + + +class ViewBuilder_1_1(ViewBuilder): + def __init__(self, addresses_builder, flavor_builder, image_builder): + ViewBuilder.__init__(self, addresses_builder) + self.flavor_builder = flavor_builder + self.image_builder = image_builder + + def _build_image(self, response, inst): + image_id = inst["image_id"] + response["imageRef"] = self.image_builder.generate_href(image_id) + + def _build_flavor(self, response, inst): + flavor_id = inst["instance_type"] + response["flavorRef"] = self.flavor_builder.generate_href(flavor_id) diff --git a/nova/auth/dbdriver.py b/nova/auth/dbdriver.py index d8dad8edd4dd..d1e3f2ed5e85 100644 --- a/nova/auth/dbdriver.py +++ b/nova/auth/dbdriver.py @@ -162,6 +162,8 @@ class DbDriver(object): values['description'] = description db.project_update(context.get_admin_context(), project_id, values) + if not self.is_in_project(manager_uid, project_id): + self.add_to_project(manager_uid, project_id) def add_to_project(self, uid, project_id): """Add user to project""" diff --git a/nova/auth/fakeldap.py b/nova/auth/fakeldap.py index 4466051f0e6b..79afb91090f7 100644 --- a/nova/auth/fakeldap.py +++ b/nova/auth/fakeldap.py @@ -90,12 +90,12 @@ MOD_DELETE = 1 MOD_REPLACE = 2 -class NO_SUCH_OBJECT(Exception): # pylint: disable-msg=C0103 +class NO_SUCH_OBJECT(Exception): # pylint: disable=C0103 """Duplicate exception class from real LDAP module.""" pass -class OBJECT_CLASS_VIOLATION(Exception): # pylint: disable-msg=C0103 +class OBJECT_CLASS_VIOLATION(Exception): # pylint: disable=C0103 """Duplicate exception class from real LDAP module.""" pass @@ -268,7 +268,7 @@ class FakeLDAP(object): # get the attributes from the store attrs = store.hgetall(key) # turn the values from the store into lists - # pylint: disable-msg=E1103 + # pylint: disable=E1103 attrs = dict([(k, _from_json(v)) for k, v in attrs.iteritems()]) # filter the objects by query @@ -277,12 +277,12 @@ class FakeLDAP(object): attrs = dict([(k, v) for k, v in attrs.iteritems() if not fields or k in fields]) objects.append((key[len(self.__prefix):], attrs)) - # pylint: enable-msg=E1103 + # pylint: enable=E1103 if objects == []: raise NO_SUCH_OBJECT() return objects @property - def __prefix(self): # pylint: disable-msg=R0201 + def __prefix(self): # pylint: disable=R0201 """Get the prefix to use for all keys.""" return 'ldap:' diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 5da7751a015a..fcac555100f1 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -275,6 +275,8 @@ class LdapDriver(object): attr.append((self.ldap.MOD_REPLACE, 'description', description)) dn = self.__project_to_dn(project_id) self.conn.modify_s(dn, attr) + if not self.is_in_project(manager_uid, project_id): + self.add_to_project(manager_uid, project_id) @sanitize def add_to_project(self, uid, project_id): @@ -632,6 +634,6 @@ class LdapDriver(object): class FakeLdapDriver(LdapDriver): """Fake Ldap Auth driver""" - def __init__(self): # pylint: disable-msg=W0231 + def __init__(self): # pylint: disable=W0231 __import__('nova.auth.fakeldap') self.ldap = sys.modules['nova.auth.fakeldap'] diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 450ab803a24e..486845399334 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -22,7 +22,7 @@ Nova authentication management import os import shutil -import string # pylint: disable-msg=W0402 +import string # pylint: disable=W0402 import tempfile import uuid import zipfile @@ -96,10 +96,19 @@ class AuthBase(object): class User(AuthBase): - """Object representing a user""" + """Object representing a user + + The following attributes are defined: + :id: A system identifier for the user. A string (for LDAP) + :name: The user name, potentially in some more friendly format + :access: The 'username' for EC2 authentication + :secret: The 'password' for EC2 authenticatoin + :admin: ??? + """ def __init__(self, id, name, access, secret, admin): AuthBase.__init__(self) + assert isinstance(id, basestring) self.id = id self.name = name self.access = access diff --git a/nova/compute/api.py b/nova/compute/api.py index 077068963551..dbf99e7c5bac 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -489,19 +489,20 @@ class API(base.Base): def resize(self, context, instance_id, flavor_id): """Resize a running instance.""" instance = self.db.instance_get(context, instance_id) - LOG.debug(_("Resizing instance %s to flavor %d") % - (instance.name, flavor_id)) + LOG.debug(_("Resizing instance %(instance_type['name'] to flavor" + "%(flavor_id)") % locals()) current_instance_type = self.db.instance_type_get_by_name( context, instance['instance_type']) new_instance_type = self.db.instance_type_get_by_flavor_id( context, flavor_id) - LOG.debug(_("Old instance type %s -> New instance type %s") % + LOG.debug(_("Old instance type %s -> New instance type %s"), (current_instance_type['name'], new_instance_type['name'])) if not new_instance_type: raise exception.ApiError(_("Requested flavor does not exist")) - if current_instance_type['memory_mb'] > new_instance_type['memory_mb']: + if current_instance_type['memory_mb'] >= \ + new_instance_type['memory_mb']: raise exception.ApiError(_("Invalid flavor: cannot downsize" "instances")) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 0178c30a3fcb..78ef33ac25fb 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -39,6 +39,7 @@ import os import random import string import socket +import sys import tempfile import time import functools @@ -114,7 +115,13 @@ class ComputeManager(manager.Manager): # and redocument the module docstring if not compute_driver: compute_driver = FLAGS.compute_driver - self.driver = utils.import_object(compute_driver) + + try: + self.driver = utils.import_object(compute_driver) + except ImportError: + LOG.error("Unable to load the virtualization driver.") + sys.exit(1) + self.network_manager = utils.import_object(FLAGS.network_manager) self.volume_manager = utils.import_object(FLAGS.volume_manager) super(ComputeManager, self).__init__(*args, **kwargs) @@ -220,9 +227,10 @@ class ComputeManager(manager.Manager): self.db.instance_update(context, instance_id, {'launched_at': now}) - except Exception: # pylint: disable-msg=W0702 - LOG.exception(_("instance %s: Failed to spawn"), instance_id, - context=context) + except Exception: # pylint: disable=W0702 + LOG.exception(_("Instance '%s' failed to spawn. Is virtualization" + " enabled in the BIOS?"), instance_id, + context=context) self.db.instance_set_state(context, instance_id, power_state.SHUTDOWN) @@ -539,7 +547,7 @@ class ComputeManager(manager.Manager): local_gb=instance_type['local_gb'])) # reload the updated instance ref - # FIXME: is there reload functionality? + # FIXME(mdietz): is there reload functionality? instance_ref = self.db.instance_get(context, instance_id) self.driver.finish_resize(instance_ref, disk_info) @@ -723,7 +731,7 @@ class ComputeManager(manager.Manager): volume_id, instance_id, mountpoint) - except Exception as exc: # pylint: disable-msg=W0702 + except Exception as exc: # pylint: disable=W0702 # NOTE(vish): The inline callback eats the exception info so we # log the traceback here and reraise the same # ecxception below. diff --git a/nova/db/api.py b/nova/db/api.py index 3cb0e58113f8..add5bd83e57b 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -608,7 +608,7 @@ def network_get_all(context): return IMPL.network_get_all(context) -# pylint: disable-msg=C0103 +# pylint: disable=C0103 def network_get_associated_fixed_ips(context, network_id): """Get all network's ips that have been associated.""" return IMPL.network_get_associated_fixed_ips(context, network_id) @@ -1118,7 +1118,7 @@ def instance_type_create(context, values): return IMPL.instance_type_create(context, values) -def instance_type_get_all(context, inactive=0): +def instance_type_get_all(context, inactive=False): """Get all instance types""" return IMPL.instance_type_get_all(context, inactive) diff --git a/nova/db/base.py b/nova/db/base.py index 1d1e80866bdf..a0f2180c6569 100644 --- a/nova/db/base.py +++ b/nova/db/base.py @@ -33,4 +33,4 @@ class Base(object): def __init__(self, db_driver=None): if not db_driver: db_driver = FLAGS.db_driver - self.db = utils.import_object(db_driver) # pylint: disable-msg=C0103 + self.db = utils.import_object(db_driver) # pylint: disable=C0103 diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index a0afbe5b25bd..ac7f7cbf1054 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -806,6 +806,11 @@ def instance_destroy(context, instance_id): update({'deleted': 1, 'deleted_at': datetime.datetime.utcnow(), 'updated_at': literal_column('updated_at')}) + session.query(models.InstanceMetadata).\ + filter_by(instance_id=instance_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) @require_context @@ -1249,7 +1254,7 @@ def network_get_all(context): # NOTE(vish): pylint complains because of the long method name, but # it fits with the names of the rest of the methods -# pylint: disable-msg=C0103 +# pylint: disable=C0103 @require_admin_context @@ -2337,7 +2342,7 @@ def instance_type_create(_context, values): @require_context -def instance_type_get_all(context, inactive=0): +def instance_type_get_all(context, inactive=False): """ Returns a dict describing all instance_types with name as key. """ @@ -2348,7 +2353,7 @@ def instance_type_get_all(context, inactive=0): all() else: inst_types = session.query(models.InstanceTypes).\ - filter_by(deleted=inactive).\ + filter_by(deleted=False).\ order_by("name").\ all() if inst_types: @@ -2392,7 +2397,7 @@ def instance_type_destroy(context, name): session = get_session() instance_type_ref = session.query(models.InstanceTypes).\ filter_by(name=name) - records = instance_type_ref.update(dict(deleted=1)) + records = instance_type_ref.update(dict(deleted=True)) if records == 0: raise exception.NotFound else: diff --git a/nova/db/sqlalchemy/migrate_repo/versions/008_add_instance_types.py b/nova/db/sqlalchemy/migrate_repo/versions/008_add_instance_types.py index 66609054e642..5e2cb69d939a 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/008_add_instance_types.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/008_add_instance_types.py @@ -55,7 +55,7 @@ def upgrade(migrate_engine): try: instance_types.create() except Exception: - logging.info(repr(table)) + logging.info(repr(instance_types)) logging.exception('Exception while creating instance_types table') raise @@ -72,11 +72,11 @@ def upgrade(migrate_engine): # FIXME(kpepple) should we be seeding created_at / updated_at ? # now = datetime.datatime.utcnow() i.execute({'name': name, 'memory_mb': values["memory_mb"], - 'vcpus': values["vcpus"], 'deleted': 0, + 'vcpus': values["vcpus"], 'deleted': False, 'local_gb': values["local_gb"], 'flavorid': values["flavorid"]}) except Exception: - logging.info(repr(table)) + logging.info(repr(instance_types)) logging.exception('Exception while seeding instance_types table') raise diff --git a/nova/db/sqlalchemy/migrate_repo/versions/012_add_flavors_to_migrations.py b/nova/db/sqlalchemy/migrate_repo/versions/012_add_flavors_to_migrations.py index 412caedd0e12..e677ba14de85 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/012_add_flavors_to_migrations.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/012_add_flavors_to_migrations.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2010 OpenStack LLC. +# Copyright 2011 OpenStack LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -42,3 +42,8 @@ def upgrade(migrate_engine): meta.bind = migrate_engine migrations.create_column(old_flavor_id) migrations.create_column(new_flavor_id) + +def downgrade(migrate_engine): + meta.bind = migrate_engine + migrations.drop_column(old_flavor_id) + migrations.drop_column(new_flavor_id) diff --git a/nova/exception.py b/nova/exception.py index 93c5fe3d7794..4e2bbdbaf956 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -46,7 +46,7 @@ class Error(Exception): class ApiError(Error): - def __init__(self, message='Unknown', code='Unknown'): + def __init__(self, message='Unknown', code='ApiError'): self.message = message self.code = code super(ApiError, self).__init__('%s: %s' % (code, message)) diff --git a/nova/image/local.py b/nova/image/local.py index c4ac3baaa4de..609d6c42a669 100644 --- a/nova/image/local.py +++ b/nova/image/local.py @@ -20,8 +20,9 @@ import os.path import random import shutil -from nova import flags from nova import exception +from nova import flags +from nova import log as logging from nova.image import service @@ -29,6 +30,8 @@ FLAGS = flags.FLAGS flags.DEFINE_string('images_path', '$state_path/images', 'path to decrypted images') +LOG = logging.getLogger('nova.image.local') + class LocalImageService(service.BaseImageService): """Image service storing images to local disk. @@ -47,7 +50,17 @@ class LocalImageService(service.BaseImageService): def _ids(self): """The list of all image ids.""" - return [int(i, 16) for i in os.listdir(self._path)] + images = [] + for image_dir in os.listdir(self._path): + try: + unhexed_image_id = int(image_dir, 16) + except ValueError: + LOG.error( + _("%s is not in correct directory naming format"\ + % image_dir)) + else: + images.append(unhexed_image_id) + return images def index(self, context): return [dict(image_id=i['id'], name=i.get('name')) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 7106e6164ed8..ee36407a628f 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -557,6 +557,7 @@ def get_dhcp_hosts(context, network_id): # NOTE(ja): Sending a HUP only reloads the hostfile, so any # configuration options (like dchp-range, vlan, ...) # aren't reloaded. +@utils.synchronized('dnsmasq_start') def update_dhcp(context, network_id): """(Re)starts a dnsmasq server for a given network @@ -582,7 +583,7 @@ def update_dhcp(context, network_id): try: _execute('sudo', 'kill', '-HUP', pid) return - except Exception as exc: # pylint: disable-msg=W0703 + except Exception as exc: # pylint: disable=W0703 LOG.debug(_("Hupping dnsmasq threw %s"), exc) else: LOG.debug(_("Pid %d is stale, relaunching dnsmasq"), pid) @@ -626,7 +627,7 @@ interface %s if conffile in out: try: _execute('sudo', 'kill', pid) - except Exception as exc: # pylint: disable-msg=W0703 + except Exception as exc: # pylint: disable=W0703 LOG.debug(_("killing radvd threw %s"), exc) else: LOG.debug(_("Pid %d is stale, relaunching radvd"), pid) @@ -713,7 +714,7 @@ def _stop_dnsmasq(network): if pid: try: _execute('sudo', 'kill', '-TERM', pid) - except Exception as exc: # pylint: disable-msg=W0703 + except Exception as exc: # pylint: disable=W0703 LOG.debug(_("Killing dnsmasq threw %s"), exc) diff --git a/nova/network/manager.py b/nova/network/manager.py index 3dfc48934b6b..91519a2abd0e 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -73,7 +73,7 @@ flags.DEFINE_string('flat_interface', None, flags.DEFINE_string('flat_network_dhcp_start', '10.0.0.2', 'Dhcp start for FlatDhcp') flags.DEFINE_integer('vlan_start', 100, 'First VLAN for private networks') -flags.DEFINE_integer('num_networks', 1000, 'Number of networks to support') +flags.DEFINE_integer('num_networks', 1, 'Number of networks to support') flags.DEFINE_string('vpn_ip', '$my_ip', 'Public IP for the cloudpipe VPN servers') flags.DEFINE_integer('vpn_start', 1000, 'First Vpn port for private networks') @@ -322,12 +322,12 @@ class NetworkManager(manager.Manager): self._create_fixed_ips(context, network_ref['id']) @property - def _bottom_reserved_ips(self): # pylint: disable-msg=R0201 + def _bottom_reserved_ips(self): # pylint: disable=R0201 """Number of reserved ips at the bottom of the range.""" return 2 # network, gateway @property - def _top_reserved_ips(self): # pylint: disable-msg=R0201 + def _top_reserved_ips(self): # pylint: disable=R0201 """Number of reserved ips at the top of the range.""" return 1 # broadcast diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index 05ddace4b73f..554c72848f45 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -167,7 +167,7 @@ class S3(ErrorHandlingResource): def __init__(self): ErrorHandlingResource.__init__(self) - def getChild(self, name, request): # pylint: disable-msg=C0103 + def getChild(self, name, request): # pylint: disable=C0103 """Returns either the image or bucket resource""" request.context = get_context(request) if name == '': @@ -177,7 +177,7 @@ class S3(ErrorHandlingResource): else: return BucketResource(name) - def render_GET(self, request): # pylint: disable-msg=R0201 + def render_GET(self, request): # pylint: disable=R0201 """Renders the GET request for a list of buckets as XML""" LOG.debug(_('List of buckets requested'), context=request.context) buckets = [b for b in bucket.Bucket.all() @@ -355,7 +355,7 @@ class ImagesResource(resource.Resource): else: return ImageResource(name) - def render_GET(self, request): # pylint: disable-msg=R0201 + def render_GET(self, request): # pylint: disable=R0201 """ returns a json listing of all images that a user has permissions to see """ @@ -384,7 +384,7 @@ class ImagesResource(resource.Resource): request.finish() return server.NOT_DONE_YET - def render_PUT(self, request): # pylint: disable-msg=R0201 + def render_PUT(self, request): # pylint: disable=R0201 """ create a new registered image """ image_id = get_argument(request, 'image_id', u'') @@ -413,7 +413,7 @@ class ImagesResource(resource.Resource): p.start() return '' - def render_POST(self, request): # pylint: disable-msg=R0201 + def render_POST(self, request): # pylint: disable=R0201 """Update image attributes: public/private""" # image_id required for all requests @@ -441,7 +441,7 @@ class ImagesResource(resource.Resource): image_object.update_user_editable_fields(clean_args) return '' - def render_DELETE(self, request): # pylint: disable-msg=R0201 + def render_DELETE(self, request): # pylint: disable=R0201 """Delete a registered image""" image_id = get_argument(request, "image_id", u"") image_object = image.Image(image_id) @@ -471,7 +471,7 @@ def get_application(): application = service.Application("objectstore") # Disabled because of lack of proper introspection in Twisted # or possibly different versions of twisted? - # pylint: disable-msg=E1101 + # pylint: disable=E1101 objectStoreService = internet.TCPServer(FLAGS.s3_port, factory, interface=FLAGS.s3_listen_host) objectStoreService.setServiceParent(application) diff --git a/nova/rpc.py b/nova/rpc.py index fbb90299b848..5935e1fb3ab5 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -62,7 +62,7 @@ class Connection(carrot_connection.BrokerConnection): params['backend_cls'] = fakerabbit.Backend # NOTE(vish): magic is fun! - # pylint: disable-msg=W0142 + # pylint: disable=W0142 if new: return cls(**params) else: @@ -114,7 +114,7 @@ class Consumer(messaging.Consumer): if self.failed_connection: # NOTE(vish): connection is defined in the parent class, we can # recreate it as long as we create the backend too - # pylint: disable-msg=W0201 + # pylint: disable=W0201 self.connection = Connection.recreate() self.backend = self.connection.create_backend() self.declare() @@ -125,7 +125,7 @@ class Consumer(messaging.Consumer): # NOTE(vish): This is catching all errors because we really don't # want exceptions to be logged 10 times a second if some # persistent failure occurs. - except Exception: # pylint: disable-msg=W0703 + except Exception: # pylint: disable=W0703 if not self.failed_connection: LOG.exception(_("Failed to fetch message from queue")) self.failed_connection = True @@ -311,7 +311,7 @@ def _pack_context(msg, context): def call(context, topic, msg): """Sends a message on a topic and wait for a response""" - LOG.debug(_("Making asynchronous call...")) + LOG.debug(_("Making asynchronous call on %s ..."), topic) msg_id = uuid.uuid4().hex msg.update({'_msg_id': msg_id}) LOG.debug(_("MSG_ID is %s") % (msg_id)) @@ -352,7 +352,7 @@ def call(context, topic, msg): def cast(context, topic, msg): """Sends a message on a topic without waiting for a response""" - LOG.debug(_("Making asynchronous cast...")) + LOG.debug(_("Making asynchronous cast on %s..."), topic) _pack_context(msg, context) conn = Connection.instance() publisher = TopicPublisher(connection=conn, topic=topic) diff --git a/nova/service.py b/nova/service.py index d60df987ce67..52bb15ad7777 100644 --- a/nova/service.py +++ b/nova/service.py @@ -217,7 +217,7 @@ class Service(object): logging.error(_("Recovered model server connection!")) # TODO(vish): this should probably only catch connection errors - except Exception: # pylint: disable-msg=W0702 + except Exception: # pylint: disable=W0702 if not getattr(self, "model_disconnected", False): self.model_disconnected = True logging.exception(_("model server went away")) diff --git a/nova/tests/api/openstack/__init__.py b/nova/tests/api/openstack/__init__.py index e18120285748..bac7181f7199 100644 --- a/nova/tests/api/openstack/__init__.py +++ b/nova/tests/api/openstack/__init__.py @@ -20,7 +20,7 @@ from nova import test from nova import context from nova import flags -from nova.api.openstack.ratelimiting import RateLimitingMiddleware +from nova.api.openstack.limits import RateLimitingMiddleware from nova.api.openstack.common import limited from nova.tests.api.openstack import fakes from webob import Request diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 0bbb1c890ee6..75eade4d0699 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -34,7 +34,7 @@ from nova import utils import nova.api.openstack.auth from nova.api import openstack from nova.api.openstack import auth -from nova.api.openstack import ratelimiting +from nova.api.openstack import limits from nova.auth.manager import User, Project from nova.image import glance from nova.image import local @@ -77,8 +77,9 @@ def wsgi_app(inner_application=None): inner_application = openstack.APIRouter() mapper = urlmap.URLMap() api = openstack.FaultWrapper(auth.AuthMiddleware( - ratelimiting.RateLimitingMiddleware(inner_application))) + limits.RateLimitingMiddleware(inner_application))) mapper['/v1.0'] = api + mapper['/v1.1'] = api mapper['/'] = openstack.FaultWrapper(openstack.Versions()) return mapper @@ -115,13 +116,13 @@ def stub_out_auth(stubs): def stub_out_rate_limiting(stubs): def fake_rate_init(self, app): - super(ratelimiting.RateLimitingMiddleware, self).__init__(app) + super(limits.RateLimitingMiddleware, self).__init__(app) self.application = app - stubs.Set(nova.api.openstack.ratelimiting.RateLimitingMiddleware, + stubs.Set(nova.api.openstack.limits.RateLimitingMiddleware, '__init__', fake_rate_init) - stubs.Set(nova.api.openstack.ratelimiting.RateLimitingMiddleware, + stubs.Set(nova.api.openstack.limits.RateLimitingMiddleware, '__call__', fake_wsgi) @@ -233,52 +234,57 @@ class FakeAuthDatabase(object): class FakeAuthManager(object): - auth_data = {} + #NOTE(justinsb): Accessing static variables through instances is FUBAR + #NOTE(justinsb): This should also be private! + auth_data = [] projects = {} @classmethod def clear_fakes(cls): - cls.auth_data = {} + cls.auth_data = [] cls.projects = {} @classmethod def reset_fake_data(cls): - cls.auth_data = dict(acc1=User('guy1', 'guy1', 'acc1', - 'fortytwo!', False)) + u1 = User('id1', 'guy1', 'acc1', 'secret1', False) + cls.auth_data = [u1] cls.projects = dict(testacct=Project('testacct', 'testacct', - 'guy1', + 'id1', 'test', [])) - def add_user(self, key, user): - FakeAuthManager.auth_data[key] = user + def add_user(self, user): + FakeAuthManager.auth_data.append(user) def get_users(self): - return FakeAuthManager.auth_data.values() + return FakeAuthManager.auth_data def get_user(self, uid): - for k, v in FakeAuthManager.auth_data.iteritems(): - if v.id == uid: - return v + for user in FakeAuthManager.auth_data: + if user.id == uid: + return user + return None + + def get_user_from_access_key(self, key): + for user in FakeAuthManager.auth_data: + if user.access == key: + return user return None def delete_user(self, uid): - for k, v in FakeAuthManager.auth_data.items(): - if v.id == uid: - del FakeAuthManager.auth_data[k] + for user in FakeAuthManager.auth_data: + if user.id == uid: + FakeAuthManager.auth_data.remove(user) return None def create_user(self, name, access=None, secret=None, admin=False): u = User(name, name, access, secret, admin) - FakeAuthManager.auth_data[access] = u + FakeAuthManager.auth_data.append(u) return u def modify_user(self, user_id, access=None, secret=None, admin=None): - user = None - for k, v in FakeAuthManager.auth_data.iteritems(): - if v.id == user_id: - user = v + user = self.get_user(user_id) if user: user.access = access user.secret = secret @@ -325,12 +331,6 @@ class FakeAuthManager(object): if (user.id in p.member_ids) or (user.id == p.project_manager_id)] - def get_user_from_access_key(self, key): - try: - return FakeAuthManager.auth_data[key] - except KeyError: - raise exc.NotFound - class FakeRateLimiter(object): def __init__(self, application): diff --git a/nova/tests/api/openstack/test_accounts.py b/nova/tests/api/openstack/test_accounts.py index 60edce7694d4..64abcf48ce33 100644 --- a/nova/tests/api/openstack/test_accounts.py +++ b/nova/tests/api/openstack/test_accounts.py @@ -19,11 +19,9 @@ import json import stubout import webob -import nova.api -import nova.api.openstack.auth -from nova import context from nova import flags from nova import test +from nova.api.openstack import accounts from nova.auth.manager import User from nova.tests.api.openstack import fakes @@ -44,9 +42,9 @@ class AccountsTest(test.TestCase): def setUp(self): super(AccountsTest, self).setUp() self.stubs = stubout.StubOutForTesting() - self.stubs.Set(nova.api.openstack.accounts.Controller, '__init__', + self.stubs.Set(accounts.Controller, '__init__', fake_init) - self.stubs.Set(nova.api.openstack.accounts.Controller, '_check_admin', + self.stubs.Set(accounts.Controller, '_check_admin', fake_admin_check) fakes.FakeAuthManager.clear_fakes() fakes.FakeAuthDatabase.data = {} @@ -57,10 +55,10 @@ class AccountsTest(test.TestCase): self.allow_admin = FLAGS.allow_admin_api FLAGS.allow_admin_api = True fakemgr = fakes.FakeAuthManager() - joeuser = User('guy1', 'guy1', 'acc1', 'fortytwo!', False) - superuser = User('guy2', 'guy2', 'acc2', 'swordfish', True) - fakemgr.add_user(joeuser.access, joeuser) - fakemgr.add_user(superuser.access, superuser) + joeuser = User('id1', 'guy1', 'acc1', 'secret1', False) + superuser = User('id2', 'guy2', 'acc2', 'secret2', True) + fakemgr.add_user(joeuser) + fakemgr.add_user(superuser) fakemgr.create_project('test1', joeuser) fakemgr.create_project('test2', superuser) @@ -76,7 +74,7 @@ class AccountsTest(test.TestCase): self.assertEqual(res_dict['account']['id'], 'test1') self.assertEqual(res_dict['account']['name'], 'test1') - self.assertEqual(res_dict['account']['manager'], 'guy1') + self.assertEqual(res_dict['account']['manager'], 'id1') self.assertEqual(res.status_int, 200) def test_account_delete(self): @@ -88,7 +86,7 @@ class AccountsTest(test.TestCase): def test_account_create(self): body = dict(account=dict(description='test account', - manager='guy1')) + manager='id1')) req = webob.Request.blank('/v1.0/accounts/newacct') req.headers["Content-Type"] = "application/json" req.method = 'PUT' @@ -101,14 +99,14 @@ class AccountsTest(test.TestCase): self.assertEqual(res_dict['account']['id'], 'newacct') self.assertEqual(res_dict['account']['name'], 'newacct') self.assertEqual(res_dict['account']['description'], 'test account') - self.assertEqual(res_dict['account']['manager'], 'guy1') + self.assertEqual(res_dict['account']['manager'], 'id1') self.assertTrue('newacct' in fakes.FakeAuthManager.projects) self.assertEqual(len(fakes.FakeAuthManager.projects.values()), 3) def test_account_update(self): body = dict(account=dict(description='test account', - manager='guy2')) + manager='id2')) req = webob.Request.blank('/v1.0/accounts/test1') req.headers["Content-Type"] = "application/json" req.method = 'PUT' @@ -121,5 +119,5 @@ class AccountsTest(test.TestCase): self.assertEqual(res_dict['account']['id'], 'test1') self.assertEqual(res_dict['account']['name'], 'test1') self.assertEqual(res_dict['account']['description'], 'test account') - self.assertEqual(res_dict['account']['manager'], 'guy2') + self.assertEqual(res_dict['account']['manager'], 'id2') self.assertEqual(len(fakes.FakeAuthManager.projects.values()), 2) diff --git a/nova/tests/api/openstack/test_adminapi.py b/nova/tests/api/openstack/test_adminapi.py index 4568cb9f5c0e..e87255b186df 100644 --- a/nova/tests/api/openstack/test_adminapi.py +++ b/nova/tests/api/openstack/test_adminapi.py @@ -23,7 +23,6 @@ from paste import urlmap from nova import flags from nova import test from nova.api import openstack -from nova.api.openstack import ratelimiting from nova.api.openstack import auth from nova.tests.api.openstack import fakes diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index 0448ed7013b0..21596fb25e84 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -39,7 +39,7 @@ class Test(test.TestCase): self.stubs.Set(nova.api.openstack.auth.AuthMiddleware, '__init__', fakes.fake_auth_init) self.stubs.Set(context, 'RequestContext', fakes.FakeRequestContext) - fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.clear_fakes() fakes.FakeAuthDatabase.data = {} fakes.stub_out_rate_limiting(self.stubs) fakes.stub_out_networking(self.stubs) @@ -51,8 +51,8 @@ class Test(test.TestCase): def test_authorize_user(self): f = fakes.FakeAuthManager() - f.add_user('user1_key', - nova.auth.manager.User(1, 'user1', None, None, None)) + user = nova.auth.manager.User('id1', 'user1', 'user1_key', None, None) + f.add_user(user) req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'user1' @@ -66,9 +66,9 @@ class Test(test.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - u = nova.auth.manager.User(1, 'user1', None, None, None) - f.add_user('user1_key', u) - f.create_project('user1_project', u) + user = nova.auth.manager.User('id1', 'user1', 'user1_key', None, None) + f.add_user(user) + f.create_project('user1_project', user) req = webob.Request.blank('/v1.0/', {'HTTP_HOST': 'foo'}) req.headers['X-Auth-User'] = 'user1' @@ -124,8 +124,8 @@ class Test(test.TestCase): def test_bad_user_good_key(self): f = fakes.FakeAuthManager() - u = nova.auth.manager.User(1, 'user1', None, None, None) - f.add_user('user1_key', u) + user = nova.auth.manager.User('id1', 'user1', 'user1_key', None, None) + f.add_user(user) req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'unknown_user' @@ -179,7 +179,7 @@ class TestLimiter(test.TestCase): self.stubs.Set(nova.api.openstack.auth.AuthMiddleware, '__init__', fakes.fake_auth_init) self.stubs.Set(context, 'RequestContext', fakes.FakeRequestContext) - fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.clear_fakes() fakes.FakeAuthDatabase.data = {} fakes.stub_out_networking(self.stubs) @@ -190,9 +190,9 @@ class TestLimiter(test.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - u = nova.auth.manager.User(1, 'user1', None, None, None) - f.add_user('user1_key', u) - f.create_project('test', u) + user = nova.auth.manager.User('id1', 'user1', 'user1_key', None, None) + f.add_user(user) + f.create_project('test', user) req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'user1' diff --git a/nova/tests/api/openstack/test_flavors.py b/nova/tests/api/openstack/test_flavors.py index 30326dc50aef..4f504808cabc 100644 --- a/nova/tests/api/openstack/test_flavors.py +++ b/nova/tests/api/openstack/test_flavors.py @@ -22,11 +22,32 @@ import webob from nova import test import nova.api from nova import context -from nova import db from nova.api.openstack import flavors +from nova import db from nova.tests.api.openstack import fakes +def stub_flavor(flavorid, name, memory_mb="256", local_gb="10"): + return { + "flavorid": str(flavorid), + "name": name, + "memory_mb": memory_mb, + "local_gb": local_gb, + } + + +def return_instance_type_by_flavor_id(context, flavorid): + return stub_flavor(flavorid, "flavor %s" % (flavorid,)) + + +def return_instance_types(context, num=2): + instance_types = {} + for i in xrange(1, num + 1): + name = "flavor %s" % (i,) + instance_types[name] = stub_flavor(i, name) + return instance_types + + class FlavorsTest(test.TestCase): def setUp(self): super(FlavorsTest, self).setUp() @@ -36,6 +57,10 @@ class FlavorsTest(test.TestCase): fakes.stub_out_networking(self.stubs) fakes.stub_out_rate_limiting(self.stubs) fakes.stub_out_auth(self.stubs) + self.stubs.Set(nova.db.api, "instance_type_get_all", + return_instance_types) + self.stubs.Set(nova.db.api, "instance_type_get_by_flavor_id", + return_instance_type_by_flavor_id) self.context = context.get_admin_context() def tearDown(self): @@ -46,10 +71,49 @@ class FlavorsTest(test.TestCase): req = webob.Request.blank('/v1.0/flavors') res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) + flavors = json.loads(res.body)["flavors"] + expected = [ + { + "id": "1", + "name": "flavor 1", + }, + { + "id": "2", + "name": "flavor 2", + }, + ] + self.assertEqual(flavors, expected) - def test_get_flavor_by_id(self): - req = webob.Request.blank('/v1.0/flavors/1') + def test_get_flavor_list_detail(self): + req = webob.Request.blank('/v1.0/flavors/detail') res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) - body = json.loads(res.body) - self.assertEqual(body['flavor']['id'], 1) + flavors = json.loads(res.body)["flavors"] + expected = [ + { + "id": "1", + "name": "flavor 1", + "ram": "256", + "disk": "10", + }, + { + "id": "2", + "name": "flavor 2", + "ram": "256", + "disk": "10", + }, + ] + self.assertEqual(flavors, expected) + + def test_get_flavor_by_id(self): + req = webob.Request.blank('/v1.0/flavors/12') + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(res.status_int, 200) + flavor = json.loads(res.body)["flavor"] + expected = { + "id": "12", + "name": "flavor 12", + "ram": "256", + "disk": "10", + } + self.assertEqual(flavor, expected) diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index 76f7589294e7..a674ccefe8b7 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -22,6 +22,7 @@ and as a WSGI layer import json import datetime +import os import shutil import tempfile @@ -151,6 +152,17 @@ class LocalImageServiceTest(test.TestCase, self.stubs.UnsetAll() super(LocalImageServiceTest, self).tearDown() + def test_get_all_ids_with_incorrect_directory_formats(self): + # create some old-style image directories (starting with 'ami-') + for x in [1, 2, 3]: + tempfile.mkstemp(prefix='ami-', dir=self.tempdir) + # create some valid image directories names + for x in ["1485baed", "1a60f0ee", "3123a73d"]: + os.makedirs(os.path.join(self.tempdir, x)) + found_image_ids = self.service._ids() + self.assertEqual(True, isinstance(found_image_ids, list)) + self.assertEqual(3, len(found_image_ids), len(found_image_ids)) + class GlanceImageServiceTest(test.TestCase, BaseImageServiceTests): diff --git a/nova/tests/api/openstack/test_limits.py b/nova/tests/api/openstack/test_limits.py new file mode 100644 index 000000000000..05cfacc60585 --- /dev/null +++ b/nova/tests/api/openstack/test_limits.py @@ -0,0 +1,584 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Tests dealing with HTTP rate-limiting. +""" + +import httplib +import json +import StringIO +import stubout +import time +import unittest +import webob + +from xml.dom.minidom import parseString + +from nova.api.openstack import limits +from nova.api.openstack.limits import Limit + + +TEST_LIMITS = [ + Limit("GET", "/delayed", "^/delayed", 1, limits.PER_MINUTE), + Limit("POST", "*", ".*", 7, limits.PER_MINUTE), + Limit("POST", "/servers", "^/servers", 3, limits.PER_MINUTE), + Limit("PUT", "*", "", 10, limits.PER_MINUTE), + Limit("PUT", "/servers", "^/servers", 5, limits.PER_MINUTE), +] + + +class BaseLimitTestSuite(unittest.TestCase): + """Base test suite which provides relevant stubs and time abstraction.""" + + def setUp(self): + """Run before each test.""" + self.time = 0.0 + self.stubs = stubout.StubOutForTesting() + self.stubs.Set(limits.Limit, "_get_time", self._get_time) + + def tearDown(self): + """Run after each test.""" + self.stubs.UnsetAll() + + def _get_time(self): + """Return the "time" according to this test suite.""" + return self.time + + +class LimitsControllerTest(BaseLimitTestSuite): + """ + Tests for `limits.LimitsController` class. + """ + + def setUp(self): + """Run before each test.""" + BaseLimitTestSuite.setUp(self) + self.controller = limits.LimitsController() + + def _get_index_request(self, accept_header="application/json"): + """Helper to set routing arguments.""" + request = webob.Request.blank("/") + request.accept = accept_header + request.environ["wsgiorg.routing_args"] = (None, { + "action": "index", + "controller": "", + }) + return request + + def _populate_limits(self, request): + """Put limit info into a request.""" + _limits = [ + Limit("GET", "*", ".*", 10, 60).display(), + Limit("POST", "*", ".*", 5, 60 * 60).display(), + ] + request.environ["nova.limits"] = _limits + return request + + def test_empty_index_json(self): + """Test getting empty limit details in JSON.""" + request = self._get_index_request() + response = request.get_response(self.controller) + expected = { + "limits": { + "rate": [], + "absolute": {}, + }, + } + body = json.loads(response.body) + self.assertEqual(expected, body) + + def test_index_json(self): + """Test getting limit details in JSON.""" + request = self._get_index_request() + request = self._populate_limits(request) + response = request.get_response(self.controller) + expected = { + "limits": { + "rate": [{ + "regex": ".*", + "resetTime": 0, + "URI": "*", + "value": 10, + "verb": "GET", + "remaining": 10, + "unit": "MINUTE", + }, + { + "regex": ".*", + "resetTime": 0, + "URI": "*", + "value": 5, + "verb": "POST", + "remaining": 5, + "unit": "HOUR", + }], + "absolute": {}, + }, + } + body = json.loads(response.body) + self.assertEqual(expected, body) + + def test_empty_index_xml(self): + """Test getting limit details in XML.""" + request = self._get_index_request("application/xml") + response = request.get_response(self.controller) + + expected = "" + body = response.body.replace("\n", "").replace(" ", "") + + self.assertEqual(expected, body) + + def test_index_xml(self): + """Test getting limit details in XML.""" + request = self._get_index_request("application/xml") + request = self._populate_limits(request) + response = request.get_response(self.controller) + + expected = parseString(""" + + + + + + + + """.replace(" ", "")) + body = parseString(response.body.replace(" ", "")) + + self.assertEqual(expected.toxml(), body.toxml()) + + +class LimitMiddlewareTest(BaseLimitTestSuite): + """ + Tests for the `limits.RateLimitingMiddleware` class. + """ + + @webob.dec.wsgify + def _empty_app(self, request): + """Do-nothing WSGI app.""" + pass + + def setUp(self): + """Prepare middleware for use through fake WSGI app.""" + BaseLimitTestSuite.setUp(self) + _limits = [ + Limit("GET", "*", ".*", 1, 60), + ] + self.app = limits.RateLimitingMiddleware(self._empty_app, _limits) + + def test_good_request(self): + """Test successful GET request through middleware.""" + request = webob.Request.blank("/") + response = request.get_response(self.app) + self.assertEqual(200, response.status_int) + + def test_limited_request_json(self): + """Test a rate-limited (403) GET request through middleware.""" + request = webob.Request.blank("/") + response = request.get_response(self.app) + self.assertEqual(200, response.status_int) + + request = webob.Request.blank("/") + response = request.get_response(self.app) + self.assertEqual(response.status_int, 403) + + body = json.loads(response.body) + expected = "Only 1 GET request(s) can be made to * every minute." + value = body["overLimitFault"]["details"].strip() + self.assertEqual(value, expected) + + def test_limited_request_xml(self): + """Test a rate-limited (403) response as XML""" + request = webob.Request.blank("/") + response = request.get_response(self.app) + self.assertEqual(200, response.status_int) + + request = webob.Request.blank("/") + request.accept = "application/xml" + response = request.get_response(self.app) + self.assertEqual(response.status_int, 403) + + root = parseString(response.body).childNodes[0] + expected = "Only 1 GET request(s) can be made to * every minute." + + details = root.getElementsByTagName("details") + self.assertEqual(details.length, 1) + + value = details.item(0).firstChild.data.strip() + self.assertEqual(value, expected) + + +class LimitTest(BaseLimitTestSuite): + """ + Tests for the `limits.Limit` class. + """ + + def test_GET_no_delay(self): + """Test a limit handles 1 GET per second.""" + limit = Limit("GET", "*", ".*", 1, 1) + delay = limit("GET", "/anything") + self.assertEqual(None, delay) + self.assertEqual(0, limit.next_request) + self.assertEqual(0, limit.last_request) + + def test_GET_delay(self): + """Test two calls to 1 GET per second limit.""" + limit = Limit("GET", "*", ".*", 1, 1) + delay = limit("GET", "/anything") + self.assertEqual(None, delay) + + delay = limit("GET", "/anything") + self.assertEqual(1, delay) + self.assertEqual(1, limit.next_request) + self.assertEqual(0, limit.last_request) + + self.time += 4 + + delay = limit("GET", "/anything") + self.assertEqual(None, delay) + self.assertEqual(4, limit.next_request) + self.assertEqual(4, limit.last_request) + + +class LimiterTest(BaseLimitTestSuite): + """ + Tests for the in-memory `limits.Limiter` class. + """ + + def setUp(self): + """Run before each test.""" + BaseLimitTestSuite.setUp(self) + self.limiter = limits.Limiter(TEST_LIMITS) + + def _check(self, num, verb, url, username=None): + """Check and yield results from checks.""" + for x in xrange(num): + yield self.limiter.check_for_delay(verb, url, username)[0] + + def _check_sum(self, num, verb, url, username=None): + """Check and sum results from checks.""" + results = self._check(num, verb, url, username) + return sum(item for item in results if item) + + def test_no_delay_GET(self): + """ + Simple test to ensure no delay on a single call for a limit verb we + didn"t set. + """ + delay = self.limiter.check_for_delay("GET", "/anything") + self.assertEqual(delay, (None, None)) + + def test_no_delay_PUT(self): + """ + Simple test to ensure no delay on a single call for a known limit. + """ + delay = self.limiter.check_for_delay("PUT", "/anything") + self.assertEqual(delay, (None, None)) + + def test_delay_PUT(self): + """ + Ensure the 11th PUT will result in a delay of 6.0 seconds until + the next request will be granced. + """ + expected = [None] * 10 + [6.0] + results = list(self._check(11, "PUT", "/anything")) + + self.assertEqual(expected, results) + + def test_delay_POST(self): + """ + Ensure the 8th POST will result in a delay of 6.0 seconds until + the next request will be granced. + """ + expected = [None] * 7 + results = list(self._check(7, "POST", "/anything")) + self.assertEqual(expected, results) + + expected = 60.0 / 7.0 + results = self._check_sum(1, "POST", "/anything") + self.failUnlessAlmostEqual(expected, results, 8) + + def test_delay_GET(self): + """ + Ensure the 11th GET will result in NO delay. + """ + expected = [None] * 11 + results = list(self._check(11, "GET", "/anything")) + + self.assertEqual(expected, results) + + def test_delay_PUT_servers(self): + """ + Ensure PUT on /servers limits at 5 requests, and PUT elsewhere is still + OK after 5 requests...but then after 11 total requests, PUT limiting + kicks in. + """ + # First 6 requests on PUT /servers + expected = [None] * 5 + [12.0] + results = list(self._check(6, "PUT", "/servers")) + self.assertEqual(expected, results) + + # Next 5 request on PUT /anything + expected = [None] * 4 + [6.0] + results = list(self._check(5, "PUT", "/anything")) + self.assertEqual(expected, results) + + def test_delay_PUT_wait(self): + """ + Ensure after hitting the limit and then waiting for the correct + amount of time, the limit will be lifted. + """ + expected = [None] * 10 + [6.0] + results = list(self._check(11, "PUT", "/anything")) + self.assertEqual(expected, results) + + # Advance time + self.time += 6.0 + + expected = [None, 6.0] + results = list(self._check(2, "PUT", "/anything")) + self.assertEqual(expected, results) + + def test_multiple_delays(self): + """ + Ensure multiple requests still get a delay. + """ + expected = [None] * 10 + [6.0] * 10 + results = list(self._check(20, "PUT", "/anything")) + self.assertEqual(expected, results) + + self.time += 1.0 + + expected = [5.0] * 10 + results = list(self._check(10, "PUT", "/anything")) + self.assertEqual(expected, results) + + def test_multiple_users(self): + """ + Tests involving multiple users. + """ + # User1 + expected = [None] * 10 + [6.0] * 10 + results = list(self._check(20, "PUT", "/anything", "user1")) + self.assertEqual(expected, results) + + # User2 + expected = [None] * 10 + [6.0] * 5 + results = list(self._check(15, "PUT", "/anything", "user2")) + self.assertEqual(expected, results) + + self.time += 1.0 + + # User1 again + expected = [5.0] * 10 + results = list(self._check(10, "PUT", "/anything", "user1")) + self.assertEqual(expected, results) + + self.time += 1.0 + + # User1 again + expected = [4.0] * 5 + results = list(self._check(5, "PUT", "/anything", "user2")) + self.assertEqual(expected, results) + + +class WsgiLimiterTest(BaseLimitTestSuite): + """ + Tests for `limits.WsgiLimiter` class. + """ + + def setUp(self): + """Run before each test.""" + BaseLimitTestSuite.setUp(self) + self.app = limits.WsgiLimiter(TEST_LIMITS) + + def _request_data(self, verb, path): + """Get data decribing a limit request verb/path.""" + return json.dumps({"verb": verb, "path": path}) + + def _request(self, verb, url, username=None): + """Make sure that POSTing to the given url causes the given username + to perform the given action. Make the internal rate limiter return + delay and make sure that the WSGI app returns the correct response. + """ + if username: + request = webob.Request.blank("/%s" % username) + else: + request = webob.Request.blank("/") + + request.method = "POST" + request.body = self._request_data(verb, url) + response = request.get_response(self.app) + + if "X-Wait-Seconds" in response.headers: + self.assertEqual(response.status_int, 403) + return response.headers["X-Wait-Seconds"] + + self.assertEqual(response.status_int, 204) + + def test_invalid_methods(self): + """Only POSTs should work.""" + requests = [] + for method in ["GET", "PUT", "DELETE", "HEAD", "OPTIONS"]: + request = webob.Request.blank("/") + request.body = self._request_data("GET", "/something") + response = request.get_response(self.app) + self.assertEqual(response.status_int, 405) + + def test_good_url(self): + delay = self._request("GET", "/something") + self.assertEqual(delay, None) + + def test_escaping(self): + delay = self._request("GET", "/something/jump%20up") + self.assertEqual(delay, None) + + def test_response_to_delays(self): + delay = self._request("GET", "/delayed") + self.assertEqual(delay, None) + + delay = self._request("GET", "/delayed") + self.assertEqual(delay, '60.00') + + def test_response_to_delays_usernames(self): + delay = self._request("GET", "/delayed", "user1") + self.assertEqual(delay, None) + + delay = self._request("GET", "/delayed", "user2") + self.assertEqual(delay, None) + + delay = self._request("GET", "/delayed", "user1") + self.assertEqual(delay, '60.00') + + delay = self._request("GET", "/delayed", "user2") + self.assertEqual(delay, '60.00') + + +class FakeHttplibSocket(object): + """ + Fake `httplib.HTTPResponse` replacement. + """ + + def __init__(self, response_string): + """Initialize new `FakeHttplibSocket`.""" + self._buffer = StringIO.StringIO(response_string) + + def makefile(self, _mode, _other): + """Returns the socket's internal buffer.""" + return self._buffer + + +class FakeHttplibConnection(object): + """ + Fake `httplib.HTTPConnection`. + """ + + def __init__(self, app, host): + """ + Initialize `FakeHttplibConnection`. + """ + self.app = app + self.host = host + + def request(self, method, path, body="", headers={}): + """ + Requests made via this connection actually get translated and routed + into our WSGI app, we then wait for the response and turn it back into + an `httplib.HTTPResponse`. + """ + req = webob.Request.blank(path) + req.method = method + req.headers = headers + req.host = self.host + req.body = body + + resp = str(req.get_response(self.app)) + resp = "HTTP/1.0 %s" % resp + sock = FakeHttplibSocket(resp) + self.http_response = httplib.HTTPResponse(sock) + self.http_response.begin() + + def getresponse(self): + """Return our generated response from the request.""" + return self.http_response + + +def wire_HTTPConnection_to_WSGI(host, app): + """Monkeypatches HTTPConnection so that if you try to connect to host, you + are instead routed straight to the given WSGI app. + + After calling this method, when any code calls + + httplib.HTTPConnection(host) + + the connection object will be a fake. Its requests will be sent directly + to the given WSGI app rather than through a socket. + + Code connecting to hosts other than host will not be affected. + + This method may be called multiple times to map different hosts to + different apps. + """ + class HTTPConnectionDecorator(object): + """Wraps the real HTTPConnection class so that when you instantiate + the class you might instead get a fake instance.""" + + def __init__(self, wrapped): + self.wrapped = wrapped + + def __call__(self, connection_host, *args, **kwargs): + if connection_host == host: + return FakeHttplibConnection(app, host) + else: + return self.wrapped(connection_host, *args, **kwargs) + + httplib.HTTPConnection = HTTPConnectionDecorator(httplib.HTTPConnection) + + +class WsgiLimiterProxyTest(BaseLimitTestSuite): + """ + Tests for the `limits.WsgiLimiterProxy` class. + """ + + def setUp(self): + """ + Do some nifty HTTP/WSGI magic which allows for WSGI to be called + directly by something like the `httplib` library. + """ + BaseLimitTestSuite.setUp(self) + self.app = limits.WsgiLimiter(TEST_LIMITS) + wire_HTTPConnection_to_WSGI("169.254.0.1:80", self.app) + self.proxy = limits.WsgiLimiterProxy("169.254.0.1:80") + + def test_200(self): + """Successful request test.""" + delay = self.proxy.check_for_delay("GET", "/anything") + self.assertEqual(delay, (None, None)) + + def test_403(self): + """Forbidden request test.""" + delay = self.proxy.check_for_delay("GET", "/delayed") + self.assertEqual(delay, (None, None)) + + delay, error = self.proxy.check_for_delay("GET", "/delayed") + error = error.strip() + + expected = ("60.00", "403 Forbidden\n\nOnly 1 GET request(s) can be "\ + "made to /delayed every minute.") + + self.assertEqual((delay, error), expected) diff --git a/nova/tests/api/openstack/test_ratelimiting.py b/nova/tests/api/openstack/test_ratelimiting.py deleted file mode 100644 index 9ae90ee205e5..000000000000 --- a/nova/tests/api/openstack/test_ratelimiting.py +++ /dev/null @@ -1,243 +0,0 @@ -import httplib -import StringIO -import time -import webob - -from nova import test -import nova.api.openstack.ratelimiting as ratelimiting - - -class LimiterTest(test.TestCase): - - def setUp(self): - super(LimiterTest, self).setUp() - self.limits = { - 'a': (5, ratelimiting.PER_SECOND), - 'b': (5, ratelimiting.PER_MINUTE), - 'c': (5, ratelimiting.PER_HOUR), - 'd': (1, ratelimiting.PER_SECOND), - 'e': (100, ratelimiting.PER_SECOND)} - self.rl = ratelimiting.Limiter(self.limits) - - def exhaust(self, action, times_until_exhausted, **kwargs): - for i in range(times_until_exhausted): - when = self.rl.perform(action, **kwargs) - self.assertEqual(when, None) - num, period = self.limits[action] - delay = period * 1.0 / num - # Verify that we are now thoroughly delayed - for i in range(10): - when = self.rl.perform(action, **kwargs) - self.assertAlmostEqual(when, delay, 2) - - def test_second(self): - self.exhaust('a', 5) - time.sleep(0.2) - self.exhaust('a', 1) - time.sleep(1) - self.exhaust('a', 5) - - def test_minute(self): - self.exhaust('b', 5) - - def test_one_per_period(self): - def allow_once_and_deny_once(): - when = self.rl.perform('d') - self.assertEqual(when, None) - when = self.rl.perform('d') - self.assertAlmostEqual(when, 1, 2) - return when - time.sleep(allow_once_and_deny_once()) - time.sleep(allow_once_and_deny_once()) - allow_once_and_deny_once() - - def test_we_can_go_indefinitely_if_we_spread_out_requests(self): - for i in range(200): - when = self.rl.perform('e') - self.assertEqual(when, None) - time.sleep(0.01) - - def test_users_get_separate_buckets(self): - self.exhaust('c', 5, username='alice') - self.exhaust('c', 5, username='bob') - self.exhaust('c', 5, username='chuck') - self.exhaust('c', 0, username='chuck') - self.exhaust('c', 0, username='bob') - self.exhaust('c', 0, username='alice') - - -class FakeLimiter(object): - """Fake Limiter class that you can tell how to behave.""" - - def __init__(self, test): - self._action = self._username = self._delay = None - self.test = test - - def mock(self, action, username, delay): - self._action = action - self._username = username - self._delay = delay - - def perform(self, action, username): - self.test.assertEqual(action, self._action) - self.test.assertEqual(username, self._username) - return self._delay - - -class WSGIAppTest(test.TestCase): - - def setUp(self): - super(WSGIAppTest, self).setUp() - self.limiter = FakeLimiter(self) - self.app = ratelimiting.WSGIApp(self.limiter) - - def test_invalid_methods(self): - requests = [] - for method in ['GET', 'PUT', 'DELETE']: - req = webob.Request.blank('/limits/michael/breakdance', - dict(REQUEST_METHOD=method)) - requests.append(req) - for req in requests: - self.assertEqual(req.get_response(self.app).status_int, 405) - - def test_invalid_urls(self): - requests = [] - for prefix in ['limit', '', 'limiter2', 'limiter/limits', 'limiter/1']: - req = webob.Request.blank('/%s/michael/breakdance' % prefix, - dict(REQUEST_METHOD='POST')) - requests.append(req) - for req in requests: - self.assertEqual(req.get_response(self.app).status_int, 404) - - def verify(self, url, username, action, delay=None): - """Make sure that POSTing to the given url causes the given username - to perform the given action. Make the internal rate limiter return - delay and make sure that the WSGI app returns the correct response. - """ - req = webob.Request.blank(url, dict(REQUEST_METHOD='POST')) - self.limiter.mock(action, username, delay) - resp = req.get_response(self.app) - if not delay: - self.assertEqual(resp.status_int, 200) - else: - self.assertEqual(resp.status_int, 403) - self.assertEqual(resp.headers['X-Wait-Seconds'], "%.2f" % delay) - - def test_good_urls(self): - self.verify('/limiter/michael/hoot', 'michael', 'hoot') - - def test_escaping(self): - self.verify('/limiter/michael/jump%20up', 'michael', 'jump up') - - def test_response_to_delays(self): - self.verify('/limiter/michael/hoot', 'michael', 'hoot', 1) - self.verify('/limiter/michael/hoot', 'michael', 'hoot', 1.56) - self.verify('/limiter/michael/hoot', 'michael', 'hoot', 1000) - - -class FakeHttplibSocket(object): - """a fake socket implementation for httplib.HTTPResponse, trivial""" - - def __init__(self, response_string): - self._buffer = StringIO.StringIO(response_string) - - def makefile(self, _mode, _other): - """Returns the socket's internal buffer""" - return self._buffer - - -class FakeHttplibConnection(object): - """A fake httplib.HTTPConnection - - Requests made via this connection actually get translated and routed into - our WSGI app, we then wait for the response and turn it back into - an httplib.HTTPResponse. - """ - def __init__(self, app, host, is_secure=False): - self.app = app - self.host = host - - def request(self, method, path, data='', headers={}): - req = webob.Request.blank(path) - req.method = method - req.body = data - req.headers = headers - req.host = self.host - # Call the WSGI app, get the HTTP response - resp = str(req.get_response(self.app)) - # For some reason, the response doesn't have "HTTP/1.0 " prepended; I - # guess that's a function the web server usually provides. - resp = "HTTP/1.0 %s" % resp - sock = FakeHttplibSocket(resp) - self.http_response = httplib.HTTPResponse(sock) - self.http_response.begin() - - def getresponse(self): - return self.http_response - - -def wire_HTTPConnection_to_WSGI(host, app): - """Monkeypatches HTTPConnection so that if you try to connect to host, you - are instead routed straight to the given WSGI app. - - After calling this method, when any code calls - - httplib.HTTPConnection(host) - - the connection object will be a fake. Its requests will be sent directly - to the given WSGI app rather than through a socket. - - Code connecting to hosts other than host will not be affected. - - This method may be called multiple times to map different hosts to - different apps. - """ - class HTTPConnectionDecorator(object): - """Wraps the real HTTPConnection class so that when you instantiate - the class you might instead get a fake instance.""" - - def __init__(self, wrapped): - self.wrapped = wrapped - - def __call__(self, connection_host, *args, **kwargs): - if connection_host == host: - return FakeHttplibConnection(app, host) - else: - return self.wrapped(connection_host, *args, **kwargs) - - httplib.HTTPConnection = HTTPConnectionDecorator(httplib.HTTPConnection) - - -class WSGIAppProxyTest(test.TestCase): - - def setUp(self): - """Our WSGIAppProxy is going to call across an HTTPConnection to a - WSGIApp running a limiter. The proxy will send input, and the proxy - should receive that same input, pass it to the limiter who gives a - result, and send the expected result back. - - The HTTPConnection isn't real -- it's monkeypatched to point straight - at the WSGIApp. And the limiter isn't real -- it's a fake that - behaves the way we tell it to. - """ - super(WSGIAppProxyTest, self).setUp() - self.limiter = FakeLimiter(self) - app = ratelimiting.WSGIApp(self.limiter) - wire_HTTPConnection_to_WSGI('100.100.100.100:80', app) - self.proxy = ratelimiting.WSGIAppProxy('100.100.100.100:80') - - def test_200(self): - self.limiter.mock('conquer', 'caesar', None) - when = self.proxy.perform('conquer', 'caesar') - self.assertEqual(when, None) - - def test_403(self): - self.limiter.mock('grumble', 'proletariat', 1.5) - when = self.proxy.perform('grumble', 'proletariat') - self.assertEqual(when, 1.5) - - def test_failure(self): - def shouldRaise(): - self.limiter.mock('murder', 'brutus', None) - self.proxy.perform('stab', 'brutus') - self.assertRaises(AssertionError, shouldRaise) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 07ebfdd882e1..3a804c649c9e 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -24,6 +24,7 @@ from xml.dom import minidom import stubout import webob +from nova import context from nova import db from nova import flags from nova import test @@ -81,7 +82,7 @@ def stub_instance(id, user_id=1, private_address=None, public_addresses=None): "admin_pass": "", "user_id": user_id, "project_id": "", - "image_id": 10, + "image_id": "10", "kernel_id": "", "ramdisk_id": "", "launch_index": 0, @@ -94,7 +95,7 @@ def stub_instance(id, user_id=1, private_address=None, public_addresses=None): "local_gb": 0, "hostname": "", "host": None, - "instance_type": "", + "instance_type": "1", "user_data": "", "reservation_id": "", "mac_address": "", @@ -179,6 +180,25 @@ class ServersTest(test.TestCase): self.assertEqual(len(addresses["private"]), 1) self.assertEqual(addresses["private"][0], private) + def test_get_server_by_id_with_addresses_v1_1(self): + private = "192.168.0.3" + public = ["1.2.3.4"] + new_return_server = return_server_with_addresses(private, public) + self.stubs.Set(nova.db.api, 'instance_get', new_return_server) + req = webob.Request.blank('/v1.1/servers/1') + req.environ['api.version'] = '1.1' + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + self.assertEqual(res_dict['server']['id'], '1') + self.assertEqual(res_dict['server']['name'], 'server1') + addresses = res_dict['server']['addresses'] + self.assertEqual(len(addresses["public"]), len(public)) + self.assertEqual(addresses["public"][0], + {"version": 4, "addr": public[0]}) + self.assertEqual(len(addresses["private"]), 1) + self.assertEqual(addresses["private"][0], + {"version": 4, "addr": private}) + def test_get_server_list(self): req = webob.Request.blank('/v1.0/servers') res = req.get_response(fakes.wsgi_app()) @@ -339,19 +359,32 @@ class ServersTest(test.TestCase): res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '404 Not Found') - def test_get_all_server_details(self): + def test_get_all_server_details_v1_0(self): req = webob.Request.blank('/v1.0/servers/detail') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) - i = 0 - for s in res_dict['servers']: + for i, s in enumerate(res_dict['servers']): self.assertEqual(s['id'], i) self.assertEqual(s['hostId'], '') self.assertEqual(s['name'], 'server%d' % i) - self.assertEqual(s['imageId'], 10) + self.assertEqual(s['imageId'], '10') + self.assertEqual(s['flavorId'], '1') + self.assertEqual(s['metadata']['seq'], i) + + def test_get_all_server_details_v1_1(self): + req = webob.Request.blank('/v1.1/servers/detail') + req.environ['api.version'] = '1.1' + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + for i, s in enumerate(res_dict['servers']): + self.assertEqual(s['id'], i) + self.assertEqual(s['hostId'], '') + self.assertEqual(s['name'], 'server%d' % i) + self.assertEqual(s['imageRef'], 'http://localhost/v1.1/images/10') + self.assertEqual(s['flavorRef'], 'http://localhost/v1.1/flavors/1') self.assertEqual(s['metadata']['seq'], i) - i += 1 def test_get_all_server_details_with_host(self): ''' @@ -1095,6 +1128,15 @@ class TestServerInstanceCreation(test.TestCase): self.assertEquals(response.status_int, 400) self.assertEquals(injected_files, None) + def test_create_instance_with_null_personality(self): + personality = None + body_dict = self._create_personality_request_dict(personality) + body_dict['server']['personality'] = None + request = self._get_create_request_json(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + self.assertEquals(response.status_int, 200) + def test_create_instance_with_three_personalities(self): files = [ ('/etc/sudoers', 'ALL ALL=NOPASSWD: ALL\n'), @@ -1134,7 +1176,3 @@ class TestServerInstanceCreation(test.TestCase): server = dom.childNodes[0] self.assertEquals(server.nodeName, 'server') self.assertTrue(server.getAttribute('adminPass').startswith('fake')) - - -if __name__ == "__main__": - unittest.main() diff --git a/nova/tests/api/openstack/test_users.py b/nova/tests/api/openstack/test_users.py index 2dda4319bc59..effb2f5926dc 100644 --- a/nova/tests/api/openstack/test_users.py +++ b/nova/tests/api/openstack/test_users.py @@ -18,11 +18,10 @@ import json import stubout import webob -import nova.api -import nova.api.openstack.auth -from nova import context from nova import flags from nova import test +from nova import utils +from nova.api.openstack import users from nova.auth.manager import User, Project from nova.tests.api.openstack import fakes @@ -43,14 +42,14 @@ class UsersTest(test.TestCase): def setUp(self): super(UsersTest, self).setUp() self.stubs = stubout.StubOutForTesting() - self.stubs.Set(nova.api.openstack.users.Controller, '__init__', + self.stubs.Set(users.Controller, '__init__', fake_init) - self.stubs.Set(nova.api.openstack.users.Controller, '_check_admin', + self.stubs.Set(users.Controller, '_check_admin', fake_admin_check) - fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.clear_fakes() fakes.FakeAuthManager.projects = dict(testacct=Project('testacct', 'testacct', - 'guy1', + 'id1', 'test', [])) fakes.FakeAuthDatabase.data = {} @@ -61,10 +60,8 @@ class UsersTest(test.TestCase): self.allow_admin = FLAGS.allow_admin_api FLAGS.allow_admin_api = True fakemgr = fakes.FakeAuthManager() - fakemgr.add_user('acc1', User('guy1', 'guy1', 'acc1', - 'fortytwo!', False)) - fakemgr.add_user('acc2', User('guy2', 'guy2', 'acc2', - 'swordfish', True)) + fakemgr.add_user(User('id1', 'guy1', 'acc1', 'secret1', False)) + fakemgr.add_user(User('id2', 'guy2', 'acc2', 'secret2', True)) def tearDown(self): self.stubs.UnsetAll() @@ -80,28 +77,44 @@ class UsersTest(test.TestCase): self.assertEqual(len(res_dict['users']), 2) def test_get_user_by_id(self): - req = webob.Request.blank('/v1.0/users/guy2') + req = webob.Request.blank('/v1.0/users/id2') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) - self.assertEqual(res_dict['user']['id'], 'guy2') + self.assertEqual(res_dict['user']['id'], 'id2') self.assertEqual(res_dict['user']['name'], 'guy2') - self.assertEqual(res_dict['user']['secret'], 'swordfish') + self.assertEqual(res_dict['user']['secret'], 'secret2') self.assertEqual(res_dict['user']['admin'], True) self.assertEqual(res.status_int, 200) def test_user_delete(self): - req = webob.Request.blank('/v1.0/users/guy1') - req.method = 'DELETE' + # Check the user exists + req = webob.Request.blank('/v1.0/users/id1') res = req.get_response(fakes.wsgi_app()) - self.assertTrue('guy1' not in [u.id for u in - fakes.FakeAuthManager.auth_data.values()]) + res_dict = json.loads(res.body) + + self.assertEqual(res_dict['user']['id'], 'id1') self.assertEqual(res.status_int, 200) + # Delete the user + req = webob.Request.blank('/v1.0/users/id1') + req.method = 'DELETE' + res = req.get_response(fakes.wsgi_app()) + self.assertTrue('id1' not in [u.id for u in + fakes.FakeAuthManager.auth_data]) + self.assertEqual(res.status_int, 200) + + # Check the user is not returned (and returns 404) + req = webob.Request.blank('/v1.0/users/id1') + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + self.assertEqual(res.status_int, 404) + def test_user_create(self): + secret = utils.generate_password() body = dict(user=dict(name='test_guy', access='acc3', - secret='invasionIsInNormandy', + secret=secret, admin=True)) req = webob.Request.blank('/v1.0/users') req.headers["Content-Type"] = "application/json" @@ -112,20 +125,25 @@ class UsersTest(test.TestCase): res_dict = json.loads(res.body) self.assertEqual(res.status_int, 200) + + # NOTE(justinsb): This is a questionable assertion in general + # fake sets id=name, but others might not... self.assertEqual(res_dict['user']['id'], 'test_guy') + self.assertEqual(res_dict['user']['name'], 'test_guy') self.assertEqual(res_dict['user']['access'], 'acc3') - self.assertEqual(res_dict['user']['secret'], 'invasionIsInNormandy') + self.assertEqual(res_dict['user']['secret'], secret) self.assertEqual(res_dict['user']['admin'], True) self.assertTrue('test_guy' in [u.id for u in - fakes.FakeAuthManager.auth_data.values()]) - self.assertEqual(len(fakes.FakeAuthManager.auth_data.values()), 3) + fakes.FakeAuthManager.auth_data]) + self.assertEqual(len(fakes.FakeAuthManager.auth_data), 3) def test_user_update(self): + new_secret = utils.generate_password() body = dict(user=dict(name='guy2', access='acc2', - secret='invasionIsInNormandy')) - req = webob.Request.blank('/v1.0/users/guy2') + secret=new_secret)) + req = webob.Request.blank('/v1.0/users/id2') req.headers["Content-Type"] = "application/json" req.method = 'PUT' req.body = json.dumps(body) @@ -134,8 +152,8 @@ class UsersTest(test.TestCase): res_dict = json.loads(res.body) self.assertEqual(res.status_int, 200) - self.assertEqual(res_dict['user']['id'], 'guy2') + self.assertEqual(res_dict['user']['id'], 'id2') self.assertEqual(res_dict['user']['name'], 'guy2') self.assertEqual(res_dict['user']['access'], 'acc2') - self.assertEqual(res_dict['user']['secret'], 'invasionIsInNormandy') + self.assertEqual(res_dict['user']['secret'], new_secret) self.assertEqual(res_dict['user']['admin'], True) diff --git a/nova/tests/api/test_wsgi.py b/nova/tests/api/test_wsgi.py index b1a849cf912e..1ecdd1cfb27f 100644 --- a/nova/tests/api/test_wsgi.py +++ b/nova/tests/api/test_wsgi.py @@ -80,7 +80,7 @@ class ControllerTest(test.TestCase): "attributes": { "test": ["id"]}}} - def show(self, req, id): # pylint: disable-msg=W0622,C0103 + def show(self, req, id): # pylint: disable=W0622,C0103 return {"test": {"id": id}} def __init__(self): diff --git a/nova/tests/hyperv_unittest.py b/nova/tests/hyperv_unittest.py index 3980ae3cb2f9..042819b9c3a8 100644 --- a/nova/tests/hyperv_unittest.py +++ b/nova/tests/hyperv_unittest.py @@ -51,7 +51,7 @@ class HyperVTestCase(test.TestCase): instance_ref = db.instance_create(self.context, instance) conn = hyperv.get_connection(False) - conn._create_vm(instance_ref) # pylint: disable-msg=W0212 + conn._create_vm(instance_ref) # pylint: disable=W0212 found = [n for n in conn.list_instances() if n == instance_ref['name']] self.assertTrue(len(found) == 1) diff --git a/nova/tests/objectstore_unittest.py b/nova/tests/objectstore_unittest.py index 5a1be08ebafd..4e2ac205e4eb 100644 --- a/nova/tests/objectstore_unittest.py +++ b/nova/tests/objectstore_unittest.py @@ -179,7 +179,7 @@ class ObjectStoreTestCase(test.TestCase): class TestHTTPChannel(http.HTTPChannel): """Dummy site required for twisted.web""" - def checkPersistence(self, _, __): # pylint: disable-msg=C0103 + def checkPersistence(self, _, __): # pylint: disable=C0103 """Otherwise we end up with an unclean reactor.""" return False @@ -209,10 +209,10 @@ class S3APITestCase(test.TestCase): root = S3() self.site = TestSite(root) - # pylint: disable-msg=E1101 + # pylint: disable=E1101 self.listening_port = reactor.listenTCP(0, self.site, interface='127.0.0.1') - # pylint: enable-msg=E1101 + # pylint: enable=E1101 self.tcp_port = self.listening_port.getHost().port if not boto.config.has_section('Boto'): @@ -231,11 +231,11 @@ class S3APITestCase(test.TestCase): self.conn.get_http_connection = get_http_connection - def _ensure_no_buckets(self, buckets): # pylint: disable-msg=C0111 + def _ensure_no_buckets(self, buckets): # pylint: disable=C0111 self.assertEquals(len(buckets), 0, "Bucket list was not empty") return True - def _ensure_one_bucket(self, buckets, name): # pylint: disable-msg=C0111 + def _ensure_one_bucket(self, buckets, name): # pylint: disable=C0111 self.assertEquals(len(buckets), 1, "Bucket list didn't have exactly one element in it") self.assertEquals(buckets[0].name, name, "Wrong name") diff --git a/nova/tests/test_api.py b/nova/tests/test_api.py index d5c54a1c3c60..fa0e56597ddb 100644 --- a/nova/tests/test_api.py +++ b/nova/tests/test_api.py @@ -20,6 +20,7 @@ import boto from boto.ec2 import regioninfo +from boto.exception import EC2ResponseError import datetime import httplib import random @@ -124,7 +125,7 @@ class ApiEc2TestCase(test.TestCase): self.mox.StubOutWithMock(self.ec2, 'new_http_connection') self.http = FakeHttplibConnection( self.app, '%s:8773' % (self.host), False) - # pylint: disable-msg=E1103 + # pylint: disable=E1103 self.ec2.new_http_connection(host, is_secure).AndReturn(self.http) return self.http @@ -177,6 +178,17 @@ class ApiEc2TestCase(test.TestCase): self.manager.delete_project(project) self.manager.delete_user(user) + def test_terminate_invalid_instance(self): + """Attempt to terminate an invalid instance""" + self.expect_http() + self.mox.ReplayAll() + user = self.manager.create_user('fake', 'fake', 'fake') + project = self.manager.create_project('fake', 'fake', 'fake') + self.assertRaises(EC2ResponseError, self.ec2.terminate_instances, + "i-00000005") + self.manager.delete_project(project) + self.manager.delete_user(user) + def test_get_all_key_pairs(self): """Test that, after creating a user and project and generating a key pair, that the API call to list key pairs works properly""" diff --git a/nova/tests/test_auth.py b/nova/tests/test_auth.py index 2a7817032a67..885596f563d8 100644 --- a/nova/tests/test_auth.py +++ b/nova/tests/test_auth.py @@ -299,6 +299,13 @@ class AuthManagerTestCase(object): self.assertEqual('test2', project.project_manager_id) self.assertEqual('new desc', project.description) + def test_modify_project_adds_new_manager(self): + with user_and_project_generator(self.manager): + with user_generator(self.manager, name='test2'): + self.manager.modify_project('testproj', 'test2', 'new desc') + project = self.manager.get_project('testproj') + self.assertTrue('test2' in project.member_ids) + def test_can_delete_project(self): with user_generator(self.manager): self.manager.create_project('testproj', 'test1') diff --git a/nova/tests/test_middleware.py b/nova/tests/test_middleware.py index 9d49167baf8b..6564a6955950 100644 --- a/nova/tests/test_middleware.py +++ b/nova/tests/test_middleware.py @@ -40,12 +40,12 @@ def conditional_forbid(req): class LockoutTestCase(test.TestCase): """Test case for the Lockout middleware.""" - def setUp(self): # pylint: disable-msg=C0103 + def setUp(self): # pylint: disable=C0103 super(LockoutTestCase, self).setUp() utils.set_time_override() self.lockout = ec2.Lockout(conditional_forbid) - def tearDown(self): # pylint: disable-msg=C0103 + def tearDown(self): # pylint: disable=C0103 utils.clear_time_override() super(LockoutTestCase, self).tearDown() diff --git a/nova/tests/test_utils.py b/nova/tests/test_utils.py index 34a407f1aabd..e08d229b0de5 100644 --- a/nova/tests/test_utils.py +++ b/nova/tests/test_utils.py @@ -14,11 +14,89 @@ # License for the specific language governing permissions and limitations # under the License. +import os +import tempfile + from nova import test from nova import utils from nova import exception +class ExecuteTestCase(test.TestCase): + def test_retry_on_failure(self): + fd, tmpfilename = tempfile.mkstemp() + _, tmpfilename2 = tempfile.mkstemp() + try: + fp = os.fdopen(fd, 'w+') + fp.write('''#!/bin/sh +# If stdin fails to get passed during one of the runs, make a note. +if ! grep -q foo +then + echo 'failure' > "$1" +fi +# If stdin has failed to get passed during this or a previous run, exit early. +if grep failure "$1" +then + exit 1 +fi +runs="$(cat $1)" +if [ -z "$runs" ] +then + runs=0 +fi +runs=$(($runs + 1)) +echo $runs > "$1" +exit 1 +''') + fp.close() + os.chmod(tmpfilename, 0755) + self.assertRaises(exception.ProcessExecutionError, + utils.execute, + tmpfilename, tmpfilename2, attempts=10, + process_input='foo', + delay_on_retry=False) + fp = open(tmpfilename2, 'r+') + runs = fp.read() + fp.close() + self.assertNotEquals(runs.strip(), 'failure', 'stdin did not ' + 'always get passed ' + 'correctly') + runs = int(runs.strip()) + self.assertEquals(runs, 10, + 'Ran %d times instead of 10.' % (runs,)) + finally: + os.unlink(tmpfilename) + os.unlink(tmpfilename2) + + def test_unknown_kwargs_raises_error(self): + self.assertRaises(exception.Error, + utils.execute, + '/bin/true', this_is_not_a_valid_kwarg=True) + + def test_no_retry_on_success(self): + fd, tmpfilename = tempfile.mkstemp() + _, tmpfilename2 = tempfile.mkstemp() + try: + fp = os.fdopen(fd, 'w+') + fp.write('''#!/bin/sh +# If we've already run, bail out. +grep -q foo "$1" && exit 1 +# Mark that we've run before. +echo foo > "$1" +# Check that stdin gets passed correctly. +grep foo +''') + fp.close() + os.chmod(tmpfilename, 0755) + utils.execute(tmpfilename, + tmpfilename2, + process_input='foo', + attempts=2) + finally: + os.unlink(tmpfilename) + os.unlink(tmpfilename2) + + class GetFromPathTestCase(test.TestCase): def test_tolerates_nones(self): f = utils.get_from_path diff --git a/nova/tests/test_volume.py b/nova/tests/test_volume.py index 1b1d72092c90..5d68ca2ae1a8 100644 --- a/nova/tests/test_volume.py +++ b/nova/tests/test_volume.py @@ -336,8 +336,8 @@ class ISCSITestCase(DriverTestCase): self.mox.StubOutWithMock(self.volume.driver, '_execute') for i in volume_id_list: tid = db.volume_get_iscsi_target_num(self.context, i) - self.volume.driver._execute("sudo ietadm --op show --tid=%(tid)d" - % locals()) + self.volume.driver._execute("sudo", "ietadm", "--op", "show", + "--tid=%(tid)d" % locals()) self.stream.truncate(0) self.mox.ReplayAll() @@ -355,8 +355,9 @@ class ISCSITestCase(DriverTestCase): # the first vblade process isn't running tid = db.volume_get_iscsi_target_num(self.context, volume_id_list[0]) self.mox.StubOutWithMock(self.volume.driver, '_execute') - self.volume.driver._execute("sudo ietadm --op show --tid=%(tid)d" - % locals()).AndRaise(exception.ProcessExecutionError()) + self.volume.driver._execute("sudo", "ietadm", "--op", "show", + "--tid=%(tid)d" % locals() + ).AndRaise(exception.ProcessExecutionError()) self.mox.ReplayAll() self.assertRaises(exception.ProcessExecutionError, diff --git a/nova/utils.py b/nova/utils.py index 24b8da9eaae6..499af203918b 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -133,13 +133,14 @@ def fetchfile(url, target): def execute(*cmd, **kwargs): - process_input = kwargs.get('process_input', None) - addl_env = kwargs.get('addl_env', None) - check_exit_code = kwargs.get('check_exit_code', 0) - stdin = kwargs.get('stdin', subprocess.PIPE) - stdout = kwargs.get('stdout', subprocess.PIPE) - stderr = kwargs.get('stderr', subprocess.PIPE) - attempts = kwargs.get('attempts', 1) + process_input = kwargs.pop('process_input', None) + addl_env = kwargs.pop('addl_env', None) + check_exit_code = kwargs.pop('check_exit_code', 0) + delay_on_retry = kwargs.pop('delay_on_retry', True) + attempts = kwargs.pop('attempts', 1) + if len(kwargs): + raise exception.Error(_('Got unknown keyword args ' + 'to utils.execute: %r') % kwargs) cmd = map(str, cmd) while attempts > 0: @@ -149,8 +150,11 @@ def execute(*cmd, **kwargs): env = os.environ.copy() if addl_env: env.update(addl_env) - obj = subprocess.Popen(cmd, stdin=stdin, - stdout=stdout, stderr=stderr, env=env) + obj = subprocess.Popen(cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env) result = None if process_input != None: result = obj.communicate(process_input) @@ -176,7 +180,8 @@ def execute(*cmd, **kwargs): raise else: LOG.debug(_("%r failed. Retrying."), cmd) - greenthread.sleep(random.randint(20, 200) / 100.0) + if delay_on_retry: + greenthread.sleep(random.randint(20, 200) / 100.0) def ssh_execute(ssh, cmd, process_input=None, diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 0a85da541d0e..e80b9fbdfa4f 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -991,24 +991,35 @@ class LibvirtConnection(object): + xml.serialize()) cpu_info = dict() - cpu_info['arch'] = xml.xpathEval('//host/cpu/arch')[0].getContent() - cpu_info['model'] = xml.xpathEval('//host/cpu/model')[0].getContent() - cpu_info['vendor'] = xml.xpathEval('//host/cpu/vendor')[0].getContent() - topology_node = xml.xpathEval('//host/cpu/topology')[0]\ - .get_properties() + arch_nodes = xml.xpathEval('//host/cpu/arch') + if arch_nodes: + cpu_info['arch'] = arch_nodes[0].getContent() + + model_nodes = xml.xpathEval('//host/cpu/model') + if model_nodes: + cpu_info['model'] = model_nodes[0].getContent() + + vendor_nodes = xml.xpathEval('//host/cpu/vendor') + if vendor_nodes: + cpu_info['vendor'] = vendor_nodes[0].getContent() + + topology_nodes = xml.xpathEval('//host/cpu/topology') topology = dict() - while topology_node: - name = topology_node.get_name() - topology[name] = topology_node.getContent() - topology_node = topology_node.get_next() + if topology_nodes: + topology_node = topology_nodes[0].get_properties() + while topology_node: + name = topology_node.get_name() + topology[name] = topology_node.getContent() + topology_node = topology_node.get_next() - keys = ['cores', 'sockets', 'threads'] - tkeys = topology.keys() - if set(tkeys) != set(keys): - ks = ', '.join(keys) - raise exception.Invalid(_("Invalid xml: topology(%(topology)s) " - "must have %(ks)s") % locals()) + keys = ['cores', 'sockets', 'threads'] + tkeys = topology.keys() + if set(tkeys) != set(keys): + ks = ', '.join(keys) + raise exception.Invalid(_("Invalid xml: topology" + "(%(topology)s) must have " + "%(ks)s") % locals()) feature_nodes = xml.xpathEval('//host/cpu/feature') features = list() diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 7dbca321f66c..28ce215d829a 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -234,11 +234,11 @@ class VMHelper(HelperBase): @classmethod def create_vif(cls, session, vm_ref, network_ref, mac_address, - dev="0", rxtx_cap=0): + dev, rxtx_cap=0): """Create a VIF record. Returns a Deferred that gives the new VIF reference.""" vif_rec = {} - vif_rec['device'] = dev + vif_rec['device'] = str(dev) vif_rec['network'] = network_ref vif_rec['VM'] = vm_ref vif_rec['MAC'] = mac_address diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 4dca26f61cfb..1f5d2d155a2e 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -92,12 +92,13 @@ class VMOps(object): instance.image_id, user, project, disk_image_type) return vdi_uuid - def spawn(self, instance): + def spawn(self, instance, network_info=None): vdi_uuid = self._create_disk(instance) - vm_ref = self._create_vm(instance, vdi_uuid) + vm_ref = self._create_vm(instance, vdi_uuid, network_info) self._spawn(instance, vm_ref) - def _create_vm(self, instance, vdi_uuid): + def _create_vm(self, instance, vdi_uuid, network_info=None): + """Create VM instance""" instance_name = instance.name vm_ref = VMHelper.lookup(self._session, instance_name) if vm_ref is not None: @@ -139,9 +140,12 @@ class VMOps(object): VMHelper.create_vbd(session=self._session, vm_ref=vm_ref, vdi_ref=vdi_ref, userdevice=0, bootable=True) - # inject_network_info and create vifs - networks = self.inject_network_info(instance) - self.create_vifs(instance, networks) + # TODO(tr3buchet) - check to make sure we have network info, otherwise + # create it now. This goes away once nova-multi-nic hits. + if network_info is None: + network_info = self._get_network_info(instance) + self.create_vifs(vm_ref, network_info) + self.inject_network_info(instance, vm_ref, network_info) return vm_ref def _spawn(self, instance, vm_ref): @@ -196,7 +200,7 @@ class VMOps(object): timer.f = _wait_for_boot # call to reset network to configure network from xenstore - self.reset_network(instance) + self.reset_network(instance, vm_ref) return timer.start(interval=0.5, now=True) @@ -383,7 +387,7 @@ class VMOps(object): #The new disk size must be in bytes new_disk_size = str(instance.local_gb * 1024 * 1024 * 1024) - LOG.debug(_("Resizing VDI %s for instance %s. Expanding to %sGB") % + LOG.debug(_("Resizing VDI %s for instance %s. Expanding to %sGB"), (vdi_uuid, instance.name, instance.local_gb)) vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) self._session.call_xenapi('VDI.resize_online', vdi_ref, new_disk_size) @@ -709,24 +713,17 @@ class VMOps(object): # TODO: implement this! return 'http://fakeajaxconsole/fake_url' - def inject_network_info(self, instance): - """ - Generate the network info and make calls to place it into the - xenstore and the xenstore param list - - """ - # TODO(tr3buchet) - remove comment in multi-nic - # I've decided to go ahead and consider multiple IPs and networks - # at this stage even though they aren't implemented because these will - # be needed for multi-nic and there was no sense writing it for single - # network/single IP and then having to turn around and re-write it - vm_ref = self._get_vm_opaque_ref(instance.id) - logging.debug(_("injecting network info to xenstore for vm: |%s|"), - vm_ref) + # TODO(tr3buchet) - remove this function after nova multi-nic + def _get_network_info(self, instance): + """creates network info list for instance""" admin_context = context.get_admin_context() - IPs = db.fixed_ip_get_all_by_instance(admin_context, instance['id']) + IPs = db.fixed_ip_get_all_by_instance(admin_context, + instance['id']) networks = db.network_get_all_by_instance(admin_context, instance['id']) + flavor = db.instance_type_get_by_name(admin_context, + instance['instance_type']) + network_info = [] for network in networks: network_IPs = [ip for ip in IPs if ip.network_id == network.id] @@ -743,67 +740,64 @@ class VMOps(object): "gateway": ip6.gatewayV6, "enabled": "1"} - mac_id = instance.mac_address.replace(':', '') - location = 'vm-data/networking/%s' % mac_id - mapping = { + info = { 'label': network['label'], 'gateway': network['gateway'], 'mac': instance.mac_address, + 'rxtx_cap': flavor['rxtx_cap'], 'dns': [network['dns']], 'ips': [ip_dict(ip) for ip in network_IPs], 'ip6s': [ip6_dict(ip) for ip in network_IPs]} + network_info.append((network, info)) + return network_info - self.write_to_param_xenstore(vm_ref, {location: mapping}) + def inject_network_info(self, instance, vm_ref, network_info): + """ + Generate the network info and make calls to place it into the + xenstore and the xenstore param list + """ + logging.debug(_("injecting network info to xs for vm: |%s|"), vm_ref) + # this function raises if vm_ref is not a vm_opaque_ref + self._session.get_xenapi().VM.get_record(vm_ref) + + for (network, info) in network_info: + location = 'vm-data/networking/%s' % info['mac'].replace(':', '') + self.write_to_param_xenstore(vm_ref, {location: info}) try: - self.write_to_xenstore(vm_ref, location, mapping['location']) + # TODO(tr3buchet): fix function call after refactor + #self.write_to_xenstore(vm_ref, location, info) + self._make_plugin_call('xenstore.py', 'write_record', instance, + location, {'value': json.dumps(info)}, + vm_ref) except KeyError: # catch KeyError for domid if instance isn't running pass - return networks - - def create_vifs(self, instance, networks=None): - """ - Creates vifs for an instance - - """ - vm_ref = self._get_vm_opaque_ref(instance['id']) - admin_context = context.get_admin_context() - flavor = db.instance_type_get_by_name(admin_context, - instance.instance_type) + def create_vifs(self, vm_ref, network_info): + """Creates vifs for an instance""" logging.debug(_("creating vif(s) for vm: |%s|"), vm_ref) - rxtx_cap = flavor['rxtx_cap'] - if networks is None: - networks = db.network_get_all_by_instance(admin_context, - instance['id']) - # TODO(tr3buchet) - remove comment in multi-nic - # this bit here about creating the vifs will be updated - # in multi-nic to handle multiple IPs on the same network - # and multiple networks - # for now it works as there is only one of each - for network in networks: + + # this function raises if vm_ref is not a vm_opaque_ref + self._session.get_xenapi().VM.get_record(vm_ref) + + for device, (network, info) in enumerate(network_info): + mac_address = info['mac'] bridge = network['bridge'] + rxtx_cap = info.pop('rxtx_cap') network_ref = \ NetworkHelper.find_network_with_bridge(self._session, bridge) - if network_ref: - try: - device = "1" if instance._rescue else "0" - except AttributeError: - device = "0" + VMHelper.create_vif(self._session, vm_ref, network_ref, + mac_address, device, rxtx_cap) - VMHelper.create_vif(self._session, vm_ref, network_ref, - instance.mac_address, device, - rxtx_cap=rxtx_cap) - - def reset_network(self, instance): - """ - Creates uuid arg to pass to make_agent_call and calls it. - - """ + def reset_network(self, instance, vm_ref): + """Creates uuid arg to pass to make_agent_call and calls it.""" args = {'id': str(uuid.uuid4())} - resp = self._make_agent_call('resetnetwork', instance, '', args) + # TODO(tr3buchet): fix function call after refactor + #resp = self._make_agent_call('resetnetwork', instance, '', args) + resp = self._make_plugin_call('agent', 'resetnetwork', instance, '', + args, vm_ref) def list_from_xenstore(self, vm, path): """Runs the xenstore-ls command to get a listing of all records @@ -844,25 +838,26 @@ class VMOps(object): """ self._make_xenstore_call('delete_record', vm, path) - def _make_xenstore_call(self, method, vm, path, addl_args={}): + def _make_xenstore_call(self, method, vm, path, addl_args=None): """Handles calls to the xenstore xenapi plugin.""" return self._make_plugin_call('xenstore.py', method=method, vm=vm, path=path, addl_args=addl_args) - def _make_agent_call(self, method, vm, path, addl_args={}): + def _make_agent_call(self, method, vm, path, addl_args=None): """Abstracts out the interaction with the agent xenapi plugin.""" return self._make_plugin_call('agent', method=method, vm=vm, path=path, addl_args=addl_args) - def _make_plugin_call(self, plugin, method, vm, path, addl_args={}): + def _make_plugin_call(self, plugin, method, vm, path, addl_args=None, + vm_ref=None): """Abstracts out the process of calling a method of a xenapi plugin. Any errors raised by the plugin will in turn raise a RuntimeError here. """ instance_id = vm.id - vm_ref = self._get_vm_opaque_ref(vm) + vm_ref = vm_ref or self._get_vm_opaque_ref(vm) vm_rec = self._session.get_xenapi().VM.get_record(vm_ref) args = {'dom_id': vm_rec['domid'], 'path': path} - args.update(addl_args) + args.update(addl_args or {}) try: task = self._session.async_call_plugin(plugin, method, args) ret = self._session.wait_for_task(task, instance_id) diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 7b4bacdec4e9..779b467551fd 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -207,8 +207,8 @@ class AOEDriver(VolumeDriver): (shelf_id, blade_id) = self.db.volume_get_shelf_and_blade(context, _volume['id']) - self._execute("sudo aoe-discover") - out, err = self._execute("sudo aoe-stat", check_exit_code=False) + self._execute('sudo', 'aoe-discover') + out, err = self._execute('sudo', 'aoe-stat', check_exit_code=False) device_path = 'e%(shelf_id)d.%(blade_id)d' % locals() if out.find(device_path) >= 0: return "/dev/etherd/%s" % device_path @@ -224,8 +224,8 @@ class AOEDriver(VolumeDriver): (shelf_id, blade_id) = self.db.volume_get_shelf_and_blade(context, volume_id) - cmd = "sudo vblade-persist ls --no-header" - out, _err = self._execute(cmd) + cmd = ('sudo', 'vblade-persist', 'ls', '--no-header') + out, _err = self._execute(*cmd) exported = False for line in out.split('\n'): param = line.split(' ') @@ -318,8 +318,8 @@ class ISCSIDriver(VolumeDriver): iscsi_name = "%s%s" % (FLAGS.iscsi_target_prefix, volume['name']) volume_path = "/dev/%s/%s" % (FLAGS.volume_group, volume['name']) self._execute('sudo', 'ietadm', '--op', 'new', - '--tid=%s --params Name=%s' % - (iscsi_target, iscsi_name)) + '--tid=%s' % iscsi_target, + '--params', 'Name=%s' % iscsi_name) self._execute('sudo', 'ietadm', '--op', 'new', '--tid=%s' % iscsi_target, '--lun=0', '--params', @@ -500,7 +500,8 @@ class ISCSIDriver(VolumeDriver): tid = self.db.volume_get_iscsi_target_num(context, volume_id) try: - self._execute("sudo ietadm --op show --tid=%(tid)d" % locals()) + self._execute('sudo', 'ietadm', '--op', 'show', + '--tid=%(tid)d' % locals()) except exception.ProcessExecutionError, e: # Instances remount read-only in this case. # /etc/init.d/iscsitarget restart and rebooting nova-volume @@ -551,7 +552,7 @@ class RBDDriver(VolumeDriver): def delete_volume(self, volume): """Deletes a logical volume.""" self._try_execute('rbd', '--pool', FLAGS.rbd_pool, - 'rm', voluname['name']) + 'rm', volume['name']) def local_path(self, volume): """Returns the path of the rbd volume.""" diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index db39cb0f428b..0a45f387331a 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -216,8 +216,7 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port, os_type): 'x-image-meta-status': 'queued', 'x-image-meta-disk-format': 'vhd', 'x-image-meta-container-format': 'ovf', - 'x-image-meta-property-os-type': os_type, - } + 'x-image-meta-property-os-type': os_type} for header, value in headers.iteritems(): conn.putheader(header, value) diff --git a/po/nova.pot b/po/nova.pot index ce88d731b9a3..58140302d073 100644 --- a/po/nova.pot +++ b/po/nova.pot @@ -300,7 +300,7 @@ msgstr "" msgid "instance %s: starting..." msgstr "" -#. pylint: disable-msg=W0702 +#. pylint: disable=W0702 #: ../nova/compute/manager.py:219 #, python-format msgid "instance %s: Failed to spawn" @@ -440,7 +440,7 @@ msgid "" "instance %(instance_id)s: attaching volume %(volume_id)s to %(mountpoint)s" msgstr "" -#. pylint: disable-msg=W0702 +#. pylint: disable=W0702 #. NOTE(vish): The inline callback eats the exception info so we #. log the traceback here and reraise the same #. ecxception below. @@ -591,7 +591,7 @@ msgstr "" msgid "Starting Bridge interface for %s" msgstr "" -#. pylint: disable-msg=W0703 +#. pylint: disable=W0703 #: ../nova/network/linux_net.py:314 #, python-format msgid "Hupping dnsmasq threw %s" @@ -602,7 +602,7 @@ msgstr "" msgid "Pid %d is stale, relaunching dnsmasq" msgstr "" -#. pylint: disable-msg=W0703 +#. pylint: disable=W0703 #: ../nova/network/linux_net.py:358 #, python-format msgid "killing radvd threw %s" @@ -613,7 +613,7 @@ msgstr "" msgid "Pid %d is stale, relaunching radvd" msgstr "" -#. pylint: disable-msg=W0703 +#. pylint: disable=W0703 #: ../nova/network/linux_net.py:449 #, python-format msgid "Killing dnsmasq threw %s" diff --git a/pylintrc b/pylintrc index f07b1498093b..135eea4d57dd 100644 --- a/pylintrc +++ b/pylintrc @@ -1,8 +1,12 @@ +# The format of this file isn't really documented; just use --generate-rcfile + [Messages Control] +# NOTE(justinsb): We might want to have a 2nd strict pylintrc in future +# C0111: Don't require docstrings on every method # W0511: TODOs in code comments are fine. # W0142: *args and **kwargs are fine. # W0622: Redefining id is fine. -disable-msg=W0511,W0142,W0622 +disable=C0111,W0511,W0142,W0622 [Basic] # Variable names can be 1 to 31 characters long, with lowercase and underscores @@ -25,3 +29,10 @@ no-docstring-rgx=((__.*__)|([tT]est.*)|setUp|tearDown)$ max-public-methods=100 min-public-methods=0 max-args=6 + +[Variables] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +# _ is used by our localization +additional-builtins=_ diff --git a/run_tests.py b/run_tests.py index 3c8d410e1cce..d5d8acd16222 100644 --- a/run_tests.py +++ b/run_tests.py @@ -60,6 +60,8 @@ import os import unittest import sys +gettext.install('nova', unicode=1) + from nose import config from nose import core from nose import result diff --git a/smoketests/base.py b/smoketests/base.py index 204b4a1eb3df..3e2446c9a0a1 100644 --- a/smoketests/base.py +++ b/smoketests/base.py @@ -31,17 +31,24 @@ from smoketests import flags SUITE_NAMES = '[image, instance, volume]' FLAGS = flags.FLAGS flags.DEFINE_string('suite', None, 'Specific test suite to run ' + SUITE_NAMES) +flags.DEFINE_integer('ssh_tries', 3, 'Numer of times to try ssh') boto_v6 = None class SmokeTestCase(unittest.TestCase): def connect_ssh(self, ip, key_name): - # TODO(devcamcar): set a more reasonable connection timeout time key = paramiko.RSAKey.from_private_key_file('/tmp/%s.pem' % key_name) - client = paramiko.SSHClient() - client.set_missing_host_key_policy(paramiko.WarningPolicy()) - client.connect(ip, username='root', pkey=key) - return client + tries = 0 + while(True): + try: + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.WarningPolicy()) + client.connect(ip, username='root', pkey=key, timeout=5) + return client + except (paramiko.AuthenticationException, paramiko.SSHException): + tries += 1 + if tries == FLAGS.ssh_tries: + raise def can_ping(self, ip, command="ping"): """Attempt to ping the specified IP, and give up after 1 second.""" @@ -147,8 +154,8 @@ class SmokeTestCase(unittest.TestCase): except: pass - def bundle_image(self, image, kernel=False): - cmd = 'euca-bundle-image -i %s' % image + def bundle_image(self, image, tempdir='/tmp', kernel=False): + cmd = 'euca-bundle-image -i %s -d %s' % (image, tempdir) if kernel: cmd += ' --kernel true' status, output = commands.getstatusoutput(cmd) @@ -157,9 +164,9 @@ class SmokeTestCase(unittest.TestCase): raise Exception(output) return True - def upload_image(self, bucket_name, image): + def upload_image(self, bucket_name, image, tempdir='/tmp'): cmd = 'euca-upload-bundle -b ' - cmd += '%s -m /tmp/%s.manifest.xml' % (bucket_name, image) + cmd += '%s -m %s/%s.manifest.xml' % (bucket_name, tempdir, image) status, output = commands.getstatusoutput(cmd) if status != 0: print '%s -> \n %s' % (cmd, output) @@ -183,29 +190,3 @@ class UserSmokeTestCase(SmokeTestCase): global TEST_DATA self.conn = self.connection_for_env() self.data = TEST_DATA - - -def run_tests(suites): - argv = FLAGS(sys.argv) - if FLAGS.use_ipv6: - global boto_v6 - boto_v6 = __import__('boto_v6') - - if not os.getenv('EC2_ACCESS_KEY'): - print >> sys.stderr, 'Missing EC2 environment variables. Please ' \ - 'source the appropriate novarc file before ' \ - 'running this test.' - return 1 - - if FLAGS.suite: - try: - suite = suites[FLAGS.suite] - except KeyError: - print >> sys.stderr, 'Available test suites:', \ - ', '.join(suites.keys()) - return 1 - - unittest.TextTestRunner(verbosity=2).run(suite) - else: - for suite in suites.itervalues(): - unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/smoketests/proxy.sh b/smoketests/proxy.sh index 9b3f3108afb1..b9057fe9dec6 100755 --- a/smoketests/proxy.sh +++ b/smoketests/proxy.sh @@ -11,12 +11,19 @@ mkfifo backpipe1 mkfifo backpipe2 +if nc -h 2>&1 | grep -i openbsd +then + NC_LISTEN="nc -l" +else + NC_LISTEN="nc -l -p" +fi + # NOTE(vish): proxy metadata on port 80 while true; do - nc -l -p 80 0backpipe1 + $NC_LISTEN 80 0backpipe1 done & # NOTE(vish): proxy google on port 8080 while true; do - nc -l -p 8080 0backpipe2 + $NC_LISTEN 8080 0backpipe2 done & diff --git a/smoketests/public_network_smoketests.py b/smoketests/public_network_smoketests.py index 5a4c67642d8c..0ba477b7ca21 100644 --- a/smoketests/public_network_smoketests.py +++ b/smoketests/public_network_smoketests.py @@ -19,10 +19,8 @@ import commands import os import random -import socket import sys import time -import unittest # If ../nova/__init__.py exists, add ../ to Python search path, so that # it will override what happens to be installed in /usr/(local/)lib/python... @@ -181,7 +179,3 @@ class InstanceTestsFromPublic(base.UserSmokeTestCase): self.conn.delete_security_group(security_group_name) if 'instance_id' in self.data: self.conn.terminate_instances([self.data['instance_id']]) - -if __name__ == "__main__": - suites = {'instance': unittest.makeSuite(InstanceTestsFromPublic)} - sys.exit(base.run_tests(suites)) diff --git a/smoketests/run_tests.py b/smoketests/run_tests.py new file mode 100644 index 000000000000..62bdfbec61a1 --- /dev/null +++ b/smoketests/run_tests.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Colorizer Code is borrowed from Twisted: +# Copyright (c) 2001-2010 Twisted Matrix Laboratories. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Unittest runner for Nova. + +To run all tests + python run_tests.py + +To run a single test: + python run_tests.py test_compute:ComputeTestCase.test_run_terminate + +To run a single test module: + python run_tests.py test_compute + + or + + python run_tests.py api.test_wsgi + +""" + +import gettext +import os +import unittest +import sys + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + + +gettext.install('nova', unicode=1) + +from nose import config +from nose import core +from nose import result + +from smoketests import flags +FLAGS = flags.FLAGS + + +class _AnsiColorizer(object): + """ + A colorizer is an object that loosely wraps around a stream, allowing + callers to write text to the stream in a particular color. + + Colorizer classes must implement C{supported()} and C{write(text, color)}. + """ + _colors = dict(black=30, red=31, green=32, yellow=33, + blue=34, magenta=35, cyan=36, white=37) + + def __init__(self, stream): + self.stream = stream + + def supported(cls, stream=sys.stdout): + """ + A class method that returns True if the current platform supports + coloring terminal output using this method. Returns False otherwise. + """ + if not stream.isatty(): + return False # auto color only on TTYs + try: + import curses + except ImportError: + return False + else: + try: + try: + return curses.tigetnum("colors") > 2 + except curses.error: + curses.setupterm() + return curses.tigetnum("colors") > 2 + except: + raise + # guess false in case of error + return False + supported = classmethod(supported) + + def write(self, text, color): + """ + Write the given text to the stream in the given color. + + @param text: Text to be written to the stream. + + @param color: A string label for a color. e.g. 'red', 'white'. + """ + color = self._colors[color] + self.stream.write('\x1b[%s;1m%s\x1b[0m' % (color, text)) + + +class _Win32Colorizer(object): + """ + See _AnsiColorizer docstring. + """ + def __init__(self, stream): + from win32console import GetStdHandle, STD_OUT_HANDLE, \ + FOREGROUND_RED, FOREGROUND_BLUE, FOREGROUND_GREEN, \ + FOREGROUND_INTENSITY + red, green, blue, bold = (FOREGROUND_RED, FOREGROUND_GREEN, + FOREGROUND_BLUE, FOREGROUND_INTENSITY) + self.stream = stream + self.screenBuffer = GetStdHandle(STD_OUT_HANDLE) + self._colors = { + 'normal': red | green | blue, + 'red': red | bold, + 'green': green | bold, + 'blue': blue | bold, + 'yellow': red | green | bold, + 'magenta': red | blue | bold, + 'cyan': green | blue | bold, + 'white': red | green | blue | bold + } + + def supported(cls, stream=sys.stdout): + try: + import win32console + screenBuffer = win32console.GetStdHandle( + win32console.STD_OUT_HANDLE) + except ImportError: + return False + import pywintypes + try: + screenBuffer.SetConsoleTextAttribute( + win32console.FOREGROUND_RED | + win32console.FOREGROUND_GREEN | + win32console.FOREGROUND_BLUE) + except pywintypes.error: + return False + else: + return True + supported = classmethod(supported) + + def write(self, text, color): + color = self._colors[color] + self.screenBuffer.SetConsoleTextAttribute(color) + self.stream.write(text) + self.screenBuffer.SetConsoleTextAttribute(self._colors['normal']) + + +class _NullColorizer(object): + """ + See _AnsiColorizer docstring. + """ + def __init__(self, stream): + self.stream = stream + + def supported(cls, stream=sys.stdout): + return True + supported = classmethod(supported) + + def write(self, text, color): + self.stream.write(text) + + +class NovaTestResult(result.TextTestResult): + def __init__(self, *args, **kw): + result.TextTestResult.__init__(self, *args, **kw) + self._last_case = None + self.colorizer = None + # NOTE(vish): reset stdout for the terminal check + stdout = sys.stdout + sys.stdout = sys.__stdout__ + for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]: + if colorizer.supported(): + self.colorizer = colorizer(self.stream) + break + sys.stdout = stdout + + def getDescription(self, test): + return str(test) + + # NOTE(vish): copied from unittest with edit to add color + def addSuccess(self, test): + unittest.TestResult.addSuccess(self, test) + if self.showAll: + self.colorizer.write("OK", 'green') + self.stream.writeln() + elif self.dots: + self.stream.write('.') + self.stream.flush() + + # NOTE(vish): copied from unittest with edit to add color + def addFailure(self, test, err): + unittest.TestResult.addFailure(self, test, err) + if self.showAll: + self.colorizer.write("FAIL", 'red') + self.stream.writeln() + elif self.dots: + self.stream.write('F') + self.stream.flush() + + # NOTE(vish): copied from nose with edit to add color + def addError(self, test, err): + """Overrides normal addError to add support for + errorClasses. If the exception is a registered class, the + error will be added to the list for that class, not errors. + """ + stream = getattr(self, 'stream', None) + ec, ev, tb = err + try: + exc_info = self._exc_info_to_string(err, test) + except TypeError: + # 2.3 compat + exc_info = self._exc_info_to_string(err) + for cls, (storage, label, isfail) in self.errorClasses.items(): + if result.isclass(ec) and issubclass(ec, cls): + if isfail: + test.passed = False + storage.append((test, exc_info)) + # Might get patched into a streamless result + if stream is not None: + if self.showAll: + message = [label] + detail = result._exception_detail(err[1]) + if detail: + message.append(detail) + stream.writeln(": ".join(message)) + elif self.dots: + stream.write(label[:1]) + return + self.errors.append((test, exc_info)) + test.passed = False + if stream is not None: + if self.showAll: + self.colorizer.write("ERROR", 'red') + self.stream.writeln() + elif self.dots: + stream.write('E') + + def startTest(self, test): + unittest.TestResult.startTest(self, test) + current_case = test.test.__class__.__name__ + + if self.showAll: + if current_case != self._last_case: + self.stream.writeln(current_case) + self._last_case = current_case + + self.stream.write( + ' %s' % str(test.test._testMethodName).ljust(60)) + self.stream.flush() + + +class NovaTestRunner(core.TextTestRunner): + def _makeResult(self): + return NovaTestResult(self.stream, + self.descriptions, + self.verbosity, + self.config) + + +if __name__ == '__main__': + if not os.getenv('EC2_ACCESS_KEY'): + print _('Missing EC2 environment variables. Please ' \ + 'source the appropriate novarc file before ' \ + 'running this test.') + sys.exit(1) + + argv = FLAGS(sys.argv) + testdir = os.path.abspath("./") + c = config.Config(stream=sys.stdout, + env=os.environ, + verbosity=3, + workingDir=testdir, + plugins=core.DefaultPluginManager()) + + runner = NovaTestRunner(stream=c.stream, + verbosity=c.verbosity, + config=c) + sys.exit(not core.run(config=c, testRunner=runner, argv=argv)) diff --git a/smoketests/admin_smoketests.py b/smoketests/test_admin.py similarity index 93% rename from smoketests/admin_smoketests.py rename to smoketests/test_admin.py index 86a7f600d37a..46e5b2233ace 100644 --- a/smoketests/admin_smoketests.py +++ b/smoketests/test_admin.py @@ -35,10 +35,7 @@ from smoketests import flags from smoketests import base -SUITE_NAMES = '[user]' - FLAGS = flags.FLAGS -flags.DEFINE_string('suite', None, 'Specific test suite to run ' + SUITE_NAMES) # TODO(devamcar): Use random tempfile ZIP_FILENAME = '/tmp/nova-me-x509.zip' @@ -92,7 +89,3 @@ class UserTests(AdminSmokeTestCase): os.remove(ZIP_FILENAME) except: pass - -if __name__ == "__main__": - suites = {'user': unittest.makeSuite(UserTests)} - sys.exit(base.run_tests(suites)) diff --git a/smoketests/netadmin_smoketests.py b/smoketests/test_netadmin.py similarity index 93% rename from smoketests/netadmin_smoketests.py rename to smoketests/test_netadmin.py index 38beb8fdc7ed..60086f0651ea 100644 --- a/smoketests/netadmin_smoketests.py +++ b/smoketests/test_netadmin.py @@ -21,7 +21,6 @@ import os import random import sys import time -import unittest # If ../nova/__init__.py exists, add ../ to Python search path, so that # it will override what happens to be installed in /usr/(local/)lib/python... @@ -74,8 +73,10 @@ class AddressTests(base.UserSmokeTestCase): groups = self.conn.get_all_security_groups(['default']) for rule in groups[0].rules: if (rule.ip_protocol == 'tcp' and - rule.from_port <= 22 and rule.to_port >= 22): + int(rule.from_port) <= 22 and + int(rule.to_port) >= 22): ssh_authorized = True + break if not ssh_authorized: self.conn.authorize_security_group('default', ip_protocol='tcp', @@ -137,11 +138,6 @@ class SecurityGroupTests(base.UserSmokeTestCase): if not self.wait_for_running(self.data['instance']): self.fail('instance failed to start') self.data['instance'].update() - if not self.wait_for_ping(self.data['instance'].private_dns_name): - self.fail('could not ping instance') - if not self.wait_for_ssh(self.data['instance'].private_dns_name, - TEST_KEY): - self.fail('could not ssh to instance') def test_003_can_authorize_security_group_ingress(self): self.assertTrue(self.conn.authorize_security_group(TEST_GROUP, @@ -185,10 +181,3 @@ class SecurityGroupTests(base.UserSmokeTestCase): self.assertFalse(TEST_GROUP in [group.name for group in groups]) self.conn.terminate_instances([self.data['instance'].id]) self.assertTrue(self.conn.release_address(self.data['public_ip'])) - - -if __name__ == "__main__": - suites = {'address': unittest.makeSuite(AddressTests), - 'security_group': unittest.makeSuite(SecurityGroupTests) - } - sys.exit(base.run_tests(suites)) diff --git a/smoketests/sysadmin_smoketests.py b/smoketests/test_sysadmin.py similarity index 90% rename from smoketests/sysadmin_smoketests.py rename to smoketests/test_sysadmin.py index e3b84d3d3ac2..9bed1e092631 100644 --- a/smoketests/sysadmin_smoketests.py +++ b/smoketests/test_sysadmin.py @@ -16,12 +16,12 @@ # License for the specific language governing permissions and limitations # under the License. -import commands import os import random import sys import time -import unittest +import tempfile +import shutil # If ../nova/__init__.py exists, add ../ to Python search path, so that # it will override what happens to be installed in /usr/(local/)lib/python... @@ -34,8 +34,6 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): from smoketests import flags from smoketests import base - - FLAGS = flags.FLAGS flags.DEFINE_string('bundle_kernel', 'openwrt-x86-vmlinuz', 'Local kernel file to use for bundling tests') @@ -46,12 +44,22 @@ TEST_PREFIX = 'test%s' % int(random.random() * 1000000) TEST_BUCKET = '%s_bucket' % TEST_PREFIX TEST_KEY = '%s_key' % TEST_PREFIX TEST_GROUP = '%s_group' % TEST_PREFIX + + class ImageTests(base.UserSmokeTestCase): def test_001_can_bundle_image(self): - self.assertTrue(self.bundle_image(FLAGS.bundle_image)) + self.data['tempdir'] = tempfile.mkdtemp() + self.assertTrue(self.bundle_image(FLAGS.bundle_image, + self.data['tempdir'])) def test_002_can_upload_image(self): - self.assertTrue(self.upload_image(TEST_BUCKET, FLAGS.bundle_image)) + try: + self.assertTrue(self.upload_image(TEST_BUCKET, + FLAGS.bundle_image, + self.data['tempdir'])) + finally: + if os.path.exists(self.data['tempdir']): + shutil.rmtree(self.data['tempdir']) def test_003_can_register_image(self): image_id = self.conn.register_image('%s/%s.manifest.xml' % @@ -148,7 +156,8 @@ class InstanceTests(base.UserSmokeTestCase): self.fail('could not ping instance') if FLAGS.use_ipv6: - if not self.wait_for_ping(self.data['instance'].ip_v6, "ping6"): + if not self.wait_for_ping(self.data['instance'].dns_name_v6, + "ping6"): self.fail('could not ping instance v6') def test_005_can_ssh_to_private_ip(self): @@ -157,7 +166,7 @@ class InstanceTests(base.UserSmokeTestCase): self.fail('could not ssh to instance') if FLAGS.use_ipv6: - if not self.wait_for_ssh(self.data['instance'].ip_v6, + if not self.wait_for_ssh(self.data['instance'].dns_name_v6, TEST_KEY): self.fail('could not ssh to instance v6') @@ -191,7 +200,7 @@ class VolumeTests(base.UserSmokeTestCase): self.assertEqual(volume.size, 1) self.data['volume'] = volume # Give network time to find volume. - time.sleep(10) + time.sleep(5) def test_002_can_attach_volume(self): volume = self.data['volume'] @@ -204,6 +213,8 @@ class VolumeTests(base.UserSmokeTestCase): else: self.fail('cannot attach volume with state %s' % volume.status) + # Give volume some time to be ready. + time.sleep(5) volume.attach(self.data['instance'].id, self.device) # wait @@ -218,7 +229,7 @@ class VolumeTests(base.UserSmokeTestCase): self.assertTrue(volume.status.startswith('in-use')) # Give instance time to recognize volume. - time.sleep(10) + time.sleep(5) def test_003_can_mount_volume(self): ip = self.data['instance'].private_dns_name @@ -255,12 +266,13 @@ class VolumeTests(base.UserSmokeTestCase): ip = self.data['instance'].private_dns_name conn = self.connect_ssh(ip, TEST_KEY) stdin, stdout, stderr = conn.exec_command( - "df -h | grep %s | awk {'print $2'}" % self.device) - out = stdout.read() + "blockdev --getsize64 %s" % self.device) + out = stdout.read().strip() conn.close() - if not out.strip() == '1007.9M': - self.fail('Volume is not the right size: %s %s' % - (out, stderr.read())) + expected_size = 1024 * 1024 * 1024 + self.assertEquals('%s' % (expected_size,), out, + 'Volume is not the right size: %s %s. Expected: %s' % + (out, stderr.read(), expected_size)) def test_006_me_can_umount_volume(self): ip = self.data['instance'].private_dns_name @@ -283,11 +295,3 @@ class VolumeTests(base.UserSmokeTestCase): def test_999_tearDown(self): self.conn.terminate_instances([self.data['instance'].id]) self.conn.delete_key_pair(TEST_KEY) - - -if __name__ == "__main__": - suites = {'image': unittest.makeSuite(ImageTests), - 'instance': unittest.makeSuite(InstanceTests), - 'volume': unittest.makeSuite(VolumeTests) - } - sys.exit(base.run_tests(suites)) diff --git a/tools/euca-get-ajax-console b/tools/euca-get-ajax-console index e407dd566d28..3df3dcb53b8d 100755 --- a/tools/euca-get-ajax-console +++ b/tools/euca-get-ajax-console @@ -1,5 +1,5 @@ #!/usr/bin/env python -# pylint: disable-msg=C0103 +# pylint: disable=C0103 # vim: tabstop=4 shiftwidth=4 softtabstop=4 # Copyright 2010 United States Government as represented by the