diff --git a/doc/source/topics/settings.rst b/doc/source/topics/settings.rst index e598a30469..c453e1e12d 100644 --- a/doc/source/topics/settings.rst +++ b/doc/source/topics/settings.rst @@ -1662,6 +1662,42 @@ Ignore all listed Nova extensions, and behave as if they were unsupported. Can be used to selectively disable certain costly extensions for performance reasons. +``OPENSTACK_PROFILER`` +---------------------- + +.. versionadded:: 11.0.0(Ocata) + +Default: ``{"enabled": False}`` + +Various settings related to integration with osprofiler library. Since it is a +developer feature, it starts as disabled. To enable it, more than a single +``"enabled"`` key should be specified. Additional keys that should be specified +in that dictionary are: + +* ``"keys"`` is a list of strings, which are secret keys used to encode/decode + the profiler data contained in request headers. Encryption is used for security + purposes, other OpenStack components that are expected to profile themselves + with osprofiler using the data from the request that Horizon initiated must + share a common set of keys with the ones in Horizon config. List of keys is + used so that security keys could be changed in non-obtrusive manner for every + component in the cloud. Example: ``"keys": ["SECRET_KEY", "MORE_SECRET_KEY"]``. + For more details see `osprofiler documentation`_. +* ``"notifier_connection_string"`` is a url to which trace messages are sent by + Horizon. For other components it is usually the only URL specified in config, + because other components act mostly as traces producers. Example: + ``"notifier_connection_string": "mongodb://%s' % OPENSTACK_HOST"``. +* ``"receiver_connection_string"`` is a url from which traces are retrieved by + Horizon, needed because Horizon is not only the traces producer, but also a + consumer. Having 2 settings which usually contain the same value is legacy + feature from older versions of osprofiler when OpenStack components could use + oslo.messaging for notifications and the trace client used ceilometer as a + receiver backend. By default Horizon uses the same URL pointing to a MongoDB + cluster for both purposes, since ceilometer was too slow for using with UI. + Example: ``"receiver_connection_string": "mongodb://%s" % OPENSTACK_HOST``. + +.. _osprofiler documentation: http://docs.openstack.org/developer/osprofiler/integration.html#how-to-initialize-profiler-to-get-one-trace-across-all-services + + ``ALLOWED_PRIVATE_SUBNET_CIDR`` ------------------------------- diff --git a/openstack_dashboard/contrib/developer/profiler/__init__.py b/openstack_dashboard/contrib/developer/profiler/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openstack_dashboard/contrib/developer/profiler/api.py b/openstack_dashboard/contrib/developer/profiler/api.py new file mode 100644 index 0000000000..965d71bcc1 --- /dev/null +++ b/openstack_dashboard/contrib/developer/profiler/api.py @@ -0,0 +1,99 @@ +# Copyright 2016 Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import contextlib + +from django.conf import settings +from osprofiler.drivers.base import get_driver as profiler_get_driver +from osprofiler import notifier +from osprofiler import profiler +from six.moves.urllib.parse import urlparse + + +PROFILER_SETTINGS = getattr(settings, 'OPENSTACK_PROFILER', {}) + + +def init_notifier(connection_str, host="localhost"): + _notifier = notifier.create( + connection_str, project='horizon', service='horizon', host=host) + notifier.set(_notifier) + + +@contextlib.contextmanager +def traced(request, name, info=None): + if info is None: + info = {} + profiler_instance = profiler.get() + if profiler_instance is not None: + trace_id = profiler_instance.get_base_id() + info['user_id'] = request.user.id + with profiler.Trace(name, info=info): + yield trace_id + else: + yield + + +def _get_engine_kwargs(request, connection_str): + from openstack_dashboard.api import base + engines_kwargs = { + # NOTE(tsufiev): actually Horizon doesn't use ceilometer backend (too + # slow for UI), but since osprofiler still supports it (due to API + # deprecation cycle limitations), Horizon also should support this + # option + 'ceilometer': lambda req: { + 'endpoint': base.url_for(req, 'metering'), + 'insecure': getattr(settings, 'OPENSTACK_SSL_NO_VERIFY', False), + 'cacert': getattr(settings, 'OPENSTACK_SSL_CACERT', None), + 'token': (lambda: req.user.token.id), + 'ceilometer_api_version': '2' + } + } + + engine = urlparse(connection_str).scheme + return engines_kwargs.get(engine, lambda req: {})(request) + + +def _get_engine(request): + connection_str = PROFILER_SETTINGS.get( + 'receiver_connection_string', "mongodb://") + kwargs = _get_engine_kwargs(request, connection_str) + return profiler_get_driver(connection_str, **kwargs) + + +def list_traces(request): + engine = _get_engine(request) + query = {"info.user_id": request.user.id} + fields = ['base_id', 'timestamp', 'info.request.path'] + traces = engine.list_traces(query, fields) + return [{'id': trace['base_id'], + 'timestamp': trace['timestamp'], + 'origin': trace['info']['request']['path']} for trace in traces] + + +def get_trace(request, trace_id): + def rec(_data, level=0): + _data['level'] = level + _data['is_leaf'] = not len(_data['children']) + _data['visible'] = True + _data['childrenVisible'] = True + for child in _data['children']: + rec(child, level + 1) + return _data + + engine = _get_engine(request) + trace = engine.get_report(trace_id) + # throw away toplevel node which is dummy and doesn't contain any info, + # use its first and only child as the toplevel node + return rec(trace['children'][0]) diff --git a/openstack_dashboard/contrib/developer/profiler/middleware.py b/openstack_dashboard/contrib/developer/profiler/middleware.py new file mode 100644 index 0000000000..ba4dcf73e0 --- /dev/null +++ b/openstack_dashboard/contrib/developer/profiler/middleware.py @@ -0,0 +1,118 @@ +# Copyright 2016 Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from django.conf import settings +from django.core import exceptions +from django.core.urlresolvers import reverse +from django.utils import safestring +from django.utils.translation import ugettext_lazy as _ +from osprofiler import _utils as profiler_utils +from osprofiler import profiler +from osprofiler import web +import six + +from horizon import messages +from openstack_dashboard.contrib.developer.profiler import api + +_REQUIRED_KEYS = ("base_id", "hmac_key") +_OPTIONAL_KEYS = ("parent_id",) + +PROFILER_SETTINGS = getattr(settings, 'OPENSTACK_PROFILER', {}) + + +class ProfilerClientMiddleware(object): + def process_request(self, request): + if 'profile_page' in request.COOKIES: + hmac_key = PROFILER_SETTINGS.get('keys')[0] + profiler.init(hmac_key) + for hdr_key, hdr_value in web.get_trace_id_headers().items(): + request.META[hdr_key] = hdr_value + return None + + +class ProfilerMiddleware(object): + def __init__(self): + self.name = PROFILER_SETTINGS.get('facility_name', 'horizon') + self.hmac_keys = PROFILER_SETTINGS.get('keys') + self._enabled = PROFILER_SETTINGS.get('enabled', False) + if self._enabled: + api.init_notifier(PROFILER_SETTINGS.get( + 'notifier_connection_string', 'mongodb://')) + else: + raise exceptions.MiddlewareNotUsed() + + @staticmethod + def is_authenticated(request): + return hasattr(request, "user") and request.user.is_authenticated() + + def is_enabled(self, request): + return self.is_authenticated(request) and settings.DEBUG + + @staticmethod + def _trace_is_valid(trace_info): + if not isinstance(trace_info, dict): + return False + trace_keys = set(six.iterkeys(trace_info)) + if not all(k in trace_keys for k in _REQUIRED_KEYS): + return False + if trace_keys.difference(_REQUIRED_KEYS + _OPTIONAL_KEYS): + return False + return True + + def process_view(self, request, view_func, view_args, view_kwargs): + # do not profile ajax requests for now + if not self.is_enabled(request) or request.is_ajax(): + return None + + trace_info = profiler_utils.signed_unpack( + request.META.get('X-Trace-Info'), + request.META.get('X-Trace-HMAC'), + self.hmac_keys) + + if not self._trace_is_valid(trace_info): + return None + + profiler.init(**trace_info) + info = { + 'request': { + 'path': request.path, + 'query': request.GET.urlencode(), + 'method': request.method, + 'scheme': request.scheme + } + } + with api.traced(request, view_func.__name__, info) as trace_id: + response = view_func(request, *view_args, **view_kwargs) + url = reverse('horizon:developer:profiler:index') + message = safestring.mark_safe( + _('Traced with id %(id)s. Go to page') % + {'id': trace_id, 'url': url}) + messages.info(request, message) + return response + + @staticmethod + def clear_profiling_cookies(request, response): + """Expire any cookie that initiated profiling request.""" + if 'profile_page' in request.COOKIES: + path = request.path[:-1] + response.set_cookie('profile_page', max_age=0, path=path) + + def process_response(self, request, response): + self.clear_profiling_cookies(request, response) + # do not profile ajax requests for now + if not self.is_enabled(request) or request.is_ajax(): + return response + + return response diff --git a/openstack_dashboard/local/local_settings.d/_9030_profiler_settings.py.example b/openstack_dashboard/local/local_settings.d/_9030_profiler_settings.py.example new file mode 100644 index 0000000000..d42bd30df1 --- /dev/null +++ b/openstack_dashboard/local/local_settings.d/_9030_profiler_settings.py.example @@ -0,0 +1,6 @@ +OPENSTACK_PROFILER.update({ + 'enabled': True, + 'keys': ['SECRET_KEY'], + 'notifier_connection_string': 'mongodb://%s' % OPENSTACK_HOST, + 'receiver_connection_string': 'mongodb://%s' % OPENSTACK_HOST +}) diff --git a/openstack_dashboard/settings.py b/openstack_dashboard/settings.py index 800fee0454..6cdcb7ea3d 100644 --- a/openstack_dashboard/settings.py +++ b/openstack_dashboard/settings.py @@ -113,6 +113,10 @@ MIDDLEWARE_CLASSES = ( 'horizon.themes.ThemeMiddleware', 'django.middleware.locale.LocaleMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'openstack_dashboard.contrib.developer.profiler.middleware.' + 'ProfilerClientMiddleware', + 'openstack_dashboard.contrib.developer.profiler.middleware.' + 'ProfilerMiddleware', ) CACHED_TEMPLATE_LOADERS = [ @@ -319,6 +323,10 @@ ANGULAR_FEATURES = { # Notice all customizable configurations should be above this line XSTATIC_MODULES = settings_utils.BASE_XSTATIC_MODULES +OPENSTACK_PROFILER = { + 'enabled': False +} + try: from local.local_settings import * # noqa except ImportError: diff --git a/requirements.txt b/requirements.txt index c049c668aa..fe4b21e74e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,6 +24,8 @@ oslo.i18n>=2.1.0 # Apache-2.0 oslo.policy>=1.15.0 # Apache-2.0 oslo.serialization>=1.10.0 # Apache-2.0 oslo.utils>=3.18.0 # Apache-2.0 +osprofiler>=1.4.0 # Apache-2.0 +pymongo>=3.0.2,!=3.1 # Apache-2.0 pyScss!=1.3.5,>=1.3.4 # MIT License python-ceilometerclient>=2.5.0 # Apache-2.0 python-cinderclient!=1.7.0,!=1.7.1,>=1.6.0 # Apache-2.0