diff --git a/doc/source/test_plans/index.rst b/doc/source/test_plans/index.rst index dc72e79..b66b07b 100644 --- a/doc/source/test_plans/index.rst +++ b/doc/source/test_plans/index.rst @@ -15,3 +15,4 @@ Test Plans provisioning/plan tenant_networking/plan container_repositories/plan + keystone/plan diff --git a/doc/source/test_plans/keystone/plan.rst b/doc/source/test_plans/keystone/plan.rst new file mode 100644 index 0000000..8b6a7a9 --- /dev/null +++ b/doc/source/test_plans/keystone/plan.rst @@ -0,0 +1,418 @@ +============================ +Keystone Performance testing +============================ + +:status: **ready** +:version: 1 + +:Abstract: + + This document describes a test plan for measuring OpenStack Identity service + (Keystone) performance, including primary analysis of database and cache + operations effectiveness. This test plan assumes to use OSprofiler library + for cross-project OpenStack requests profiling. + +Test Plan +========= + +Keystone is an OpenStack project that provides Identity, Token, Catalog and +Policy services for use specifically by projects in the OpenStack family. +It implements OpenStack’s Identity API and is widely used by almost all +OpenStack services, therefore its performance is a key. To evaluate it this +test plan proposes to use OSprofiler. + +OSprofiler is an Oslo library that provides Python wrappers to trace +operations on the Python level. It provides several ways to wrap +separated methods, all methods inside one specific class and all methods +encapsulated under Python classes having common ancestor. + +For every profiled method, notifications are sent about operation start time +and end time, including information about parent operation. Right now these +notifications are stored as OpenStack Telemetry (Ceilometer) events, so on +high level OpenStack profiling process can be presented as follows: + +.. image:: profiling_workflow.png + :width: 650px + +Test Environment +---------------- + +This section describes the setup for Keystone testing. It can be either +a single (all-in-one) or a multi-node installation. + +A single-node setup requires just one node to be up and running. It has +both compute and controller roles and all OpenStack services run on this node. +This setup does not support test cases, related to the performance +measurements, combined with the components HA testing. + +A basic multi-node setup with Keystone comprises 4 physical nodes: + * One node for a compute node. This node simulates activity which is + typical for OpenStack compute components. + * Three nodes for a controller nodes. These node simulate activity which + is typical for OpenStack control plane services, including running three + MySQL instances managed by Galera cluster and memcached cluster for + Keystone caching. + +Preparation +^^^^^^^^^^^ + +**Common preparation steps** + +No matter if you're running single node or multi node setup, you will require +several steps to be completed before running any profiling tasks: + * Ensure OSprofiler library_ is installed to all environment nodes to present + most full information in the trace. + * OSprofiler requires persistent profiling events storage to be presented in + the environment. For now, the only supported variant is to use OpenStack + Telemetry (Ceilometer) project, that will consume profiling events via + message queue notifications from the affected OpenStack services. + * OSprofiler integration to the OpenStack projects is ongoing initiative, + so please ensure that all OpenStack services you would like to profile + support it in your environment. Here is a project to release mapping that + can be useful: + * Cinder OSprofiler support - OpenStack Juno release + * Glance OSprofiler support - OpenStack Juno release + * Heat OSprofiler support - OpenStack Juno release + * Trove OSprofiler support - OpenStack Juno release + * Nova_ OSprofiler support - [planned] OpenStack Newton release + * Neutron_ OSprofiler support - [planned] OpenStack Newton release + * Keystone_ OSprofiler support - [planned] OpenStack Newton release + At the time of this document composing Nova, Neutron and Keystone changes + need to be applied manually to trace these projects usage. + * Please make sure that all OpenStack services are properly configured to + allow cross-project request profiling. In case of single node installation + using DevStack this will be automatically managed by OSprofiler DevStack + plugin, but for multi node environment this needs to be tracked separately. + Please pay attention to the appropriate subsection under `Multi node + installation` section. + * Test cases described in this document suppose to collect information for + comparison analysis of Keystone database and cache operations + effectiveness, that requires Keystone reconfiguration depending on if + caching mechanism is used at the moment or not. This is tracked via + specialized `keystone.conf` file section:: + + [cache] + enabled = True|False + backend = oslo_cache.memcache_pool + memcache_servers = :[,:] + expiration_time = 600 + +**Single node installation** + +For single node installation the one can use DevStack_ tool that is targeted +at developers and CI systems to use upstream code. It makes many choices that +are not appropriate for production systems, but for the all-in-one purposes +this can fit ok. + +At the time of document writing, DevStack's `local.conf` should be looking like +as following to have and all-in-one OpenStack installation with Neutron enabled +and OSprofiler installation included with all needed fixes to the OpenStack +services:: + + [[local|localrc]] + ADMIN_PASSWORD=password + DATABASE_PASSWORD=$ADMIN_PASSWORD + RABBIT_PASSWORD=$ADMIN_PASSWORD + SERVICE_PASSWORD=$ADMIN_PASSWORD + + LIBS_FROM_GIT=osprofiler,python-openstackclient + + NOVA_REPO=https://review.openstack.org/p/openstack/nova + NOVA_BRANCH=refs/changes/03/254703/39 + + KEYSTONE_REPO=https://review.openstack.org/p/openstack/keystone + KEYSTONE_BRANCH=refs/changes/35/294535/2 + + NEUTRON_REPO=https://review.openstack.org/p/openstack/neutron + NEUTRON_BRANCH=refs/changes/51/273951/12 + + disable_service n-net horizon + enable_service q-svc q-dhcp q-meta q-agt q-l3 neutron + + enable_plugin ceilometer https://git.openstack.org/openstack/ceilometer.git + enable_plugin osprofiler https://github.com/openstack/osprofiler.git + +To add Fernet tokens usage (as for the time of document writing, default token +format for DevStack is still UUID), the next line needs to be added +explicitly:: + + KEYSTONE_TOKEN_FORMAT=fernet + + +Please make sure to have identical cache configuration for Keystone authtoken +middleware. For example, cache might be external (memcached) and appropriate +configuration section will look like this in this case:: + + [keystone_authtoken] + memcache_servers = :[,:] + signing_dir = + cafile = + auth_uri = + project_domain_id = + project_name = + user_domain_id = + password = + username = + auth_url = + auth_plugin = + +Sadly there is no simple way to setup specific patches to be applyed on the +python libraries used (OpenStack clients, OSProfiler, etc.) so they require +manual patching in this case: + +* `OSprofiler changes`_ + +**Multi node installation** + +Multi node environment installation depends much on the chosen set of OpenStack +deployment tools. Whatever instrument will be used, please consider to ensure +the following patches to be applied against main OpenStack services to be +profiled (Nova, Neutron, Keystone) and the appropriate libraries: + +* `OSprofiler changes`_ +* Nova_ OSprofiler integration +* Neutron_ OSprofiler integration +* Keystone_ OSprofiler integration + +*OpenStack services configuration* + +Several OpenStack configuration files need to be modified to enable appropriate +OSprofiler workability. + +First of all, the one needs to enable `Ceilometer` profiling events storage via +adding the following lines to the `event_definitions.yaml` file, declaratively +announcing the wish to consume them:: + + - event_type: profiler.* + traits: + project: + fields: payload.project + service: + fields: payload.service + name: + fields: payload.name + base_id: + fields: payload.base_id + trace_id: + fields: payload.trace_id + parent_id: + fields: payload.parent_id + timestamp: + fields: payload.timestamp + host: + fields: payload.info.host + path: + fields: payload.info.request.path + query: + fields: payload.info.request.query + method: + fields: payload.info.request.method + scheme: + fields: payload.info.request.scheme + db.statement: + fields: payload.info.db.statement + db.params: + fields: payload.info.db.params + +Also, for extended tracing information providing it's useful (although, not +mandatory if you need only high-level traces) to add the following lines to the +`ceilometer.conf` configuration file:: + + [event] + store_raw=info + +.. note:: Please pay attention to the fact, that the configuration parameter + defined above will store raw events for **all** `info` level event + notifications coming to the Ceilometer queue. Please make sure either + to turn off not needed events in `event_definitions.yaml` or save + enough place for the Ceilometer storage backend. + +Also for every project you would like to trace it's required to enable its +profiling via service configuration files and add the following section:: + + [profiler] + enabled = True + trace_sqlalchemy = True + hmac_keys = SECRET_KEY + +In this test plan it's supposed to turn profiling on for Cinder, Glance, Nova, +Neutron and Keystone. + +.. _library: https://pypi.python.org/pypi/osprofiler +.. _Nova: https://review.openstack.org/#/q/status:open+branch:master+topic:bp/osprofiler-support-in-nova +.. _Neutron: https://review.openstack.org/#/q/status:open++branch:master+topic:bug/1335640 +.. _Keystone: https://review.openstack.org/#/q/status:open+branch:master+topic:osprofiler-support-in-keystone +.. _DevStack: http://devstack.org +.. _OSprofiler changes: https://review.openstack.org/#/c/294516/ + +Environment description +^^^^^^^^^^^^^^^^^^^^^^^ + +The environment description includes hardware specification of servers, +network parameters, operation system and OpenStack deployment characteristics. + +Hardware +~~~~~~~~ + +This section contains list of all types of hardware nodes. + ++-----------+-------+----------------------------------------------------+ +| Parameter | Value | Comments | ++-----------+-------+----------------------------------------------------+ +| model | | e.g. Supermicro X9SRD-F | ++-----------+-------+----------------------------------------------------+ +| CPU | | e.g. 6 x Intel(R) Xeon(R) CPU E5-2620 v2 @ 2.10GHz | ++-----------+-------+----------------------------------------------------+ + +Network +~~~~~~~ + +This section contains list of interfaces and network parameters. +For complicated cases this section may include topology diagram and switch +parameters. + ++------------------+-------+-------------------------+ +| Parameter | Value | Comments | ++------------------+-------+-------------------------+ +| card model | | e.g. Intel | ++------------------+-------+-------------------------+ +| driver | | e.g. ixgbe | ++------------------+-------+-------------------------+ +| speed | | e.g. 10G or 1G | ++------------------+-------+-------------------------+ + +Software +~~~~~~~~ + +This section describes installed software. + ++-----------------+--------+---------------------------+ +| Parameter | Value | Comments | ++-----------------+--------+---------------------------+ +| OS | | e.g. Ubuntu 14.04.3 | ++-----------------+--------+---------------------------+ +| Keystone DB | | e.g. MySQL 5.6 | ++-----------------+--------+---------------------------+ +| Keystone Cache | on/off | e.g. memcached v1.4.25 | ++-----------------+--------+---------------------------+ +| HA mode | | e.g. Cluster | ++-----------------+--------+---------------------------+ + +Test Case 1: Keystone DB / cache operations analysis +---------------------------------------------------- + +Description +^^^^^^^^^^^ + +This test records all HTTP, RPC and DB calls happening during selected list of +OpenStack control plane operations, including Keystone operations and their +duration via OSprofiler. Human-readable report would be automatically generated +to evaluate overall number of DB / cache calls, as well as raw information +ported to the JSON format. + +Let's focus on the following control plane operations: + +* Keystone token get (token issue) +* Keystone user list +* Keystone endpoint list +* Keystone service list +* Nova instance boot (server create) + +OSprofiler adds an opportunity to call CLI command, generating report on the +profiled control plane operation in one of the chosen formats - either JSON +or HTML. + +To initiate OpenStack request tracing `--profile ` option needs to +be added to the CLI command, triggering this specific action. This key needs +to present one of the secret keys defined in .conf configuration file +with `hmac_keys` option under the `[profiler]` configuration section. In case +if all OpenStack projects have shared HMAC_KEY defined in their configuration +files, it will be possible to generate a report, containing tracing points from +all services taking part in the request processing. + +To initiate VM creation tracing the following command should be used from the +CLI:: + + openstack --profile SECRET_KEY server create --image --flavor + +Without `--profile SECRET_KEY` option trace generation won't be triggered even +if it's enabled in the OpenStack services configuration files. + +At the end of output there will be message printed with , and to +plot nice HTML graphs the following command should be used:: + + osprofiler trace show --html --out result.html + + All other chosen control plane operations request can be traced via similar + approach. + +Parameters +^^^^^^^^^^ + +=========================== ==================================================== +Parameter name Value +=========================== ==================================================== +OpenStack release Liberty, Mitaka +Cache on, off +Token type UUID, fernet +Environment characteristics Single node, multi node (clusterized DB / memcached) +=========================== ==================================================== + +List of performance metrics +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Test case result is presented as a weighted tree structure with operations +as nodes and time spent on them as node weights for every control plane +operation under the test. This information is automatically gathered in +Ceilometer and can be gracefully transformed to the human-friendly report via +OSprofiler. + +======== ============== ================= ========================================= +Priority Value Measurement Units Description +======== ============== ================= ========================================= +1 Operation time milliseconds Time spent on every HTTP/RPC/DB operation +======== ============== ================= ========================================= + +.. note:: Please keep in mind that OSprofiler uses python code and libraries + to listen on specific events to happen - in common case that will be + `before method starts` and `after method starts` via Python + decorators usage, therefore time it collects includes time spent on + Python operations inside. For DB calls tracing OSprofiler uses + `before_cursor_execute` and `after_cursor_execute` events defined in + SQLAlchemy library. + +Test Case 2: Keystone DB / cache operations analysis (HA version) +----------------------------------------------------------------- + +Description +^^^^^^^^^^^ + +This test case should provide almost the same analysis as the previous one, +the difference is in adding failover testing component to the research. First +test run assumes to be the same as for previous case, the second one should +happen after turning off one of the distributed components used by Keystone, +e.g. stop one of memcached instances and run the same control plane operations +tracing. + +Parameters +^^^^^^^^^^ + +=========================== ==================================================== +Parameter name Value +=========================== ==================================================== +OpenStack release Liberty, Mitaka +Cache on, off +Token type UUID, fernet +Environment characteristics Single node, multi node (clusterized DB / memcached) +Memcached cluster status 3 nodes, 2 nodes, 1 node +Galera cluster status 3 nodes, 2 nodes +=========================== ==================================================== + +List of performance metrics +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +======== ============== ================= ========================================= +Priority Value Measurement Units Description +======== ============== ================= ========================================= +1 Operation time milliseconds Time spent on every HTTP/RPC/DB operation +======== ============== ================= ========================================= diff --git a/doc/source/test_plans/keystone/profiling_workflow.png b/doc/source/test_plans/keystone/profiling_workflow.png new file mode 100644 index 0000000..57fdf61 Binary files /dev/null and b/doc/source/test_plans/keystone/profiling_workflow.png differ diff --git a/doc/source/test_results/index.rst b/doc/source/test_results/index.rst index c091692..d84a180 100644 --- a/doc/source/test_results/index.rst +++ b/doc/source/test_results/index.rst @@ -13,3 +13,4 @@ Test Results mq_ha/index container_repositories/index db/mysql/index + keystone/index diff --git a/doc/source/test_results/keystone/all-in-one/index.rst b/doc/source/test_results/keystone/all-in-one/index.rst new file mode 100644 index 0000000..53ff62b --- /dev/null +++ b/doc/source/test_results/keystone/all-in-one/index.rst @@ -0,0 +1,59 @@ +Keystone DB / cache operations analysis +--------------------------------------- + +Environment description +^^^^^^^^^^^^^^^^^^^^^^^ + +The test plan (test case #1) is executed at the all-in-one virtual environment, +created with Oracle VM VirtualBox Manager. + +Hardware +~~~~~~~~ + +All-in-one installation on virtual environment. + ++-----------+------------------------------------------------------------+ +| Parameter | Value | ++-----------+------------------------------------------------------------+ +| CPU | 4 CPU out of 8 (2,7 GHz Intel Core i5) | ++-----------+------------------------------------------------------------+ +| RAM | 4 Gb | ++-----------+------------------------------------------------------------+ + +Software +~~~~~~~~ + +This section describes installed software. + ++-----------------+--------------------------------------------+ +| Parameter | Value | ++-----------------+--------------------------------------------+ +| OS | Ubuntu 15.04 | ++-----------------+--------------------------------------------+ + +Execution +^^^^^^^^^ + +To process all control plane requests against the environment the following +set of commands need to be processed:: + + openstack --profile SECRET_KEY token issue + openstack --profile SECRET_KEY user list + openstack --profile SECRET_KEY endpoint list + openstack --profile SECRET_KEY service list + openstack --profile SECRET_KEY server create --image --flavor + +Reports +^^^^^^^ + +.. toctree:: + :maxdepth: 1 + + liberty_fernet_cache_off + liberty_fernet_cache_on + liberty_uuid_cache_off + liberty_uuid_cache_on + mitaka_fernet_cache_off + mitaka_fernet_cache_on + mitaka_uuid_cache_off + mitaka_uuid_cache_on diff --git a/doc/source/test_results/keystone/all-in-one/liberty_fernet_cache_off.rst b/doc/source/test_results/keystone/all-in-one/liberty_fernet_cache_off.rst new file mode 100644 index 0000000..69464fa --- /dev/null +++ b/doc/source/test_results/keystone/all-in-one/liberty_fernet_cache_off.rst @@ -0,0 +1,773 @@ +Running profiling against Liberty Keystone (Fernet tokens, cache turned off) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. contents:: + +Parameters +~~~~~~~~~~ + +=========================== =========== +Parameter name Value +=========================== =========== +OpenStack release Liberty +Cache off +Token type Fernet +Environment characteristics Single node +=========================== =========== + +Endpoint list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 328 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 2570 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 164 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 1312 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 164 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 1258 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 164 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 1258 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 10 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 8 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 23 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 15 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 14 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT domain.id AS domain_id, domain.name AS domain_name, domain.enabled AS domain_enabled, | 14 | +| domain.extra AS domain_extra | | +| FROM domain | | +| WHERE domain.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 3 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 21 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 16 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 23 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 9 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.name = %s AND user.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 24 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +Server create request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 190 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 1515 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 95 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 714 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 95 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 801 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 95 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 801 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 10 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 28 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 23 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 10 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 12 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT domain.id AS domain_id, domain.name AS domain_name, domain.enabled AS domain_enabled, | 52 | +| domain.extra AS domain_extra | | +| FROM domain | | +| WHERE domain.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 17 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 26 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 11 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 16 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 9 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.name = %s AND user.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 22 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +Service list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 112 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 605 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 56 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 308 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 56 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 297 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 56 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 297 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 10 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 3 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 11 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 9 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 8 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT domain.id AS domain_id, domain.name AS domain_name, domain.enabled AS domain_enabled, | 7 | +| domain.extra AS domain_extra | | +| FROM domain | | +| WHERE domain.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 3 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 3 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 7 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 14 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 7 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.name = %s AND user.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 10 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +Token issue request stats +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 44 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 316 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 22 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 183 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 22 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 133 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 22 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 133 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 10 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 3 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 9 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 9 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 8 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT domain.id AS domain_id, domain.name AS domain_name, domain.enabled AS domain_enabled, | 3 | +| domain.extra AS domain_extra | | +| FROM domain | | +| WHERE domain.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 10 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 3 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 3 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 14 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 8 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.name = %s AND user.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 14 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +User list request stats +~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 112 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 675 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 56 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 297 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 56 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 378 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 56 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 378 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 10 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 8 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 9 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 14 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 4 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 12 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT domain.id AS domain_id, domain.name AS domain_name, domain.enabled AS domain_enabled, | 12 | +| domain.extra AS domain_extra | | +| FROM domain | | +| WHERE domain.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 9 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 8 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 4 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 12 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 8 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.name = %s AND user.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 13 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ diff --git a/doc/source/test_results/keystone/all-in-one/liberty_fernet_cache_on.rst b/doc/source/test_results/keystone/all-in-one/liberty_fernet_cache_on.rst new file mode 100644 index 0000000..e1ad747 --- /dev/null +++ b/doc/source/test_results/keystone/all-in-one/liberty_fernet_cache_on.rst @@ -0,0 +1,563 @@ +Running profiling against Liberty Keystone (Fernet tokens, cache turned on) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. contents:: + +Parameters +~~~~~~~~~~ + +=========================== =========== +Parameter name Value +=========================== =========== +OpenStack release Liberty +Cache on +Token type Fernet +Environment characteristics Single node +=========================== =========== + +Endpoint list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 70 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 466 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 35 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 241 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 35 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 225 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 35 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 225 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 9 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 8 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 10 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 4 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 9 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +Server create request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 60 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 980 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 30 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 691 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 30 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 289 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 30 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 289 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 9 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 16 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 8 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 21 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 18 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 4 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 17 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 9 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +Service list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 54 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 400 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 27 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 190 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 27 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 210 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 27 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 210 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 9 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 16 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 46 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 21 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 18 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 15 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 11 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 9 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +Token issue request stats +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 28 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 219 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 14 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 150 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 14 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 69 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 14 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 69 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 9 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 16 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 46 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 3 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 9 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 6 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 7 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 11 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 9 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +User list request stats +~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 52 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 332 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 26 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 157 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 26 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 175 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 26 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 175 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 9 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 16 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 10 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 3 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 8 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 4 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 9 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 9 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ diff --git a/doc/source/test_results/keystone/all-in-one/liberty_uuid_cache_off.rst b/doc/source/test_results/keystone/all-in-one/liberty_uuid_cache_off.rst new file mode 100644 index 0000000..e61855b --- /dev/null +++ b/doc/source/test_results/keystone/all-in-one/liberty_uuid_cache_off.rst @@ -0,0 +1,512 @@ +Running profiling against Liberty Keystone (UUID tokens, cache turned off) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. contents:: + +Parameters +~~~~~~~~~~ + +=========================== =========== +Parameter name Value +=========================== =========== +OpenStack release Liberty +Cache off +Token type UUID +Environment characteristics Single node +=========================== =========== + +Endpoint list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 92 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 534 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 46 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 259 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 46 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 275 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 44 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 266 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 9 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 9 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 8 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 7 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 24 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 10 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT domain.id AS domain_id, domain.name AS domain_name, domain.enabled AS domain_enabled, | 9 | +| domain.extra AS domain_extra | | +| FROM domain | | +| WHERE domain.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 16 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +Server create request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. note:: This control plane request numbers were not collected yet due to + environmental issues. + +Service list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 56 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 369 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 28 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 177 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 28 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 192 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 26 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 174 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 18 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 14 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 8 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 4 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 4 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.name = %s AND user.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 4 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 9 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 11 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT domain.id AS domain_id, domain.name AS domain_name, domain.enabled AS domain_enabled, | 9 | +| domain.extra AS domain_extra | | +| FROM domain | | +| WHERE domain.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 17 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +Token issue request stats +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 26 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 174 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 13 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 91 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 13 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 83 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 12 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 79 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 1 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 4 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 14 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 8 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 10 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 4 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.name = %s AND user.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 4 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 9 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 9 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT domain.id AS domain_id, domain.name AS domain_name, domain.enabled AS domain_enabled, | 9 | +| domain.extra AS domain_extra | | +| FROM domain | | +| WHERE domain.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 9 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +User list request stats +~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 56 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 360 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 28 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 212 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 28 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 148 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 26 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 139 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 9 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 14 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 8 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 10 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 3 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.name = %s AND user.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 4 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 9 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 13 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT domain.id AS domain_id, domain.name AS domain_name, domain.enabled AS domain_enabled, | 3 | +| domain.extra AS domain_extra | | +| FROM domain | | +| WHERE domain.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 9 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ diff --git a/doc/source/test_results/keystone/all-in-one/liberty_uuid_cache_on.rst b/doc/source/test_results/keystone/all-in-one/liberty_uuid_cache_on.rst new file mode 100644 index 0000000..1067f05 --- /dev/null +++ b/doc/source/test_results/keystone/all-in-one/liberty_uuid_cache_on.rst @@ -0,0 +1,426 @@ +Running profiling against Liberty Keystone (UUID tokens, cache turned on) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. contents:: + +Parameters +~~~~~~~~~~ + +=========================== =========== +Parameter name Value +=========================== =========== +OpenStack release Liberty +Cache on +Token type UUID +Environment characteristics Single node +=========================== =========== + +Endpoint list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 44 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 465 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 22 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 230 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 22 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 235 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 20 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 204 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 31 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 5 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 18 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 15 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 25 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 13 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +Server create request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. note:: This control plane request numbers were not collected yet due to + environmental issues. + + +Service list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 36 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 227 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 18 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 101 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 18 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 126 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 16 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 108 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 18 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 13 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 12 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 5 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 16 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 18 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 13 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 15 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +Token issue request stats +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 22 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 107 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 11 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 45 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 11 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 62 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 10 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 55 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 1 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 7 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 13 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 12 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 5 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 16 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 18 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 13 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 15 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +User list request stats +~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 26 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 165 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 13 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 61 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 13 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 104 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 11 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 90 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 14 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 13 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 12 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 22 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 16 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 18 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 13 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 15 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ diff --git a/doc/source/test_results/keystone/all-in-one/mitaka_fernet_cache_off.rst b/doc/source/test_results/keystone/all-in-one/mitaka_fernet_cache_off.rst new file mode 100644 index 0000000..2fe1ff4 --- /dev/null +++ b/doc/source/test_results/keystone/all-in-one/mitaka_fernet_cache_off.rst @@ -0,0 +1,1366 @@ +Running profiling against Mitaka Keystone (Fernet tokens, cache turned off) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. contents:: + +Parameters +~~~~~~~~~~ + +=========================== =========== +Parameter name Value +=========================== =========== +OpenStack release Mitaka +Cache off +Token type Fernet +Environment characteristics Single node +=========================== =========== + +Endpoint list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 508 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 3975 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 207 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 1667 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 301 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 2308 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 301 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 2308 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 18 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 22 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 11 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 19 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 12 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 12 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 17 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 19 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 17 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 3 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 15 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 11 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 6 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 6 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 9 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Server create request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 4822 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 52521 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 1962 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 20748 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 2860 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 31773 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 2860 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 31773 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 42 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 53 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 29 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 29 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 19 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %(name_1)s AND project.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 35 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 35 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 60 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 44 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 39 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 16 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 38 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 34 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 222 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 41 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 34 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 38 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 17 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 6 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 6 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 6 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 7 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 11 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 11 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 16 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 25 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 12 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 60 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 29 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 14 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 17 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 21 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Service list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 140 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 1298 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 55 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 463 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 85 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 835 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 85 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 835 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 20 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 53 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 18 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 29 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 19 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %(name_1)s AND project.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 35 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 16 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 60 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 14 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 39 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 15 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 33 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 16 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 222 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 41 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 25 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 15 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 17 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 10 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 9 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 3 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Token issue request stats +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 47 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 442 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 18 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 153 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 29 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 289 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 29 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 289 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 20 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 53 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 18 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 29 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 24 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %(name_1)s AND project.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 35 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 16 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 60 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 14 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 39 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 15 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 33 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 20 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 21 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 41 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 25 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 15 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 17 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 7 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 6 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +User list request stats +~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 150 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 1119 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 55 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 442 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 95 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 677 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 95 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 677 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 3 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 11 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 18 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 11 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 11 | +| user.default_project_id AS user_default_project_id | | +| FROM user LEFT OUTER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 24 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %(name_1)s AND project.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 25 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 16 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 60 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 14 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 16 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 11 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 33 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 12 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 21 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 41 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 15 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 11 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 17 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 11 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 7 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ diff --git a/doc/source/test_results/keystone/all-in-one/mitaka_fernet_cache_on.rst b/doc/source/test_results/keystone/all-in-one/mitaka_fernet_cache_on.rst new file mode 100644 index 0000000..058c2f8 --- /dev/null +++ b/doc/source/test_results/keystone/all-in-one/mitaka_fernet_cache_on.rst @@ -0,0 +1,1269 @@ +Running profiling against Mitaka Keystone (Fernet tokens, cache turned on) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. contents:: + +Parameters +~~~~~~~~~~ + +=========================== =========== +Parameter name Value +=========================== =========== +OpenStack release Mitaka +Cache on +Token type Fernet +Environment characteristics Single node +=========================== =========== + +Endpoint list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 398 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 4107 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 152 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 1477 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 246 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 2630 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 246 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 2630 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 29 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 18 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 24 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 42 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 19 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 4 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 12 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 24 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 29 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 52 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 24 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 19 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 14 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 29 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 15 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 4 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 9 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Server create request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 2108 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 19003 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 807 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 6813 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 1301 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 12190 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 1301 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 12190 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 16 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 27 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 27 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 18 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 26 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 26 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 4 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 12 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 23 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 26 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 196 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 15 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 50 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 18 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 18 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 18 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 16 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 6 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 9 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Service list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 110 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 1048 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 40 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 389 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 70 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 659 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 70 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 659 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 12 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 4 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 27 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 18 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 13 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 5 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 17 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 4 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 12 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 23 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 4 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 15 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 14 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 16 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 13 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 17 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 11 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 12 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 7 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 10 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Token issue request stats +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 37 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 308 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 13 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 120 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 24 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 188 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 24 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 188 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 12 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 9 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 27 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 18 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 9 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 8 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 17 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 4 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 12 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 23 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 4 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 15 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 9 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 8 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 14 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 16 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 10 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 19 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +User list request stats +~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 120 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 906 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 40 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 270 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 80 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 636 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 80 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 636 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 4 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 14 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 15 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 15 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 9 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user LEFT OUTER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id | | +| = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 9 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 11 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 4 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 12 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 23 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 4 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 12 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 14 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 11 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 24 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 4 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 4 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 4 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 7 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 7 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ diff --git a/doc/source/test_results/keystone/all-in-one/mitaka_uuid_cache_off.rst b/doc/source/test_results/keystone/all-in-one/mitaka_uuid_cache_off.rst new file mode 100644 index 0000000..1ed9381 --- /dev/null +++ b/doc/source/test_results/keystone/all-in-one/mitaka_uuid_cache_off.rst @@ -0,0 +1,1362 @@ +Running profiling against Mitaka Keystone (UUID tokens, cache turned off) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. contents:: + +Parameters +~~~~~~~~~~ + +=========================== =========== +Parameter name Value +=========================== =========== +OpenStack release Mitaka +Cache off +Token type UUID +Environment characteristics Single node +=========================== =========== + +Endpoint list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 152 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 1348 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 65 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 551 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 87 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 797 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 85 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 781 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 16 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 5 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 25 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 13 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 5 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 20 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 5 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 43 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 17 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 5 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 25 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 3 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 9 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 16 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 14 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 4 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 9 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 3 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 20 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 7 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 8 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 17 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Server create request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 1670 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 14784 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 712 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 5853 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 958 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 8931 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 956 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 8908 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 23 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 88 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 24 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 69 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 24 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 13 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 5 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 75 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 3 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 12 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 22 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 50 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 33 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 16 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 51 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 87 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 22 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 26 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 17 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 4 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 13 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 12 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Service list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 104 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 945 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 41 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 346 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 63 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 599 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 61 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 588 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 11 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 13 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 24 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 69 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 24 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 11 | +| service.extra AS service_extra | | +| FROM service | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 32 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 5 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 75 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 3 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 14 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 11 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 16 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 15 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 3 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 51 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 87 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 15 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 74 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 17 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 32 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 31 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 6 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 14 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Token issue request stats +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 49 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 324 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 19 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 121 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 30 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 203 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 29 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 197 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 1 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 6 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 13 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 10 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 69 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 24 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 11 | +| service.extra AS service_extra | | +| FROM service | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 32 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 5 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 11 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 3 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 14 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 14 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 16 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 15 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 3 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 51 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 87 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 15 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 74 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 17 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 4 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 6 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +User list request stats +~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 114 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 798 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 41 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 267 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 73 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 531 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 71 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 511 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 20 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 8 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 5 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 4 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 9 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 17 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 12 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 22 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 15 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 32 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 8 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 13 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 8 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %(name_1)s AND project.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 69 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 9 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 87 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 3 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 9 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user LEFT OUTER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = | | +| local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 11 | +| service.extra AS service_extra | | +| FROM service | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 14 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 4 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 17 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 7 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ diff --git a/doc/source/test_results/keystone/all-in-one/mitaka_uuid_cache_on.rst b/doc/source/test_results/keystone/all-in-one/mitaka_uuid_cache_on.rst new file mode 100644 index 0000000..e1f5c59 --- /dev/null +++ b/doc/source/test_results/keystone/all-in-one/mitaka_uuid_cache_on.rst @@ -0,0 +1,1368 @@ +Running profiling against Mitaka Keystone (UUID tokens, cache turned on) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. contents:: + +Parameters +~~~~~~~~~~ + +=========================== =========== +Parameter name Value +=========================== =========== +OpenStack release Mitaka +Cache on +Token type UUID +Environment characteristics Single node +=========================== =========== + +Endpoint list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 132 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 1240 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 55 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 506 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 77 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 734 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 75 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 722 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 12 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 2 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 4 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 24 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 15 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 13 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 17 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 40 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 4 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 4 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 7 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 8 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 11 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 13 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Server create request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 1140 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 9581 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 451 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 3500 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 689 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 6081 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 687 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 6067 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 14 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 39 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 61 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 24 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 27 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 12 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 40 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 8 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %(name_1)s AND project.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 9 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 18 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 63 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 27 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 9 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 40 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 16 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 22 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 9 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 14 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 28 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 25 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 33 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 9 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 9 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 6 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Service list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 84 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 619 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 31 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 207 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 53 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 412 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 51 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 393 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 19 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 9 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 40 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 10 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 5 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 9 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 16 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 15 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 11 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 24 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 12 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 18 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 12 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 9 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %(name_1)s AND project.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 27 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 4 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 33 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 22 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 9 | +| service.extra AS service_extra | | +| FROM service | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 9 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 19 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 9 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 6 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 12 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Token issue request stats +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 39 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 448 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 14 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 179 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 25 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 269 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 24 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 263 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 1 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 6 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 20 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 40 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 15 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 5 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 9 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 16 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 15 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 11 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 24 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 12 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 18 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 27 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 9 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %(name_1)s AND project.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 27 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 4 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 33 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 22 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 9 | +| service.extra AS service_extra | | +| FROM service | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 9 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 19 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 10 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +User list request stats +~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 94 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 700 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 31 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 240 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 63 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 460 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 61 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 450 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 10 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 20 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 40 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 15 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 5 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 9 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 16 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 16 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 11 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user LEFT OUTER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id | | +| = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 11 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 24 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 12 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 18 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 13 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 9 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %(name_1)s AND project.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 27 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 4 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 33 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 22 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 9 | +| service.extra AS service_extra | | +| FROM service | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 18 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 19 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 6 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 7 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 9 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ diff --git a/doc/source/test_results/keystone/index.rst b/doc/source/test_results/keystone/index.rst new file mode 100644 index 0000000..4ac92da --- /dev/null +++ b/doc/source/test_results/keystone/index.rst @@ -0,0 +1,11 @@ +================================= +Keystone Performance Test Reports +================================= + +Reports +^^^^^^^ + +.. toctree:: + :maxdepth: 2 + + all-in-one/index diff --git a/doc/source/test_results/keystone/parse_json_res.py b/doc/source/test_results/keystone/parse_json_res.py new file mode 100644 index 0000000..c213952 --- /dev/null +++ b/doc/source/test_results/keystone/parse_json_res.py @@ -0,0 +1,305 @@ +import argparse +import collections +import copy +import itertools +import json +import numpy as np +import os +import prettytable + + +NODES_TEMPLATE = { + "DB queries": { + "total": 0, + "total_time_spent": 0, + "select1": 0, + "select1_time_spent": 0, + "real": 0, + "real_time_spent": 0, + "SELECT": { + "total": 0, + "INNER JOIN": 0, + "LEFT JOIN": 0, + "RIGHT JOIN": 0, + "FULL JOIN": 0, + "time_spent": collections.OrderedDict(), + }, + "INSERT": { + "total": 0, + "time_spent": collections.OrderedDict(), + }, + "UPDATE": { + "total": 0, + "time_spent": collections.OrderedDict(), + }, + "DELETE": { + "total": 0, + "time_spent": collections.OrderedDict(), + }, + "red_flag": { + "2joins": { + "total": 0, + "queries": [], + "time_spent": collections.OrderedDict(), + }, + "3+joins": { + "total": 0, + "queries": [], + "time_spent": collections.OrderedDict(), + } + } + }, + "Cached operations": {}, + "Cached time spent": collections.OrderedDict(), +} + +NODES = copy.deepcopy(NODES_TEMPLATE) +OUTLIER_QUERIES = {} + + +def define_node(node): + if node["info"]["project"] != "keystone": + return + + if node["info"]["name"] not in ["db", "cache"]: + return + + time_spent = node["info"]["finished"] - node["info"]["started"] + + if node["info"]["name"] == "db": + process_db_calls(node, time_spent) + elif node["info"]["name"] == "cache": + if not node["children"]: + process_cache_calls(node, time_spent) + else: + for child in node["children"]: + define_node(child) + + +def process_cache_calls(node, time_spent): + cache_info = node["info"]["meta.raw_payload.cache-start"][ + "info"]["fn_info"] + if not NODES["Cached operations"].get(cache_info): + NODES["Cached operations"][cache_info] = 0 + NODES["Cached operations"][cache_info] += 1 + if not NODES["Cached time spent"].get(time_spent): + NODES["Cached time spent"][time_spent] = [] + NODES["Cached time spent"][time_spent].append(cache_info) + + +def process_db_calls(node, time_spent): + NODES["DB queries"]["total"] += 1 + NODES["DB queries"]["total_time_spent"] += time_spent + statement = node[ + "info"]["meta.raw_payload.db-start"]["info"]["db"]["statement"] + if statement.startswith("SELECT 1"): + NODES["DB queries"]["select1"] += 1 + NODES["DB queries"]["select1_time_spent"] += time_spent + else: + NODES["DB queries"]["real"] += 1 + NODES["DB queries"]["real_time_spent"] += time_spent + + if statement.startswith("SELECT"): + process_selects(statement, time_spent) + elif statement.startswith("UPDATE"): + process_base_db_calls("UPDATE", statement, time_spent) + elif statement.startswith("INSERT"): + process_base_db_calls("INSERT", statement, time_spent) + elif statement.startswith("DELETE"): + process_base_db_calls("DELETE", statement, time_spent) + + +def process_base_db_calls(command, statement, time_spent): + NODES["DB queries"][command]["total"] += 1 + if not NODES["DB queries"][command]["time_spent"].get(time_spent): + NODES["DB queries"][command]["time_spent"][time_spent] = [] + NODES["DB queries"][command]["time_spent"][time_spent].append( + statement) + + +def process_selects(statement, time_spent): + process_base_db_calls("SELECT", statement, time_spent) + + ij = statement.count("INNER JOIN") + lj = statement.count("LEFT JOIN") + rj = statement.count("RIGHT JOIN") + fj = statement.count("FULL JOIN") + NODES["DB queries"]["SELECT"]["INNER JOIN"] += ij + NODES["DB queries"]["SELECT"]["LEFT JOIN"] += lj + NODES["DB queries"]["SELECT"]["RIGHT JOIN"] += rj + NODES["DB queries"]["SELECT"]["FULL JOIN"] += fj + + # raise red flags if too many JOINS met + if ij + lj + rj + fj == 2: + NODES["DB queries"]["red_flag"]["2joins"]["total"] += 1 + NODES["DB queries"]["red_flag"]["2joins"][ + "queries"].append(statement) + if not NODES["DB queries"]["red_flag"]["2joins"][ + "time_spent"].get(time_spent): + NODES["DB queries"]["red_flag"]["2joins"]["time_spent"][ + time_spent] = [] + NODES["DB queries"]["red_flag"]["2joins"]["time_spent"][ + time_spent].append(statement) + elif ij + lj + rj + fj >= 3: + NODES["DB queries"]["red_flag"]["3+joins"]["total"] += 1 + NODES["DB queries"]["red_flag"]["3+joins"][ + "queries"].append(statement) + if not NODES["DB queries"]["red_flag"]["3+joins"][ + "time_spent"].get(time_spent): + NODES["DB queries"]["red_flag"]["3+joins"]["time_spent"][ + time_spent] = [] + NODES["DB queries"]["red_flag"]["3+joins"]["time_spent"][ + time_spent].append(statement) + + +def define_nodes(data): + for child in data["children"]: + if not child["children"]: + define_node(child) + else: + define_nodes(child) + + +def sort_dicts(dictionary): + new_nodes = copy.deepcopy(dictionary) + for key in ["SELECT", "INSERT", "DELETE", "UPDATE"]: + new_nodes["DB queries"][key]["time_spent"] = \ + sum([k*len(v) for k, v + in dictionary["DB queries"][key]["time_spent"].iteritems()]) + for key in ["2joins", "3+joins"]: + new_nodes["DB queries"]["red_flag"][key]["time_spent"] = \ + sum([k*len(v) for k, v + in dictionary["DB queries"]["red_flag"][key][ + "time_spent"].iteritems()]) + new_nodes["Cached time spent"] = \ + sum([k*len(v) for k, v + in dictionary["Cached time spent"].iteritems()]) + return new_nodes + + +def detect_outliers(data, m=2.): + full_time_set = list(itertools.chain(*[[k] * len(v) for k, v + in data.iteritems()])) + dat = np.abs(full_time_set - np.median(full_time_set)) + mdev = np.median(dat) + sss = dat/mdev if mdev else 0. + if mdev: + for idx, val in enumerate((sss < m).tolist()): + if not val: + for query in data[full_time_set[idx]]: + OUTLIER_QUERIES[query] = full_time_set[idx] + + +def prepare_tables(nodes): + # prepare table with common information + common_info_table = prettytable.PrettyTable(["**Metric**", "**Value**"]) + common_info_table.align["**Metric**"] = "l" + common_info_table.align["**Value**"] = "l" + common_info_table.padding_width = 1 + common_info_table.max_width = 100 + common_info_table.header = True + common_info_table.hrules = prettytable.ALL + + common_info_table.add_row(["Total (*) Keystone DB queries count", + nodes["DB queries"]["total"]]) + common_info_table.add_row(["Total (*) Keystone DB queries time spent, ms", + nodes["DB queries"]["total_time_spent"]]) + common_info_table.add_row([ + "Infrastructure (SELECT 1) Keystone DB queries count", + nodes["DB queries"]["select1"]]) + common_info_table.add_row([ + "Infrastructure (SELECT 1) Keystone DB queries time spent, ms", + nodes["DB queries"]["select1_time_spent"]]) + common_info_table.add_row(["Real Keystone DB queries count", + nodes["DB queries"]["real"]]) + common_info_table.add_row(["Real Keystone DB queries time spent, ms", + nodes["DB queries"]["real_time_spent"]]) + + db_query_tmpl = "%s\n\n|" + + for key in ["SELECT", "INSERT", "DELETE", "UPDATE"]: + if nodes["DB queries"][key]["total"]: + common_info_table.add_row([ + "%s Keystone DB queries count" % key, + nodes["DB queries"][key]["total"]]) + common_info_table.add_row([ + "%s Keystone DB queries time spent, ms" % key, + nodes["DB queries"][key]["time_spent"]]) + detect_outliers(NODES["DB queries"][key]["time_spent"]) + + # prepare table with outliers information + outliers_table = prettytable.PrettyTable(["**DB query**", + "**Time spent, ms**"]) + outliers_table.align["**DB query**"] = "l" + outliers_table.align["**Time spent, ms**"] = "l" + outliers_table.max_width = 100 + outliers_table.header = True + outliers_table.hrules = prettytable.ALL + + for query in OUTLIER_QUERIES: + outliers_table.add_row([db_query_tmpl % query, OUTLIER_QUERIES[query]]) + + # prepare table with information about DB requests containing multiple + # JOIN statements inside + multi_join_queries = prettytable.PrettyTable(["**DB query**", + "**Time spent, ms**"]) + multi_join_queries.align["**DB query**"] = "l" + multi_join_queries.align["**Time spent, ms**"] = "l" + multi_join_queries.max_width = 100 + multi_join_queries.header = True + multi_join_queries.hrules = prettytable.ALL + + for key in ["2joins", "3+joins"]: + for ts in NODES["DB queries"]["red_flag"][key]["time_spent"]: + for query in NODES["DB queries"]["red_flag"][key][ + "time_spent"][ts]: + multi_join_queries.add_row([db_query_tmpl % query, ts]) + return common_info_table, multi_join_queries, outliers_table + + +def main(): + parser = argparse.ArgumentParser(description='Process JSON file with ' + 'OSprofiler output.') + parser.add_argument('path', type=str, + help='Path to the JSON file / directory with list of ' + 'JSON files with OSprofiler output') + + args = parser.parse_args() + global NODES + if os.path.isfile(args.path): + with open(args.path) as data_file: + data = json.load(data_file) + define_nodes(data) + nodes = sort_dicts(NODES) + common_info_table, multi_join_queries, outliers_table = \ + prepare_tables(nodes) + print(common_info_table) + print(outliers_table) + print(multi_join_queries) + elif os.path.isdir(args.path): + for item in os.listdir(args.path): + if item.endswith(".txt"): + with open(os.path.join(args.path, item)) as data_file: + data = json.load(data_file) + NODES = copy.deepcopy(NODES_TEMPLATE) + define_nodes(data) + nodes = sort_dicts(NODES) + common_info_table, multi_join_queries, outliers_table = \ + prepare_tables(nodes) + item_name = \ + item.split(".")[0].replace("_", " ").capitalize() + \ + " request stats" + print(item_name) + print(len(item_name) * "~" + "\n") + print("**%s**\n" % "Control plane request overlook") + print(common_info_table) + print("\n**%s**\n" % "Keystone DB queries outliers") + print(outliers_table) + print("\n**%s**\n" % "Keystone DB queries with multi " + "JOINs inside") + print(multi_join_queries) + print("\n") + + +if __name__ == "__main__": + main()