From 6a8001d72c1b2c178447f3f14a8f1d8e70f9db35 Mon Sep 17 00:00:00 2001 From: Dina Belova Date: Wed, 30 Mar 2016 15:13:32 +0300 Subject: [PATCH] Add Keystone Performance testing reports Change-Id: Ifa1b5c1a91558eda5c5fca0c4df0577464cc6047 --- doc/source/test_plans/index.rst | 1 + doc/source/test_plans/keystone/plan.rst | 418 +++++ .../keystone/profiling_workflow.png | Bin 0 -> 44870 bytes doc/source/test_results/index.rst | 1 + .../keystone/all-in-one/index.rst | 59 + .../all-in-one/liberty_fernet_cache_off.rst | 773 ++++++++++ .../all-in-one/liberty_fernet_cache_on.rst | 563 +++++++ .../all-in-one/liberty_uuid_cache_off.rst | 512 ++++++ .../all-in-one/liberty_uuid_cache_on.rst | 426 +++++ .../all-in-one/mitaka_fernet_cache_off.rst | 1366 ++++++++++++++++ .../all-in-one/mitaka_fernet_cache_on.rst | 1269 +++++++++++++++ .../all-in-one/mitaka_uuid_cache_off.rst | 1362 ++++++++++++++++ .../all-in-one/mitaka_uuid_cache_on.rst | 1368 +++++++++++++++++ doc/source/test_results/keystone/index.rst | 11 + .../test_results/keystone/parse_json_res.py | 305 ++++ 15 files changed, 8434 insertions(+) create mode 100644 doc/source/test_plans/keystone/plan.rst create mode 100644 doc/source/test_plans/keystone/profiling_workflow.png create mode 100644 doc/source/test_results/keystone/all-in-one/index.rst create mode 100644 doc/source/test_results/keystone/all-in-one/liberty_fernet_cache_off.rst create mode 100644 doc/source/test_results/keystone/all-in-one/liberty_fernet_cache_on.rst create mode 100644 doc/source/test_results/keystone/all-in-one/liberty_uuid_cache_off.rst create mode 100644 doc/source/test_results/keystone/all-in-one/liberty_uuid_cache_on.rst create mode 100644 doc/source/test_results/keystone/all-in-one/mitaka_fernet_cache_off.rst create mode 100644 doc/source/test_results/keystone/all-in-one/mitaka_fernet_cache_on.rst create mode 100644 doc/source/test_results/keystone/all-in-one/mitaka_uuid_cache_off.rst create mode 100644 doc/source/test_results/keystone/all-in-one/mitaka_uuid_cache_on.rst create mode 100644 doc/source/test_results/keystone/index.rst create mode 100644 doc/source/test_results/keystone/parse_json_res.py diff --git a/doc/source/test_plans/index.rst b/doc/source/test_plans/index.rst index dc72e79..b66b07b 100644 --- a/doc/source/test_plans/index.rst +++ b/doc/source/test_plans/index.rst @@ -15,3 +15,4 @@ Test Plans provisioning/plan tenant_networking/plan container_repositories/plan + keystone/plan diff --git a/doc/source/test_plans/keystone/plan.rst b/doc/source/test_plans/keystone/plan.rst new file mode 100644 index 0000000..8b6a7a9 --- /dev/null +++ b/doc/source/test_plans/keystone/plan.rst @@ -0,0 +1,418 @@ +============================ +Keystone Performance testing +============================ + +:status: **ready** +:version: 1 + +:Abstract: + + This document describes a test plan for measuring OpenStack Identity service + (Keystone) performance, including primary analysis of database and cache + operations effectiveness. This test plan assumes to use OSprofiler library + for cross-project OpenStack requests profiling. + +Test Plan +========= + +Keystone is an OpenStack project that provides Identity, Token, Catalog and +Policy services for use specifically by projects in the OpenStack family. +It implements OpenStack’s Identity API and is widely used by almost all +OpenStack services, therefore its performance is a key. To evaluate it this +test plan proposes to use OSprofiler. + +OSprofiler is an Oslo library that provides Python wrappers to trace +operations on the Python level. It provides several ways to wrap +separated methods, all methods inside one specific class and all methods +encapsulated under Python classes having common ancestor. + +For every profiled method, notifications are sent about operation start time +and end time, including information about parent operation. Right now these +notifications are stored as OpenStack Telemetry (Ceilometer) events, so on +high level OpenStack profiling process can be presented as follows: + +.. image:: profiling_workflow.png + :width: 650px + +Test Environment +---------------- + +This section describes the setup for Keystone testing. It can be either +a single (all-in-one) or a multi-node installation. + +A single-node setup requires just one node to be up and running. It has +both compute and controller roles and all OpenStack services run on this node. +This setup does not support test cases, related to the performance +measurements, combined with the components HA testing. + +A basic multi-node setup with Keystone comprises 4 physical nodes: + * One node for a compute node. This node simulates activity which is + typical for OpenStack compute components. + * Three nodes for a controller nodes. These node simulate activity which + is typical for OpenStack control plane services, including running three + MySQL instances managed by Galera cluster and memcached cluster for + Keystone caching. + +Preparation +^^^^^^^^^^^ + +**Common preparation steps** + +No matter if you're running single node or multi node setup, you will require +several steps to be completed before running any profiling tasks: + * Ensure OSprofiler library_ is installed to all environment nodes to present + most full information in the trace. + * OSprofiler requires persistent profiling events storage to be presented in + the environment. For now, the only supported variant is to use OpenStack + Telemetry (Ceilometer) project, that will consume profiling events via + message queue notifications from the affected OpenStack services. + * OSprofiler integration to the OpenStack projects is ongoing initiative, + so please ensure that all OpenStack services you would like to profile + support it in your environment. Here is a project to release mapping that + can be useful: + * Cinder OSprofiler support - OpenStack Juno release + * Glance OSprofiler support - OpenStack Juno release + * Heat OSprofiler support - OpenStack Juno release + * Trove OSprofiler support - OpenStack Juno release + * Nova_ OSprofiler support - [planned] OpenStack Newton release + * Neutron_ OSprofiler support - [planned] OpenStack Newton release + * Keystone_ OSprofiler support - [planned] OpenStack Newton release + At the time of this document composing Nova, Neutron and Keystone changes + need to be applied manually to trace these projects usage. + * Please make sure that all OpenStack services are properly configured to + allow cross-project request profiling. In case of single node installation + using DevStack this will be automatically managed by OSprofiler DevStack + plugin, but for multi node environment this needs to be tracked separately. + Please pay attention to the appropriate subsection under `Multi node + installation` section. + * Test cases described in this document suppose to collect information for + comparison analysis of Keystone database and cache operations + effectiveness, that requires Keystone reconfiguration depending on if + caching mechanism is used at the moment or not. This is tracked via + specialized `keystone.conf` file section:: + + [cache] + enabled = True|False + backend = oslo_cache.memcache_pool + memcache_servers = :[,:] + expiration_time = 600 + +**Single node installation** + +For single node installation the one can use DevStack_ tool that is targeted +at developers and CI systems to use upstream code. It makes many choices that +are not appropriate for production systems, but for the all-in-one purposes +this can fit ok. + +At the time of document writing, DevStack's `local.conf` should be looking like +as following to have and all-in-one OpenStack installation with Neutron enabled +and OSprofiler installation included with all needed fixes to the OpenStack +services:: + + [[local|localrc]] + ADMIN_PASSWORD=password + DATABASE_PASSWORD=$ADMIN_PASSWORD + RABBIT_PASSWORD=$ADMIN_PASSWORD + SERVICE_PASSWORD=$ADMIN_PASSWORD + + LIBS_FROM_GIT=osprofiler,python-openstackclient + + NOVA_REPO=https://review.openstack.org/p/openstack/nova + NOVA_BRANCH=refs/changes/03/254703/39 + + KEYSTONE_REPO=https://review.openstack.org/p/openstack/keystone + KEYSTONE_BRANCH=refs/changes/35/294535/2 + + NEUTRON_REPO=https://review.openstack.org/p/openstack/neutron + NEUTRON_BRANCH=refs/changes/51/273951/12 + + disable_service n-net horizon + enable_service q-svc q-dhcp q-meta q-agt q-l3 neutron + + enable_plugin ceilometer https://git.openstack.org/openstack/ceilometer.git + enable_plugin osprofiler https://github.com/openstack/osprofiler.git + +To add Fernet tokens usage (as for the time of document writing, default token +format for DevStack is still UUID), the next line needs to be added +explicitly:: + + KEYSTONE_TOKEN_FORMAT=fernet + + +Please make sure to have identical cache configuration for Keystone authtoken +middleware. For example, cache might be external (memcached) and appropriate +configuration section will look like this in this case:: + + [keystone_authtoken] + memcache_servers = :[,:] + signing_dir = + cafile = + auth_uri = + project_domain_id = + project_name = + user_domain_id = + password = + username = + auth_url = + auth_plugin = + +Sadly there is no simple way to setup specific patches to be applyed on the +python libraries used (OpenStack clients, OSProfiler, etc.) so they require +manual patching in this case: + +* `OSprofiler changes`_ + +**Multi node installation** + +Multi node environment installation depends much on the chosen set of OpenStack +deployment tools. Whatever instrument will be used, please consider to ensure +the following patches to be applied against main OpenStack services to be +profiled (Nova, Neutron, Keystone) and the appropriate libraries: + +* `OSprofiler changes`_ +* Nova_ OSprofiler integration +* Neutron_ OSprofiler integration +* Keystone_ OSprofiler integration + +*OpenStack services configuration* + +Several OpenStack configuration files need to be modified to enable appropriate +OSprofiler workability. + +First of all, the one needs to enable `Ceilometer` profiling events storage via +adding the following lines to the `event_definitions.yaml` file, declaratively +announcing the wish to consume them:: + + - event_type: profiler.* + traits: + project: + fields: payload.project + service: + fields: payload.service + name: + fields: payload.name + base_id: + fields: payload.base_id + trace_id: + fields: payload.trace_id + parent_id: + fields: payload.parent_id + timestamp: + fields: payload.timestamp + host: + fields: payload.info.host + path: + fields: payload.info.request.path + query: + fields: payload.info.request.query + method: + fields: payload.info.request.method + scheme: + fields: payload.info.request.scheme + db.statement: + fields: payload.info.db.statement + db.params: + fields: payload.info.db.params + +Also, for extended tracing information providing it's useful (although, not +mandatory if you need only high-level traces) to add the following lines to the +`ceilometer.conf` configuration file:: + + [event] + store_raw=info + +.. note:: Please pay attention to the fact, that the configuration parameter + defined above will store raw events for **all** `info` level event + notifications coming to the Ceilometer queue. Please make sure either + to turn off not needed events in `event_definitions.yaml` or save + enough place for the Ceilometer storage backend. + +Also for every project you would like to trace it's required to enable its +profiling via service configuration files and add the following section:: + + [profiler] + enabled = True + trace_sqlalchemy = True + hmac_keys = SECRET_KEY + +In this test plan it's supposed to turn profiling on for Cinder, Glance, Nova, +Neutron and Keystone. + +.. _library: https://pypi.python.org/pypi/osprofiler +.. _Nova: https://review.openstack.org/#/q/status:open+branch:master+topic:bp/osprofiler-support-in-nova +.. _Neutron: https://review.openstack.org/#/q/status:open++branch:master+topic:bug/1335640 +.. _Keystone: https://review.openstack.org/#/q/status:open+branch:master+topic:osprofiler-support-in-keystone +.. _DevStack: http://devstack.org +.. _OSprofiler changes: https://review.openstack.org/#/c/294516/ + +Environment description +^^^^^^^^^^^^^^^^^^^^^^^ + +The environment description includes hardware specification of servers, +network parameters, operation system and OpenStack deployment characteristics. + +Hardware +~~~~~~~~ + +This section contains list of all types of hardware nodes. + ++-----------+-------+----------------------------------------------------+ +| Parameter | Value | Comments | ++-----------+-------+----------------------------------------------------+ +| model | | e.g. Supermicro X9SRD-F | ++-----------+-------+----------------------------------------------------+ +| CPU | | e.g. 6 x Intel(R) Xeon(R) CPU E5-2620 v2 @ 2.10GHz | ++-----------+-------+----------------------------------------------------+ + +Network +~~~~~~~ + +This section contains list of interfaces and network parameters. +For complicated cases this section may include topology diagram and switch +parameters. + ++------------------+-------+-------------------------+ +| Parameter | Value | Comments | ++------------------+-------+-------------------------+ +| card model | | e.g. Intel | ++------------------+-------+-------------------------+ +| driver | | e.g. ixgbe | ++------------------+-------+-------------------------+ +| speed | | e.g. 10G or 1G | ++------------------+-------+-------------------------+ + +Software +~~~~~~~~ + +This section describes installed software. + ++-----------------+--------+---------------------------+ +| Parameter | Value | Comments | ++-----------------+--------+---------------------------+ +| OS | | e.g. Ubuntu 14.04.3 | ++-----------------+--------+---------------------------+ +| Keystone DB | | e.g. MySQL 5.6 | ++-----------------+--------+---------------------------+ +| Keystone Cache | on/off | e.g. memcached v1.4.25 | ++-----------------+--------+---------------------------+ +| HA mode | | e.g. Cluster | ++-----------------+--------+---------------------------+ + +Test Case 1: Keystone DB / cache operations analysis +---------------------------------------------------- + +Description +^^^^^^^^^^^ + +This test records all HTTP, RPC and DB calls happening during selected list of +OpenStack control plane operations, including Keystone operations and their +duration via OSprofiler. Human-readable report would be automatically generated +to evaluate overall number of DB / cache calls, as well as raw information +ported to the JSON format. + +Let's focus on the following control plane operations: + +* Keystone token get (token issue) +* Keystone user list +* Keystone endpoint list +* Keystone service list +* Nova instance boot (server create) + +OSprofiler adds an opportunity to call CLI command, generating report on the +profiled control plane operation in one of the chosen formats - either JSON +or HTML. + +To initiate OpenStack request tracing `--profile ` option needs to +be added to the CLI command, triggering this specific action. This key needs +to present one of the secret keys defined in .conf configuration file +with `hmac_keys` option under the `[profiler]` configuration section. In case +if all OpenStack projects have shared HMAC_KEY defined in their configuration +files, it will be possible to generate a report, containing tracing points from +all services taking part in the request processing. + +To initiate VM creation tracing the following command should be used from the +CLI:: + + openstack --profile SECRET_KEY server create --image --flavor + +Without `--profile SECRET_KEY` option trace generation won't be triggered even +if it's enabled in the OpenStack services configuration files. + +At the end of output there will be message printed with , and to +plot nice HTML graphs the following command should be used:: + + osprofiler trace show --html --out result.html + + All other chosen control plane operations request can be traced via similar + approach. + +Parameters +^^^^^^^^^^ + +=========================== ==================================================== +Parameter name Value +=========================== ==================================================== +OpenStack release Liberty, Mitaka +Cache on, off +Token type UUID, fernet +Environment characteristics Single node, multi node (clusterized DB / memcached) +=========================== ==================================================== + +List of performance metrics +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Test case result is presented as a weighted tree structure with operations +as nodes and time spent on them as node weights for every control plane +operation under the test. This information is automatically gathered in +Ceilometer and can be gracefully transformed to the human-friendly report via +OSprofiler. + +======== ============== ================= ========================================= +Priority Value Measurement Units Description +======== ============== ================= ========================================= +1 Operation time milliseconds Time spent on every HTTP/RPC/DB operation +======== ============== ================= ========================================= + +.. note:: Please keep in mind that OSprofiler uses python code and libraries + to listen on specific events to happen - in common case that will be + `before method starts` and `after method starts` via Python + decorators usage, therefore time it collects includes time spent on + Python operations inside. For DB calls tracing OSprofiler uses + `before_cursor_execute` and `after_cursor_execute` events defined in + SQLAlchemy library. + +Test Case 2: Keystone DB / cache operations analysis (HA version) +----------------------------------------------------------------- + +Description +^^^^^^^^^^^ + +This test case should provide almost the same analysis as the previous one, +the difference is in adding failover testing component to the research. First +test run assumes to be the same as for previous case, the second one should +happen after turning off one of the distributed components used by Keystone, +e.g. stop one of memcached instances and run the same control plane operations +tracing. + +Parameters +^^^^^^^^^^ + +=========================== ==================================================== +Parameter name Value +=========================== ==================================================== +OpenStack release Liberty, Mitaka +Cache on, off +Token type UUID, fernet +Environment characteristics Single node, multi node (clusterized DB / memcached) +Memcached cluster status 3 nodes, 2 nodes, 1 node +Galera cluster status 3 nodes, 2 nodes +=========================== ==================================================== + +List of performance metrics +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +======== ============== ================= ========================================= +Priority Value Measurement Units Description +======== ============== ================= ========================================= +1 Operation time milliseconds Time spent on every HTTP/RPC/DB operation +======== ============== ================= ========================================= diff --git a/doc/source/test_plans/keystone/profiling_workflow.png b/doc/source/test_plans/keystone/profiling_workflow.png new file mode 100644 index 0000000000000000000000000000000000000000..57fdf61119bc1e773e5a12f494d3864e81f36a08 GIT binary patch literal 44870 zcmeFY_gm9T*FGAM0D_206A)3{Al*Xmpdu<=dLR^OLg>9W+eRrCdKaYkk^rFv5s(g2 zLI@#|sz3-ebU2^*?ESp&`JU@s=N~xy#P!LVnKf(Hthv`cYohgZG?*B;7(gHp)8j`E zo`OJUP#_TXFg-2s#mb+v4g`t-J$|5O7%;VpW7aCiLrMGNNmlnaJJ@e%>s906-`=r@ z|9Dv}X`k@&VS$|Vq?$de#L;Xt|BX+CN6oV8`D|<&0^Vgy%f-($)4j#Grg(|@bc1JS zY#y$ilkacoKliZf9cQ!Jtrqonv@0W7#txx_Clza*9kn)RxF^GBD(x1oki*Ee*^b1P ztd`cHgQY8gUjP3eV1Wp8<2i$_LWgXmsPCcV^WPP^EJ*^PY1&`ZvyEAt`U-J;1@MGj zdMCMslmv&|xvMqazvBn1I|rLSRQ|DFLh@_Dzlrb&_vu0>MtTlW>*=Z-FMQ#gAQ3x< z)%VyRwhUHb5TM)oG9%carJ2fCfJivr&A(y*A)MTnZ`%?49=ByJ!p~e6e=(bO^%!zC ziwfvAC*a4Wi&Ug^06{GN079Ma_-8d^gwt0ogA zX_gf9VEVcOrQv3QRIn8JL&czK7-urnsv*7%_zLPZsO>tqO4oaxibVd4UyY= z^s$L}J`kl*uMDVlX|%boBADS?<^~vU(AFAJlCnf z!uQ6wOJmxjcjBWBOwOG|9}NGydom|Vho|p#2Mk^0tRg7ds|=guqweAINKGy)>juf` zW{}7l$lKrYvCn!NTtwCyzzyc)meB`pkU8X#y?J=?q*p6w0Wfq}PGuq_0xK$LY~nOf z5^VjL>a75ps}}UIyayig*CTyLe4a9+&v>78N8$ZsR}6I9sW;0#|5@KYHN~uYPgI7v zZk36b+-JXB$_VeWSN6uZjft>d5t$tMC<$GgmoF<4Me66yoWYR;bsJV?w~QD)#&b)P zaDj|IaP2p`JO7-%}v+yco9NTIU++EP{ zZtX1TE8u!(I8-;JqLe)SrEhS!EJF5vygEG zE``!^_A)Io>bU`{4%pWK)}Dxj>>@4D5b#%T_V?xHRRv$%5JRh{3gi&_EJrcz2TsF3 zR9P>)9&;e^6e9Vu%(Z(@=EkzTw4UHbJmWEeTIx&n&z71mE@(DaEVL>iY_LMSypv(C zp%whf8R1L|ueZQ^p};0OQcbQv3cg>hi&)cPm5nn;$?$;VN&|oBus#wP4dZKQc9z!) zj2}p_O;fs=!y7}rsP&4jCV_j9BWD^2I?R%wUMBR}f7}8@>6PME5xmbMygK%&|B+Dz z4Ay6$DD%hV{`t-LO8Mi~MI9a4Le4o2@dn;@hGEWf$rnZ_jVtHV)i?YM+UKnGNZ4>AF$Eyk%21A_PlXiz?ware2fF%Iu z23%sv9Ku@V)Vy{t5-XxQ?SHfTENO4-itDF|w6t3w71 zglkRdb{#mHgDK)B^q>Jd%+q+oGsd3<>k6s5;_tXt`Fh8XA7Kt> z?fZ-RfNliunfCaKZ7V1M7k6j>fX&~_?B;QJ(hALgS2JIPCWe_|(ztzbRB&CZ;2kZ> ziOMBJi$$G2!f{A$5$R2B4Mw~Pv{cXl^^-&>zJ4Yl-Ak{}ceYfrII-*7L`E$xR{^t} zI;2ibD3_onRP%vR8p(asPpH6=v+8L~#ca^OGB8B}*gFY{>9 z1jF^$Cmr@{-%mcI@-&E~@;E|Xm@GN;eI9C@JkNA!;r#9b$Z@rqoh1@+l8x{&@H(Ec z3_faAz}%b3nB=9p{l^4F@4dXae-B5!tuy>ZegkerCyNq#4L5vqJKgAi9gW)u;HdRu zvoqrnj1n@urt6JvSgi~e&>Kuj{i@XUfaWDH*xAadFnF7SP~@gPwKZqlpeD8w6Wsp* z`!c$o?2_J~x!3W{o15Jb&ktJMW&4khYu}MHz5OYP0oQtxvGr7S+h6%uGkImgd;lF? zb)HlQ_TCi>(DDmhi$RREDMc@(i-F2DSA)MN#&Z`ATkqHP_U3Vg6Z_1#0;Xh#>2$6oE zD-fxsEtfnqgZdIhkfk3c+67{ewc^)^aaQY0+5!v8P&^#^m8vSre83P%)QNXALKSj8 z>fWM34UjET`txrtC>@CYeV*mVH_h;0$C6FIW0X{ub#w$-x~7*m&IX5?XL;+(%-_N4 z6yxo?Wt+a8Z105z>uMtn!ga1nA-BwXOs0FJ9*znudit%t5UPy7d64?(Y~7Z2kVML6 z!;JU$K~m&`yBH+`Q*?;{$u@m?_`LIqu3{xBV0}mGsguCpKnMT1nqJ%_6O{ar)p(^<)1arzFSd^t1a(N^&}iGtc)A#|GVT2s7`6 zJ7ye!%l>COSOtSbz1d}7<&@*8t0xjb4HMKF@Q?|9@~$X@V7+U)=@l1eES;kw{=;6= z;Ec?g+m#92l>wG!x;b)~=LNh&ymrotk;iQNG%959Apfd_U!?g;5 zJjZ@cX8vr129Xqq)p*nwP5ip~I4J&(e~?_tg0Fb}45U6NS`Tb5#6#j(`QE>s?yDblwtg5FHb!xN5qE`q%O9VPo*1Wa@*c zNQV}6{e9Dt*C(A$p-=SZlU=iQalb1Xj@j#+!nsbTQt2;eL#E){lKfSfg114I1K!P! zzRaz*l{|8`>BCG-hf!4PCAjC06bd#NX;Dyd8E)UH6C`l7+@c)YAR5%oVz`vt4WGN9 z^Hg~CiPnLJ>2C&3VlB??hKxi@zIDTO<8*qiC6+)X165GjZfHQXVo5#WxXkSbBSISo zK0~$`z5QQTE_HY*dSZQ4l|Pr~5&Gz;BAOW2^n!$NI=i4XJ&+!!! zW$Q?XYuk4poe@W?%d)gAy;KMnBRD9S@W&dt)uEkZacBI478i_@xoEWHG!Oh#PkJpDYEl!F%qE3##4Mf~72nramNR{O4aK-ve&&8IJw4?DV((+> zalT9SsMmA2M{;4lUuQG#n6qRDoN$2~Zq;lps+q(DDe@nZw0-lxr0n~&AB z|Ds1#sZ&)MQ$4?{;8f1!)(rh7FpOdTW38gdH3L{a#46Yz>Nc@nA}AEvdtxrSe!kl* zeJRD%vg~z|G z4=#-^m=A1_8?hqXdFm$npTuA?75@Ut>%G-I-Umf(SYl zwjt1a9PZ$6%%V&i9{_OpZ#vST!)2A>S(vO%{XahU9{@^88~~8|-wywQfq!=Ie*osc zeSidIfD`1uN49efNk<&7&i7$Z_s~L^k|;3)*u3=}fb}H4Pi+nS_viC9N^%Lb`}d>pCNRdo9}VBgXUBb~y#NBhT|%~RwtNq5j`bPk zfe%c7Bm`CbLm?Z*uggG#`4f1tY-A6^e?$v|l?M(DHH|@mTuPsq7n1E@)G4ubl5 zl}R0{%)QXpmSnXaws)r$+9UF5>U~Y=;RUpRWVnHzBq(J&SRkZgtS={WZ7(|v-Onb6 zruky7UIbBqh-%_Ue?8>Do}c0Tw*L*Y7k869F=oT#3#X_;Fk@8ex$_J25u(6g4a!X7 z3fG6rz#Yp1^2AY|;PZk;+Ux{n56oG-AMrG(EN)dT_1PsrG=h68~uLBgEA32_#9r*}Pth7p(vyHJ(h?yo~}to+f~fRnEuOt?p=_`CmPNULGrB)YA@X)yO9> zW84`{20h8fi4^c{XKjZg{vTtQ-J&6C=+50q56r|Z_^SAUr2SC+M`#gs8)-CIbnjRs znK;M{sC+4ZXC{aKy#@s7$2nN zML2(_7KXFmGPN#KUH*Q8tIK(!wjGM~`)*kp>SG$w*)ab#3Zs>93iX?_8KX7^S1ixA z+63};36!L7`ms(g7j!+^=Da|lPTo@4+qQ2i031H2cO2r8Uop`lj)ogYu4jPlR@lpg ziqnY_ho^B$yirSLnk@QUb&?mrFR#Nf!QI6vEMsF|R#kL}Lfv1+czAYGGP@0Z!bJQA zx%FGWu@G0Ww%mx!&5cN+Csy`6 zz%X`zJ}!8##K0ax4s@3k+x)tA{!jjM#G672b=K9@ZEwv}d)(m4y(YcQJ`!8u@d?(R zxyt#WIgl;rp-a=KlEzxocDk15vhQyt>3hAbci{P}ZMPOueo*he8QJe_ahz40eA07< zc)#Q7J@-`OC`X{%Z|x+7`D zZO_WThg-7Y`MP}#pOhSFpSlT%YP8obIuHL9kgnefYr0+i88V9H*#fd5kRhTwp7*qo z+B*9rVjA?+DJERewmAwrAG2}20pi(scJg5!p(IunVdrKjI6SWRY1q{QSH(fxVw4e{ zO`W}I;a0Hc(0v=oRw6d%U(6(jfI?iT)>TFrVuaClY84QVap6+-2eIYx=yS|e7h$hW z{Bo{LFfglmrG(p{TbZ%!lBH~c-&+Be1u&eTz49Kejv({J)MSS=VLsW+f#mkulC!Dg{b^oNHmSV-$Yx@R!L-74k8R)@#SZI>`wT zFEuqaEwX|@4wwNm8kjuyqTzp1ZtcxZA9G%ekPT&g}R`~sbfZuXbD@Q3`aN{%4YU>O_wrBvd*RhK=QW9UzSh~4RfdMYW%jps?SS(V#p&`p z?{ucnj4`77__Xjl=k(oyhco}f@kS`NyQAm2<7L>6UK?BSf|&PCRsQnVqo!KV&11VP z__C|l{m)^V!2=rF2L+f)sHg;O7Y$Go&Y$keH@>(;AHlmPse9pE2KSLOV*AdY16DBv zh5dB~_d+*x*}(mW=!EAh z4IwD}HmKKyZFozI%ib6NnRJGb(`TO?&`7H3YXOZ{U~V!QjXbq+4Wu9ske!oL@#FGl zXE6CRPWZBpVq1|^WRFn7BakRuId<<0rq_`rF)Ro2{8#78HLpk`b(O(eYQy0}9$N(uu#S z#dD-n#uV4d~zJR ziF@xN^z=^;=aqkzkDMEVEI$e_6{(B*Iq%2F$hZSg^w>`@!Ji=6;oZ7!d&gMgs+WKsv=N2y6vhIpr4eG?#C2kfcJz%LtR)sCJ>Wy@(NcLl zE!CY9#|9N}4(w!RSAp{4%GA#m`C}^jk@~f6c!M3}i0tm7xiFCOf)elJGT^Hpm|fEI zuI*+;%_>r@EAfKjKX?mvfMg4j;NpxCR22;oGpMu2-zvA~aYNQ6+E5H6$SYmC6>U9{q$;m?<9S7`C#o*)^G6^X zg0r*Y;n&om1-&~uc@kcH?%xhSonoo?(=)XzBX5VG4_qDgc5(Xl=s@7YMD%{ z%nd$Mh|!$CUf|sZw#y=9!Yo*9_Pdvc0*P-SQ9bI00icZdB$H);ru-SqKDmmicKVph z#uzsq3m=N&NwO>c7BR&O8})#2~SvURj62 zRxwUS<6tq5%qoI$`|3kh!qrJU^0BNvc3FN(>yI(imXWJ3&`i8E@r8#OwsLSlsNcyTE)Jz1%RR>BhX$#N|0~cyB-9(=(uQF9 zM)l9ZJ*B0k99$}neL_6*+y>|V9Rxi{SJ(22rj1=t*UImn$mx`kZ?!2TK05i?`S_6w>_2K8`F;=paq(L zq*#^TNRx9syx4X;ZI9Q*WwW`IDr|G6`)+Lf+c2?oB5F#ty>g`?FCS=gSC zBYSyQNNx?WcGvxjR7m)8Vc0T@)vmAAZvHpd zA{9U=Qsx=KH+*@@GFyPn5n#9~HXuj_>7~`iH_ETvP|i#^e+;`R*oi5rfEY;h9{o{9 ztR1!FARXFlw{iwsXtE7p=SK7L@|buJhTnvpVq1yWlLMp5pm99rWNixJ@7ktTPlWBj zD?^FfB9+aXae$Iv6VaW&q6XN4hc@bB(o$0PM%@nz+=E=O{Wmv%%TB$SQCV9q z-wnIaQlS*&CSRRhEm)aQui)3(5@H0wHVX>)K|Qb}p%T8=!F%$x)N-ff*28W;>!k~W>TCr9jE33BQI>#A>$zy=GDbhg zS<^UN&)TFyQg|$av?mAtc4mnNb-nEuwYK`zKlA|W9e_S#`6iI9N&DR7Yy{m?GEa=O zq~uXJep3v;x%!I3a_?SU{1!Ch;rTHl2YEPTrDUKAjYi@yn)rwpLogZ^fcd7Pef1q+4N7{?KDi*yR zi{FdIr+=@vTYzEXOv*bo#L=6gTy%;qr}#%SC>RaOaK4B74?pFm405X?l#4K&-_0X< zV5l3pG&#;jthnGmi+CIlovaR>ERllohsvi+_;sf2;QfVUHVE~+RUWnzcs$+^%k0e_L{f>W1O<3oO-NL<# z)0bg-y?V-}5^}jEyURRYo$3U*K+Y(|FF;aZ`z25R*|{cp#xsq-NCN&fF!xHZ_Y>#w zRV|u24R%w%FU@}6p0@5X9L`jhmL48$Z*Nj2JJB9v>f1D_t8RISv;7 zPM7s77b~+x_Ya4B{C5L9b&BR=G_Hq_>3@j&tq>NjAjA3o2SW@)*>^_^zhkG^78Cza zCMHu4jN1{%9p}o@_^YEHubq@FDM&D)#HPG87{J!M@7T&9=q06Wq}!%N8C!rZl~T5- za1LK(^)}aNrRQo|vBlgVJi)YC(&V!=QE?U(IFVUP_CH%RXG*y~0GSl<$4XA13(KFy zj~HpLYWs%j$W@wUr{pj!SblK##$blVH?f8O|4{m!EwEaszZvs4FXBjdjIQRDGE!cwN_^OPBHTE7g z;a;#ok#8OOW4A&0!{I>78MFoAa;4YR+IkOB6Coxx6}JOhrWAyP#a0WUcI(;H0ix=6PI&BEnr#k`96#Dn~c14A|d;ZwR%! zwSpQ3sMl>ZA(vc4t7DOlrRE4oqD-!{HOMlar{TGiWF9K)`qV>c3rW4Z8MfpJ zY$D&A$sVJJ75s6Yk{efpv>oekUpWJdhzOR$^uNj8K3&znlLo zfng0SwINr#NS-namk)`dS?c>pmuwT4{y3E@Ly%r+N~KB8eDSkMk5rBlqwU==((cKE zqw-ej5al$T>?jd+V_P8@o0zPOWlqs$`M;}b>NgVBl`cldinh^E6JQ?QGu}lJuH2%N z7*C8;X;UcDy)ypAMBT(#9Z1$T(s9x%zOP~<^a}XpU{){}xEse_#fycp`RqW35w`6R zdN&xc8p_DJGvyRa^zmsf9XS;C#dwRci@Hso%-OE$hPctO7sVIQ{H(632+Ah9i}r@M z>@Lc6*Eo?vyJtwU7vJlewl0TuiTx_C1qJjC1`ku?FI3e(tmLifW_L|*|K)vEt9eKw zxS~6Y5tfWL4xm6mHU~a^DzU!=%XicWDN*f}`)QT?>7=7UQixFX;}HSh4bn!{-{lo1 z3=y5#{->^fowQWzZ0Dwe$*fhx2C!mJ#Sa*J0#p*z>Uap#VfSPVSh>`F8NXfvcRhaz zophlYKEJPETBS-XvrS4(*e$B|HhUD}*2l3IGlXniT%kT5rY5}pR$X>8>cm6*)pP>v zF%80(UGdgZqfQrA2~Loi@Qx(0P}t1|&sgJP2)x&#)ab5^*Z>4vJR@x6d{NK8vq5tP zM>?J%9iudFwQP6pw(hos{}>*=ZJQ_w*V58RENgi62r57_<%f-Dx)>EzKY<0f2~;#c zoC{lVX$r);O)Wrp#yYFeGdDBLEvVdT-eOByh7v_|mzM`>9u=*F0y--gQy-*h($v4A zMVYfV^Y36+npnD19=p`TNIWYwu1%}%P+qb@+`i@Pq?<-n(b*KV) z4Y=S57H@&=RJr2)1{=RTrbT?#P$kJ>U#a?E`*16B%}4VNWrl*B2F7aV*pdYpSslfE z&DPD`)!dtu3K%TXlcK;j$&Sc`;j@3dw=N6%)eQEPl;rI*Sxv z-RaT#`_Ac4ccB8H8F)7|>zZFx1So)<0SefOu~inn22xhO0hI== z)mcmX{JbiD8>Bq=$qERFlarY5v|Z#qPPE(4T$?&DH={_evp z3>b+(keTtPC%dP&81`m1lj@JsP1+8|d9{jD94;mPu(Q9jMs8p;njzzQ_zkn_FNjpt z?SyT?4LiFPpR4%{Uo~nCp6=QBp8Cuv>!@1$dE?DiW5K+)At^Ml3P5pD>)tP43N&d^ z-ZB?J?$FH?b(y8bt>)H^k5ubheE^AF^BQz~0)Un*>NA8e1>Vrtwn>$Fft$#36sSQ( zcjh)NMcco8Zy0mT-7hsAUpO1Fv$xH7t9iZKJ!Fpi^vCI;ZbwH)Td`*w85ylfgGzny z9?Pc0VUlHRb$WnSA(6}ObVcuSR;&8sGBY0g-rfjHE!r$!x@xHbD!YSHrnfYaaczbC z(U`c~%sHM6wSn!k0~g-Uiirv6H-wTrdaCvNO|9b=vniQ8c<0D$>^J@`D=P9RH8sIC zdB#5ETc!(qz+e1&8|-OCxXryv%R>dZXcMn8k7NaB~IN^Mrxa=Dkn2X+v8V&bIB z9Dguzx+0Oi-;duTxFHJ)9K=ej*%d%KI&W(aHv4X!+Vp-gDak3eFTB2xGQ|YOOk)IWflq7Od$vRkG>7)Gq4`UY>^aZN z@^j6zwmJw49jsKb#72PYMd1FC)AVT9^U3fD>Z-eL#)~d{66=M2_xx#(%icG;kEExn z|GL@ErAvc?``OwkEy%)3zfe&A-LWf1#)ge)Z-w=C>%I*g9Uaa4hwu8Ib|}nj#Hoy2 z(lFka7rh2r`nK9y&Y@x5J9iTO?<@pJmtwf5KnjI;rJ2vY`2#4CHBf;=w>$!le1 zrnVbB{rho3#U&8>!C_2%;3&VqmZl4fz&fL*ijb8U>Jo9SAFRKc*}j#&)cW67k-LOur;Y)RWCg%H|rGj`LL$hZ1qV& zP9R`R*r&;_%)4`Na-mG9^%=z-)4?iOV`&77y#!79ZduG41SI`iXYj^w`lS?UbIxhs zo8}z8p#c|KmYn%gyxE@*B1gU1w3+Yb}Q|9J7|Do9wx~>Bb;QZ9kerr zZYAUM4gc1hnVo&sA@|9T%5Yv^95PK5b^k-Hbrx=vC=;3;HP7Li22g0&3EBOWeERzZ z?-7CHZfd`KgA1eY{nSNRp+m2OB9IN_#e8AH(c?<66!4Yf;>2Ucn=xSaj#L-B(PE^s z!J@KpJfm-GJNL2AZv-Z^q!on}TPSJ@Y)WMkmdw7y^AS~2YuCPyySJeBU!Kufbb9+Bg;Omb)M{O_T z`;^1k{yvO+pz7k`e@Aq6hoNQU4GU>gRebM?{e;onIZT8p&KGZ^i0g*xe(1t>hfalw8@e_ zdvw&f_Ns|RC4X(Lfw82h!dyo1Hs%*C7;pNox=~3< zNog*}mf)7sWrxx`Z88D$o*>bsR)5K#jGQN_jD(mX$>m@x|O|U~w&DEuE#n>%QMmt#EZdyHhMJX=Vn%oL*6eFi~2nOJ) z2(q?IyjQJaqt?YsO@HGN3El>luGoPH_P(Ka)2NgmtHWg%&;6rd#K~8M z!%03IEA-^OMts?mTUVxk5r40J3DM{K6YASo(aMq()h=k7sjcX52~xb+)q<$6KYdn5 z?3X)?b_(xfZ_t?!temgrxjE=M9%a=s6qL2w_z2VtJ49ZJvi`@F4EcQ>$!eqs=D&?y zTu5iKa&n>)w$aIP8u(1bdj!wyq7p0=>7&)Y8kryH?{%pxG`I!n?f%s?R#P2XfR~*?GP2?ycl!qn`Dp z#OgbJyR2hSLVgQ-mF0sPMxVxT73b|ty$fZWZ9;bp=cyI2J z^f;EV^kL(P!Mc4CYR(A-2I9rE;M2F3&_K&KCSB+5oAV(X=cNPL=b&bNS7u zP2*et(R`=O{t#7|#n`MBStBs_boXbZCUCRqgf@8=<~-pJZkGZ;0-Et`t{CXf%m!y zkl&={n3T8dZktLizVaPt`4xb__dLdj4v!f1dC|GFD=u?II13UES4>S>%C%>d6)RI$ z(J}^H!p=Fcc5R^PbACPKg=MvUhGaF`;R7a$v`_*s z)KMDh7hq#bo_oEc-F!~a3U{O1ZY*B`sb#-m{Dc10J2_JA&~fN0KU#X z=CWo8*}qQ5wnD0Q@9WolN^(y(>ax*@fv-g9ThAILnSgOVNIN8!b5Xn?C$nifdS z^M|;gjt9mk_(kDji#H0(bNUO&c43|%mz`JuP`4egzfn1Q_555()WjRIe5}d2z;mB%-uihO# z;kZP?lf34_=)_WD@s86o8mDVWi{#%>g+=M$%KIkvQ1v``GN%YcY3%+znOx&sf@-+T zM=3#DWY@}KG4hCalJua`_O2`lBtF!oI|E3V~mlEKF;oc9-oK@=Xs+~D!VV!o&llki@9r= z30-4fVQQrNzH{~iQ%epkRJH;%HEo+BZJX^f%|Dr}j70+ZOo=rNyPv@&*UK|5fw5S7SP5Aipxv7YU-GW?Y`|uwt`xYPlRJB0-gLZt-2Udr(Sd95+~2Q1AU~97ymk}Z zz@dP$p^Bj2&wFx*W5b6|D!Kg!>@ETf7bEQ$pEO&N$Yx}FsP9Go+p_9&^mCrIVgN&5 zdSWa#Cm|tmx+Vqif|lDEn>u2sSmO_v{@NisP}$?OEZ}9vd<7#7DwF+ox3mS~6;N** z=y*-x{%*7G$}$m!=4Ll73f>MCsnvzLIXHoP{jWd^^({D?9>xQesP=dTrcWbW=uqHJ zixDU%(A%hlZ!}wFe(OyVdG`B#?T-0{{buT;r21z2-0b+0CJ=-mMo*O_nQN#X(0}n5}X<4xj|4 zcwQzyW~m(Uxa_UXEam3HisHTDr$01rg>D>^!o%sM6XSJ()rNT9G=Do*9*E5{kmmye#Ej^ua9I_0*8RRywATZ=*{FyX&BXou_w_Bha zsp*yj0-pD9Yuy)@(a-F~Nr>P@#L>17WKxk=R;rBe;sl)#E0xOb+CD|xF{OTDVb18) zz^2#CQ}7gf3=rbIhOwWMpxM%8UZ)|7w_u+w<#hd}MvR_W_;h6|rLsNAhz)0+i*oAe zuy^{{^gmC3KHDaz$3olRBOBt>)w~$h^dCD_4L!(?4Fdf?Dy#ez+(LljlF;1zw`1@? zD_GXl|V=q;_ND)S-eJJa+>2xGW^4+fLU>zus;-M7B*bESh;~DApD@-_qd^l(XR&B&_74@`}i6 zsHls$*-`+m7@=HKpKvN}UJ;YD{Vr5;(5p{m~3oDYe)0Jn%}kv6FVC zrSnET302d3T@EP11X<{IHsQK5DzIe(kV>ETJ5MUJeXpZ8^*ZKh%h0d7evoPm0hR@% zm+{08_v6RYHl&0Gi>dbESSO9KOa~-%e5#(p-?Mws+)1~N| z?r>@Uac7OvTW{565H8U*+EB*tKzIWgLSD&OiMaEYH}iAv-JRe-AM-FmmxFtN%n)RF z4|#qG7Ak(K&KH=&qXtt!`Ak$J@0aW z7^UemGS00Er$a(rzt)_krcYSe=?f|+ovWZarv@n66|!uhBPfvRVze9vh~0s~Xp zrMHAr#S-g!pFRc}kL&L%+=*oCV^w>;qb_Chr$KM|uW{%8bY>6U$vv4Qp|2*z91*AD z{iDn8ob8mX>kOZ&P^POlqZSL7CXFDL0&;9MCS$g4kIVNgmsaDz` z^1kxP;fPIb$gSH=G9Jrm^5s84)3QK;qF^jT8f5K&?H926!P1E2jb_0wUMe>D^y^+U zj~qKR{CsQw2J#G0gN<|ax>{eAuTvO%l5noq>wEho-P*3mQhQD4LeE8l3#DGE-n0D$ z#jQ|GG@>I<)w=-_RJx?QhsWY2+<^!4z(mxYhV?29tcBp4T_C6c^x9eJ0xz~VnUe?& zD!G#e$xil6YdGhdY+@29fTXT4jB|E%d!@bzfLc%kr1OdGU4_t`;Gw95GuaNu_!h%D z54MR=?>EsE5wpX4flMl&IRM;>>Ws343vBO6+ES6?z$mCs_5C}0q)~bnC{<@6CzJUVoLiY8ecrC_SH&G{l(CrybQNhTZdBWj0kqa3f6N0QdA>v z9lRV|A6ppYH)9J!0p;AjG~2sWymq2U8F~+!mu@t94)^_K+~AhRGaY3`kys0|a;)7R z@*F)2h(L!+*9N55KIU-9m{DarKD$bo=*huui{M91);id!3D1LOdApNgMKU&$LkfLFZ1V-K(nxo-n& zc}Nj0t+F}|EJXT4ox%Dj}x#J56pf& z67qJk7f8qtQ%Hgwp0*Ne`E?^QqI*64dwr{W$mYB)1^B0%X@;rkAyfOh@@9VKjbgEj@pyjY`SYwWs_{O$*Z!9q4EOs;*ErQ zot8_IKiMRDly7w(3|5+%X(go_i71E*ziA>LTh@{M-ycr;2Tbvj4NNn|&N#>lHwYJX@UUIfhDoAmu$<{^U(bb1kTL_}< zRE7TM#Er1+84*PMeB<;x>Qh_-j2y$WrJtayjc}ck;&P;G7n3f2#tXy+)df0~=k7q0 zap04~Lp8%5ZCXE|bOOg-M%&^ZeuzD_vcjYsZWpQ?$Dhtx9lg^QA ziirXX5B9 z+wQ;G7raPw0hD|p{db^wd0Wixv9larOGw7k{o{Sc{xsqt?gH44a?QAAHxw%-Qp?H6 z^(kBZ?sF~12w#iOzJ}f8{t70i?vKqgS`Q0J~Bq$x$Z2uSaQ7J3yCP!W+D0jYt| zJA|fGA@tBf=skoUAe6oM{r2qcIeYf(Kb&*P%-kvOyz|WSmP;4>7Q{R8dDZW{u{?(9 z@e1)1eZiSbU(!J1m3jUsNH`W55@$G1X9CW?#QM%hah@zcKkL^MQZh6&JX3{#=&g-2 zrlbWOZW!=YSJ~0_8KubuzXa-x2+g3MoKvaFrtP=1@kb*D7kqmhIfkAFou=#fO!QI( z#xd^9E!Px&TcQe7&2g2Ql=v*bdw87zWd?C(vZobb%R z%FlrK;j#bh$-CqY3$c}r+=AX`B-Yz36%6b<9`42;p{W)b$rnJ=rsBr@<&PL)#MoBF zc7fev{>$XT_pSqhtJHzR!Op0Kw)O{MOvx6!FBM3fJv|8#0jEm={+>2AD^7BUof@Zb z)8jv;$3>H!B>_igxjMFW0yhA7OSN+l;I)+kh%@MP%xN>|=t2uyB;uBgB&h>=ThBQl zYJXb#K-l5~u@JqBfrbpk0yXkqwIr3kVZg!WC%}vM3gD?P4+RO#v?T+c9)7flBz-XS zA&iOt3cyVUA)Qt&jx5=KbDJ9MCskksL;TQ>+QaB)0jOv3a1vS`7`&+HOs8W1=%?q} zAnbH^$$$R2zr`6k-~^3g6_8-W;WJ2zr=WK=fJy84?Y7*nd0W1?{QyY!U-X6AxfFB9 zp4a^`?G(%`K*V&qtfV99^g!-K&MU`E*MJCqlnp~jg<%b?*cZvGLGPs>-lG4NdLh}f zSK@p4Q~mKqd0^6}eQ}$-X?26mGl;{phQBY4zkzo5X?c$~dV2oh?94G3%!%SgB)gF} z>G2SxKJ8EfKex%ICE@n`hJTb9b4{T9?pr#}%=dy5Kk(Q@;Jw(t{fLZ6Eo{^&w_LGB{<GyWO+D$&-@FIFMjK{@_Z>2xp3jv1rig|O4juliZf-9L*b*^SZOivd~(<$i3QW7tigrvA|LB? z!?WUM=RPEeMiSyX-S}^Ky59H|^R3IvKsL;!qg;NHv=9fM*26lu-YJ@QwNiMl?Z|R< zp%%OFeOaP+$0##C@#vc!DzKaH%C*{yIq?|hi>cnE(Rlpy4mW;@n;Kkg$Fdk(t&ke0tThS6O@TOya85WZGyBglkAQIo zzRSGSQDXJs??ARUD_2#!<Tz;&>U2j{sIY7)vwk z9;J|LppE7c)L6qErTY%|lu3A&>aZou7g5K__9xMUp>G!`8dj+n2NA*KZO_t4d22$* z8-eX(3jSpl1$7WUH2Z!wgg^S6ME5!_=pi59o-h*)qd^TFgUOX+(icRTAzx-zG3iM# zMr$(k+>QewlvZd!&~;HpB+UYWw%NDjDy3hso}U<&?dL&IZ84T0AkY~}N$W+U0 z!04mD*a^fUIx!u*otv@DtpyY;6>Q5ri4w>!9x$*HPKy1)3LskU{TbhTpm>g*e1h-= zZ3jnZu~Sc`;UXbA3VnYQjb-x|)I59eM~|;$jos(tHHjRE(WqPc)j(RQue$Cj>w{WZ zeiM$nPKo;=g=4$UX*^}AAjA2-GxDpJ(}GG~YnOqZ_Wn1etLVu2iYGR=v=EBp{L@LA zpURWCxnkXTTsZDTy5W?KoP@qVzmKs~DGI;(g#omZ0gyc-1A|)5dsRHMq3mi*e?1Jd z9R3Ek>%(^o4YQoFIp(UATF+b_MNonSXyT(lp0*#PefQjAJnf_hH3xzU3}<8I9nl=o|Y41fX*9Aqe`x21$VcZj^HRhcy<0& z_(-`4?dMKHH)W&2#`4dukqXCRyxMZsBS&nWcLM7x=gSvLXBPhkUwG=WnUA-heAne! zQXhsY=}uk$BfQ{R*f+C?c}<(rN{3olrfAPK?yj{D(2>arZA6wIt^E};6#!|8j)8%J zz%v{eW>o+?OoVwybyvR3cNJPG<%?luvS2r~^!!6AsN;L&05xcLlzZV=#!BDnLH4A1 zvr^}YX91}XHkiQy{iJe%`MvLcUfmOQWd$X-Yt=8`Na#+p)b@mPP^Kr_Q@v3#LU8Fe zyD(rcUpq+c9##7cCU8MoHS)o=irJ8!>6YFD2us_nje=c#O0B{mBMT6?rn3V&li`W7 zTag;}sNUYwg>y&NK8zaU7d3cFE~zV;=kJZuf&vl43+}&zr!C=L9Ak7b`cU-=LHz{i z)KOh&dHG|La4`iRK4(qzP@k$f=w3UweWCJ(-U6RXwa%drN9e6v&TE8qYupBzda4d|_?Hn>E8{WtCeOfZeDI@1@uq_45I!KK!P^PmUf9ZyV#6qpL zeb;W}D@Awjy`!4zX?^2%q0KC%)aS9Z`&e={nMXA5p@|BMtyTsaj6qW|N!FovCrG=S z>cThv#-5DE`dPTRY`Mw$eN*;x4yS-y;AdpWrG@iP^2ov@FgAB26v^}3f)Ye@sMem^!zUaf=f2^O7 zIPw@--5CCqKSSW)i!a#MAv*_ z_i)~oe}ZAIFBp@bB7HIUHjCY)UE^SFXO;Ov02j?i;RKBLoA79T@U)3XjQPK;T5s15 zdXiHoFZSv}z3twfToPefHtgz<4c5-Dak#;`E4afjYlMl8@&3(ex;b2evE$#H&HR9t z-B#-T`%_iF&SJ;+GN-+-mS0TvasE;W1gd0$X$KOF$^xgSEJ;Z()Sb4w{%t$j|6F*V z%t1@A-XUlq8dbmEty<;CVpH#{(loJ9I+v(e)h>wCkJEeffH82#!l;t@lmA3({>3e6 z>I$o4gdKhC3GMz%Fk66yjj;)4Y#3t}&0FiPXSs&VyOE}MeJx5T3x7tIz^dnYN6f{2 z?!}zHh{+TO%^YrN{msw-tAXRP7Rhw}z@l;=L`kegqvv?NNbQa+-)9FP&^3#uyzN3> zg&rm)0>fzd<@nMZJLZ<|n!s2T&Cij`g=U4{zq0e*L06W4?dstM?;Ubr#QKs5RO7;| zr%|H73i;26Lxv6V=u~4>TfU7~sX@AL0eOkFvaX7ac@I;F)sfbaxG4m)EB6U!A;6=V z6VF`nTfJKNy>w$}IOsGJd3bmz0ERO-O%($_9@T@)KSGg3u^x>7P2AerB7%&E)tOZ( z1)#}41bM}$%ChtNoBZB6QUSwQhvu*A8I@kRPnpE#SotETxR#@rSiS>qJc0)tZ^Z^2 zqdQFiE)}=i7tupQ$EW9#pmZ1L~8#CA$$)>8u)cbs&Pl^tfw{N(%ElIr);~8&+|P^KCaY& zbMoUaVQmNMHMiAM=6Kf!Ma?d)k!B=`JP+DL5qCLJqaICN8Er#;kYMpWFrFZ`fLxQ_ zz6Ht72f1!c=2?hLZ$F*v7rF>kb2aQ|YM1Mp5@(Y4Y@+#5O>SVWLdlgVS(!2#vzLrd zqSNa*DUnRa`#iusT9R)J0T`B%q=kOP+DL*WEbVBwiob3W4B!=0DSMo+lb`jW*(vqA z{HC1G4tZmfUI-RM@o^gM-%r<;5B+IlUUOG$TiVDN>{9_Sh;xvkA2i z+HAa)05+sN=k&O(xbBVAP;9D!pI?#|ub}=#U!4SvfSEy$cSr&YFV;wicB3Y<*#}}ryH-r(8gZ(ip{jJFbdJ90WjC8m z_&69_9s9YkDN&nx$UZj!7uVLdK5m-R ze|4!l`K=koa?1L8O^%Hz`(G1qculV?M{tw#JWeCjN7Uax6QWwZqmznUt3$4Nxw?L> zpEjV<4l^(PIg4N2w~jWAetlO=rE*=9zWXzTbfhM3q`%?LKDhUZ>&gQuAdF&x zu4@ThRS$j=DgPR(Sli2S7vDRq2WtOT&LMW~i~oDy1+&3pQ%w zS};*RpZCgZf9WRNvN9CWelU7IrgDXGeNtp^L(3=Q6@%|zR= zoC~mfUFGYO+7jy}C~n_IPrp}$!&e3y8O<>^=ol^ZM|NHYF-VGoY~gsLOW3B_jZYPv zmB+Sq=>#}LzS*GDkqc+= zAW2oWpRitnu)oJ$?>#chAAVUb3rQFytTg!-IF+h;BsIU7&+~uP?PfY}HBp zfV~`LKgB~s-gM}6HCL!&QN8N#UbRu|rwOL65qXjc4&%8cKOUfRYroFYS*^%LY$C%H z_UbUL;@IwfrqGAg+}Kdo#~}|E-cO~~8IpRJylbBDmU*C5R3^JUD0ea#a2O(Y5E5XV zjm-&XMZ|)xyVsH!mwJKBhVd!Hdy+?RSrSi~ zQxeE|m)#E9;d`Cv8hyns6z<_QQ%{iA`fM#hiX2i>m~a z2fQ)I`6clQHVS`drc3Auu}Bohv+>`?9^R8%tK`*XqpeYTwbH5#e`6 z*JaM#vraZAy4bxWuk<;4K-{98aJW~QEW=GjC8yFJS9RjbPR47Jj}>(~|MDNzy| zvP{Xt2bfekBxzFN!=|h+i%!F6C+Xf`L#gyArVBmIbK`Y-bRP3Z2t7LCV z6|x#fMILnoNy$9U=#N@owc{i?w1H~AqaMrJ^Eq?;SA{HoJgOysm2K~ zuI+MRmqI;1$Gc!&>(f-(xi-V+-_USo3GA|gw}}jYUU%i7=UZNvsa8f%&G|d?-#7Z~ z(>t1ttz6O2mc6vTmzX=?b^YYU&{&w=1kcEgx)(AKZtgDz740n^Jz88xCK=m47R=*B z{s!K1f9dxbP0oUnRi7I0KT{QBPc)@fkBbYRlbA#qZ?-eYHMv~A(dM`hGE|U*X-^IF zz}LZ_?<6Ew=lmg^6|bg}tIpi_P0-1olV<6)ZhCSrVH0DxZ9Oq_IA%^7FuD6h1V`a z_x^IwPGCxe($-I|OquuaFep3p6Wp6m<$m0k%k;Pd@8(fy_lV;{q=ZS^ljW_K0*k3I>o7fbIfUizIu0KhAdLW z)By0EIusqZDlu!Hv@i1hUaiDr%bPW;oOP+{#5;zoO9k1)$4{IYuduRwH0P@5w4cHH zD(ri@4ipz|oN16@H)L`9hwMkk&%;Z3rhIo-@1*vxEv+B*S)zZbO;|a&W3Z*|8{>~W z?;-Tzu%n?IWNCHv^P^vFCS2$9F$QUllh#k<<-=%+f zWmf(J86|zFxZusFA1;F={^)J7?^G3Uo6ECHp1r!YxGW()S`{BfMmeBz1)({k!aKy2 z>;+v|oSYooJKaDZ;TE>b&JI1#yk+0I#7y8$cgCQ8vTJ+#0kS@mH8hmJ%5xsnr)|-2 zKU5OpXf%wi`E0)i|I`lS-SI~Wv2^45_ql<_-Z{P6v^lkHiBOZsU8p<$BP|;w{=g(m zQu@1ZQMT$j=WO~I`-Tg?Lo4#;ab`G*I3ly5gBZHQK{;KSmX>ygD>^$WI_uZrNP?k8 z->*<-(h|zW#!XO;+f$;#tH0IS=9c0JUn>Igcm0_w8V2M1p@RT+9$PG^jYY1?ae3yft&E$W4kkf~aN&-*wo+tK2oOt91EnBTK`Zf8d0or6&3#%<9m} z9SANU2XEON4~x1%`BtjWc=hM#`-HojBnQ2xd}VXU{xv&+D{q(L97MEc-@^8$_$TLT zv}kGaj4eqwpM*Iu$_^HOLT$Ch<16k0==}p$ozUNp0jGE>?-^l^Xci&Vp-FW7nB2TO z>AXTTVvL!58ou1HJJk21Bmhl8LBVl)z6}HNXT5ceTr8AFCQG3>y#f4@ep@BV$EkZ` z%l%71`1{&IXukN0N3O#+B%;vDS4?A)$d{{>8N1)%M!K29L%L}Z=BMXW$~jh4N*`|< zJ8!&F@k+_qWpv}&wt+2mQ#$+QapCl(nn|ODq7wPrMhJDzB1w zNRH{ob-H8@LG9c$G$;_O5)<8EZgT(b!hp;B*p;O>HvNBI;YOx-o`?~j{G`4lZWbY` z^~bM`cm2hHlDA~+9Xs{6HDv04hQC9mYcztf+KH*#+U_^&UH{bhD|XUjbf}sLG>vjF zUZ}IPpp8tE#y!Pn+D26rGww zqsLoT0sB^Gg+&3wzRZPY6)PXnl33L#w;G2Ey+2UeOn%Xfrc}|R0SDtAB43q2;eya@ zvyLSIl-UFCQ(sA~Fue05t-dWbUUk)`M$+BT5wPAH-Aa>esS-gVjtI2FNcS2N(+2eeS7jjV*nN$$Hw z(|l=eV2P?p-j2ih2u>xG-oF1jj9obbTCa&phdX*rsgf4E{W1A_N<;a3FV{s9f##&I z_Uoi?$b14I`xVAnsz#FEp-*H=FB|ixGrG`Wnu*#Zf}zc_hscqU z%am&wTVtOQ?#m2M@JlM?R2_=xFqXBxe*<}@SjQXsn@>-AODx-OybF|C{OJaM%E7_m zM0FT{M{gp(BSiB3Z-=TIdSGbFB{T&6)ct1e`>g&1is|CAQ=^%{gXy&W5AETMwSpeIU(*}Br938gbybV~l38iF>C!6o!=Z)bylXYm1>J?xV1H+oF8%S@ey%}%_??sFi&_+fU3!?5mI>B_syS5bt&Oj zS(U6L*QWz_9^R7YxOc0%l~F)i&cy8bKzb7+XY~_BXPLR1=)+IIWU|X3Ec}<7aymw+$j-sSV!nNGaj`MCI8{n`3Tu4Ls3kId*B>K7KHbNl zw^|CUDAXqr>fjFOgc2NeQ&FZNU0w30kbpNY9l7U39OA8gqwitiJ68Th1GQ7evu>uS zQEm%9@zRwvelM}aeru`W{bfF_2#yBC`7P+A5%eeW$6@QbcCeDF;45CxbK}B!ebKxk zLanxl2%R+h++BZeqx~13VtAeeB%8E{b(=`QlsH#cb}kZ&G4nr$@}`qFtg?mFA467& z{x>RUpRgMgXh!7IHR<>j7(eAg7NdXmAp(*nU67iJ42G6mmjjRcupC1X(U+59TMK2|QLVQ}A zZ&x0Zf2C&a(FKc6SuLb59uf#tqtAu0`SXcLAEhezF+fi0E%Xb%Tv8ylZ6>3mtO}=L59OT@s-&!Dv$ULfA_{cp| zmEi}qW_GX)JuIhVR0`Q`FXCkhVq#i)=*d1^?9t`8n>4^+|M~{5UbrE~mdsha#uHZx zo%+|C3H-+d*hkYdzp%I<^0$Fx>SOG`-?L)u$pNX0mW&xlr_C}8QU zu+M8Q4_Wj6W`vqt(Q}mdubZiiEsGGCqR6L-z3amL>Tb8kBQIIhFPHL48v=S~>hGazGjI&oIQUC^}dX5$fNqr9{C7+{Ro@g znauG!F2=@gV~XaAniDTJl6gxm#1?DQ(s>mEsiz&bjwT*w)3l*#!ZQYP+H>4C3JdeK5v(-YvKn@Zu6 zoVg?!VNXpeC|fX-K(PG4*SrNXhU9Pm`~CE%Ij_7^DHm1KO(4Xlbty9TEv-u_i!6ll z=fHFW3-+P30?zHl){27KVSTdO1)5px6sXG5ptQwK7ikH$W6{arI`dm~YyFAQTEsHEr}y zHUDqS@|biEQZ|rFrv5j?|F_2>)=h8HA@)EOvde)anobAOB>xo!DC8SvcF5(=-w%Sc zpY*{fhMyb2&jA43H}Va)OJN-4 zJj-fwL+nXqQTOPPE>H_oIs8-H7uGQIe+{Ks%2USin%+|-H=^x`LQcXTrKS1xtn5>Z za)YhCDWf-y@%7%cta;j0;Ulrw9ky!c)aEy0lpaAoq_F8NnxSCoSOH_@9pum-ICT}9 zlZBPbvA=bDR1$J`6S8;~#%g_EkBi8nDNIV?m7}JGX=r)$^|NVG+ul)IYVG}F$0M@GK_r4My zsj=X!ee(?H<%+Ff4?{Cxkm-J-B^y%7p;%Vjgl>hvq}uqy2_L_7@KjEH zsg{1O@FYv%ss8r&%ySwG*nJEKZpN@SDe^<9+{LHYt)v4+F6ZpbyP)GXQ&$J)P~25c z%XrCyB!qh-%74S-&odHpCX+!1f??-r;VS^)QTWxp(R-;>W0BHq*N7!M$L2pd@M}0b zkbg@F!*AFK^hZK{+T3|1sB~zWvgpM}R4rN~S|H%C$zo1$?-L2ddBkr#Y|d0od-kS5 zV)Svkl!F|h5+W4lcQ$PBTW+Mz>amUajWX)1>bVJ77j<$&$lso;vx3Ew*r##=wy=b> z$;!B`wHxUNn~bEJ#c5)2nSu01WBwhEN}hGvni#3`!(qq_nDW#>6^yH6d;exDenMg9 z;+gH}n#>j^@RCjT5B|QtFulh^FRsW%vvY~MY40J7d?fNJT^2jR{aV{QCBZ@w0F_T~ zq6FC#_onk8LKsQB@}9EKC;j=idwPhRD*O5ooqm!PjY&PDc)rG)famfdvTnA1rjKK6 z*ss=Tfx&0rv&wS(4yd3((egfX(HMNDG$-*;{!x;P^9o4H(6>Hp=XpHjBupP{Lv>$F znt^=U=R5tOY1b$Rne7jU=T#MT?X>+e+<;*w{+8`@7DV_B8kMy;7~y?TrcHZ3CBDr> zSZ=WZ?HwdV=0saX!0~P{APt|05Z+JTvg;T(Z5uJ#zij<)qq3Knz`U|73=jA2N)#J$ zQGQKvRE^*wpsxoiI^)z>=Jk{n{Y)uSjUow`nZ^^B)?NaR@}iBRCVgCgZScyghLf%R z*-AeyOc$&Hc)c5eikO$oRR$Weah*J?qtV?+U+RB5kfo{<*`eOi&fVyc)K&wPmdVs3 zrn!H6L)B!iQ~A6lvfZm(C5JmB8a6r3?y;=Wv+FFv?{Z&g)ks!02D5ds{~PC&wU`E}ptcJ(iIPmiVfn z(LI*=yB*psK#!`bJw=o15+N~q^(=HlxClo5-%@v0?hBg*Tv38Kz)>&;(7K2 zzm-@Mkt6^6;!dk2sJOtBPR63alUBwE(l(h}ZptzZx_3U$1pX6bW$`gtR(3RcnB$G> zpSTU)T8FA9Pz$MXW@BqkU4)EKX63W$vtJx?>{`O{C9tR>vdKa;Q`Per%;3!MKf%6K zo^e@vOC=VqpY3w`${4Y2gk__1By|osERd{TGs&qc$24GUGt-%~QXJb;eE)4^OznPH z5oEh|jcnqM!?>OhY4|@0_>4wf-aRS=msPbGhz;7thF=HS@B@uq}>`Hj4hXN>U>Sxs1{WWM)6MQcs;Aed;e$ zSFFx~%-H(COot2fZb20Azp7>ViU>iNrK+VOyh>zB+s?zzNwBV zmOICl|Daq5b7xIU3sa4lf7+?14+jXuim&KnuG?lTrJut%lmoZBXy*>tF>o&oeXcw) z6#JMnwc?#CHPotfaxo!k**iC6dgr&>1a-|lZ?HNzc1qUGg}dev8bJ1j<|S;zp6L3K z5cmK#R~8nXEAs^WKiDPF=7{x8>ZT#~)fr+(JoQiVx2rb~qz(VPigbASuS62lz*={b z06Vha$Tj`lSru(xEb%L0LFFvwZh#XZac|)bL1NERp-M`ox1_RUPu^jL-~w>$S^i0I zlW9vc(W_Z15w3RY-}$YycfH-G=eZ8~v~ok`>HPl8)v6HSIH@4bciZv_)`1veZsc1!RseKm4sG2rscqg`eKCp!XO=hPq@>&$gu>L3`GG>dhNY zRY<&^P23vJL`%JZWKMx_P6R&JZ5vzg4EKfcM0QGU9BFwoGgXiov-AcR{#KRP#M$rf zEJ+RWY?s17nc24*mev3fO~zB`UA-BvVx5ftu&Dx7I+M|*VRq&}eM!>Q}0t8H-dmHdvX&7I# z)^ME2`3Pd-H{#yzS$KzB%A9=8iKD5A+Q(l~HV+Z_jndW0R1SDU7vOtic+c{r5OXR~ zi~W$cB%Njmy4Snn? z+;_wEWAV#)vArs2r+-co;G6f>E}-lR&d|7RC-h`H{nSJFPDeKGOR^QkG%HT;A7UuF z9yoV%j0yK^PPkyMmT#9dU<4)HX+Ewn)69qp*JUAXz;nPPj7M-MojV2J#H&No(vize z-#N75H%MjDLJ7`IxQz4B-2x0#*J2ze`b70(9qsfo1stu7^Mi)Jht6^ysY~g3Q|%hR zl7m8suI>|IEl$v@9)oDFY{Fpbmh=LtGk;z{^Oq^k^IK65XW*c@S6#5FkE^|qP3Fb5 zCHMI*M9iE*)~&t?uJIdY0iwCE!7t*5opyi6#lG8JiWkmoO#J2ZZ{w~}OoJ)nS@YeD zGlE#?1*48?GF#|D##9{!6{jwU;yOi2QMc`IZRmBgy_Ndl7t3MyZB+sA&jC`>suMlN zWHPL1!dx^{)i(wa6U^XzuidD^DiuAD))={M;H}42vh~1w%7C<3Vkk23@}dLw9rxg3 zuI;%l_)gulQZnvip`lvN*|xir1ofSj`JqN?p9Q3BIJJL8Un6z#xsmD4c|)n(y+IHl ztob>-7CFC#E$)$RYAR*!=RX1Ht)x}sZNbN|iK;GNC6_yX@Rp#$Bw%voqhhvL9t-*O zrHf5jt7;}(2;QtN-*^yrwAg{xAIOvY2+xHLJvBi6d15H+#_7v*jfer*@H+z@Ppo6t zbw`;!bZ-RmR=lpn4E6>cludlv@O<66V^P8oD7w8kAcM0871qxI-?xB&ANGB`!KH^N z>L#?)wZKB=A2p5S(@NA$AU*Y#z)Ly(g@b>s5@S?w zUqDV6^Qltncr46s(Ik>Md^J#PnCb9k#lI8bh`Pz_zGjwioB)Wn?rTZr2wvPV8bsM4 zjz6#4l5QgCGt7E>9uJ2AM(rfkL(;x_85^=<;9U8a-5`{OoBS{r)6AaIhTgaf$6v~C z7mJxefm5$R>bwBd&y+^U=a{erqE*Xont!or1^f_bwWm>0EMU|~BL48BPF_MeF^r^I?H1u~ zjYiJuiytti#r5n=?r=MhW|k|PzW#5T?*e+vf!r!=XQ~&b)4*M|DZQeMbRLi{u(`U( zku#`J38!k3z@cMv>HnSb0bql4@1(kGGJe`hL8TN~1cDiuzPuT1Hbb-l>lsV>eYi4P zFIY`G?V{1z{tXPu1f6y2k^7NMy}s{htYi11gH{>V$`eXt%M*ELNE*mH3G}M#0hl=X8mXjP660#!-BBScdUC zTgpe5-|D%;5?RQckEM6|PxocLqf>3=CanYPP!+2m6MpbA2kzLG71(0gnE!1u-s^qz z5)D6()s^PTo)g$(9oGi@IE`FNBFza01y*$|U0ZmmE@{sZ+{%u0?yWTXsM8ebgbP-N zsV*oPV+YB~!ba5y5w+zC=R*?GO+!f!4S_RZ1l~kL3G=cjK&GhaI+CDg`|uBmmsT_n z@dTpHxsG%LFC@F(y+SN-c0{#o(sQ8X3E{q^Ud4r0%M28n z`;DUf(Ej$~)Y(h@vJXwo0+NS0lwnH>#QK;|jcTxNr51L}`gV6L^ndDp^@B*c+s2m2 zlh+tVf79+A?WCo_m1=(hlIlJ8x}1ci7p*pEDQS&dIEo!9o!7?opV)gvc|1Y!RzBxh zDR>t^{lDYidZrB?JvnOr`u|SOD3z6(4Tu(u`vee2+0JDa_#cJSUtphB+7rua`Tr>X z|AWiB04gh=>$@%D%4bfEh5D>TI9b!Yg_KnC=0ccjf91&waZBdcOm*4*~239`9Mg1(g4w zH$8=#o~|VLuOyr%goK2+iBg|e0OJ||1nhJ-9(Q}5Ha>0&iluJSQ|~QC0e}FB@>2zE zit9o#2=plFOOkS0b2V-VNZHK*z_#cVHb78RQ!c ze4@ZviNIMHnCvoINlIM&jnxhSOuz~6?Ustzyv-q<6htQ15=3M3=l%d!H|2`?3a!1rcm=n2vr2U)#`hXeXOoPH-q zkRd$Tr7>S1FxEO$V~k5bzZy;#T`AEbn$zC9AtcS>MS=xV%r9sIbY0`*qCcq)@XTxr&f^QMf%n-B{>nCv-+{udoj&PE?9qVY;q(c0KsK0|n(EiwTF!fMVlvH{ zu1P{~Rx(hYw@fu~R3#VL`^Yr|Irh$yXixF@5i3dMa(9UU=mijmXsc<^sT8U7OMD+k z2CQS2)7zF9eZ+acGT+mQ7jua{k9#i$yEd??llEiqet?Q7Wo zw3gAi%D2WNEbpZ#%OCZ(jbqdjQe#U5PtS`16u^*Sh%Y*!P|V9(DmP~NrX8(yN^X$! z{12&5Ob~Ddw_3V`=8a}T2a2CqvB~Frod}j4mP&5NUKvIO{X){;4M4-&*}A;wMt*P% z&*{(k{9y#Y_x%fNCDK&4YmhxTw-)WW>a;E@WyYl5u~*%)`|-AS*vhJ%y7A32(zV~YjsV;c&Cs8vf zAh6cL%KC(4b(AxA#H55HiU%Aeb%gnT%FlVB5q_7dO>W{vjH=B01LEauyn9{G8BC8I*@yE^ zz(6Ilma5u{Q?J#?E0%8BTu_0jT| zt7_l00^}P+yQCg#EG@G~@aQZ0C`GraifH1d9U30X`0`lmf|e0l(e2p_B;p(;TgoH_ zGWmK{M`xidtqxj!1@{2Y`yF9^L3{W7&~7h5x1bUm>%T`Q%DX$iOboq@t2SH&icTCK zPN_X38Lq@UX>(h*hFx8MX0{sv^~vCSc>H>$;aKzLC2q#`itx1kxVxJPk|aPq6On#@ ztd9;8L^XIzAhSl zFWqFC*OUTwBQPNYh70YFW@$kl&7hU4Tb1YHF*pyKPBP&CK&uqrUvYzm*y?8gl{)@$xW4FP3?dU zom7M<7$Hs~Zue#>zVi8TlGIm1W6owD6VB@WZWS%)zTtZ{P>Q*JKw!8{%F_B(WsjZ#u{36*D8cpa@W%!MTO}+MHutl0 zuG2NI{@}a1(*m%COI!71R!#MKH5Vq{1+dQ)1pn4RY~RXNcfGz6p4koudB?h0&D8?H zenxV`Y^opo<4%GMxsL|E=BpvWJsr_6gwHYpkgl=NSkM(BcYr$#V>ui)aetkE&67@o zy<_BBNz3z#fc5?U`sw=Ap28iaP&zVW9dj(p){<7p7)zy5FbU*&CLh0@bmupU>C5ye zcVaCsKh1FA(OX3>J#B$)<&MdI{PvN=n|@SxzTvHU^|h^*cSdF(_5w-wzCxuhiP3}h z{d$oG#HkR8*DHel$c}N%I~2N@^eD-{F@1v4*hLG@)sRCU_TdhuGVinPgzpF=v{M7K z;}gi7fwp-qxrIMY+h3#!0ioPX(+}0!xjX?T3{6*G%)$OIJUlwYaX& z8IR_#Ax(@8oi?SUF`s$mULNvgDgI1;W_T@zZ)mbw()b%yQMO`x;a;wdSR^nfaM^s}0&D0a(L8Gq#y-}TTLo9DK_4J3vf%4kqV zY1B?yMg9g}8|wlQCTI;b6GkxHH0Cd?JRWF0Phy=hN2L%k{IY zu|#|7Y2z{F9y8ohp_r%M%*g(Z$JNJ0n^HL-UW&6UTT3pxLWIjz!*utBNkA6_V7mNGaI0es(ym+nW!MOEl})uGq&` z{vXACXH=8Rw|79JND+{t(h(6fG!-dQ1w=V2MU()c1*9evL5iV@5(GqS7zAk|f|Mk* z&_feYBOO8@bficPO-h8kPdMlN-@D%T%U$=&%?H+Eo;`c^%=~8WDSP&)Stsdx(*w~o za{VWx0q2ATd%_`5hCn1O2`gMf2WDv&>2#NI3gJ=PZZS9zi7n##OuXYXZxfH}%yQVA z_xquarf=KOH5khhumic)mi6+8I=VhX%w-|4^4j)Z&I}Ir$kCJ z2l=ru`fN19qGN9F*HkX8J%pBu=Nm29;RNYs^6E^J-sz~NQ5XZK6a`T)qWMFP3q0W# zcqT6~`il{5bY0BM>1E|tu13M65iA zJajNld}N&WC~@PwH5hoI_KM}&h_BgRT zuT9!X=~NTDqn~oM)+1vck;WHkCY^5&ulkY%N&Cs}-*5QjeJ)YkAoF*xfJuN=MT|+n zhM6~SBTlsXWE&WyyLh-eKt9P|59ed<7TByF-h`#Z@lWqARydlx;)*Yoe5L9UG?Ud? zGZ(8bozwf2yP?v#PCWT9pN2+D`D~3ma|`Y>cVq-Vpvr{%@|D)NRPvy~3iNxu5n?Gf zSTaA+5KxrFA65`eZ=Jooz!>`8MD5t6rKX{&&Yt(s$J*;EIm_K*1JC622QF-n~whViR)wrWA96qWgh7HZ}S>cJAQ(9k>WgkJH-dCtzE-)7U zOX1vdx0GYnX=O~>YXe-)#T}8uA39Xu8@!7lav-__Pao{xI(Ta~s_;yGdxc{HlC9{s z&}`z>;ZE(DH#81Vy>N*gTpsiw9}e|<0Qk#pDGsJ0~E1PWWn0_)ZREB4KYfVGjXKntul|*Dh`^g$8+A zP5Jr0f~z^femEmb3O_DLQ?-tMb~mk!^U{e~O5Zryb`|Gzdh#9{T~-;kRo0hglJwyq zs9vL}(r|A~V}azt))WxmySp1Ic<)N>+)X`O`(twx7vA#HzJqJiHx2e$y#KT?b^sSY zDr4t(_{woc^|?lSSyv;TXCzyGcKa;NHxIG5r<5PoUOPD+Anz3Xk?JMq1Zzz{68|ux ziWY3p0fZG07e-$T8Q25XxT&H^q3_OcX5Agww2yPzD7ply@!fJRa|MCsWI4+b&O_Szjd3+Sp>h#eP+xWf2RuBzK|wAq zF8UW=1@(Sk);Z*V!Shie5Y^*A6kI&ARi2IV9v&{Cv&m4PK1wZoe1@?^JvWz6w6Vr1 z!4B${$*m6_xu+3UdrSTLmcNb-nlhYNQ1{&2y7A0oE%TA)^r%bL7-$E zi3e`H^3^rnw=CD2EZ%*RjBv@UYzX*;zf0@~u?nv)Y;ue)8k}T~VKS1F(^9`4Q-g|L z9F?w4CFy3o9N2_Pi7OzU~3>AAyZnso`}9lW#lXbA|;hArGmZ_oAxwn zzn-$vMX=>-(yMgZ$Uq#sOI-gVz zuz09C{O4g1DCm5Oz5CjevW%~AwlA5{6m)8FqA6|x)gUw-{==h7*IYDZ(DO~S=<9{8 zFV=#ApSVhAg_EBFLc5vJ71)0+6p$G5M4RE*h+xiEsyTX}3aC~3z zrDv$vC}SG=fqJ5?{DR7@$HH6qDz#5%*{f~bf#xqg*w!OrscZ8!ZgwXk2GIux!1lP5s_(t>FmZ60q^fg`m8&N6z3kFz}?7yod4+R#;G*gorVb zvKhB5NK;YXalDqL=u+VeaMwLT<|r^l#+?-XjM>t|e8-cNLO&hm#&xQV-kA)Ek>IR3 zn%ZmAlgRq^GYlZ_;FKdiLulW(!?feKzs#$Yzeex@(IPC#d4{3|106_#jDX~c;#|JB z)E{eLpu?O4&T@^8CExLGDngs2AB#1E%x4$3RDuQ&n;FHIBo=u zi3&R=PpSj{&v_$kdgt?YkCq``5^l5^i96%obnm z5Aalzg@N0yV>qbA%T>0jDPtTUP-i5tr9&3Bef7)q%B;24X_Cr4vfeRRUZs0OfmJGx zx=Wf;^7$>_RN#?skS6>Qv?0*!%GO^HQ*kR(0_oC#k{;ns&mpu~DEW6~2j7X#rw?ow zgBd>{T zdw_u8E%QtLne?euT)a(iNov1jEQMwT9(CS0Wf%Rhne4{37Hbvs`MjIjpEN;1>oX6= zYTd0{UMaX5&;VrUhL)rWj|Pc$nf|h!7El$_D9R*xSn-SyM#2f~BWJe;1HIzz&H)|R zi4UPH(&7iN*uHY@VFb6-Oti2`?;x?47DAtmgw#0GoXu1KF_jUrUOlm2%#)!ZA{`L& zRC0yg{nQYPD(rfUtvQ7e-frxHccw)y-#OfH93YoBS&NlN7P{H36e-rWIb-N zwI^oIw6f;H-?=z|wJuk8GXd#_Roqm0W6hy<3nuH-K1VH9e;*dCXo**s8$&fdee9ib z%dfL@Tg&(jkOPD}(NaozyP@ov12N@`9aWw!MQCrn6PQnQ)!+;9cRc64@v0E$T2M`> zeod1?^+$o+BhAb8=zPe!C2mv>Q{z$D5q=X=OAT~*%2WWL5Fc4=ulWy=W=sjVMA?8;c%bd7AXV`X92*T}mK>Gq>NkXn^88x5aSXw>L6B#4 zj`KM$r>wySi(v0RMnG_s+?~T%1f-FraRjyd017~3IT;R+8xT39$5k8Q%b`Nke0;xS zd$X?dV4Z|E9Ax}gZAi?1)y*~)5gcG&i@`N@x(};R=KN$QC6M{|FJ9Z49QgLMiM9G; zpDfcdfa*&qJh$!76hJAcX#caH_}UUgz566T2|j5Of!AqSBsuNuJ!o{1wjPU>`~#4K z>@Gbhg_+>xKMBa=oiT3*ig>y`{9-O$3o@s|WA36yqf_5>u|;{J#+lOA2y!2132dt!s5-QgIo)PwnNY;#?Aan&Kwk4) z?lJ~3IVX5G#1W=42vZ78sZ3#%Mdr_zsW(}}2U%CL(L5#tm*P^Sw_x7cSfgsAp#1oF zetaUq6N`%%aR4RFG-G-swW-OdwXHW)4d+G@j+?Br4m1wIvdovs$3p$J^yC9ERXmg%`pWw zB(a>?WYKJ~{QFLRc-meGtXKMD(mbF@foBpMek4zy?5nvK(gh~g zR2HiHv^7#j3pc|pm(!UV4>+^pxZU+<99Dg($X1_>rjXquyw`B|EpYzWyV^4H5BhE| z1F?32Lk?w|xy!|Z5gnI)29c)6$y81A4YU}{+o!~aJ?}@bZTGp310gr|aT%}o2?a}d zunVULk}ce@nEEAqQbAT6U0;&5onij^Efxr`D5CM<+4{rXahzms<@5U>jf>i!I5cg zL7ijBDL3)3d&j0&4i2Ye&OjFK%`YBWioRjPCeY^ zX^#y#iM>#RrPIA?B|-voWwtM-Gbuo0w%fXwKL8Jh*42&Py&6$lI)7AWtO51HM$;l%F>{$lm;SY$#$8;!$5NRdK9 z^l}>1wyUd!EW*-Or@kByt~m;x5C)zbq*L(oJh{~ZXV&~{wh0@TABWkW*|K2PM|T%J zrovHA6lsMRPyyf8BXp|kqPIs-^5PBoNiU{#-Zu)~twAi9zkle1 z4=Qs8*0novL&LL5&%1PYNfXsh3zR|0j}wD;w$GuoVC?k)#~jMO1eeZe`bF6-#?6Kq zC-~>~6ASrRQoH=7Hv+G9Eiwy_=;v&ZTxXt$4=Bw9xs`TJK>M@N*8m4xll5wzbIbfk zm;D*oV+rL2V<&zXCg}VW+mxe9rxL3hI}vn78Dza!Xg;PkW=~5)m9 zT6AZYS8WqFmePY*J};)=c}4R_46)O1A1otE2Ls*U1?i}(yLH#JY1-BWbHFipmtcL~ zN+xjPLXzFG0@<18F8D>Ib*>#taIJ0X>FSX5K}!TD4lxW9G(OX(vV6sNZFhvI8#~js zMXrQokmk!3OTpjQvOppXPwmURO8Na26G8}H;00K~D#}@l4l2AU4qfKBrz-~@K0*d0WSW2MDF+Ib1x=BGf^HJJAT+K%xT7faBE$z3s~ zdvS(+&-~IjY=4Y<8;tks(Yy!C!kTa^XNOJ48AyeU2R&&$%(L+8kc0J;^wm-k;!^m?*Jsra7);J$I$ zuhu?xJrdldU3_S~p_#L^qGyxM zicdYE);n+4L!X@aYeljBtav;s>L5Bru-R2JP5 z=L(H#UFdbXB@Oqzo*8&*GO*!knCf(Fob$ls1}5k^99nbwHq9vE{D)Cz+e2q<+GRyA zX!B^{dPe=PF}SIZfSokpx7-jt3~Y2VTwz?+evbn~+0t^?FrF-1u_oNi&!(4bg?)=|K=ObdP5)RV<1k_0YGnKh+1=Twd)Bxd?iO(s z&!hn5rrL5-qQ@8+pg)`YGjEV?&nnB7{aFwL3hZS8Ki!_4>N zpCQ{nNiV+R70u7}5amA0(Tfw`gxaqfsc%l@4X$Fsw=oRMaqZTV9lYP+Er{acleJ4H zYcVmiTEFK)1K0M|Vnj=~KosRXn0VFRLV#_P|AdLaIq#5d-bCTyFy1JV=k6kk|4hzb zZ;E7XToP?QUpRk$rQ`dqcGkM9=3Y<>;ye42Mi*TpMlYa-tNkM3kygul#n&jJcH>2+ zj5TSU8XCC~s=`r~QsAiRa18b*8|BZdsYei13*51SO7ioyrNObmZO|J8B}1pA;k{8#twxLe_R{SOP7GD z%AAv%eO@(zzb}lg#dRpY&#j1!Tz$2KNS~C^+V2GAQn4tT`Gzt(tNAN6>r>TxTE5Nd z`I$K?tuBb~8Gno(sXqBB51xR8iYxFZ7ao6TI0{BvGp%&4Mk-e60XcFB4f#=CCRwb? zUKxN^szpy&5`%jDjl_2G^%z>oK9R~(nHxL2t0k({MrTAyzZ9I zPz<&}Dz6-hzif7+hy7TY^F-aNm4jL;?kT>CuubG9k z4baLHQ#`Nd$~!cN*p*&LQAK8nwcCtW0WTf{(fL3b>NhL(68e=>ScAS89~^MB0#3Bw zv@sR5vP!iJt}XOl?nP7Ub^)xSX8R{5L!q7|2P`2-TI}ld|8FPhqlj)6@w$dG8|VZpxin=Kq}$EL&f@I5_V# zO@B}4ivW@$TDa;F#D%Ixe-oB_CG#Mv$1lHw4ghJ!{R8-oJ%!3fX*)|Ejv$Xl0gLdo z_&H|vBMx!aU#WxG9CL5I|L%A_d==33bdhsvzTKE&VGGeyuwHB2)RMF@kMo+i$W^z(?0G>{q zf>p1R%=nKk%y_AY3t6^f+@U8{!{qLZ;Efz1)`L+?y#b>+We5dZf z+}#9L0HbnS0oe?yr>M8hzZ{J>wl6HR*SeD9Tp&|}O1EM1{es<&UzN2ZHztk(o=tI$ zN_&Xi@D_}T`2Wpgg~yL~&0{+?{*1*!Zoo_lX^v>)$xBl-g!sC-bKwIsBZ~f9;K{;<-7i-TeU&f%lI-7V z#ZSc2?qIa}=t$&B>45BzHEz*pwQ_o++8Jt@Ily}VIf^;tGX#WJGjo&<=A8teo0lIk zS0YIGD$YJlS#o?F#d_ z=nd~|&WHNpE8rx_6j%8t%YJKugjPZSRN!I6e$zytHFfuCb@j#mrp=J# zHr=ZnL+s?6K~n_ebN0b0;LoBsywf2T;duaj**Uv*nGkFFbRF2nh5|5Ge3(0p*o zUA zk7fQ0;HXX68Acj{jDw>+DeJY0GsxTjHbOjq#I0&YRiX`)H=<3N|m#R9zR z-@pQxq+`0d)u{u%hv=&H#p}%GrNm ziaJ{FUZhlw^_joGicdUj@0FDvcwveAKdi@J>(QPm-oWCkN(#4z?oRA2V)1h&yKNtq zbjELRr@Z>7sQ(uv0HFQ`2>__SLBjPv0Mvg#0s!i7Tmsc8dN z;lc!|5a2QT7=~!#ceOU0pWNEFQJ=W%1CBK4Hq(L^ zNUSN$sh^vTUm}+; z-qYPnr1y$YBC>1tT3m6|_-~uc{=o0&wSaFRHxEybt@P@~npP|ODvF$&qZU@Ft{)Zb z>A!v+a1#b>7z$}eO9tR4vU8yj4^qA?Jb{!6TsYlPrbr^L7w<1dEpd3;J!gG*11cJ7 z17A|oH|Sy;+z=%^T*92o-#Na}NM>n>Pc-z3*p( zQ9s$&hIEP^GDXsi-YDQY3!*Ch3Wg@#aM$uwxwliYbf88+w z7a7)ieSbQN!`{kNeZtGp#Jv+}UJi4Z_Pr5wH080E6jy;vzk>M#>@vodbou=QIM|O_ zg7_!f%phyYX7tU`Dm}$iE|X!?jFbMHCRfK#$I4%K)=!$T9WS;K^znRf#qvd)pliw< zN_OCJAfxVZ2?@egsfVK_N5;z+Z!`%Ap1piq4R~*Ml3l~*)n|1}6KpE~JD;fSpDLFn z(nI#>SiEW9GYF^C(!G-X8G=gYVIPBWir$O1nNoN!+3!zA#Jxw~P=_)8b?2k<-e*B+ zSwHzLTKR1-{9f`u(hONPEp(Qv8uji5jKYB+=geQwu@~}gTvd7;Dx0%IgGL;}tHcwY-W(Nj zvHRA+TjHQvioNDytkx^Rk>~u+I|K#`psRKaot|Yq#kIGML${B8DCC3;MFEB(o`i_* zE?qx^lT>i(LfCfB+xG_#_62Kp(TTTTa7`JOi(aFi(tfEV28N+!_q+YEdqSi$+e<$7 z7k=#F`DRMmv5meq&$CS5y``RZ(=HF`2wLd*=`WJVwo+oIWZb0D@3$}1entPc{?NMi Y!iTqsCQ@R5#WQ6vx@v5I)pLmaKVB25#sB~S literal 0 HcmV?d00001 diff --git a/doc/source/test_results/index.rst b/doc/source/test_results/index.rst index c091692..d84a180 100644 --- a/doc/source/test_results/index.rst +++ b/doc/source/test_results/index.rst @@ -13,3 +13,4 @@ Test Results mq_ha/index container_repositories/index db/mysql/index + keystone/index diff --git a/doc/source/test_results/keystone/all-in-one/index.rst b/doc/source/test_results/keystone/all-in-one/index.rst new file mode 100644 index 0000000..53ff62b --- /dev/null +++ b/doc/source/test_results/keystone/all-in-one/index.rst @@ -0,0 +1,59 @@ +Keystone DB / cache operations analysis +--------------------------------------- + +Environment description +^^^^^^^^^^^^^^^^^^^^^^^ + +The test plan (test case #1) is executed at the all-in-one virtual environment, +created with Oracle VM VirtualBox Manager. + +Hardware +~~~~~~~~ + +All-in-one installation on virtual environment. + ++-----------+------------------------------------------------------------+ +| Parameter | Value | ++-----------+------------------------------------------------------------+ +| CPU | 4 CPU out of 8 (2,7 GHz Intel Core i5) | ++-----------+------------------------------------------------------------+ +| RAM | 4 Gb | ++-----------+------------------------------------------------------------+ + +Software +~~~~~~~~ + +This section describes installed software. + ++-----------------+--------------------------------------------+ +| Parameter | Value | ++-----------------+--------------------------------------------+ +| OS | Ubuntu 15.04 | ++-----------------+--------------------------------------------+ + +Execution +^^^^^^^^^ + +To process all control plane requests against the environment the following +set of commands need to be processed:: + + openstack --profile SECRET_KEY token issue + openstack --profile SECRET_KEY user list + openstack --profile SECRET_KEY endpoint list + openstack --profile SECRET_KEY service list + openstack --profile SECRET_KEY server create --image --flavor + +Reports +^^^^^^^ + +.. toctree:: + :maxdepth: 1 + + liberty_fernet_cache_off + liberty_fernet_cache_on + liberty_uuid_cache_off + liberty_uuid_cache_on + mitaka_fernet_cache_off + mitaka_fernet_cache_on + mitaka_uuid_cache_off + mitaka_uuid_cache_on diff --git a/doc/source/test_results/keystone/all-in-one/liberty_fernet_cache_off.rst b/doc/source/test_results/keystone/all-in-one/liberty_fernet_cache_off.rst new file mode 100644 index 0000000..69464fa --- /dev/null +++ b/doc/source/test_results/keystone/all-in-one/liberty_fernet_cache_off.rst @@ -0,0 +1,773 @@ +Running profiling against Liberty Keystone (Fernet tokens, cache turned off) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. contents:: + +Parameters +~~~~~~~~~~ + +=========================== =========== +Parameter name Value +=========================== =========== +OpenStack release Liberty +Cache off +Token type Fernet +Environment characteristics Single node +=========================== =========== + +Endpoint list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 328 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 2570 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 164 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 1312 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 164 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 1258 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 164 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 1258 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 10 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 8 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 23 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 15 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 14 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT domain.id AS domain_id, domain.name AS domain_name, domain.enabled AS domain_enabled, | 14 | +| domain.extra AS domain_extra | | +| FROM domain | | +| WHERE domain.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 3 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 21 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 16 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 23 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 9 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.name = %s AND user.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 24 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +Server create request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 190 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 1515 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 95 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 714 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 95 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 801 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 95 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 801 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 10 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 28 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 23 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 10 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 12 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT domain.id AS domain_id, domain.name AS domain_name, domain.enabled AS domain_enabled, | 52 | +| domain.extra AS domain_extra | | +| FROM domain | | +| WHERE domain.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 17 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 26 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 11 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 16 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 9 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.name = %s AND user.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 22 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +Service list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 112 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 605 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 56 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 308 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 56 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 297 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 56 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 297 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 10 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 3 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 11 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 9 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 8 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT domain.id AS domain_id, domain.name AS domain_name, domain.enabled AS domain_enabled, | 7 | +| domain.extra AS domain_extra | | +| FROM domain | | +| WHERE domain.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 3 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 3 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 7 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 14 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 7 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.name = %s AND user.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 10 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +Token issue request stats +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 44 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 316 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 22 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 183 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 22 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 133 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 22 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 133 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 10 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 3 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 9 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 9 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 8 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT domain.id AS domain_id, domain.name AS domain_name, domain.enabled AS domain_enabled, | 3 | +| domain.extra AS domain_extra | | +| FROM domain | | +| WHERE domain.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 10 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 3 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 3 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 14 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 8 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.name = %s AND user.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 14 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +User list request stats +~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 112 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 675 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 56 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 297 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 56 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 378 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 56 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 378 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 10 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 8 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 9 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 14 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 4 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 12 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT domain.id AS domain_id, domain.name AS domain_name, domain.enabled AS domain_enabled, | 12 | +| domain.extra AS domain_extra | | +| FROM domain | | +| WHERE domain.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 9 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 8 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 4 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 12 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 8 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.name = %s AND user.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 13 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ diff --git a/doc/source/test_results/keystone/all-in-one/liberty_fernet_cache_on.rst b/doc/source/test_results/keystone/all-in-one/liberty_fernet_cache_on.rst new file mode 100644 index 0000000..e1ad747 --- /dev/null +++ b/doc/source/test_results/keystone/all-in-one/liberty_fernet_cache_on.rst @@ -0,0 +1,563 @@ +Running profiling against Liberty Keystone (Fernet tokens, cache turned on) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. contents:: + +Parameters +~~~~~~~~~~ + +=========================== =========== +Parameter name Value +=========================== =========== +OpenStack release Liberty +Cache on +Token type Fernet +Environment characteristics Single node +=========================== =========== + +Endpoint list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 70 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 466 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 35 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 241 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 35 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 225 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 35 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 225 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 9 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 8 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 10 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 4 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 9 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +Server create request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 60 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 980 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 30 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 691 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 30 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 289 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 30 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 289 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 9 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 16 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 8 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 21 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 18 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 4 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 17 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 9 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +Service list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 54 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 400 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 27 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 190 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 27 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 210 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 27 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 210 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 9 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 16 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 46 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 21 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 18 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 15 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 11 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 9 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +Token issue request stats +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 28 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 219 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 14 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 150 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 14 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 69 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 14 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 69 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 9 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 16 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 46 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 3 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 9 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 6 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 7 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 11 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 9 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +User list request stats +~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 52 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 332 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 26 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 157 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 26 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 175 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 26 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 175 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 9 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 16 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 10 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 3 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 8 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 4 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 9 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 9 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ diff --git a/doc/source/test_results/keystone/all-in-one/liberty_uuid_cache_off.rst b/doc/source/test_results/keystone/all-in-one/liberty_uuid_cache_off.rst new file mode 100644 index 0000000..e61855b --- /dev/null +++ b/doc/source/test_results/keystone/all-in-one/liberty_uuid_cache_off.rst @@ -0,0 +1,512 @@ +Running profiling against Liberty Keystone (UUID tokens, cache turned off) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. contents:: + +Parameters +~~~~~~~~~~ + +=========================== =========== +Parameter name Value +=========================== =========== +OpenStack release Liberty +Cache off +Token type UUID +Environment characteristics Single node +=========================== =========== + +Endpoint list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 92 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 534 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 46 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 259 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 46 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 275 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 44 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 266 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 9 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 9 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 8 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 7 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 24 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 10 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT domain.id AS domain_id, domain.name AS domain_name, domain.enabled AS domain_enabled, | 9 | +| domain.extra AS domain_extra | | +| FROM domain | | +| WHERE domain.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 16 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +Server create request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. note:: This control plane request numbers were not collected yet due to + environmental issues. + +Service list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 56 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 369 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 28 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 177 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 28 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 192 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 26 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 174 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 18 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 14 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 8 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 4 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 4 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.name = %s AND user.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 4 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 9 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 11 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT domain.id AS domain_id, domain.name AS domain_name, domain.enabled AS domain_enabled, | 9 | +| domain.extra AS domain_extra | | +| FROM domain | | +| WHERE domain.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 17 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +Token issue request stats +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 26 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 174 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 13 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 91 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 13 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 83 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 12 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 79 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 1 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 4 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 14 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 8 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 10 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 4 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.name = %s AND user.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 4 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 9 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 9 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT domain.id AS domain_id, domain.name AS domain_name, domain.enabled AS domain_enabled, | 9 | +| domain.extra AS domain_extra | | +| FROM domain | | +| WHERE domain.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 9 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +User list request stats +~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 56 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 360 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 28 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 212 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 28 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 148 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 26 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 139 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 9 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 14 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 8 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 10 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 3 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.name = %s AND user.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 4 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.extra AS role_extra | 9 | +| FROM role | | +| WHERE role.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 13 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT domain.id AS domain_id, domain.name AS domain_name, domain.enabled AS domain_enabled, | 3 | +| domain.extra AS domain_extra | | +| FROM domain | | +| WHERE domain.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 8 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 9 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ diff --git a/doc/source/test_results/keystone/all-in-one/liberty_uuid_cache_on.rst b/doc/source/test_results/keystone/all-in-one/liberty_uuid_cache_on.rst new file mode 100644 index 0000000..1067f05 --- /dev/null +++ b/doc/source/test_results/keystone/all-in-one/liberty_uuid_cache_on.rst @@ -0,0 +1,426 @@ +Running profiling against Liberty Keystone (UUID tokens, cache turned on) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. contents:: + +Parameters +~~~~~~~~~~ + +=========================== =========== +Parameter name Value +=========================== =========== +OpenStack release Liberty +Cache on +Token type UUID +Environment characteristics Single node +=========================== =========== + +Endpoint list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 44 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 465 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 22 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 230 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 22 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 235 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 20 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 204 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 31 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 5 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 18 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 15 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 25 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 13 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +Server create request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. note:: This control plane request numbers were not collected yet due to + environmental issues. + + +Service list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 36 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 227 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 18 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 101 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 18 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 126 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 16 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 108 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 18 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 13 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 12 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 5 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 16 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 18 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 13 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 15 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +Token issue request stats +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 22 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 107 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 11 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 45 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 11 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 62 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 10 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 55 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 1 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 7 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 13 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 12 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 5 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 16 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 18 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 13 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 15 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ + + +User list request stats +~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 26 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 165 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 13 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 61 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 13 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 104 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 11 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 90 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 14 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 13 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 12 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %s AND project.domain_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.name AS user_name, user.domain_id AS user_domain_id, user.password | 22 | +| AS user_password, user.enabled AS user_enabled, user.extra AS user_extra, user.default_project_id AS | | +| user_default_project_id | | +| FROM user | | +| WHERE user.id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 16 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 18 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.type = %s AND assignment.actor_id = %s AND assignment.target_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 13 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 15 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++--------------+--------------------+ +| **DB query** | **Time spent, ms** | ++--------------+--------------------+ diff --git a/doc/source/test_results/keystone/all-in-one/mitaka_fernet_cache_off.rst b/doc/source/test_results/keystone/all-in-one/mitaka_fernet_cache_off.rst new file mode 100644 index 0000000..2fe1ff4 --- /dev/null +++ b/doc/source/test_results/keystone/all-in-one/mitaka_fernet_cache_off.rst @@ -0,0 +1,1366 @@ +Running profiling against Mitaka Keystone (Fernet tokens, cache turned off) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. contents:: + +Parameters +~~~~~~~~~~ + +=========================== =========== +Parameter name Value +=========================== =========== +OpenStack release Mitaka +Cache off +Token type Fernet +Environment characteristics Single node +=========================== =========== + +Endpoint list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 508 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 3975 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 207 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 1667 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 301 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 2308 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 301 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 2308 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 18 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 22 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 11 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 19 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 12 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 12 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 17 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 19 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 17 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 3 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 15 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 11 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 6 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 6 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 9 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Server create request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 4822 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 52521 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 1962 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 20748 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 2860 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 31773 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 2860 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 31773 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 42 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 53 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 29 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 29 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 19 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %(name_1)s AND project.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 35 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 35 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 60 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 44 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 39 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 16 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 38 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 34 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 222 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 41 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 34 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 38 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 17 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 6 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 6 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 6 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 7 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 11 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 11 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 16 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 25 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 12 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 60 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 29 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 14 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 17 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 21 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Service list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 140 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 1298 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 55 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 463 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 85 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 835 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 85 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 835 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 20 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 53 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 18 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 29 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 19 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %(name_1)s AND project.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 35 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 16 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 60 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 14 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 39 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 15 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 33 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 16 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 222 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 41 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 25 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 15 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 17 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 10 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 9 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 3 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Token issue request stats +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 47 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 442 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 18 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 153 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 29 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 289 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 29 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 289 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 20 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 53 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 18 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 29 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 24 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %(name_1)s AND project.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 35 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 16 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 60 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 14 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 39 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 15 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 33 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 20 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 21 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 41 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 25 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 15 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 17 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 7 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 6 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +User list request stats +~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 150 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 1119 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 55 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 442 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 95 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 677 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 95 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 677 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 3 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 11 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 18 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 11 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 11 | +| user.default_project_id AS user_default_project_id | | +| FROM user LEFT OUTER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 24 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %(name_1)s AND project.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 25 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 16 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 60 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 14 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 16 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 11 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 33 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 12 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 21 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 41 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 15 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 11 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 17 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 11 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 7 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ diff --git a/doc/source/test_results/keystone/all-in-one/mitaka_fernet_cache_on.rst b/doc/source/test_results/keystone/all-in-one/mitaka_fernet_cache_on.rst new file mode 100644 index 0000000..058c2f8 --- /dev/null +++ b/doc/source/test_results/keystone/all-in-one/mitaka_fernet_cache_on.rst @@ -0,0 +1,1269 @@ +Running profiling against Mitaka Keystone (Fernet tokens, cache turned on) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. contents:: + +Parameters +~~~~~~~~~~ + +=========================== =========== +Parameter name Value +=========================== =========== +OpenStack release Mitaka +Cache on +Token type Fernet +Environment characteristics Single node +=========================== =========== + +Endpoint list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 398 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 4107 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 152 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 1477 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 246 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 2630 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 246 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 2630 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 29 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 18 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 24 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 42 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 19 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 4 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 12 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 24 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 29 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 52 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 24 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 19 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 14 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 29 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 15 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 4 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 9 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Server create request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 2108 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 19003 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 807 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 6813 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 1301 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 12190 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 1301 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 12190 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 16 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 27 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 27 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 18 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 26 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 26 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 4 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 12 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 23 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 26 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 196 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 15 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 50 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 18 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 18 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 18 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 16 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 6 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 9 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Service list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 110 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 1048 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 40 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 389 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 70 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 659 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 70 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 659 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 12 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 4 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 27 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 18 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 13 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 5 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 17 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 4 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 12 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 23 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 4 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 15 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 14 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 16 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 13 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 17 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 11 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 12 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 7 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 10 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Token issue request stats +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 37 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 308 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 13 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 120 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 24 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 188 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 24 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 188 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 12 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 9 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 27 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 18 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 9 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 8 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 17 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 4 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 12 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 23 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 4 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 15 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 9 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 8 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 14 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 16 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 10 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 19 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +User list request stats +~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 120 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 906 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 40 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 270 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 80 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 636 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 80 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 636 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 4 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 14 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 15 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 15 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 9 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user LEFT OUTER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id | | +| = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 9 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 11 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 4 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 12 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 23 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 4 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 12 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 14 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 11 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 24 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 4 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 4 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 4 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 7 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 7 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ diff --git a/doc/source/test_results/keystone/all-in-one/mitaka_uuid_cache_off.rst b/doc/source/test_results/keystone/all-in-one/mitaka_uuid_cache_off.rst new file mode 100644 index 0000000..1ed9381 --- /dev/null +++ b/doc/source/test_results/keystone/all-in-one/mitaka_uuid_cache_off.rst @@ -0,0 +1,1362 @@ +Running profiling against Mitaka Keystone (UUID tokens, cache turned off) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. contents:: + +Parameters +~~~~~~~~~~ + +=========================== =========== +Parameter name Value +=========================== =========== +OpenStack release Mitaka +Cache off +Token type UUID +Environment characteristics Single node +=========================== =========== + +Endpoint list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 152 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 1348 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 65 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 551 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 87 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 797 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 85 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 781 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 16 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 5 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 25 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 13 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 5 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 20 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 5 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 43 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 17 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 5 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 25 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 3 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 9 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 16 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 14 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 4 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 9 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 3 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 20 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 7 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 8 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 17 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Server create request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 1670 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 14784 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 712 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 5853 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 958 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 8931 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 956 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 8908 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 23 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 88 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 24 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 69 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 24 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 13 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 5 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 75 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 3 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 12 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 22 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 50 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 33 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 16 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 51 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 87 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 22 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 26 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 17 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 4 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 13 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 12 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Service list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 104 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 945 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 41 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 346 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 63 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 599 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 61 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 588 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 11 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 13 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 24 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 69 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 24 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 11 | +| service.extra AS service_extra | | +| FROM service | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 32 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 5 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 75 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 3 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 14 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 11 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 16 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 15 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 3 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 51 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 87 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 15 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 74 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 17 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 32 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 31 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 6 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 14 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Token issue request stats +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 49 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 324 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 19 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 121 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 30 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 203 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 29 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 197 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 1 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 6 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 13 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 10 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 69 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 24 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 11 | +| service.extra AS service_extra | | +| FROM service | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 32 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 5 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 11 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 3 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 14 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 14 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 16 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 15 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 3 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 51 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 87 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 15 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 74 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 17 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 4 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 6 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +User list request stats +~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 114 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 798 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 41 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 267 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 73 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 531 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 71 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 511 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 20 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 8 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 5 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 4 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 9 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 17 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 12 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 22 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 15 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 32 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 8 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 13 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 8 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %(name_1)s AND project.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 69 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 9 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 87 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 3 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 9 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user LEFT OUTER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = | | +| local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 11 | +| service.extra AS service_extra | | +| FROM service | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 14 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 4 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 17 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 7 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ diff --git a/doc/source/test_results/keystone/all-in-one/mitaka_uuid_cache_on.rst b/doc/source/test_results/keystone/all-in-one/mitaka_uuid_cache_on.rst new file mode 100644 index 0000000..e1f5c59 --- /dev/null +++ b/doc/source/test_results/keystone/all-in-one/mitaka_uuid_cache_on.rst @@ -0,0 +1,1368 @@ +Running profiling against Mitaka Keystone (UUID tokens, cache turned on) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. contents:: + +Parameters +~~~~~~~~~~ + +=========================== =========== +Parameter name Value +=========================== =========== +OpenStack release Mitaka +Cache on +Token type UUID +Environment characteristics Single node +=========================== =========== + +Endpoint list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 132 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 1240 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 55 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 506 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 77 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 734 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 75 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 722 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 12 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 2 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 4 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 24 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 15 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 13 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 17 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 40 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 4 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 4 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 7 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 8 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 11 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 13 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Server create request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 1140 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 9581 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 451 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 3500 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 689 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 6081 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 687 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 6067 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 14 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 39 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 61 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 24 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 27 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 12 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 40 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 8 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %(name_1)s AND project.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 9 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 18 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 63 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 27 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 9 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 40 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 16 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 22 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 9 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 14 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 28 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 25 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 33 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 9 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 9 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 6 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Service list request stats +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 84 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 619 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 31 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 207 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 53 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 412 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 51 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 393 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 19 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 9 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 40 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 10 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 5 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 9 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 16 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 15 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 11 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 24 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 12 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 18 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 12 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 9 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %(name_1)s AND project.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 27 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 4 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 33 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 22 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 9 | +| service.extra AS service_extra | | +| FROM service | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 9 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 19 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 9 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 6 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 12 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +Token issue request stats +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 39 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 448 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 14 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 179 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 25 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 269 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 24 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 263 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 1 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 6 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 20 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 40 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 15 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 5 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 9 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 16 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 15 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 11 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 24 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 12 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 18 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 27 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 9 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %(name_1)s AND project.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 27 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 4 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 33 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 22 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 9 | +| service.extra AS service_extra | | +| FROM service | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 9 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 19 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 10 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + + +User list request stats +~~~~~~~~~~~~~~~~~~~~~~~ + +**Control plane request overlook** + ++--------------------------------------------------------------+-----------+ +| **Metric** | **Value** | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries count | 94 | ++--------------------------------------------------------------+-----------+ +| Total (*) Keystone DB queries time spent, ms | 700 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries count | 31 | ++--------------------------------------------------------------+-----------+ +| Infrastructure (SELECT 1) Keystone DB queries time spent, ms | 240 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries count | 63 | ++--------------------------------------------------------------+-----------+ +| Real Keystone DB queries time spent, ms | 460 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries count | 61 | ++--------------------------------------------------------------+-----------+ +| SELECT Keystone DB queries time spent, ms | 450 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries count | 2 | ++--------------------------------------------------------------+-----------+ +| INSERT Keystone DB queries time spent, ms | 10 | ++--------------------------------------------------------------+-----------+ + +.. note:: (*) OSprofiler uses specific SQLalchemy cursor events to track + what's going on with the DB layer. This number includes non-real + DB requests "SELECT 1", processed by SQLalchemy itself to make + sure that connection to the database is still in place. + + +.. note:: Number of DB queries is much bigger for this request comparing with + similar Liberty environment. The difference is in + `keystone_authtoken` middleware (and, more specifically, its cache) + usage. Although all environments that took part in the research had + exactly the same `keystone_authtoken` middleware configuration, + described in the test plan and containing `memcache_servers` + parameter set up, Mitaka environments profiling shows that all + OpenStack services that used `keystone_authtoken` middleware did not + use the external Memcached cached token copy. All of them used + Keystone API every time REST API request was coming to the OpenStack + services. This behaviour needs to be investigated separately. + + +**Keystone DB queries outliers** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 20 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 40 | +| service.extra AS service_extra | | +| FROM service | | +| WHERE service.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT implied_role.prior_role_id AS implied_role_prior_role_id, implied_role.implied_role_id AS | 15 | +| implied_role_implied_role_id | | +| FROM implied_role | | +| WHERE implied_role.prior_role_id = %(prior_role_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 5 | +| user.default_project_id AS user_default_project_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT user.id AS user_id, user.enabled AS user_enabled, user.extra AS user_extra, | 9 | +| user.default_project_id AS user_default_project_id | | +| FROM user | | +| WHERE user.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT role.id AS role_id, role.name AS role_name, role.domain_id AS role_domain_id, role.extra AS | 16 | +| role_extra | | +| FROM role | | +| WHERE role.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 16 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id = | | +| federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 11 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user LEFT OUTER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN federated_user ON anon_1.user_id | | +| = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT password.id AS password_id, password.local_user_id AS password_local_user_id, | 11 | +| password.password AS password_password | | +| FROM password | | +| WHERE %(param_1)s = password.local_user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT assignment.type AS assignment_type, assignment.actor_id AS assignment_actor_id, | 24 | +| assignment.target_id AS assignment_target_id, assignment.role_id AS assignment_role_id, | | +| assignment.inherited AS assignment_inherited | | +| FROM assignment | | +| WHERE assignment.actor_id IN (%(actor_id_1)s) AND assignment.target_id IN (%(target_id_1)s) AND | | +| assignment.type IN (%(type_1)s) AND assignment.inherited = false | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 12 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT `group`.id AS group_id, `group`.name AS group_name, `group`.domain_id AS group_domain_id, | 18 | +| `group`.description AS group_description, `group`.extra AS group_extra | | +| FROM `group` INNER JOIN user_group_membership ON `group`.id = user_group_membership.group_id | | +| WHERE user_group_membership.user_id = %(user_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 13 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT project.id AS project_id, project.name AS project_name, project.domain_id AS | 9 | +| project_domain_id, project.description AS project_description, project.enabled AS project_enabled, | | +| project.extra AS project_extra, project.parent_id AS project_parent_id, project.is_domain AS | | +| project_is_domain | | +| FROM project | | +| WHERE project.name = %(name_1)s AND project.domain_id = %(domain_id_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT token.id AS token_id, token.expires AS token_expires, token.extra AS token_extra, token.valid | 27 | +| AS token_valid, token.user_id AS token_user_id, token.trust_id AS token_trust_id | | +| FROM token | | +| WHERE token.id = %(param_1)s | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT revocation_event.id AS revocation_event_id, revocation_event.domain_id AS | 4 | +| revocation_event_domain_id, revocation_event.project_id AS revocation_event_project_id, | | +| revocation_event.user_id AS revocation_event_user_id, revocation_event.role_id AS | | +| revocation_event_role_id, revocation_event.trust_id AS revocation_event_trust_id, | | +| revocation_event.consumer_id AS revocation_event_consumer_id, revocation_event.access_token_id AS | | +| revocation_event_access_token_id, revocation_event.issued_before AS revocation_event_issued_before, | | +| revocation_event.expires_at AS revocation_event_expires_at, revocation_event.revoked_at AS | | +| revocation_event_revoked_at, revocation_event.audit_id AS revocation_event_audit_id, | | +| revocation_event.audit_chain_id AS revocation_event_audit_chain_id | | +| FROM revocation_event ORDER BY revocation_event.revoked_at | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 33 | +| service.extra AS service_extra, endpoint_1.id AS endpoint_1_id, endpoint_1.legacy_endpoint_id AS | | +| endpoint_1_legacy_endpoint_id, endpoint_1.interface AS endpoint_1_interface, endpoint_1.region_id AS | | +| endpoint_1_region_id, endpoint_1.service_id AS endpoint_1_service_id, endpoint_1.url AS | | +| endpoint_1_url, endpoint_1.enabled AS endpoint_1_enabled, endpoint_1.extra AS endpoint_1_extra | | +| FROM service LEFT OUTER JOIN endpoint AS endpoint_1 ON service.id = endpoint_1.service_id | | +| WHERE service.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service_provider.id AS service_provider_id, service_provider.enabled AS | 22 | +| service_provider_enabled, service_provider.description AS service_provider_description, | | +| service_provider.auth_url AS service_provider_auth_url, service_provider.sp_url AS | | +| service_provider_sp_url, service_provider.relay_state_prefix AS service_provider_relay_state_prefix | | +| FROM service_provider | | +| WHERE service_provider.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT service.id AS service_id, service.type AS service_type, service.enabled AS service_enabled, | 9 | +| service.extra AS service_extra | | +| FROM service | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 5 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 18 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user | | +| WHERE user.id = %(param_1)s) AS anon_1 INNER JOIN local_user ON anon_1.user_id = local_user.user_id | | +| ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 12 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra | | +| FROM endpoint | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT endpoint.id AS endpoint_id, endpoint.legacy_endpoint_id AS endpoint_legacy_endpoint_id, | 19 | +| endpoint.interface AS endpoint_interface, endpoint.region_id AS endpoint_region_id, | | +| endpoint.service_id AS endpoint_service_id, endpoint.url AS endpoint_url, endpoint.enabled AS | | +| endpoint_enabled, endpoint.extra AS endpoint_extra, service_1.id AS service_1_id, service_1.type AS | | +| service_1_type, service_1.enabled AS service_1_enabled, service_1.extra AS service_1_extra | | +| FROM endpoint LEFT OUTER JOIN service AS service_1 ON service_1.id = endpoint.service_id | | +| WHERE endpoint.enabled = true | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ + +**Keystone DB queries with multi JOINs inside** + ++------------------------------------------------------------------------------------------------------+--------------------+ +| **DB query** | **Time spent, ms** | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 6 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 7 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT federated_user.id AS federated_user_id, federated_user.user_id AS federated_user_user_id, | 9 | +| federated_user.idp_id AS federated_user_idp_id, federated_user.protocol_id AS | | +| federated_user_protocol_id, federated_user.unique_id AS federated_user_unique_id, | | +| federated_user.display_name AS federated_user_display_name, anon_1.user_id AS anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| federated_user ON anon_1.user_id = federated_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ +| SELECT local_user.id AS local_user_id, local_user.user_id AS local_user_user_id, | 5 | +| local_user.domain_id AS local_user_domain_id, local_user.name AS local_user_name, anon_1.user_id AS | | +| anon_1_user_id | | +| FROM (SELECT user.id AS user_id | | +| FROM user INNER JOIN local_user ON user.id = local_user.user_id | | +| WHERE local_user.name = %(name_1)s AND local_user.domain_id = %(domain_id_1)s) AS anon_1 INNER JOIN | | +| local_user ON anon_1.user_id = local_user.user_id ORDER BY anon_1.user_id | | +| | | +| | | | ++------------------------------------------------------------------------------------------------------+--------------------+ diff --git a/doc/source/test_results/keystone/index.rst b/doc/source/test_results/keystone/index.rst new file mode 100644 index 0000000..4ac92da --- /dev/null +++ b/doc/source/test_results/keystone/index.rst @@ -0,0 +1,11 @@ +================================= +Keystone Performance Test Reports +================================= + +Reports +^^^^^^^ + +.. toctree:: + :maxdepth: 2 + + all-in-one/index diff --git a/doc/source/test_results/keystone/parse_json_res.py b/doc/source/test_results/keystone/parse_json_res.py new file mode 100644 index 0000000..c213952 --- /dev/null +++ b/doc/source/test_results/keystone/parse_json_res.py @@ -0,0 +1,305 @@ +import argparse +import collections +import copy +import itertools +import json +import numpy as np +import os +import prettytable + + +NODES_TEMPLATE = { + "DB queries": { + "total": 0, + "total_time_spent": 0, + "select1": 0, + "select1_time_spent": 0, + "real": 0, + "real_time_spent": 0, + "SELECT": { + "total": 0, + "INNER JOIN": 0, + "LEFT JOIN": 0, + "RIGHT JOIN": 0, + "FULL JOIN": 0, + "time_spent": collections.OrderedDict(), + }, + "INSERT": { + "total": 0, + "time_spent": collections.OrderedDict(), + }, + "UPDATE": { + "total": 0, + "time_spent": collections.OrderedDict(), + }, + "DELETE": { + "total": 0, + "time_spent": collections.OrderedDict(), + }, + "red_flag": { + "2joins": { + "total": 0, + "queries": [], + "time_spent": collections.OrderedDict(), + }, + "3+joins": { + "total": 0, + "queries": [], + "time_spent": collections.OrderedDict(), + } + } + }, + "Cached operations": {}, + "Cached time spent": collections.OrderedDict(), +} + +NODES = copy.deepcopy(NODES_TEMPLATE) +OUTLIER_QUERIES = {} + + +def define_node(node): + if node["info"]["project"] != "keystone": + return + + if node["info"]["name"] not in ["db", "cache"]: + return + + time_spent = node["info"]["finished"] - node["info"]["started"] + + if node["info"]["name"] == "db": + process_db_calls(node, time_spent) + elif node["info"]["name"] == "cache": + if not node["children"]: + process_cache_calls(node, time_spent) + else: + for child in node["children"]: + define_node(child) + + +def process_cache_calls(node, time_spent): + cache_info = node["info"]["meta.raw_payload.cache-start"][ + "info"]["fn_info"] + if not NODES["Cached operations"].get(cache_info): + NODES["Cached operations"][cache_info] = 0 + NODES["Cached operations"][cache_info] += 1 + if not NODES["Cached time spent"].get(time_spent): + NODES["Cached time spent"][time_spent] = [] + NODES["Cached time spent"][time_spent].append(cache_info) + + +def process_db_calls(node, time_spent): + NODES["DB queries"]["total"] += 1 + NODES["DB queries"]["total_time_spent"] += time_spent + statement = node[ + "info"]["meta.raw_payload.db-start"]["info"]["db"]["statement"] + if statement.startswith("SELECT 1"): + NODES["DB queries"]["select1"] += 1 + NODES["DB queries"]["select1_time_spent"] += time_spent + else: + NODES["DB queries"]["real"] += 1 + NODES["DB queries"]["real_time_spent"] += time_spent + + if statement.startswith("SELECT"): + process_selects(statement, time_spent) + elif statement.startswith("UPDATE"): + process_base_db_calls("UPDATE", statement, time_spent) + elif statement.startswith("INSERT"): + process_base_db_calls("INSERT", statement, time_spent) + elif statement.startswith("DELETE"): + process_base_db_calls("DELETE", statement, time_spent) + + +def process_base_db_calls(command, statement, time_spent): + NODES["DB queries"][command]["total"] += 1 + if not NODES["DB queries"][command]["time_spent"].get(time_spent): + NODES["DB queries"][command]["time_spent"][time_spent] = [] + NODES["DB queries"][command]["time_spent"][time_spent].append( + statement) + + +def process_selects(statement, time_spent): + process_base_db_calls("SELECT", statement, time_spent) + + ij = statement.count("INNER JOIN") + lj = statement.count("LEFT JOIN") + rj = statement.count("RIGHT JOIN") + fj = statement.count("FULL JOIN") + NODES["DB queries"]["SELECT"]["INNER JOIN"] += ij + NODES["DB queries"]["SELECT"]["LEFT JOIN"] += lj + NODES["DB queries"]["SELECT"]["RIGHT JOIN"] += rj + NODES["DB queries"]["SELECT"]["FULL JOIN"] += fj + + # raise red flags if too many JOINS met + if ij + lj + rj + fj == 2: + NODES["DB queries"]["red_flag"]["2joins"]["total"] += 1 + NODES["DB queries"]["red_flag"]["2joins"][ + "queries"].append(statement) + if not NODES["DB queries"]["red_flag"]["2joins"][ + "time_spent"].get(time_spent): + NODES["DB queries"]["red_flag"]["2joins"]["time_spent"][ + time_spent] = [] + NODES["DB queries"]["red_flag"]["2joins"]["time_spent"][ + time_spent].append(statement) + elif ij + lj + rj + fj >= 3: + NODES["DB queries"]["red_flag"]["3+joins"]["total"] += 1 + NODES["DB queries"]["red_flag"]["3+joins"][ + "queries"].append(statement) + if not NODES["DB queries"]["red_flag"]["3+joins"][ + "time_spent"].get(time_spent): + NODES["DB queries"]["red_flag"]["3+joins"]["time_spent"][ + time_spent] = [] + NODES["DB queries"]["red_flag"]["3+joins"]["time_spent"][ + time_spent].append(statement) + + +def define_nodes(data): + for child in data["children"]: + if not child["children"]: + define_node(child) + else: + define_nodes(child) + + +def sort_dicts(dictionary): + new_nodes = copy.deepcopy(dictionary) + for key in ["SELECT", "INSERT", "DELETE", "UPDATE"]: + new_nodes["DB queries"][key]["time_spent"] = \ + sum([k*len(v) for k, v + in dictionary["DB queries"][key]["time_spent"].iteritems()]) + for key in ["2joins", "3+joins"]: + new_nodes["DB queries"]["red_flag"][key]["time_spent"] = \ + sum([k*len(v) for k, v + in dictionary["DB queries"]["red_flag"][key][ + "time_spent"].iteritems()]) + new_nodes["Cached time spent"] = \ + sum([k*len(v) for k, v + in dictionary["Cached time spent"].iteritems()]) + return new_nodes + + +def detect_outliers(data, m=2.): + full_time_set = list(itertools.chain(*[[k] * len(v) for k, v + in data.iteritems()])) + dat = np.abs(full_time_set - np.median(full_time_set)) + mdev = np.median(dat) + sss = dat/mdev if mdev else 0. + if mdev: + for idx, val in enumerate((sss < m).tolist()): + if not val: + for query in data[full_time_set[idx]]: + OUTLIER_QUERIES[query] = full_time_set[idx] + + +def prepare_tables(nodes): + # prepare table with common information + common_info_table = prettytable.PrettyTable(["**Metric**", "**Value**"]) + common_info_table.align["**Metric**"] = "l" + common_info_table.align["**Value**"] = "l" + common_info_table.padding_width = 1 + common_info_table.max_width = 100 + common_info_table.header = True + common_info_table.hrules = prettytable.ALL + + common_info_table.add_row(["Total (*) Keystone DB queries count", + nodes["DB queries"]["total"]]) + common_info_table.add_row(["Total (*) Keystone DB queries time spent, ms", + nodes["DB queries"]["total_time_spent"]]) + common_info_table.add_row([ + "Infrastructure (SELECT 1) Keystone DB queries count", + nodes["DB queries"]["select1"]]) + common_info_table.add_row([ + "Infrastructure (SELECT 1) Keystone DB queries time spent, ms", + nodes["DB queries"]["select1_time_spent"]]) + common_info_table.add_row(["Real Keystone DB queries count", + nodes["DB queries"]["real"]]) + common_info_table.add_row(["Real Keystone DB queries time spent, ms", + nodes["DB queries"]["real_time_spent"]]) + + db_query_tmpl = "%s\n\n|" + + for key in ["SELECT", "INSERT", "DELETE", "UPDATE"]: + if nodes["DB queries"][key]["total"]: + common_info_table.add_row([ + "%s Keystone DB queries count" % key, + nodes["DB queries"][key]["total"]]) + common_info_table.add_row([ + "%s Keystone DB queries time spent, ms" % key, + nodes["DB queries"][key]["time_spent"]]) + detect_outliers(NODES["DB queries"][key]["time_spent"]) + + # prepare table with outliers information + outliers_table = prettytable.PrettyTable(["**DB query**", + "**Time spent, ms**"]) + outliers_table.align["**DB query**"] = "l" + outliers_table.align["**Time spent, ms**"] = "l" + outliers_table.max_width = 100 + outliers_table.header = True + outliers_table.hrules = prettytable.ALL + + for query in OUTLIER_QUERIES: + outliers_table.add_row([db_query_tmpl % query, OUTLIER_QUERIES[query]]) + + # prepare table with information about DB requests containing multiple + # JOIN statements inside + multi_join_queries = prettytable.PrettyTable(["**DB query**", + "**Time spent, ms**"]) + multi_join_queries.align["**DB query**"] = "l" + multi_join_queries.align["**Time spent, ms**"] = "l" + multi_join_queries.max_width = 100 + multi_join_queries.header = True + multi_join_queries.hrules = prettytable.ALL + + for key in ["2joins", "3+joins"]: + for ts in NODES["DB queries"]["red_flag"][key]["time_spent"]: + for query in NODES["DB queries"]["red_flag"][key][ + "time_spent"][ts]: + multi_join_queries.add_row([db_query_tmpl % query, ts]) + return common_info_table, multi_join_queries, outliers_table + + +def main(): + parser = argparse.ArgumentParser(description='Process JSON file with ' + 'OSprofiler output.') + parser.add_argument('path', type=str, + help='Path to the JSON file / directory with list of ' + 'JSON files with OSprofiler output') + + args = parser.parse_args() + global NODES + if os.path.isfile(args.path): + with open(args.path) as data_file: + data = json.load(data_file) + define_nodes(data) + nodes = sort_dicts(NODES) + common_info_table, multi_join_queries, outliers_table = \ + prepare_tables(nodes) + print(common_info_table) + print(outliers_table) + print(multi_join_queries) + elif os.path.isdir(args.path): + for item in os.listdir(args.path): + if item.endswith(".txt"): + with open(os.path.join(args.path, item)) as data_file: + data = json.load(data_file) + NODES = copy.deepcopy(NODES_TEMPLATE) + define_nodes(data) + nodes = sort_dicts(NODES) + common_info_table, multi_join_queries, outliers_table = \ + prepare_tables(nodes) + item_name = \ + item.split(".")[0].replace("_", " ").capitalize() + \ + " request stats" + print(item_name) + print(len(item_name) * "~" + "\n") + print("**%s**\n" % "Control plane request overlook") + print(common_info_table) + print("\n**%s**\n" % "Keystone DB queries outliers") + print(outliers_table) + print("\n**%s**\n" % "Keystone DB queries with multi " + "JOINs inside") + print(multi_join_queries) + print("\n") + + +if __name__ == "__main__": + main()