openstack/tuning-box
Code Issues Proposed changes

Fetching effective resource values optimized

Browse Source 
Now only required resource values are fetched on calculating
effective values.
Calculation of effective values optimised to use indexed resource
values insead of going throw nested for loop.
Test for fetching values and effective values for 500 items
(nodes) added. Each resource value contains over than 10000 keys.
Logging added to the resource values fetching.

Change-Id: Icea7ece8a2bc6957bcb00c9d90d8cf59e5ca1da0
Closes-Bug: #1626147
This commit is contained in:
Alexander Kislitsky 2016-09-21 17:17:33 +03:00
parent 3a86b593f4
commit 3705aa1526
6 changed files with 150 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tuning_box/app.py
View File

@ -134,8 +134,8 @@ def build_app(configure_logging=True, with_keystone=True):
handle_keys_operation_error)
db.db.init_app(app)
if configure_logging:
log_level = app.config.get('LOG_LEVEL', 'INFO')
logger.init_logger(log_level)
log_level = app.config.get('LOG_LEVEL', 'DEBUG')
logger.init_logger(app, log_level)
if with_keystone:
app.wsgi_app = keystone.KeystoneMiddleware(app)
return app

6
tuning_box/library/hierarchy_levels.py
View File

@ -11,6 +11,7 @@
# under the License.
import flask
from flask import current_app as app
import flask_restful
from flask_restful import fields
@ -20,7 +21,12 @@ from tuning_box import errors
def iter_environment_level_values(environment, levels):
app.logger.debug("Getting environment level values. Env: %s, "
"levels: %s", environment.id, levels)
env_levels = db.EnvironmentHierarchyLevel.get_for_environment(environment)
app.logger.debug("Environment levels got. Env: %s, levels: %s",
environment.id, [l.name for l in env_levels])
if len(env_levels) < len(levels):
raise errors.TuningboxNotFound(
"Levels {0} can't be matched with "

97
tuning_box/library/resource_values.py
View File

@ -11,8 +11,10 @@
# under the License.
import flask
from flask import current_app as app
import flask_restful
import itertools
import six
from sqlalchemy import or_
from tuning_box import db
from tuning_box import library
@ -39,8 +41,28 @@ class ResourceValues(flask_restful.Resource):
esv.values = flask.request.json
return None, 204
def _calculate_effective_values(self, result, level_value,
resource_values_idx, show_lookup,
lookup_path):
level_value_id = getattr(level_value, 'id', None)
if level_value_id in resource_values_idx:
resource_value = resource_values_idx[level_value_id]
if show_lookup:
values = ((k, (v, lookup_path)) for k, v in
six.iteritems(resource_value.values))
overrides = ((k, (v, lookup_path)) for k, v in
six.iteritems(resource_value.overrides))
else:
values = resource_value.values
overrides = resource_value.overrides
result.update(values)
result.update(overrides)
@db.with_transaction
def get(self, environment_id, resource_id_or_name, levels):
app.logger.debug("Getting resource value. Env: %s, "
"resource: %s, levels: %s", environment_id,
resource_id_or_name, levels)
environment = db.Environment.query.get_or_404(environment_id)
res_def = library.get_resource_definition(
resource_id_or_name, environment_id)
@ -48,37 +70,48 @@ class ResourceValues(flask_restful.Resource):
level_values = list(hierarchy_levels.iter_environment_level_values(
environment, levels))
if 'effective' in flask.request.args:
show_lookup = 'show_lookup' in flask.request.args
resource_values = db.ResourceValues.query.filter_by(
resource_definition=res_def,
environment=environment,
).all()
result = {}
lookup_path = ''
for level_value in itertools.chain([None], level_values):
if level_value is not None:
name = level_value.level.name
value = level_value.value
lookup_path += name + '/' + value + '/'
else:
lookup_path += '/'
level_values_ids = [l.id for l in level_values]
app.logger.debug("Got level values ids: %s", level_values_ids)
for resource_value in resource_values:
if resource_value.level_value == level_value:
if show_lookup:
values = {}
for k, v in resource_value.values.items():
values[k] = (v, lookup_path)
overrides = {}
for k, v in resource_value.overrides.items():
overrides[k] = (v, lookup_path)
else:
values = resource_value.values
overrides = resource_value.overrides
result.update(values)
result.update(overrides)
break
if 'effective' in flask.request.args:
app.logger.debug("Getting effective resource value. Env: %s, "
"resource: %s, levels: %s", environment_id,
resource_id_or_name, levels)
show_lookup = 'show_lookup' in flask.request.args
resource_values = db.ResourceValues.query.filter(
or_(
db.ResourceValues.level_value_id.in_(level_values_ids),
db.ResourceValues.level_value_id.is_(None)
),
db.ResourceValues.resource_definition == res_def,
db.ResourceValues.environment == environment
).all()
app.logger.debug("Processing values for resource: %s, env: %s. "
"Loaded resource values: %s",
res_def.id, environment.id, len(resource_values))
# Creating index of resource_values by level_value_id
resource_values_idx = {r.level_value_id: r
for r in resource_values}
app.logger.debug("Resource values index size: %s",
len(resource_values_idx))
result = {}
lookup_path = '/'
self._calculate_effective_values(
result, None, resource_values_idx, show_lookup,
lookup_path)
for level_value in level_values:
name = level_value.level.name
value = level_value.value
lookup_path += name + '/' + value + '/'
self._calculate_effective_values(
result, level_value, resource_values_idx, show_lookup,
lookup_path)
app.logger.debug("Effective values got for resource: "
"%s, env: %s", res_def.id, environment.id)
return result
else:
if not level_values:
@ -90,6 +123,8 @@ class ResourceValues(flask_restful.Resource):
environment=environment,
level_value=level_value,
).one_or_none()
app.logger.debug("Values got for resource: "
"%s, env: %s", res_def.id, environment.id)
if not resource_values:
return {}
return resource_values.values

7
tuning_box/logger.py
View File

@ -20,9 +20,8 @@ def get_formatter():
return logging.Formatter(fmt=log_format, datefmt=date_format)
def init_logger(log_level):
def init_logger(app, log_level):
handler = logging.StreamHandler()
handler.setFormatter(get_formatter())
logger = logging.getLogger()
logger.addHandler(handler)
logger.setLevel(log_level)
app.logger.addHandler(handler)
app.logger.setLevel(log_level)

4
tuning_box/tests/library/test_resource_overrides.py
View File

@ -278,8 +278,8 @@ class TestResourceOverrides(BaseTest):
# Checking lookup info
res = self.client.get(
'/environments/9/lvl1/1/lvl2/2/resources/5/values?'
'effective&show_lookup',
'/environments/9/lvl1/1/lvl2/2/resources/5/values',
query_string={'effective': 1, 'show_lookup': 1}
)
self.assertEqual(res.status_code, 200)
expected = {

71
tuning_box/tests/library/test_resource_values.py
View File

@ -11,6 +11,9 @@
# under the License.
import itertools
import uuid
import six
from tuning_box import db
from tuning_box.tests.test_app import BaseTest
@ -365,3 +368,71 @@ class TestResourceValues(BaseTest):
'key3': ['root_value_3', '/']
}
self.assertEqual(expected, res.json)
def generate_values(self, size):
result = {}
for i in six.moves.range(size):
result[six.text_type(uuid.uuid4())] = i
return result
def test_get_resource_values_effective_lot_of_data(self):
self._fixture()
env_id = 9
res_id = 5
keys_on_root = 10000
keys_on_lvl1 = 15000
keys_on_lvl2 = 20000
values_on_level = 500
# Adding values on the root level
self._add_resource_values(
env_id, res_id, (), self.generate_values(keys_on_root))
# Adding values on the level lvl1 and lvl2
lvl_1_values = self.generate_values(keys_on_lvl1)
lvl_2_values = self.generate_values(keys_on_lvl2)
for lvl_val in six.moves.range(values_on_level):
lvl_val = six.text_type(lvl_val)
self._add_resource_values(
env_id, res_id, (('lvl1', lvl_val),), lvl_1_values)
self._add_resource_values(
env_id, res_id, (('lvl1', lvl_val), ('lvl2', lvl_val)),
lvl_2_values)
with self.app.app_context():
res_vals_count = db.ResourceValues.query.count()
self.assertEqual(1 + values_on_level * 2, res_vals_count)
# Check keys num on root level
obj_url = self.object_url.format(
env_id, self.get_levels_path(()),
res_id
)
res = self.client.get(obj_url)
self.assertEqual(keys_on_root, len(res.json))
res = self.client.get(obj_url, query_string={'effective': 1})
self.assertEqual(keys_on_root, len(res.json))
# Check keys num on lvl1
obj_url = self.object_url.format(
env_id, self.get_levels_path((('lvl1', '1'),)),
res_id
)
res = self.client.get(obj_url)
self.assertEqual(keys_on_lvl1, len(res.json))
res = self.client.get(obj_url, query_string={'effective': 1})
self.assertEqual(keys_on_root + keys_on_lvl1, len(res.json))
# Check keys num on lvl2
obj_url = self.object_url.format(
env_id, self.get_levels_path((('lvl1', '1'), ('lvl2', '2'))),
res_id
)
res = self.client.get(obj_url)
self.assertEqual(keys_on_lvl2, len(res.json))
res = self.client.get(obj_url, query_string={'effective': 1})
self.assertEqual(keys_on_root + keys_on_lvl1 + keys_on_lvl2,
len(res.json))