x/stackalytics
Code Issues Proposed changes

Show warning if data is older than configured value

Browse Source 
The patch introduces config param 'age-warn' with default value
in 2 days. The warning message will be useful for cases when
processor stuck or data was recovered from the dump.

Change-Id: I910b87da5c664fbda7d25e77ee51554f01ad9bc2
This commit is contained in:
Ilya Shakhat 2015-06-02 16:35:56 +03:00
parent ebaaa7e414
commit 4ee02f0b6c
7 changed files with 37 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
doc/source/tools/stackalytics-dashboard.txt
View File

@ -47,14 +47,14 @@ optional arguments:
--syslog-log-facility SYSLOG_LOG_FACILITY
Syslog facility to receive log lines.
--use-syslog Use syslog for logging. Existing syslog format is
DEPRECATED during I, and will change in J to honor
DEPRECATED during I, and changed in J to honor
RFC5424.
--use-syslog-rfc-format
(Optional) Enables or disables syslog rfc5424 format
for logging. If enabled, prefixes the MSG part of the
syslog message with APP-NAME (RFC5424). The format
without the APP-NAME is deprecated in I, and will be
removed in J.
without the APP-NAME is deprecated in K, and will be
removed in M, along with this option.
--verbose, -v Print more verbose output (set logging level to INFO
instead of default WARNING level).
--version show program's version number and exit

9
doc/source/tools/stackalytics-dump.txt
View File

@ -1,4 +1,4 @@
usage: stackalytics-dump [-h]
usage: stackalytics-dump [-h] [--age-warn AGE_WARN]
[--collect-profiler-stats COLLECT_PROFILER_STATS]
[--config-dir DIR] [--config-file PATH]
[--corrections-uri CORRECTIONS_URI]
@ -31,6 +31,7 @@ usage: stackalytics-dump [-h]
optional arguments:
-h, --help show this help message and exit
--age-warn AGE_WARN Warn if the age of data is more than this value, sec
--collect-profiler-stats COLLECT_PROFILER_STATS
Name of file to store python profiler data. This
option works for dashboard only
@ -114,14 +115,14 @@ optional arguments:
--syslog-log-facility SYSLOG_LOG_FACILITY
Syslog facility to receive log lines.
--use-syslog Use syslog for logging. Existing syslog format is
DEPRECATED during I, and will change in J to honor
DEPRECATED during I, and changed in J to honor
RFC5424.
--use-syslog-rfc-format
(Optional) Enables or disables syslog rfc5424 format
for logging. If enabled, prefixes the MSG part of the
syslog message with APP-NAME (RFC5424). The format
without the APP-NAME is deprecated in I, and will be
removed in J.
without the APP-NAME is deprecated in K, and will be
removed in M, along with this option.
--verbose, -v Print more verbose output (set logging level to INFO
instead of default WARNING level).
--version show program's version number and exit

9
doc/source/tools/stackalytics-processor.txt
View File

@ -1,4 +1,4 @@
usage: stackalytics-processor [-h]
usage: stackalytics-processor [-h] [--age-warn AGE_WARN]
[--collect-profiler-stats COLLECT_PROFILER_STATS]
[--config-dir DIR] [--config-file PATH]
[--corrections-uri CORRECTIONS_URI]
@ -31,6 +31,7 @@ usage: stackalytics-processor [-h]
optional arguments:
-h, --help show this help message and exit
--age-warn AGE_WARN Warn if the age of data is more than this value, sec
--collect-profiler-stats COLLECT_PROFILER_STATS
Name of file to store python profiler data. This
option works for dashboard only
@ -109,14 +110,14 @@ optional arguments:
--syslog-log-facility SYSLOG_LOG_FACILITY
Syslog facility to receive log lines.
--use-syslog Use syslog for logging. Existing syslog format is
DEPRECATED during I, and will change in J to honor
DEPRECATED during I, and changed in J to honor
RFC5424.
--use-syslog-rfc-format
(Optional) Enables or disables syslog rfc5424 format
for logging. If enabled, prefixes the MSG part of the
syslog message with APP-NAME (RFC5424). The format
without the APP-NAME is deprecated in I, and will be
removed in J.
without the APP-NAME is deprecated in K, and will be
removed in M, along with this option.
--verbose, -v Print more verbose output (set logging level to INFO
instead of default WARNING level).
--version show program's version number and exit

18
etc/stackalytics.conf
View File

@ -39,14 +39,16 @@
#log_dir = <None>
# Use syslog for logging. Existing syslog format is DEPRECATED during I, and
# will change in J to honor RFC5424. (boolean value)
# changed in J to honor RFC5424. (boolean value)
#use_syslog = false
# (Optional) Enables or disables syslog rfc5424 format for logging. If enabled,
# prefixes the MSG part of the syslog message with APP-NAME (RFC5424). The
# format without the APP-NAME is deprecated in I, and will be removed in J.
# (boolean value)
#use_syslog_rfc_format = false
# format without the APP-NAME is deprecated in K, and will be removed in M,
# along with this option. (boolean value)
# This option is deprecated for removal.
# Its value may be silently ignored in the future.
#use_syslog_rfc_format = true
# Syslog facility to receive log lines. (string value)
#syslog_log_facility = LOG_USER
@ -64,7 +66,7 @@
#logging_debug_format_suffix = %(funcName)s %(pathname)s:%(lineno)d
# Prefix each line of exception output with this format. (string value)
#logging_exception_prefix = %(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s
#logging_exception_prefix = %(asctime)s.%(msecs)03d %(process)d ERROR %(name)s %(instance)s
# List of logger=LEVEL pairs. (list value)
#default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,requests.packages.urllib3.util.retry=WARN,urllib3.util.retry=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN
@ -83,6 +85,9 @@
# value)
#instance_uuid_format = "[instance: %(uuid)s] "
# Enables or disables fatal status of deprecations. (boolean value)
#fatal_deprecations = false
#
# From stackalytics.processor.config
#
@ -148,3 +153,6 @@
# How many member profiles to look ahead after the last (integer value)
#members_look_ahead = 250
# Warn if the age of data is more than this value, sec (integer value)
#age_warn = 172800

5
stackalytics/dashboard/templates/layout.html
View File

@ -24,9 +24,12 @@ Stackalytics {% if page_title %}| {{ page_title }} {% endif %}
<div class="page">
{% if not runtime_storage_update_time %}
<div class="banner">Please note! The data is being loaded now and may be not complete.</div>
<div class="banner">The data is being loaded now and is not complete</div>
{% set update_time_title = '' %}
{% else %}
{% if runtime_storage_update_time is too_old %}
<div class="banner">The data was last updated on {{ runtime_storage_update_time_str }}</div>
{% endif %}
{% set update_time_title = 'Last updated on ' + runtime_storage_update_time_str %}
{% endif %}
<div class="aheader">

15
stackalytics/dashboard/web.py
View File

@ -16,7 +16,6 @@
import collections
import operator
import os
import re
import time
import flask
@ -639,15 +638,11 @@ def timeline(records, **kwargs):
return [array_commits, array_commits_hl, array_loc]
@app.template_filter('make_url')
def to_url_params(dict_params, base_url):
return base_url + '?' + '&'.join(
['%s=%s' % (k, v) for k, v in six.iteritems(dict_params)])
@app.template_filter('remove_ctrl_chars')
def remove_ctrl_chars(text):
return re.sub(r'[\W]', '_', text)
@app.template_test()
def too_old(timestamp):
age = cfg.CONF.age_warn
now = time.time()
return timestamp + age < now
def main():

2
stackalytics/processor/config.py
View File

@ -69,6 +69,8 @@ OPTS = [
'works for dashboard only'),
cfg.IntOpt('members-look-ahead', default=250,
help='How many member profiles to look ahead after the last'),
cfg.IntOpt('age-warn', default=2 * 24 * 60 * 60,
help='Warn if the age of data is more than this value, sec'),
]