Show warning if data is older than configured value

The patch introduces config param 'age-warn' with default value
in 2 days. The warning message will be useful for cases when
processor stuck or data was recovered from the dump.

Change-Id: I910b87da5c664fbda7d25e77ee51554f01ad9bc2

doc/source/tools/stackalytics-dashboard.txt

@@ -47,14 +47,14 @@ optional arguments:
   --syslog-log-facility SYSLOG_LOG_FACILITY
                         Syslog facility to receive log lines.
   --use-syslog          Use syslog for logging. Existing syslog format is
-                        DEPRECATED during I, and will change in J to honor
+                        DEPRECATED during I, and changed in J to honor
                         RFC5424.
   --use-syslog-rfc-format
                         (Optional) Enables or disables syslog rfc5424 format
                         for logging. If enabled, prefixes the MSG part of the
                         syslog message with APP-NAME (RFC5424). The format
-                        without the APP-NAME is deprecated in I, and will be
+                        without the APP-NAME is deprecated in K, and will be
-                        removed in J.
+                        removed in M, along with this option.
   --verbose, -v         Print more verbose output (set logging level to INFO
                         instead of default WARNING level).
   --version             show program's version number and exit

doc/source/tools/stackalytics-dump.txt

@@ -1,4 +1,4 @@
-usage: stackalytics-dump [-h]
+usage: stackalytics-dump [-h] [--age-warn AGE_WARN]
                          [--collect-profiler-stats COLLECT_PROFILER_STATS]
                          [--config-dir DIR] [--config-file PATH]
                          [--corrections-uri CORRECTIONS_URI]
@@ -31,6 +31,7 @@ usage: stackalytics-dump [-h]
 
 optional arguments:
   -h, --help            show this help message and exit
+  --age-warn AGE_WARN   Warn if the age of data is more than this value, sec
   --collect-profiler-stats COLLECT_PROFILER_STATS
                         Name of file to store python profiler data. This
                         option works for dashboard only
@@ -114,14 +115,14 @@ optional arguments:
   --syslog-log-facility SYSLOG_LOG_FACILITY
                         Syslog facility to receive log lines.
   --use-syslog          Use syslog for logging. Existing syslog format is
-                        DEPRECATED during I, and will change in J to honor
+                        DEPRECATED during I, and changed in J to honor
                         RFC5424.
   --use-syslog-rfc-format
                         (Optional) Enables or disables syslog rfc5424 format
                         for logging. If enabled, prefixes the MSG part of the
                         syslog message with APP-NAME (RFC5424). The format
-                        without the APP-NAME is deprecated in I, and will be
+                        without the APP-NAME is deprecated in K, and will be
-                        removed in J.
+                        removed in M, along with this option.
   --verbose, -v         Print more verbose output (set logging level to INFO
                         instead of default WARNING level).
   --version             show program's version number and exit

doc/source/tools/stackalytics-processor.txt

@@ -1,4 +1,4 @@
-usage: stackalytics-processor [-h]
+usage: stackalytics-processor [-h] [--age-warn AGE_WARN]
                               [--collect-profiler-stats COLLECT_PROFILER_STATS]
                               [--config-dir DIR] [--config-file PATH]
                               [--corrections-uri CORRECTIONS_URI]
@@ -31,6 +31,7 @@ usage: stackalytics-processor [-h]
 
 optional arguments:
   -h, --help            show this help message and exit
+  --age-warn AGE_WARN   Warn if the age of data is more than this value, sec
   --collect-profiler-stats COLLECT_PROFILER_STATS
                         Name of file to store python profiler data. This
                         option works for dashboard only
@@ -109,14 +110,14 @@ optional arguments:
   --syslog-log-facility SYSLOG_LOG_FACILITY
                         Syslog facility to receive log lines.
   --use-syslog          Use syslog for logging. Existing syslog format is
-                        DEPRECATED during I, and will change in J to honor
+                        DEPRECATED during I, and changed in J to honor
                         RFC5424.
   --use-syslog-rfc-format
                         (Optional) Enables or disables syslog rfc5424 format
                         for logging. If enabled, prefixes the MSG part of the
                         syslog message with APP-NAME (RFC5424). The format
-                        without the APP-NAME is deprecated in I, and will be
+                        without the APP-NAME is deprecated in K, and will be
-                        removed in J.
+                        removed in M, along with this option.
   --verbose, -v         Print more verbose output (set logging level to INFO
                         instead of default WARNING level).
   --version             show program's version number and exit

etc/stackalytics.conf

@@ -39,14 +39,16 @@
 #log_dir = <None>
 
 # Use syslog for logging. Existing syslog format is DEPRECATED during I, and
-# will change in J to honor RFC5424. (boolean value)
+# changed in J to honor RFC5424. (boolean value)
 #use_syslog = false
 
 # (Optional) Enables or disables syslog rfc5424 format for logging. If enabled,
 # prefixes the MSG part of the syslog message with APP-NAME (RFC5424). The
-# format without the APP-NAME is deprecated in I, and will be removed in J.
+# format without the APP-NAME is deprecated in K, and will be removed in M,
-# (boolean value)
+# along with this option. (boolean value)
-#use_syslog_rfc_format = false
+# This option is deprecated for removal.
+# Its value may be silently ignored in the future.
+#use_syslog_rfc_format = true
 
 # Syslog facility to receive log lines. (string value)
 #syslog_log_facility = LOG_USER
@@ -64,7 +66,7 @@
 #logging_debug_format_suffix = %(funcName)s %(pathname)s:%(lineno)d
 
 # Prefix each line of exception output with this format. (string value)
-#logging_exception_prefix = %(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s
+#logging_exception_prefix = %(asctime)s.%(msecs)03d %(process)d ERROR %(name)s %(instance)s
 
 # List of logger=LEVEL pairs. (list value)
 #default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,requests.packages.urllib3.util.retry=WARN,urllib3.util.retry=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN
@@ -83,6 +85,9 @@
 # value)
 #instance_uuid_format = "[instance: %(uuid)s] "
 
+# Enables or disables fatal status of deprecations. (boolean value)
+#fatal_deprecations = false
+
 #
 # From stackalytics.processor.config
 #
@@ -148,3 +153,6 @@
 
 # How many member profiles to look ahead after the last (integer value)
 #members_look_ahead = 250
+
+# Warn if the age of data is more than this value, sec (integer value)
+#age_warn = 172800

stackalytics/dashboard/templates/layout.html

@@ -24,9 +24,12 @@ Stackalytics {% if page_title %}| {{ page_title }} {% endif %}
 
     <div class="page">
         {% if not runtime_storage_update_time %}
-            <div class="banner">Please note! The data is being loaded now and may be not complete.</div>
+            <div class="banner">The data is being loaded now and is not complete</div>
             {% set update_time_title = '' %}
         {% else %}
+            {% if runtime_storage_update_time is too_old %}
+                <div class="banner">The data was last updated on {{ runtime_storage_update_time_str }}</div>
+            {% endif %}
             {% set update_time_title = 'Last updated on ' + runtime_storage_update_time_str %}
         {% endif %}
     <div class="aheader">

stackalytics/dashboard/web.py

@@ -16,7 +16,6 @@
 import collections
 import operator
 import os
-import re
 import time
 
 import flask
@@ -639,15 +638,11 @@ def timeline(records, **kwargs):
     return [array_commits, array_commits_hl, array_loc]
 
 
-@app.template_filter('make_url')
+@app.template_test()
-def to_url_params(dict_params, base_url):
+def too_old(timestamp):
-    return base_url + '?' + '&'.join(
+    age = cfg.CONF.age_warn
-        ['%s=%s' % (k, v) for k, v in six.iteritems(dict_params)])
+    now = time.time()
-
+    return timestamp + age < now
-
-@app.template_filter('remove_ctrl_chars')
-def remove_ctrl_chars(text):
-    return re.sub(r'[\W]', '_', text)
 
 
 def main():

stackalytics/processor/config.py

@@ -69,6 +69,8 @@ OPTS = [
                     'works for dashboard only'),
     cfg.IntOpt('members-look-ahead', default=250,
                help='How many member profiles to look ahead after the last'),
+    cfg.IntOpt('age-warn', default=2 * 24 * 60 * 60,
+               help='Warn if the age of data is more than this value, sec'),
 ]