diff --git a/bin/dashboard b/bin/dashboard deleted file mode 100755 index d7862984c..000000000 --- a/bin/dashboard +++ /dev/null @@ -1,11 +0,0 @@ -#!.venv/bin/python -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -import os -import sys -sys.path.insert(0, os.getcwd()) - -from dashboard.web import app - - -app.run() diff --git a/bin/processor b/bin/processor deleted file mode 100755 index 415b67946..000000000 --- a/bin/processor +++ /dev/null @@ -1,11 +0,0 @@ -#!.venv/bin/python -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -import os -import sys -sys.path.insert(0, os.getcwd()) - -from stackalytics.processor.main import main - - -main() diff --git a/dashboard/web.py b/dashboard/web.py index 618d77f3c..8e1216715 100644 --- a/dashboard/web.py +++ b/dashboard/web.py @@ -23,9 +23,12 @@ import urllib import flask from flask.ext import gravatar as gravatar_ext +from oslo.config import cfg import time from dashboard import memory_storage +from stackalytics.openstack.common import log as logging +from stackalytics.processor import config from stackalytics.processor import persistent_storage from stackalytics.processor import runtime_storage from stackalytics.processor import user_utils @@ -33,10 +36,6 @@ from stackalytics.processor import user_utils # Constants and Parameters --------- -DEBUG = True -RUNTIME_STORAGE_URI = 'memcached://127.0.0.1:11211' -PERSISTENT_STORAGE_URI = 'mongodb://localhost' - DEFAULTS = { 'metric': 'commits', 'release': 'havana', @@ -62,16 +61,33 @@ app = flask.Flask(__name__) app.config.from_object(__name__) app.config.from_envvar('DASHBOARD_CONF', silent=True) +LOG = logging.getLogger(__name__) + +conf = cfg.CONF +conf.register_opts(config.OPTS) +logging.setup('dashboard') +LOG.info('Logging enabled') + +conf_file = os.getenv('STACKALYTICS_CONF') +if (conf_file is None) or (not os.path.isfile(conf_file)): + conf_file = '/etc/stackalytics/stackalytics.conf' + +if os.path.isfile(conf_file): + conf(default_config_files=[conf_file]) + app.config['DEBUG'] = cfg.CONF.debug +else: + LOG.warn('Conf file is empty or not exist') + def get_vault(): vault = getattr(app, 'stackalytics_vault', None) if not vault: vault = {} vault['runtime_storage'] = runtime_storage.get_runtime_storage( - RUNTIME_STORAGE_URI) + cfg.CONF.runtime_storage_uri) vault['persistent_storage'] = ( persistent_storage.get_persistent_storage( - PERSISTENT_STORAGE_URI)) + cfg.CONF.persistent_storage_uri)) vault['memory_storage'] = memory_storage.get_memory_storage( memory_storage.MEMORY_STORAGE_CACHED, vault['runtime_storage'].get_update(os.getpid())) @@ -208,7 +224,7 @@ def exception_handler(): try: return f(*args, **kwargs) except Exception as e: - print e + LOG.debug(e) flask.abort(404) return decorated_function @@ -510,5 +526,9 @@ def make_commit_message(record): gravatar = gravatar_ext.Gravatar(app, size=100, rating='g', default='wavatar') + +def main(): + app.run(cfg.CONF.listen_host, cfg.CONF.listen_port) + if __name__ == '__main__': - app.run('0.0.0.0') + main() diff --git a/etc/dashboard.conf b/etc/dashboard.conf deleted file mode 100644 index 90826e6f3..000000000 --- a/etc/dashboard.conf +++ /dev/null @@ -1,5 +0,0 @@ -# -# Configuration of stackalytics dashboard -# - -DEBUG = True \ No newline at end of file diff --git a/etc/stackalytics.conf b/etc/stackalytics.conf index c4dcae893..bbe46bb04 100644 --- a/etc/stackalytics.conf +++ b/etc/stackalytics.conf @@ -3,10 +3,10 @@ # debug = False # Default data -# default-data = etc/default_data.json +# default-data = /etc/stackalytics/default_data.json # The folder that holds all project sources to analyze -# sources_root = ../metric-root-tmp +# sources_root = /var/run/stackalytics # Runtime storage URI # runtime_storage_uri = memcached://127.0.0.1:11211 @@ -21,4 +21,10 @@ # repo_poll_period = 300 # Address of update handler -# frontend_update_address = http://user:user@localhost/update/%s \ No newline at end of file +# frontend_update_address = http://user:user@localhost/update/%s + +# Hostname where dashboard listens on +# listen_host = 127.0.0.1 + +# Port where dashboard listens on +# listen_port = 8080 diff --git a/setup.cfg b/setup.cfg index 42eeee4ab..be4db8631 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ version = 0.1 summary = OpenStack analytics dashboard description-file = README.rst -author = OpenStack Stackalytics Project +author = OpenStack author-email = openstack-dev@lists.openstack.org home-page = http://www.openstack.org/ classifier = @@ -21,6 +21,10 @@ classifier = packages = dashboard stackalytics +data_files = + etc/stackalytics = + etc/stackalytics.conf + etc/default_data.json [global] setup-hooks = @@ -28,4 +32,5 @@ setup-hooks = [entry_points] console_scripts = - stackalytics-dashboard = dashboard.dashboard:main + stackalytics-dashboard = dashboard.web:main + stackalytics-processor = stackalytics.processor.main:main diff --git a/stackalytics/processor/commit_processor.py b/stackalytics/processor/commit_processor.py index fde5ca6ef..d1fd4e6f4 100644 --- a/stackalytics/processor/commit_processor.py +++ b/stackalytics/processor/commit_processor.py @@ -79,11 +79,11 @@ class CachedProcessor(CommitProcessor): def _unknown_user_email(self, email): lp_profile = None - if not re.match(r'[^@]+@[^@]+\.[^@]+', email): + if not re.match(r'[\w\d_\.-]+@([\w\d_\.-]+\.)+[\w]+', email): LOG.debug('User email is not valid %s' % email) else: LOG.debug('Lookup user email %s at Launchpad' % email) - lp = launchpad.Launchpad.login_anonymously(cfg.CONF.launchpad_user) + lp = launchpad.Launchpad.login_anonymously('stackalytics') try: lp_profile = lp.people.getByEmail(email=email) except Exception as error: diff --git a/stackalytics/processor/config.py b/stackalytics/processor/config.py new file mode 100644 index 000000000..9b020eb69 --- /dev/null +++ b/stackalytics/processor/config.py @@ -0,0 +1,44 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo.config import cfg + +OPTS = [ + cfg.StrOpt('default-data', default='etc/default_data.json', + help='Default data'), + cfg.StrOpt('sources-root', default='/var/run/stackalytics', + help='The folder that holds all project sources to analyze'), + cfg.StrOpt('runtime-storage-uri', default='memcached://127.0.0.1:11211', + help='Storage URI'), + cfg.StrOpt('frontend-update-address', + default='http://user:user@localhost/update/%s', + help='Address of update handler'), + cfg.StrOpt('repo-poll-period', default='300', + help='Repo poll period in seconds'), + cfg.StrOpt('persistent-storage-uri', default='mongodb://localhost', + help='URI of persistent storage'), + cfg.BoolOpt('sync-default-data', default=False, + help='Update persistent storage with default data. ' + 'Existing data is not overwritten'), + cfg.BoolOpt('force-sync-default-data', default=False, + help='Completely overwrite persistent storage with the ' + 'default data'), + cfg.BoolOpt('filter-robots', default=True, + help='Filter out commits from robots'), + cfg.StrOpt('listen-host', default='127.0.0.1', + help='The address dashboard listens on'), + cfg.IntOpt('listen-port', default=8080, + help='The port dashboard listens on'), +] diff --git a/stackalytics/processor/main.py b/stackalytics/processor/main.py index 64f416419..b44a0ae98 100644 --- a/stackalytics/processor/main.py +++ b/stackalytics/processor/main.py @@ -19,6 +19,7 @@ from psutil import _error from stackalytics.openstack.common import log as logging from stackalytics.processor import commit_processor +from stackalytics.processor import config from stackalytics.processor import persistent_storage from stackalytics.processor import runtime_storage from stackalytics.processor import vcs @@ -26,27 +27,6 @@ from stackalytics.processor import vcs LOG = logging.getLogger(__name__) -OPTS = [ - cfg.StrOpt('default-data', default='etc/default_data.json', - help='Default data'), - cfg.StrOpt('sources-root', default=None, required=True, - help='The folder that holds all project sources to analyze'), - cfg.StrOpt('runtime-storage-uri', default='memcached://127.0.0.1:11211', - help='Storage URI'), - cfg.StrOpt('persistent-storage-uri', default='mongodb://localhost', - help='URI of persistent storage'), - cfg.BoolOpt('sync-default-data', default=False, - help='Update persistent storage with default data. ' - 'Existing data is not overwritten'), - cfg.BoolOpt('force-sync-default-data', default=False, - help='Completely overwrite persistent storage with the ' - 'default data'), - cfg.StrOpt('launchpad-user', default='stackalytics-bot', - help='User to access Launchpad'), - cfg.BoolOpt('filter-robots', default=True, - help='Filter out commits from robots'), -] - def get_pids(): uwsgi_dict = {} @@ -107,8 +87,8 @@ def update_repos(runtime_storage, persistent_storage): def main(): # init conf and logging conf = cfg.CONF - conf.register_cli_opts(OPTS) - conf.register_opts(OPTS) + conf.register_cli_opts(config.OPTS) + conf.register_opts(config.OPTS) conf() logging.setup('stackalytics')