Decouple log processing from logstash

As part of the move to logstash 2.0 we are relying on upstream packaging
for logstash. This packaging replaces a lot of the micromanagement of
users and groups and dirs that was done in puppet for logstash. This is
great news because its less work for us but means that the log
processors can't rely on puppet resources for those items and we don't
actually want to install logstash package everywhere we run log
processor daemons.

Since the log processors don't need a logstash service running and
actually don't need any of the logstash stuff at all decouple them
completely and have the log processor daemons use their own user, group,
log dir, config dir, etc. With this in place we can easily switch to
using the logstash packages only where we actually need logstash to be
running.

Change-Id: I2354fbe9d3ab25134c52bfe58f562dfdf9ff6786

files/jenkins-log-client.init

@@ -17,9 +17,9 @@ DESC="Jenkins Log Client"
 NAME=jenkins-log-client
 DAEMON=/usr/local/bin/log-gearman-client.py
 PIDFILE=/var/run/$NAME/$NAME.pid
-DAEMON_ARGS="-c /etc/logstash/jenkins-log-client.yaml -d /var/log/logstash/log-client-debug.log -p $PIDFILE"
+DAEMON_ARGS="-c /etc/logprocessor/jenkins-log-client.yaml -d /var/log/logprocessor/log-client-debug.log -p $PIDFILE"
 SCRIPTNAME=/etc/init.d/$NAME
-USER=logstash
+USER=logprocessor
 
 # Exit if the package is not installed
 [ -x "$DAEMON" ] || exit 0

manifests/client.pp

@@ -20,13 +20,13 @@ class log_processor::client (
   $statsd_host = undef,
 ) {
 
-  file { '/etc/logstash/jenkins-log-client.yaml':
+  file { '/etc/logprocessor/jenkins-log-client.yaml':
     ensure  => present,
     owner   => 'root',
     group   => 'root',
     mode    => '0555',
     source  => $config_file,
-    require => File['/etc/logstash'],
+    require => File['/etc/logprocessor'],
   }
 
   file { '/etc/init.d/jenkins-log-client':
@@ -37,7 +37,7 @@ class log_processor::client (
     source  => 'puppet:///modules/log_processor/jenkins-log-client.init',
     require => [
       File['/usr/local/bin/log-gearman-client.py'],
-      File['/etc/logstash/jenkins-log-client.yaml'],
+      File['/etc/logprocessor/jenkins-log-client.yaml'],
       File['/etc/default/jenkins-log-client'],
     ],
   }
@@ -53,13 +53,13 @@ class log_processor::client (
   service { 'jenkins-log-client':
     enable     => true,
     hasrestart => true,
-    subscribe  => File['/etc/logstash/jenkins-log-client.yaml'],
+    subscribe  => File['/etc/logprocessor/jenkins-log-client.yaml'],
     require    => File['/etc/init.d/jenkins-log-client'],
   }
 
   include ::logrotate
   logrotate::file { 'log-client-debug.log':
-    log     => '/var/log/logstash/log-client-debug.log',
+    log     => '/var/log/logprocessor/log-client-debug.log',
     options => [
       'compress',
       'copytruncate',

manifests/init.pp

@@ -23,6 +23,35 @@ class log_processor (
     }
   }
 
+  group { 'logprocessor':
+    ensure => present,
+  }
+  user { 'logprocessor':
+    ensure     => present,
+    comment    => 'Log Processor User',
+    home       => '/etc/logprocessor',
+    gid        => 'logprocessor',
+    shell      => '/bin/bash',
+    membership => 'minimum',
+    require    => Group['logprocessor'],
+  }
+
+  file { '/etc/logprocessor':
+    ensure  => directory,
+    owner   => 'logprocessor',
+    group   => 'logprocessor',
+    mode    => '0755',
+    require => User['logprocessor'],
+  }
+
+  file { '/var/log/logprocessor':
+    ensure  => directory,
+    owner   => 'logprocessor',
+    group   => 'logprocessor',
+    mode    => '0755',
+    require => User['logprocessor'],
+  }
+
   package { 'python-zmq':
     ensure => present,
   }
@@ -50,9 +79,9 @@ class log_processor (
 
   file { '/var/lib/crm114':
     ensure  => directory,
-    owner   => 'logstash',
-    group   => 'logstash',
-    require => User['logstash'],
+    owner   => 'logprocessor',
+    group   => 'logprocessor',
+    require => User['logprocessor'],
   }
 
   file { '/usr/local/bin/classify-log.crm':

manifests/worker.pp

@@ -20,13 +20,13 @@ define log_processor::worker (
 ) {
   $suffix = "-${name}"
 
-  file { "/etc/logstash/jenkins-log-worker${suffix}.yaml":
+  file { "/etc/logprocessor/jenkins-log-worker${suffix}.yaml":
     ensure  => present,
     owner   => 'root',
     group   => 'root',
     mode    => '0555',
     source  => $config_file,
-    require => Class['logstash::indexer'],
+    require => User['logprocessor'],
   }
 
   file { "/etc/init.d/jenkins-log-worker${suffix}":
@@ -37,23 +37,20 @@ define log_processor::worker (
     content => template('log_processor/jenkins-log-worker.init.erb'),
     require => [
       File['/usr/local/bin/log-gearman-worker.py'],
-      File["/etc/logstash/jenkins-log-worker${suffix}.yaml"],
+      File["/etc/logprocessor/jenkins-log-worker${suffix}.yaml"],
     ],
   }
 
   service { "jenkins-log-worker${suffix}":
     enable     => true,
     hasrestart => true,
-    subscribe  => File["/etc/logstash/jenkins-log-worker${suffix}.yaml"],
-    require    => [
-      Class['logstash::indexer'],
-      File["/etc/init.d/jenkins-log-worker${suffix}"],
-    ],
+    subscribe  => File["/etc/logprocessor/jenkins-log-worker${suffix}.yaml"],
+    require    => File["/etc/init.d/jenkins-log-worker${suffix}"],
   }
 
   include ::logrotate
   logrotate::file { "log-worker${suffix}-debug.log":
-    log     => "/var/log/logstash/log-worker${suffix}-debug.log",
+    log     => "/var/log/logprocessor/log-worker${suffix}-debug.log",
     options => [
       'compress',
       'copytruncate',

metadata.json

@@ -8,7 +8,6 @@
   "project_page": "http://docs.openstack.org/infra/system-config/",
   "issues_url": "https://storyboard.openstack.org/#!/project/801",
   "dependencies": [
-    {"name":"openstackci/logstash","version_requirement":">= 0.0.1"},
     {"name":"openstackci/pip","version_requirement":">= 0.0.1"},
     {"name":"openstackci/logrotate","version_requirement":">= 0.0.1"}
   ]

templates/jenkins-log-worker.init.erb

@@ -17,9 +17,9 @@ DESC="Jenkins Log Worker"
 NAME=jenkins-log-worker<%= suffix %>
 DAEMON=/usr/local/bin/log-gearman-worker.py
 PIDFILE=/var/run/$NAME/$NAME.pid
-DAEMON_ARGS="-c /etc/logstash/jenkins-log-worker<%= suffix %>.yaml -d /var/log/logstash/log-worker<%= suffix %>-debug.log -p $PIDFILE"
+DAEMON_ARGS="-c /etc/logprocessor/jenkins-log-worker<%= suffix %>.yaml -d /var/log/logprocessor/log-worker<%= suffix %>-debug.log -p $PIDFILE"
 SCRIPTNAME=/etc/init.d/$NAME
-USER=logstash
+USER=logprocessor
 
 # Exit if the package is not installed
 [ -x "$DAEMON" ] || exit 0