From a798af1228ec5d3f9f5d9865c140bf69fcf8ec68 Mon Sep 17 00:00:00 2001
From: Jeremy Stanley <fungi@yuggoth.org>
Date: Fri, 21 Sep 2012 17:49:52 +0000
Subject: [PATCH] Check for out-of-memory condition during tox runs.

* modules/jenkins/files/slave_scripts/jenkins-oom-grep.sh: Script to
compare "pre" and "post" snapshots of dmesg output, looking for any
oom-killer kernel messages which occurred between. If it finds at least
one, all kernel messages which arrived during the test are printed to
stdout and the script subsequently ends with an exit code of 1. If none
are found, no output is generated and the script exits normally.

* modules/jenkins/files/slave_scripts/run-tox.sh: Add "pre" and "post"
calls to jenkins-oom-grep.sh, printing a notice about out-of-memory
conditions if the "post" invocation exits nonzero.

Change-Id: Ib28f528fe1c4b3ccc2b3669e1f602609d7560e2e
Reviewed-on: https://review.openstack.org/13477
Reviewed-by: Clark Boylan <clark.boylan@gmail.com>
Approved: Monty Taylor <mordred@inaugust.com>
Reviewed-by: Monty Taylor <mordred@inaugust.com>
Tested-by: Jenkins
---
 .../files/slave_scripts/jenkins-oom-grep.sh   | 39 +++++++++++++++++++
 .../jenkins/files/slave_scripts/run-tox.sh    | 15 +++++++
 2 files changed, 54 insertions(+)
 create mode 100755 modules/jenkins/files/slave_scripts/jenkins-oom-grep.sh

diff --git a/modules/jenkins/files/slave_scripts/jenkins-oom-grep.sh b/modules/jenkins/files/slave_scripts/jenkins-oom-grep.sh
new file mode 100755
index 0000000000..b60fa6315c
--- /dev/null
+++ b/modules/jenkins/files/slave_scripts/jenkins-oom-grep.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Copyright 2012 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Find out if jenkins has triggered the out-of-memory killer by checking
+# the output of dmesg before and after a test run.
+
+PATTERN=" invoked oom-killer: "
+
+case "$1" in
+    pre)
+        rm -fr /tmp/jenkins-oom-log
+        mkdir /tmp/jenkins-oom-log
+        dmesg > /tmp/jenkins-oom-log/pre
+        exit 0
+        ;;
+    post)
+        dmesg > /tmp/jenkins-oom-log/post
+        diff /tmp/jenkins-oom-log/{pre,post} \
+            | grep "^> " | sed "s/^> //" > /tmp/jenkins-oom-log/diff
+        if grep -q "$PATTERN" /tmp/jenkins-oom-log/diff
+        then
+            cat /tmp/jenkins-oom-log/diff
+            exit 1
+        fi
+        ;;
+esac
diff --git a/modules/jenkins/files/slave_scripts/run-tox.sh b/modules/jenkins/files/slave_scripts/run-tox.sh
index 8a5a6c3c30..6b0c53c7b2 100755
--- a/modules/jenkins/files/slave_scripts/run-tox.sh
+++ b/modules/jenkins/files/slave_scripts/run-tox.sh
@@ -26,6 +26,8 @@ export NOSE_WITH_XUNIT=1
 export NOSE_WITH_HTML_OUTPUT=1
 export NOSE_HTML_OUT_FILE='nose_results.html'
 
+/usr/local/jenkins/slave_scripts/jenkins-oom-grep.sh pre
+
 sudo /usr/local/jenkins/slave_scripts/jenkins-sudo-grep.sh pre
 
 tox -e$venv
@@ -48,4 +50,17 @@ then
     exit 1
 fi
 
+/usr/local/jenkins/slave_scripts/jenkins-oom-grep.sh post
+oomresult=$?
+
+if [ $oomresult -ne "0" ]
+then
+    echo
+    echo "This test has failed because it attempted to exceed configured"
+    echo "memory limits and was killed prior to completion.  See above"
+    echo "for related kernel messages."
+    echo
+    exit 1
+fi
+
 exit $result