opendev/elastic-recheck
Code Issues Proposed changes

Wait until the most recent index is available

Browse Source 
When elastic search indexing is behind, and the day has
progressed forward to a new day,  the latest
index is not yet available for use. Exclude it from searches
until it is ready in order to avoid the ElasticHttpNotFoundError.

Add Unit tests for this case as well as for when multiple days
are specified for the search.

Change-Id: Ifd27d1ab21bebcb63b48ea164f425c4a2ac8759c
This commit is contained in:
Ramy Asselin 2016-10-18 05:13:45 -07:00
parent 3c61614411
commit de437439ad
2 changed files with 42 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
elastic_recheck/results.py
View File

@ -71,11 +71,16 @@ class SearchEngine(object):
# today's index
datefmt = self._indexfmt
now = datetime.datetime.utcnow()
indexes = [now.strftime(datefmt)]
indexes = []
latest_index = now.strftime(datefmt)
if self._is_valid_index(es, latest_index):
indexes.append(latest_index)
if recent:
lasthr = now - datetime.timedelta(hours=1)
if lasthr.strftime(datefmt) != now.strftime(datefmt):
indexes.append(lasthr.strftime(datefmt))
lasthr_index = lasthr.strftime(datefmt)
if lasthr_index != latest_index:
if self._is_valid_index(es, lasthr_index):
indexes.append(lasthr.strftime(datefmt))
for day in range(1, days):
lastday = now - datetime.timedelta(days=day)
index_name = lastday.strftime(datefmt)

39
elastic_recheck/tests/unit/test_results.py
View File

@ -145,13 +145,42 @@ class TestSearchEngine(tests.TestCase):
# The search index comparison goes back one hour and cuts off by day,
# so test that we're one hour and one second into today so we only have
# one index in the search call.
self._test_search_recent(search_mock, MockDatetimeToday,
expected_indexes=['logstash-2014.06.12'])
with mock.patch.object(
pyelasticsearch.ElasticSearch, 'status') as mock_data:
mock_data.return_value = "Not an exception"
self._test_search_recent(search_mock, MockDatetimeToday,
expected_indexes=['logstash-2014.06.12'])
def test_search_recent_multiple_indexes(self, search_mock):
# The search index comparison goes back one hour and cuts off by day,
# so test that we're 59 minutes and 59 seconds into today so that we
# have an index for today and yesterday in the search call.
self._test_search_recent(search_mock, MockDatetimeYesterday,
expected_indexes=['logstash-2014.06.12',
'logstash-2014.06.11'])
with mock.patch.object(
pyelasticsearch.ElasticSearch, 'status') as mock_data:
mock_data.return_value = "Not an exception"
self._test_search_recent(search_mock, MockDatetimeYesterday,
expected_indexes=['logstash-2014.06.12',
'logstash-2014.06.11'])
def test_search_no_indexes(self, search_mock):
# Test when no indexes are valid
with mock.patch.object(
pyelasticsearch.ElasticSearch, 'status') as mock_data:
mock_data.side_effect = pyelasticsearch.exceptions.\
ElasticHttpNotFoundError()
self._test_search_recent(search_mock, MockDatetimeYesterday,
expected_indexes=[])
def test_search_days(self, search_mock):
# Test when specific days are used.
with mock.patch.object(
pyelasticsearch.ElasticSearch, 'status') as mock_data:
mock_data.return_value = "Not an exception"
datetime.datetime = MockDatetimeYesterday
result_set = self.engine.search(self.query, size=10, days=3,
recent=False)
self.assertEqual(0, len(result_set))
search_mock.assert_called_once_with(self.query, size=10,
index=['logstash-2014.06.12',
'logstash-2014.06.11',
'logstash-2014.06.10'])