mirror of
https://github.com/outbackdingo/patroni.git
synced 2026-01-27 18:20:05 +00:00
37f03790cc
A few times we observed that Patroni HA loop was blocked for a few minutes due to not being able to write logs to stderr. This is a very rare condition which we hit so far only on k8s. This commit makes Patroni resilient to such kind of problems. All log messages first are written into the in-memory queue and later they are asynchronously flushed into the stderr or file from a separate thread. The maximum queue size is configurable and the default value is 1000. This should be enough to keep more than one hour of log messages with default settings and when Patroni cluster operates normally (without big issues). In case if we hit the maximum size of the queue further logs will be discarded until the queue size will be reduced. The number of discarded messages will be reported into the log later. In addition to that, the number of non-flushed and discarded messages (if there are any), will be reported via Patroni REST API as: ```json "logger_queue_size": X, "logger_records_lost": Y` ```
59 lines
2.1 KiB
Python
59 lines
2.1 KiB
Python
import logging
|
|
import os
|
|
import sys
|
|
import unittest
|
|
import yaml
|
|
|
|
from mock import Mock, patch
|
|
from patroni.config import Config
|
|
from patroni.log import PatroniLogger
|
|
from six.moves.queue import Queue, Full
|
|
|
|
|
|
class TestPatroniLogger(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
self._handlers = logging.getLogger().handlers[:]
|
|
|
|
def tearDown(self):
|
|
logging.getLogger().handlers[:] = self._handlers
|
|
|
|
@patch('logging.FileHandler._open', Mock())
|
|
@patch('logging.Handler.close', Mock(side_effect=Exception))
|
|
def test_patroni_logger(self):
|
|
config = {
|
|
'log': {
|
|
'max_queue_size': 5,
|
|
'dir': 'foo',
|
|
'file_size': 4096,
|
|
'file_num': 5,
|
|
'loggers': {
|
|
'foo.bar': 'INFO'
|
|
}
|
|
},
|
|
'restapi': {}, 'postgresql': {'data_dir': 'foo'}
|
|
}
|
|
sys.argv = ['patroni.py']
|
|
os.environ[Config.PATRONI_CONFIG_VARIABLE] = yaml.dump(config, default_flow_style=False)
|
|
logger = PatroniLogger()
|
|
patroni_config = Config()
|
|
logger.reload_config(patroni_config['log'])
|
|
|
|
with patch.object(logging.Handler, 'format', Mock(side_effect=Exception)):
|
|
logging.error('test')
|
|
|
|
self.assertEqual(logger._log_handler.maxBytes, config['log']['file_size'])
|
|
self.assertEqual(logger._log_handler.backupCount, config['log']['file_num'])
|
|
|
|
config['log'].pop('dir')
|
|
logger.reload_config(config['log'])
|
|
with patch.object(logging.Logger, 'makeRecord',
|
|
Mock(side_effect=[logging.LogRecord('', logging.INFO, '', 0, '', (), None), Exception])):
|
|
logging.error('test')
|
|
logging.error('test')
|
|
with patch.object(Queue, 'put_nowait', Mock(side_effect=Full)):
|
|
self.assertRaises(SystemExit, logger.shutdown)
|
|
self.assertRaises(Exception, logger.shutdown)
|
|
self.assertLessEqual(logger.queue_size, 2) # "Failed to close the old log handler" could be still in the queue
|
|
self.assertEqual(logger.records_lost, 0)
|