mirror of
https://github.com/optim-enterprises-bv/patroni.git
synced 2026-01-11 18:35:15 +00:00
26ae38960a
Test if config (file) parsed with yaml_load() contains a valid Mapping object, otherwise Patroni throws an explicit exception. It also makes the Patroni output more explicit when using that kind of "invalid" configuration. ``` console $ touch /tmp/patroni.yaml $ patroni --validate-config /tmp/patroni.yaml /tmp/patroni.yaml does not contain a dict invalid config file /tmp/patroni.yaml ``` reportUnnecessaryIsInstance is explicitly ignored since we can't determine what yaml_safeload can bring from a YAML config (list, dict,...).
287 lines
14 KiB
Python
287 lines
14 KiB
Python
import io
|
|
import os
|
|
import sys
|
|
import unittest
|
|
|
|
from copy import deepcopy
|
|
from unittest.mock import MagicMock, Mock, patch
|
|
|
|
from patroni import global_config
|
|
from patroni.config import ClusterConfig, Config, ConfigParseError
|
|
|
|
from .test_ha import get_cluster_initialized_with_only_leader
|
|
|
|
|
|
class TestConfig(unittest.TestCase):
|
|
|
|
@patch('os.path.isfile', Mock(return_value=True))
|
|
@patch('json.load', Mock(side_effect=Exception))
|
|
@patch('builtins.open', MagicMock())
|
|
def setUp(self):
|
|
sys.argv = ['patroni.py']
|
|
os.environ[Config.PATRONI_CONFIG_VARIABLE] = 'restapi: {}\npostgresql: {data_dir: foo}'
|
|
self.config = Config(None)
|
|
|
|
def test_set_dynamic_configuration(self):
|
|
with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
|
|
self.assertFalse(self.config.set_dynamic_configuration({'foo': 'bar'}))
|
|
self.assertTrue(self.config.set_dynamic_configuration({'standby_cluster': {}, 'postgresql': {
|
|
'parameters': {'cluster_name': 1, 'hot_standby': 1, 'wal_keep_size': 1,
|
|
'track_commit_timestamp': 1, 'wal_level': 1, 'max_connections': '100'}}}))
|
|
|
|
def test_reload_local_configuration(self):
|
|
os.environ.update({
|
|
'PATRONI_NAME': 'postgres0',
|
|
'PATRONI_NAMESPACE': '/patroni/',
|
|
'PATRONI_SCOPE': 'batman2',
|
|
'PATRONI_LOGLEVEL': 'ERROR',
|
|
'PATRONI_LOG_FORMAT': '["message", {"levelname": "level"}]',
|
|
'PATRONI_LOG_LOGGERS': 'patroni.postmaster: WARNING, urllib3: DEBUG',
|
|
'PATRONI_LOG_FILE_NUM': '5',
|
|
'PATRONI_LOG_MODE': '0123',
|
|
'PATRONI_CITUS_DATABASE': 'citus',
|
|
'PATRONI_CITUS_GROUP': '0',
|
|
'PATRONI_CITUS_HOST': '0',
|
|
'PATRONI_RESTAPI_USERNAME': 'username',
|
|
'PATRONI_RESTAPI_PASSWORD': 'password',
|
|
'PATRONI_RESTAPI_LISTEN': '0.0.0.0:8008',
|
|
'PATRONI_RESTAPI_CONNECT_ADDRESS': '127.0.0.1:8008',
|
|
'PATRONI_RESTAPI_CERTFILE': '/certfile',
|
|
'PATRONI_RESTAPI_KEYFILE': '/keyfile',
|
|
'PATRONI_RESTAPI_ALLOWLIST_INCLUDE_MEMBERS': 'on',
|
|
'PATRONI_POSTGRESQL_LISTEN': '0.0.0.0:5432',
|
|
'PATRONI_POSTGRESQL_CONNECT_ADDRESS': '127.0.0.1:5432',
|
|
'PATRONI_POSTGRESQL_PROXY_ADDRESS': '127.0.0.1:5433',
|
|
'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0',
|
|
'PATRONI_POSTGRESQL_CONFIG_DIR': 'data/postgres0',
|
|
'PATRONI_POSTGRESQL_PGPASS': '/tmp/pgpass0',
|
|
'PATRONI_ETCD_HOST': '127.0.0.1:2379',
|
|
'PATRONI_ETCD_URL': 'https://127.0.0.1:2379',
|
|
'PATRONI_ETCD_PROXY': 'http://127.0.0.1:2379',
|
|
'PATRONI_ETCD_SRV': 'test',
|
|
'PATRONI_ETCD_CACERT': '/cacert',
|
|
'PATRONI_ETCD_CERT': '/cert',
|
|
'PATRONI_ETCD_KEY': '/key',
|
|
'PATRONI_CONSUL_HOST': '127.0.0.1:8500',
|
|
'PATRONI_CONSUL_REGISTER_SERVICE': 'on',
|
|
'PATRONI_KUBERNETES_LABELS': 'a: b: c',
|
|
'PATRONI_KUBERNETES_SCOPE_LABEL': 'a',
|
|
'PATRONI_KUBERNETES_PORTS': '[{"name": "postgresql"}]',
|
|
'PATRONI_KUBERNETES_RETRIABLE_HTTP_CODES': '401',
|
|
'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'",
|
|
'PATRONI_EXHIBITOR_HOSTS': 'host1,host2',
|
|
'PATRONI_EXHIBITOR_PORT': '8181',
|
|
'PATRONI_RAFT_PARTNER_ADDRS': "'host1:1234','host2:1234'",
|
|
'PATRONI_foo_HOSTS': '[host1,host2', # Exception in parse_list
|
|
'PATRONI_SUPERUSER_USERNAME': 'postgres',
|
|
'PATRONI_SUPERUSER_PASSWORD': 'patroni',
|
|
'PATRONI_REPLICATION_USERNAME': 'replicator',
|
|
'PATRONI_REPLICATION_PASSWORD': 'rep-pass',
|
|
'PATRONI_admin_PASSWORD': 'admin',
|
|
'PATRONI_admin_OPTIONS': 'createrole,createdb',
|
|
'PATRONI_POSTGRESQL_BIN_POSTGRES': 'sergtsop'
|
|
})
|
|
config = Config('postgres0.yml')
|
|
self.assertEqual(config.local_configuration['log']['mode'], 0o123)
|
|
with patch.object(Config, '_load_config_file', Mock(return_value={'restapi': {}})):
|
|
with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
|
|
config.reload_local_configuration()
|
|
self.assertTrue(config.reload_local_configuration())
|
|
self.assertIsNone(config.reload_local_configuration())
|
|
|
|
@patch('tempfile.mkstemp', Mock(return_value=[3000, 'blabla']))
|
|
@patch('os.path.exists', Mock(return_value=True))
|
|
@patch('os.remove', Mock(side_effect=IOError))
|
|
@patch('os.close', Mock(side_effect=IOError))
|
|
@patch('os.chmod', Mock())
|
|
@patch('shutil.move', Mock(return_value=None))
|
|
@patch('json.dump', Mock())
|
|
def test_save_cache(self):
|
|
self.config.set_dynamic_configuration({'ttl': 30, 'postgresql': {'foo': 'bar'}})
|
|
with patch('os.fdopen', Mock(side_effect=IOError)):
|
|
self.config.save_cache()
|
|
with patch('os.fdopen', MagicMock()):
|
|
self.config.save_cache()
|
|
|
|
def test_standby_cluster_parameters(self):
|
|
dynamic_configuration = {
|
|
'standby_cluster': {
|
|
'create_replica_methods': ['wal_e', 'basebackup'],
|
|
'host': 'localhost',
|
|
'port': 5432
|
|
}
|
|
}
|
|
self.config.set_dynamic_configuration(dynamic_configuration)
|
|
for name, value in dynamic_configuration['standby_cluster'].items():
|
|
self.assertEqual(self.config['standby_cluster'][name], value)
|
|
|
|
@patch('os.path.exists', Mock(return_value=True))
|
|
@patch('os.path.isfile', Mock(side_effect=lambda fname: fname != 'postgres0'))
|
|
@patch('os.path.isdir', Mock(return_value=True))
|
|
@patch('os.listdir', Mock(return_value=['01-specific.yml', '00-base.yml']))
|
|
def test_configuration_directory(self):
|
|
def open_mock(fname, *args, **kwargs):
|
|
if fname.endswith('00-base.yml'):
|
|
return io.StringIO(
|
|
u'''
|
|
test: True
|
|
test2:
|
|
child-1: somestring
|
|
child-2: 5
|
|
child-3: False
|
|
test3: True
|
|
test4:
|
|
- abc: 3
|
|
- abc: 4
|
|
''')
|
|
elif fname.endswith('01-specific.yml'):
|
|
return io.StringIO(
|
|
u'''
|
|
test: False
|
|
test2:
|
|
child-2: 10
|
|
child-3: !!null
|
|
test4:
|
|
- ab: 5
|
|
new-attr: True
|
|
''')
|
|
|
|
with patch('builtins.open', MagicMock(side_effect=open_mock)):
|
|
config = Config('postgres0')
|
|
self.assertEqual(config._local_configuration,
|
|
{'test': False, 'test2': {'child-1': 'somestring', 'child-2': 10},
|
|
'test3': True, 'test4': [{'ab': 5}], 'new-attr': True})
|
|
|
|
@patch('os.path.exists', Mock(return_value=True))
|
|
@patch('os.path.isfile', Mock(return_value=False))
|
|
@patch('os.path.isdir', Mock(return_value=False))
|
|
def test_invalid_path(self):
|
|
self.assertRaises(ConfigParseError, Config, 'postgres0')
|
|
|
|
@patch('os.path.exists', Mock(return_value=True))
|
|
@patch('os.path.isfile', Mock(side_effect=lambda fname: fname != 'postgres0'))
|
|
@patch('os.path.isdir', Mock(return_value=True))
|
|
@patch('os.listdir', Mock(return_value=['00-empty.yml', '00-base.yml']))
|
|
@patch('patroni.config.logger')
|
|
def test_invalid_empty_config_file(self, mock_logger):
|
|
def open_mock(fname, *args, **kwargs):
|
|
if fname.endswith('00-base.yml'):
|
|
return io.StringIO(
|
|
u'''
|
|
test: True
|
|
test2:
|
|
child-1: somestring
|
|
child-2: 5
|
|
child-3: False
|
|
test3: True
|
|
test4:
|
|
- abc: 3
|
|
- abc: 4
|
|
''')
|
|
elif fname.endswith('00-empty.yml'):
|
|
return io.StringIO(u'''---''')
|
|
|
|
with patch('builtins.open', MagicMock(side_effect=open_mock)):
|
|
self.assertRaises(ConfigParseError, Config, 'postgres0')
|
|
mock_logger.error.assert_called_once_with(
|
|
'%s does not contain a dict',
|
|
'postgres0\\00-empty.yml' if sys.platform == 'win32' else 'postgres0/00-empty.yml')
|
|
|
|
@patch.object(Config, 'get')
|
|
@patch('patroni.config.logger')
|
|
def test__validate_tags(self, mock_logger, mock_get):
|
|
"""Ensures that only one of `nofailover`/`nosync' or `failover_priority`/`sync_priority` can be provided"""
|
|
tag_setup = (('nofailover', 'failover_priority'),
|
|
('nosync', 'sync_priority',))
|
|
|
|
for tag, priority_tag in tag_setup:
|
|
# Providing one tag is fine
|
|
for single_param in ({tag: True}, {priority_tag: 1}, {priority_tag: 0}):
|
|
mock_get.side_effect = [single_param] * 2
|
|
self.assertIsNone(self.config._validate_contradictory_tags())
|
|
mock_logger.warning.assert_not_called()
|
|
|
|
# Providing both tags is fine if consistent
|
|
for consistent_state in (
|
|
{tag: False, priority_tag: 1},
|
|
{tag: True, priority_tag: 0},
|
|
{tag: "False", priority_tag: 0}
|
|
):
|
|
mock_get.side_effect = [consistent_state] * 2
|
|
self.assertIsNone(self.config._validate_contradictory_tags())
|
|
mock_logger.warning.assert_not_called()
|
|
|
|
# Providing both inconsistently should log a warning
|
|
for inconsistent_state in (
|
|
{tag: False, priority_tag: 0},
|
|
{tag: True, priority_tag: 1},
|
|
{tag: "False", priority_tag: 1},
|
|
{tag: "", priority_tag: 0}
|
|
):
|
|
mock_get.side_effect = [inconsistent_state] * 2
|
|
self.assertIsNone(self.config._validate_contradictory_tags())
|
|
mock_logger.warning.assert_called_once_with(
|
|
'Conflicting configuration between %s: %s and %s: %s.'
|
|
+ ' Defaulting to %s: %s',
|
|
tag,
|
|
inconsistent_state[tag],
|
|
priority_tag,
|
|
inconsistent_state[priority_tag],
|
|
tag,
|
|
inconsistent_state[tag])
|
|
mock_logger.warning.reset_mock()
|
|
|
|
def test__process_postgresql_parameters(self):
|
|
expected_params = {
|
|
'f.oo': 'bar', # not in ConfigHandler.CMDLINE_OPTIONS
|
|
'max_connections': 100, # IntValidator
|
|
'wal_keep_size': '128MB', # IntValidator
|
|
'wal_level': 'hot_standby', # EnumValidator
|
|
}
|
|
input_params = deepcopy(expected_params)
|
|
|
|
input_params['max_connections'] = '100'
|
|
self.assertEqual(self.config._process_postgresql_parameters(input_params), expected_params)
|
|
|
|
expected_params['f.oo'] = input_params['f.oo'] = '100'
|
|
self.assertEqual(self.config._process_postgresql_parameters(input_params), expected_params)
|
|
|
|
input_params['wal_level'] = 'cold_standby'
|
|
expected_params.pop('wal_level')
|
|
self.assertEqual(self.config._process_postgresql_parameters(input_params), expected_params)
|
|
|
|
input_params['max_connections'] = 10
|
|
expected_params.pop('max_connections')
|
|
self.assertEqual(self.config._process_postgresql_parameters(input_params), expected_params)
|
|
|
|
def test__validate_and_adjust_timeouts(self):
|
|
with patch('patroni.config.logger.warning') as mock_logger:
|
|
self.config._validate_and_adjust_timeouts({'ttl': 15})
|
|
self.assertEqual(mock_logger.call_args_list[0][0],
|
|
("%s=%d can't be smaller than %d, adjusting...", 'ttl', 15, 20))
|
|
with patch('patroni.config.logger.warning') as mock_logger:
|
|
self.config._validate_and_adjust_timeouts({'loop_wait': 0})
|
|
self.assertEqual(mock_logger.call_args_list[0][0],
|
|
("%s=%d can't be smaller than %d, adjusting...", 'loop_wait', 0, 1))
|
|
with patch('patroni.config.logger.warning') as mock_logger:
|
|
self.config._validate_and_adjust_timeouts({'retry_timeout': 1})
|
|
self.assertEqual(mock_logger.call_args_list[0][0],
|
|
("%s=%d can't be smaller than %d, adjusting...", 'retry_timeout', 1, 3))
|
|
with patch('patroni.config.logger.warning') as mock_logger:
|
|
self.config._validate_and_adjust_timeouts({'ttl': 20, 'loop_wait': 11, 'retry_timeout': 5})
|
|
self.assertEqual(mock_logger.call_args_list[0][0],
|
|
('Violated the rule "loop_wait + 2*retry_timeout <= ttl", where ttl=%d '
|
|
'and retry_timeout=%d. Adjusting loop_wait from %d to %d', 20, 5, 11, 10))
|
|
with patch('patroni.config.logger.warning') as mock_logger:
|
|
self.config._validate_and_adjust_timeouts({'ttl': 20, 'loop_wait': 10, 'retry_timeout': 10})
|
|
self.assertEqual(mock_logger.call_args_list[0][0],
|
|
('Violated the rule "loop_wait + 2*retry_timeout <= ttl", where ttl=%d. Adjusting'
|
|
' loop_wait from %d to %d and retry_timeout from %d to %d', 20, 10, 1, 10, 9))
|
|
|
|
def test_global_config_is_synchronous_mode(self):
|
|
# we should ignore synchronous_mode setting in a standby cluster
|
|
config = {'standby_cluster': {'host': 'some_host'}, 'synchronous_mode': True}
|
|
cluster = get_cluster_initialized_with_only_leader(cluster_config=ClusterConfig(1, config, 1))
|
|
test_config = global_config.from_cluster(cluster)
|
|
self.assertFalse(test_config.is_synchronous_mode)
|