mirror of
https://github.com/outbackdingo/patroni.git
synced 2026-01-27 18:20:05 +00:00
0ab5b49757
Sharing a single connection between REST API and the main thread (doing heartbeats) was working mostly fine, except when Postgres becomes so slow that REST API queries start blocking the main loop. If the dedicated REST API connection isn't available we use the heartbeat connection as a fallback.
688 lines
31 KiB
Python
688 lines
31 KiB
Python
import datetime
|
|
import json
|
|
import unittest
|
|
import socket
|
|
|
|
from http.server import HTTPServer
|
|
from io import BytesIO as IO
|
|
from mock import Mock, PropertyMock, patch
|
|
from socketserver import ThreadingMixIn
|
|
|
|
from patroni.api import RestApiHandler, RestApiServer
|
|
from patroni.config import GlobalConfig
|
|
from patroni.dcs import ClusterConfig, Member
|
|
from patroni.exceptions import PostgresConnectionException
|
|
from patroni.ha import _MemberStatus
|
|
from patroni.psycopg import OperationalError
|
|
from patroni.utils import RetryFailedError, tzutc
|
|
|
|
from . import MockConnect, psycopg_connect
|
|
from .test_ha import get_cluster_initialized_without_leader
|
|
|
|
|
|
future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5)
|
|
postmaster_start_time = datetime.datetime.now(tzutc)
|
|
|
|
|
|
class MockConnection:
|
|
|
|
@staticmethod
|
|
def get(*args):
|
|
return psycopg_connect()
|
|
|
|
@staticmethod
|
|
def query(sql, *params):
|
|
return [(postmaster_start_time, 0, '', 0, '', False, postmaster_start_time, 'streaming', None,
|
|
'[{"application_name":"walreceiver","client_addr":"1.2.3.4",'
|
|
+ '"state":"streaming","sync_state":"async","sync_priority":0}]')]
|
|
|
|
|
|
class MockConnectionPool:
|
|
|
|
@staticmethod
|
|
def get(*args):
|
|
return MockConnection()
|
|
|
|
|
|
class MockPostgresql:
|
|
|
|
connection_pool = MockConnectionPool()
|
|
name = 'test'
|
|
state = 'running'
|
|
role = 'primary'
|
|
server_version = 90625
|
|
major_version = 90600
|
|
sysid = 'dummysysid'
|
|
scope = 'dummy'
|
|
pending_restart = True
|
|
wal_name = 'wal'
|
|
lsn_name = 'lsn'
|
|
wal_flush = '_flush'
|
|
POSTMASTER_START_TIME = 'pg_catalog.pg_postmaster_start_time()'
|
|
TL_LSN = 'CASE WHEN pg_catalog.pg_is_in_recovery()'
|
|
citus_handler = Mock()
|
|
|
|
@staticmethod
|
|
def postmaster_start_time():
|
|
return postmaster_start_time
|
|
|
|
@staticmethod
|
|
def replica_cached_timeline(_):
|
|
return 2
|
|
|
|
@staticmethod
|
|
def is_running():
|
|
return True
|
|
|
|
@staticmethod
|
|
def replication_state_from_parameters(*args):
|
|
return 'streaming'
|
|
|
|
|
|
class MockWatchdog(object):
|
|
is_healthy = False
|
|
|
|
|
|
class MockHa(object):
|
|
|
|
state_handler = MockPostgresql()
|
|
watchdog = MockWatchdog()
|
|
|
|
@staticmethod
|
|
def update_failsafe(*args):
|
|
return 'foo'
|
|
|
|
@staticmethod
|
|
def failsafe_is_active(*args):
|
|
return True
|
|
|
|
@staticmethod
|
|
def is_leader():
|
|
return False
|
|
|
|
@staticmethod
|
|
def reinitialize(_):
|
|
return 'reinitialize'
|
|
|
|
@staticmethod
|
|
def restart(*args, **kwargs):
|
|
return (True, '')
|
|
|
|
@staticmethod
|
|
def restart_scheduled():
|
|
return False
|
|
|
|
@staticmethod
|
|
def delete_future_restart():
|
|
return True
|
|
|
|
@staticmethod
|
|
def fetch_nodes_statuses(members):
|
|
return [_MemberStatus(None, True, None, 0, {})]
|
|
|
|
@staticmethod
|
|
def schedule_future_restart(data):
|
|
return True
|
|
|
|
@staticmethod
|
|
def is_lagging(wal):
|
|
return False
|
|
|
|
@staticmethod
|
|
def get_effective_tags():
|
|
return {'nosync': True}
|
|
|
|
@staticmethod
|
|
def wakeup():
|
|
pass
|
|
|
|
@staticmethod
|
|
def is_paused():
|
|
return True
|
|
|
|
|
|
class MockLogger(object):
|
|
|
|
NORMAL_LOG_QUEUE_SIZE = 2
|
|
queue_size = 3
|
|
records_lost = 1
|
|
|
|
|
|
class MockConfig(object):
|
|
|
|
def get_global_config(self, _):
|
|
return GlobalConfig({})
|
|
|
|
|
|
class MockPatroni(object):
|
|
|
|
ha = MockHa()
|
|
config = MockConfig()
|
|
postgresql = ha.state_handler
|
|
dcs = Mock()
|
|
logger = MockLogger()
|
|
tags = {"key1": True, "key2": False, "key3": 1, "key4": 1.4, "key5": "RandomTag"}
|
|
version = '0.00'
|
|
noloadbalance = PropertyMock(return_value=False)
|
|
scheduled_restart = {'schedule': future_restart_time,
|
|
'postmaster_start_time': postgresql.postmaster_start_time()}
|
|
|
|
@staticmethod
|
|
def sighup_handler():
|
|
pass
|
|
|
|
@staticmethod
|
|
def api_sigterm():
|
|
pass
|
|
|
|
|
|
class MockRequest(object):
|
|
|
|
def __init__(self, request):
|
|
self.request = request.encode('utf-8')
|
|
|
|
def makefile(self, *args, **kwargs):
|
|
return IO(self.request)
|
|
|
|
def sendall(self, *args, **kwargs):
|
|
pass
|
|
|
|
|
|
class MockRestApiServer(RestApiServer):
|
|
|
|
def __init__(self, Handler, request, config=None):
|
|
self.socket = 0
|
|
self.serve_forever = Mock()
|
|
MockRestApiServer._BaseServer__is_shut_down = Mock()
|
|
MockRestApiServer._BaseServer__shutdown_request = True
|
|
config = config or {'listen': '127.0.0.1:8008', 'auth': 'test:test', 'certfile': 'dumb', 'verify_client': 'a',
|
|
'http_extra_headers': {'foo': 'bar'}, 'https_extra_headers': {'foo': 'sbar'}}
|
|
super(MockRestApiServer, self).__init__(MockPatroni(), config)
|
|
Handler(MockRequest(request), ('0.0.0.0', 8080), self)
|
|
|
|
|
|
@patch('ssl.SSLContext.load_cert_chain', Mock())
|
|
@patch('ssl.SSLContext.wrap_socket', Mock(return_value=0))
|
|
@patch.object(HTTPServer, '__init__', Mock())
|
|
class TestRestApiHandler(unittest.TestCase):
|
|
|
|
_authorization = '\nAuthorization: Basic dGVzdDp0ZXN0'
|
|
|
|
def test_do_GET(self):
|
|
MockPatroni.dcs.cluster.last_lsn = 20
|
|
MockPatroni.dcs.cluster.sync.members = [MockPostgresql.name]
|
|
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=True)):
|
|
MockRestApiServer(RestApiHandler, 'GET /replica')
|
|
MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M')
|
|
MockRestApiServer(RestApiHandler, 'GET /replica?lag=10MB')
|
|
MockRestApiServer(RestApiHandler, 'GET /replica?lag=10485760')
|
|
MockRestApiServer(RestApiHandler, 'GET /read-only')
|
|
with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={})):
|
|
MockRestApiServer(RestApiHandler, 'GET /replica')
|
|
with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'primary'})):
|
|
MockRestApiServer(RestApiHandler, 'GET /replica')
|
|
with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'state': 'running'})):
|
|
MockRestApiServer(RestApiHandler, 'GET /health')
|
|
MockRestApiServer(RestApiHandler, 'GET /leader')
|
|
with patch.object(RestApiHandler, 'get_postgresql_status',
|
|
Mock(return_value={'role': 'replica', 'sync_standby': True})):
|
|
MockRestApiServer(RestApiHandler, 'GET /synchronous')
|
|
MockRestApiServer(RestApiHandler, 'GET /read-only-sync')
|
|
with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'replica'})):
|
|
MockPatroni.dcs.cluster.sync.members = []
|
|
MockRestApiServer(RestApiHandler, 'GET /asynchronous')
|
|
with patch.object(MockHa, 'is_leader', Mock(return_value=True)):
|
|
MockRestApiServer(RestApiHandler, 'GET /replica')
|
|
MockRestApiServer(RestApiHandler, 'GET /read-only-sync')
|
|
with patch.object(GlobalConfig, 'is_standby_cluster', Mock(return_value=True)):
|
|
MockRestApiServer(RestApiHandler, 'GET /standby_leader')
|
|
MockPatroni.dcs.cluster = None
|
|
with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'primary'})):
|
|
MockRestApiServer(RestApiHandler, 'GET /primary')
|
|
with patch.object(MockHa, 'restart_scheduled', Mock(return_value=True)):
|
|
MockRestApiServer(RestApiHandler, 'GET /primary')
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /primary'))
|
|
with patch.object(RestApiServer, 'query', Mock(return_value=[('', 1, '', '', '', '', False, None, None, '')])):
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
|
|
with patch.object(GlobalConfig, 'is_standby_cluster', Mock(return_value=True)), \
|
|
patch.object(GlobalConfig, 'is_paused', Mock(return_value=True)):
|
|
MockRestApiServer(RestApiHandler, 'GET /standby_leader')
|
|
|
|
# test tags
|
|
#
|
|
MockRestApiServer(RestApiHandler, 'GET /primary?lag=1M&'
|
|
'tag_key1=true&tag_key2=false&'
|
|
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
|
|
MockRestApiServer(RestApiHandler, 'GET /primary?lag=1M&'
|
|
'tag_key1=true&tag_key2=False&'
|
|
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
|
|
MockRestApiServer(RestApiHandler, 'GET /primary?lag=1M&'
|
|
'tag_key1=true&tag_key2=false&'
|
|
'tag_key3=1.0&tag_key4=1.4&tag_key5=RandomTag')
|
|
MockRestApiServer(RestApiHandler, 'GET /primary?lag=1M&'
|
|
'tag_key1=true&tag_key2=false&'
|
|
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag&tag_key6=RandomTag2')
|
|
#
|
|
with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'primary'})):
|
|
MockRestApiServer(RestApiHandler, 'GET /primary?lag=1M&'
|
|
'tag_key1=true&tag_key2=false&'
|
|
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
|
|
MockRestApiServer(RestApiHandler, 'GET /primary?lag=1M&'
|
|
'tag_key1=true&tag_key2=False&'
|
|
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
|
|
MockRestApiServer(RestApiHandler, 'GET /primary?lag=1M&'
|
|
'tag_key1=true&tag_key2=false&'
|
|
'tag_key3=1.0&tag_key4=1.4&tag_key5=RandomTag')
|
|
MockRestApiServer(RestApiHandler, 'GET /primary?lag=1M&'
|
|
'tag_key1=true&tag_key2=false&'
|
|
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag&tag_key6=RandomTag2')
|
|
#
|
|
with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'standby_leader'})):
|
|
MockRestApiServer(RestApiHandler, 'GET /standby_leader?lag=1M&'
|
|
'tag_key1=true&tag_key2=false&'
|
|
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
|
|
MockRestApiServer(RestApiHandler, 'GET /standby_leader?lag=1M&'
|
|
'tag_key1=true&tag_key2=False&'
|
|
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
|
|
MockRestApiServer(RestApiHandler, 'GET /standby_leader?lag=1M&'
|
|
'tag_key1=true&tag_key2=false&'
|
|
'tag_key3=1.0&tag_key4=1.4&tag_key5=RandomTag')
|
|
MockRestApiServer(RestApiHandler, 'GET /standby_leader?lag=1M&'
|
|
'tag_key1=true&tag_key2=false&'
|
|
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag&tag_key6=RandomTag2')
|
|
#
|
|
MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&'
|
|
'tag_key1=true&tag_key2=false&'
|
|
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
|
|
MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&'
|
|
'tag_key1=true&tag_key2=False&'
|
|
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
|
|
MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&'
|
|
'tag_key1=true&tag_key2=false&'
|
|
'tag_key3=1.0&tag_key4=1.4&tag_key5=RandomTag')
|
|
MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&'
|
|
'tag_key1=true&tag_key2=false&'
|
|
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag&tag_key6=RandomTag2')
|
|
#
|
|
with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'primary'})):
|
|
MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&'
|
|
'tag_key1=true&tag_key2=false&'
|
|
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
|
|
MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&'
|
|
'tag_key1=true&tag_key2=False&'
|
|
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
|
|
MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&'
|
|
'tag_key1=true&tag_key2=false&'
|
|
'tag_key3=1.0&tag_key4=1.4&tag_key5=RandomTag')
|
|
MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M&'
|
|
'tag_key1=true&tag_key2=false&'
|
|
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag&tag_key6=RandomTag2')
|
|
#
|
|
MockRestApiServer(RestApiHandler, 'GET /read-write?lag=1M&'
|
|
'tag_key1=true&tag_key2=false&'
|
|
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
|
|
MockRestApiServer(RestApiHandler, 'GET /read-write?lag=1M&'
|
|
'tag_key1=true&tag_key2=False&'
|
|
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag')
|
|
MockRestApiServer(RestApiHandler, 'GET /read-write?lag=1M&'
|
|
'tag_key1=true&tag_key2=false&'
|
|
'tag_key3=1.0&tag_key4=1.4&tag_key5=RandomTag')
|
|
MockRestApiServer(RestApiHandler, 'GET /read-write?lag=1M&'
|
|
'tag_key1=true&tag_key2=false&'
|
|
'tag_key3=1&tag_key4=1.4&tag_key5=RandomTag&tag_key6=RandomTag2')
|
|
|
|
def test_do_OPTIONS(self):
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'OPTIONS / HTTP/1.0'))
|
|
|
|
def test_do_HEAD(self):
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'HEAD / HTTP/1.0'))
|
|
|
|
@patch.object(MockPatroni, 'dcs')
|
|
def test_do_GET_liveness(self, mock_dcs):
|
|
mock_dcs.ttl.return_value = PropertyMock(30)
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /liveness HTTP/1.0'))
|
|
|
|
def test_do_GET_readiness(self):
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /readiness HTTP/1.0'))
|
|
with patch.object(MockHa, 'is_leader', Mock(return_value=True)):
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /readiness HTTP/1.0'))
|
|
with patch.object(MockPostgresql, 'state', PropertyMock(return_value='stopped')):
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /readiness HTTP/1.0'))
|
|
|
|
@patch.object(MockPostgresql, 'state', PropertyMock(return_value='stopped'))
|
|
def test_do_GET_patroni(self):
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
|
|
|
|
def test_basicauth(self):
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /restart HTTP/1.0'))
|
|
MockRestApiServer(RestApiHandler, 'POST /restart HTTP/1.0\nAuthorization:')
|
|
|
|
@patch.object(MockPatroni, 'dcs')
|
|
def test_do_GET_cluster(self, mock_dcs):
|
|
mock_dcs.get_cluster.return_value = get_cluster_initialized_without_leader()
|
|
mock_dcs.get_cluster.return_value.members[1].data['xlog_location'] = 11
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /cluster'))
|
|
|
|
@patch.object(MockPatroni, 'dcs')
|
|
def test_do_GET_history(self, mock_dcs):
|
|
mock_dcs.cluster = get_cluster_initialized_without_leader()
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /history'))
|
|
|
|
@patch.object(MockPatroni, 'dcs')
|
|
def test_do_GET_config(self, mock_dcs):
|
|
mock_dcs.cluster.config.data = {}
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /config'))
|
|
mock_dcs.cluster.config = None
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /config'))
|
|
|
|
@patch.object(MockPatroni, 'dcs')
|
|
def test_do_GET_metrics(self, mock_dcs):
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /metrics'))
|
|
|
|
@patch.object(MockPatroni, 'dcs')
|
|
def test_do_PATCH_config(self, mock_dcs):
|
|
config = {'postgresql': {'use_slots': False, 'use_pg_rewind': True, 'parameters': {'wal_level': 'logical'}}}
|
|
mock_dcs.get_cluster.return_value.config = ClusterConfig.from_node(1, json.dumps(config))
|
|
request = 'PATCH /config HTTP/1.0' + self._authorization
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
|
|
request += '\nContent-Length: '
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '34\n\n{"postgresql":{"use_slots":false}}'))
|
|
config['ttl'] = 5
|
|
config['postgresql'].update({'use_slots': {'foo': True}, "parameters": None})
|
|
config = json.dumps(config)
|
|
request += str(len(config)) + '\n\n' + config
|
|
MockRestApiServer(RestApiHandler, request)
|
|
mock_dcs.set_config_value.return_value = False
|
|
MockRestApiServer(RestApiHandler, request)
|
|
mock_dcs.get_cluster.return_value.config = None
|
|
MockRestApiServer(RestApiHandler, request)
|
|
|
|
@patch.object(MockPatroni, 'dcs')
|
|
def test_do_PUT_config(self, mock_dcs):
|
|
mock_dcs.get_cluster.return_value.config = ClusterConfig.from_node(1, '{}')
|
|
request = 'PUT /config HTTP/1.0' + self._authorization + '\nContent-Length: '
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '2\n\n{}'))
|
|
config = '{"foo": "bar"}'
|
|
request += str(len(config)) + '\n\n' + config
|
|
MockRestApiServer(RestApiHandler, request)
|
|
mock_dcs.set_config_value.return_value = False
|
|
MockRestApiServer(RestApiHandler, request)
|
|
mock_dcs.get_cluster.return_value.config = ClusterConfig.from_node(1, config)
|
|
MockRestApiServer(RestApiHandler, request)
|
|
|
|
@patch.object(MockPatroni, 'dcs')
|
|
def test_do_GET_failsafe(self, mock_dcs):
|
|
type(mock_dcs).failsafe = PropertyMock(return_value={'node1': 'http://foo:8080/patroni'})
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /failsafe'))
|
|
type(mock_dcs).failsafe = PropertyMock(return_value=None)
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /failsafe'))
|
|
|
|
def test_do_POST_failsafe(self):
|
|
with patch.object(MockHa, 'is_failsafe_mode', Mock(return_value=False), create=True):
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /failsafe HTTP/1.0' + self._authorization))
|
|
with patch.object(MockHa, 'is_failsafe_mode', Mock(return_value=True), create=True):
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /failsafe HTTP/1.0' + self._authorization
|
|
+ '\nContent-Length: 9\n\n{"a":"b"}'))
|
|
|
|
@patch.object(MockPatroni, 'sighup_handler', Mock())
|
|
def test_do_POST_reload(self):
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /reload HTTP/1.0' + self._authorization))
|
|
|
|
@patch('os.environ', {'BEHAVE_DEBUG': 'true'})
|
|
@patch('os.name', 'nt')
|
|
def test_do_POST_sigterm(self):
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /sigterm HTTP/1.0' + self._authorization))
|
|
|
|
def test_do_POST_restart(self):
|
|
request = 'POST /restart HTTP/1.0' + self._authorization
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
|
|
|
|
with patch.object(MockHa, 'restart', Mock(side_effect=Exception)):
|
|
MockRestApiServer(RestApiHandler, request)
|
|
|
|
post = request + '\nContent-Length: '
|
|
|
|
def make_request(request=None, **kwargs):
|
|
request = json.dumps(kwargs) if request is None else request
|
|
return '{0}{1}\n\n{2}'.format(post, len(request), request)
|
|
|
|
# empty request
|
|
request = make_request('')
|
|
MockRestApiServer(RestApiHandler, request)
|
|
# invalid request
|
|
request = make_request('foobar=baz')
|
|
MockRestApiServer(RestApiHandler, request)
|
|
# wrong role
|
|
request = make_request(schedule=future_restart_time.isoformat(), role='unknown', postgres_version='9.5.3')
|
|
MockRestApiServer(RestApiHandler, request)
|
|
# wrong version
|
|
request = make_request(schedule=future_restart_time.isoformat(), role='primary', postgres_version='9.5.3.1')
|
|
MockRestApiServer(RestApiHandler, request)
|
|
# unknown filter
|
|
request = make_request(schedule=future_restart_time.isoformat(), batman='lives')
|
|
MockRestApiServer(RestApiHandler, request)
|
|
# incorrect schedule
|
|
request = make_request(schedule='2016-08-42 12:45TZ+1', role='primary')
|
|
MockRestApiServer(RestApiHandler, request)
|
|
# everything fine, but the schedule is missing
|
|
request = make_request(role='primary', postgres_version='9.5.2')
|
|
MockRestApiServer(RestApiHandler, request)
|
|
for retval in (True, False):
|
|
with patch.object(MockHa, 'schedule_future_restart', Mock(return_value=retval)):
|
|
request = make_request(schedule=future_restart_time.isoformat())
|
|
MockRestApiServer(RestApiHandler, request)
|
|
with patch.object(MockHa, 'restart', Mock(return_value=(retval, "foo"))):
|
|
request = make_request(role='primary', postgres_version='9.5.2')
|
|
MockRestApiServer(RestApiHandler, request)
|
|
|
|
with patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
|
|
MockRestApiServer(RestApiHandler, make_request(schedule='2016-08-42 12:45TZ+1', role='primary'))
|
|
# Valid timeout
|
|
MockRestApiServer(RestApiHandler, make_request(timeout='60s'))
|
|
# Invalid timeout
|
|
MockRestApiServer(RestApiHandler, make_request(timeout='42towels'))
|
|
|
|
def test_do_DELETE_restart(self):
|
|
for retval in (True, False):
|
|
with patch.object(MockHa, 'delete_future_restart', Mock(return_value=retval)):
|
|
request = 'DELETE /restart HTTP/1.0' + self._authorization
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
|
|
|
|
@patch.object(MockPatroni, 'dcs')
|
|
def test_do_DELETE_switchover(self, mock_dcs):
|
|
request = 'DELETE /switchover HTTP/1.0' + self._authorization
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
|
|
mock_dcs.manual_failover.return_value = False
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
|
|
mock_dcs.get_cluster.return_value.failover = None
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
|
|
|
|
def test_do_POST_reinitialize(self):
|
|
request = 'POST /reinitialize HTTP/1.0' + self._authorization + '\nContent-Length: 15\n\n{"force": true}'
|
|
MockRestApiServer(RestApiHandler, request)
|
|
with patch.object(MockHa, 'reinitialize', Mock(return_value=None)):
|
|
MockRestApiServer(RestApiHandler, request)
|
|
|
|
@patch('time.sleep', Mock())
|
|
def test_RestApiServer_query(self):
|
|
with patch.object(MockConnection, 'query', Mock(side_effect=RetryFailedError('bla'))):
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
|
|
|
|
@patch('time.sleep', Mock())
|
|
@patch.object(MockPatroni, 'dcs')
|
|
def test_do_POST_switchover(self, dcs):
|
|
dcs.loop_wait = 10
|
|
cluster = dcs.get_cluster.return_value
|
|
|
|
post = 'POST /switchover HTTP/1.0' + self._authorization + '\nContent-Length: '
|
|
|
|
MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}')
|
|
|
|
request = post + '0\n\n'
|
|
MockRestApiServer(RestApiHandler, request)
|
|
|
|
cluster.leader.name = 'postgresql1'
|
|
MockRestApiServer(RestApiHandler, request)
|
|
|
|
request = post + '25\n\n{"leader": "postgresql1"}'
|
|
|
|
with patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
|
|
MockRestApiServer(RestApiHandler, request)
|
|
|
|
for is_synchronous_mode in (True, False):
|
|
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)):
|
|
MockRestApiServer(RestApiHandler, request)
|
|
|
|
cluster.leader.name = 'postgresql2'
|
|
request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
|
|
MockRestApiServer(RestApiHandler, request)
|
|
|
|
cluster.leader.name = 'postgresql1'
|
|
cluster.sync.matches.return_value = False
|
|
for is_synchronous_mode in (True, False):
|
|
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)):
|
|
MockRestApiServer(RestApiHandler, request)
|
|
|
|
cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}),
|
|
Member(0, 'postgresql2', 30, {'api_url': 'http'})]
|
|
MockRestApiServer(RestApiHandler, request)
|
|
|
|
cluster.failover = None
|
|
MockRestApiServer(RestApiHandler, request)
|
|
|
|
dcs.get_cluster.side_effect = [cluster]
|
|
MockRestApiServer(RestApiHandler, request)
|
|
|
|
cluster2 = cluster.copy()
|
|
cluster2.leader.name = 'postgresql0'
|
|
cluster2.is_unlocked.return_value = False
|
|
dcs.get_cluster.side_effect = [cluster, cluster2]
|
|
MockRestApiServer(RestApiHandler, request)
|
|
|
|
cluster2.leader.name = 'postgresql2'
|
|
dcs.get_cluster.side_effect = [cluster, cluster2]
|
|
MockRestApiServer(RestApiHandler, request)
|
|
|
|
dcs.get_cluster.side_effect = None
|
|
dcs.manual_failover.return_value = False
|
|
MockRestApiServer(RestApiHandler, request)
|
|
dcs.manual_failover.return_value = True
|
|
|
|
with patch.object(MockHa, 'fetch_nodes_statuses', Mock(return_value=[])):
|
|
MockRestApiServer(RestApiHandler, request)
|
|
|
|
# Valid future date
|
|
request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2",' +\
|
|
' "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}'
|
|
MockRestApiServer(RestApiHandler, request)
|
|
with patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)), \
|
|
patch.object(MockPatroni, 'dcs') as d:
|
|
d.manual_failover.return_value = False
|
|
MockRestApiServer(RestApiHandler, request)
|
|
|
|
# Exception: No timezone specified
|
|
request = post + '97\n\n{"leader": "postgresql1", "member": "postgresql2",' +\
|
|
' "scheduled_at": "6016-02-15T18:13:30.568224"}'
|
|
MockRestApiServer(RestApiHandler, request)
|
|
|
|
# Exception: Scheduled in the past
|
|
request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "'
|
|
MockRestApiServer(RestApiHandler, request + '1016-02-15T18:13:30.568224+01:00"}')
|
|
|
|
# Invalid date
|
|
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}'))
|
|
|
|
def test_do_POST_failover(self):
|
|
post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: '
|
|
MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}')
|
|
MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}')
|
|
|
|
@patch.object(MockHa, 'is_leader', Mock(return_value=True))
|
|
def test_do_POST_citus(self):
|
|
post = 'POST /citus HTTP/1.0' + self._authorization + '\nContent-Length: '
|
|
MockRestApiServer(RestApiHandler, post + '0\n\n')
|
|
MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}')
|
|
|
|
|
|
class TestRestApiServer(unittest.TestCase):
|
|
|
|
@patch('ssl.SSLContext.load_cert_chain', Mock())
|
|
@patch('ssl.SSLContext.set_ciphers', Mock())
|
|
@patch('ssl.SSLContext.wrap_socket', Mock(return_value=0))
|
|
@patch.object(HTTPServer, '__init__', Mock())
|
|
def setUp(self):
|
|
self.srv = MockRestApiServer(Mock(), '', {'listen': '*:8008', 'certfile': 'a', 'verify_client': 'required',
|
|
'ciphers': '!SSLv1:!SSLv2:!SSLv3:!TLSv1:!TLSv1.1',
|
|
'allowlist': ['127.0.0.1', '::1/128', '::1/zxc'],
|
|
'allowlist_include_members': True})
|
|
|
|
@patch.object(HTTPServer, '__init__', Mock())
|
|
def test_reload_config(self):
|
|
bad_config = {'listen': 'foo'}
|
|
self.assertRaises(ValueError, MockRestApiServer, None, '', bad_config)
|
|
self.assertRaises(ValueError, self.srv.reload_config, bad_config)
|
|
self.assertRaises(ValueError, self.srv.reload_config, {})
|
|
with patch.object(socket.socket, 'setsockopt', Mock(side_effect=socket.error)), \
|
|
patch.object(MockRestApiServer, 'server_close', Mock()):
|
|
self.srv.reload_config({'listen': ':8008'})
|
|
|
|
@patch.object(MockPatroni, 'dcs')
|
|
def test_check_access(self, mock_dcs):
|
|
mock_dcs.cluster = get_cluster_initialized_without_leader()
|
|
mock_dcs.cluster.members[1].data['api_url'] = 'http://127.0.0.1z:8011/patroni'
|
|
mock_dcs.cluster.members.append(Member(0, 'bad-api-url', 30, {'api_url': 123}))
|
|
mock_rh = Mock()
|
|
mock_rh.client_address = ('127.0.0.2',)
|
|
self.assertIsNot(self.srv.check_access(mock_rh), True)
|
|
mock_rh.client_address = ('127.0.0.1',)
|
|
mock_rh.request.getpeercert.return_value = None
|
|
self.assertIsNot(self.srv.check_access(mock_rh), True)
|
|
|
|
def test_handle_error(self):
|
|
try:
|
|
raise Exception()
|
|
except Exception:
|
|
self.assertIsNone(self.srv.handle_error(None, ('127.0.0.1', 55555)))
|
|
|
|
@patch.object(HTTPServer, '__init__', Mock(side_effect=socket.error))
|
|
def test_socket_error(self):
|
|
self.assertRaises(socket.error, MockRestApiServer, Mock(), '', {'listen': '*:8008'})
|
|
|
|
def __create_socket(self):
|
|
sock = socket.socket()
|
|
try:
|
|
import ssl
|
|
ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
|
|
ctx.check_hostname = False
|
|
sock = ctx.wrap_socket(sock=sock)
|
|
sock.do_handshake = Mock()
|
|
sock.unwrap = Mock(side_effect=Exception)
|
|
except Exception:
|
|
pass
|
|
return sock
|
|
|
|
@patch.object(ThreadingMixIn, 'process_request_thread', Mock())
|
|
def test_process_request_thread(self):
|
|
self.srv.process_request_thread(self.__create_socket(), ('2', 54321))
|
|
|
|
@patch.object(MockRestApiServer, 'process_request', Mock(side_effect=RuntimeError))
|
|
@patch.object(MockRestApiServer, 'get_request')
|
|
def test_process_request_error(self, mock_get_request):
|
|
mock_get_request.return_value = (self.__create_socket(), ('127.0.0.1', 55555))
|
|
self.srv._handle_request_noblock()
|
|
|
|
@patch('ssl._ssl._test_decode_cert', Mock())
|
|
def test_reload_local_certificate(self):
|
|
self.assertTrue(self.srv.reload_local_certificate())
|
|
|
|
def test_get_certificate_serial_number(self):
|
|
self.assertIsNone(self.srv.get_certificate_serial_number())
|
|
|
|
def test_query(self):
|
|
with patch.object(MockConnection, 'get', Mock(side_effect=OperationalError)):
|
|
self.assertRaises(PostgresConnectionException, self.srv.query, 'SELECT 1')
|
|
with patch.object(MockConnection, 'get', Mock(side_effect=[MockConnect(), OperationalError])), \
|
|
patch.object(MockConnection, 'query') as mock_query:
|
|
self.srv.query('SELECT 1')
|
|
mock_query.assert_called_once_with('SELECT 1')
|