github-personal/patroni
1
0
Fork 0
mirror of https://github.com/outbackdingo/patroni.git synced 2026-01-27 10:20:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Extend Postgres GUCs validator (#2671)

Browse Source 
* Use YAML files to validate Postgres GUCs through Patroni.

Patroni used to have a static list of Postgres GUCs validators in
`patroni.postgresql.validator`.

One problem with that approach, for example, is that it would not
allow GUCs from custom Postgres builds to be validated/accepted.

The idea that we had to work around that issue was to move the
validators from the source code to an external and extendable source.
With that Patroni will start reading the current validators from that
external source plus whatever custom validators are found.

From this commit onwards Patroni will read and parse all YAML files
that are found under the `patroni/postgresql/available_parameters`
directory to build its Postgres GUCs validation rules.

All the details about how this work can be found in the docstring
of the introduced function `_load_postgres_gucs_validators`.
This commit is contained in:
Israel
2023-05-31 08:54:54 -03:00
committed by GitHub
parent d11328020d
commit df18885f20
11 changed files with 2341 additions and 452 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
patroni.spec
View File

@@ -16,7 +16,10 @@ def hiddenimports():
a = Analysis(['patroni/__main__.py'],
pathex=[],
binaries=None,
datas=None,
datas=[
('patroni/postgresql/available_parameters/*.yml', 'patroni/postgresql/available_parameters'),
('patroni/postgresql/available_parameters/*.yaml', 'patroni/postgresql/available_parameters'),
],
hiddenimports=hiddenimports(),
hookspath=[],
runtime_hooks=[],

16
patroni/postgresql/__init__.py
View File

@@ -26,6 +26,7 @@ from .slots import SlotsHandler
from .sync import SyncHandler
from .. import psycopg
from ..async_executor import CriticalTask
from ..collections import CaseInsensitiveSet
from ..dcs import Cluster, Leader, Member
from ..exceptions import PostgresConnectionException
from ..utils import Retry, RetryFailedError, polling_loop, data_directory_is_empty, parse_int
@@ -211,6 +212,11 @@ class Postgresql(object):
return ("SELECT " + self.TL_LSN + ", {2}").format(self.wal_name, self.lsn_name, extra)
@property
def available_gucs(self) -> CaseInsensitiveSet:
"""GUCs available in this Postgres server."""
return self._get_gucs()
def _version_file_exists(self) -> bool:
return not self.data_directory_empty() and os.path.isfile(self._version_file)
@@ -1265,3 +1271,13 @@ class Postgresql(object):
self.slots_handler.schedule()
self.citus_handler.schedule_cache_rebuild()
self._sysid = ''
def _get_gucs(self) -> CaseInsensitiveSet:
"""Get all available GUCs based on ``postgres --describe-config`` output.
:returns: all available GUCs in the local Postgres server.
"""
cmd = [self.pgcommand('postgres'), '--describe-config']
return CaseInsensitiveSet({
line.split('\t')[0] for line in subprocess.check_output(cmd).decode('utf-8').strip().split('\n')
})

1710
patroni/postgresql/available_parameters/0_postgres.yml Normal file
View File

File diff suppressed because it is too large Load Diff

6
patroni/postgresql/config.py
View File

@@ -412,7 +412,8 @@ class ConfigHandler(object):
include = self._config.get('custom_conf') or self._postgresql_base_conf_name
f.writeline("include '{0}'\n".format(ConfigWriter.escape(include)))
for name, value in sorted((configuration).items()):
value = transform_postgresql_parameter_value(self._postgresql.major_version, name, value)
value = transform_postgresql_parameter_value(self._postgresql.major_version, name, value,
self._postgresql.available_gucs)
if value is not None and\
(name != 'hba_file' or not self._postgresql.bootstrap.running_custom_bootstrap):
f.write_param(name, value)
@@ -534,7 +535,8 @@ class ConfigHandler(object):
self._passfile_mtime = mtime(self._pgpass)
value = self.format_dsn(value)
else:
value = transform_recovery_parameter_value(self._postgresql.major_version, name, value)
value = transform_recovery_parameter_value(self._postgresql.major_version, name, value,
self._postgresql.available_gucs)
if value is None:
continue
fd.write_param(name, value)

813
patroni/postgresql/validator.py
View File

@@ -1,9 +1,13 @@
import abc
from copy import deepcopy
import logging
import os
import yaml
from typing import Any, MutableMapping, Optional, Tuple, Union
from typing import Any, Dict, Iterator, List, MutableMapping, Optional, Tuple, Type, Union
from ..collections import CaseInsensitiveDict
from ..collections import CaseInsensitiveDict, CaseInsensitiveSet
from ..exceptions import PatroniException
from ..utils import parse_bool, parse_int, parse_real
logger = logging.getLogger(__name__)
@@ -11,10 +15,20 @@ logger = logging.getLogger(__name__)
class _Transformable(abc.ABC):
def __init__(self, version_from: int, version_till: Optional[int]) -> None:
def __init__(self, version_from: int, version_till: Optional[int] = None) -> None:
self.__version_from = version_from
self.__version_till = version_till
@classmethod
def get_subclasses(cls) -> Iterator[Type['_Transformable']]:
"""Recursively get all subclasses of :class:`_Transformable`.
:yields: each subclass of :class:`_Transformable`.
"""
for subclass in cls.__subclasses__():
yield from subclass.get_subclasses()
yield subclass
@property
def version_from(self) -> int:
return self.__version_from
@@ -43,8 +57,8 @@ class Bool(_Transformable):
class Number(_Transformable):
def __init__(self, version_from: int, version_till: Optional[int],
min_val: Union[int, float], max_val: Union[int, float], unit: Optional[str]) -> None:
def __init__(self, *, version_from: int, version_till: Optional[int] = None, min_val: Union[int, float],
max_val: Union[int, float], unit: Optional[str] = None) -> None:
super(Number, self).__init__(version_from, version_till)
self.__min_val = min_val
self.__max_val = max_val
@@ -99,7 +113,8 @@ class Real(Number):
class Enum(_Transformable):
def __init__(self, version_from: int, version_till: Optional[int], possible_values: Tuple[str, ...]) -> None:
def __init__(self, *, version_from: int, version_till: Optional[int] = None,
possible_values: Tuple[str, ...]) -> None:
super(Enum, self).__init__(version_from, version_till)
self.__possible_values = possible_values
@@ -128,456 +143,366 @@ class String(_Transformable):
# Format:
# key - parameter name
# value - tuple or multiple tuples if something was changing in GUC across postgres versions
parameters = CaseInsensitiveDict({
'allow_in_place_tablespaces': Bool(150000, None),
'allow_system_table_mods': Bool(90300, None),
'application_name': String(90300, None),
'archive_command': String(90300, None),
'archive_library': String(150000, None),
'archive_mode': (
Bool(90300, 90500),
EnumBool(90500, None, ('always',))
),
'archive_timeout': Integer(90300, None, 0, 1073741823, 's'),
'array_nulls': Bool(90300, None),
'authentication_timeout': Integer(90300, None, 1, 600, 's'),
'autovacuum': Bool(90300, None),
'autovacuum_analyze_scale_factor': Real(90300, None, 0, 100, None),
'autovacuum_analyze_threshold': Integer(90300, None, 0, 2147483647, None),
'autovacuum_freeze_max_age': Integer(90300, None, 100000, 2000000000, None),
'autovacuum_max_workers': (
Integer(90300, 90600, 1, 8388607, None),
Integer(90600, None, 1, 262143, None)
),
'autovacuum_multixact_freeze_max_age': Integer(90300, None, 10000, 2000000000, None),
'autovacuum_naptime': Integer(90300, None, 1, 2147483, 's'),
'autovacuum_vacuum_cost_delay': (
Integer(90300, 120000, -1, 100, 'ms'),
Real(120000, None, -1, 100, 'ms')
),
'autovacuum_vacuum_cost_limit': Integer(90300, None, -1, 10000, None),
'autovacuum_vacuum_insert_scale_factor': Real(130000, None, 0, 100, None),
'autovacuum_vacuum_insert_threshold': Integer(130000, None, -1, 2147483647, None),
'autovacuum_vacuum_scale_factor': Real(90300, None, 0, 100, None),
'autovacuum_vacuum_threshold': Integer(90300, None, 0, 2147483647, None),
'autovacuum_work_mem': Integer(90400, None, -1, 2147483647, 'kB'),
'backend_flush_after': Integer(90600, None, 0, 256, '8kB'),
'backslash_quote': EnumBool(90300, None, ('safe_encoding',)),
'backtrace_functions': String(130000, None),
'bgwriter_delay': Integer(90300, None, 10, 10000, 'ms'),
'bgwriter_flush_after': Integer(90600, None, 0, 256, '8kB'),
'bgwriter_lru_maxpages': (
Integer(90300, 100000, 0, 1000, None),
Integer(100000, None, 0, 1073741823, None)
),
'bgwriter_lru_multiplier': Real(90300, None, 0, 10, None),
'bonjour': Bool(90300, None),
'bonjour_name': String(90300, None),
'bytea_output': Enum(90300, None, ('escape', 'hex')),
'check_function_bodies': Bool(90300, None),
'checkpoint_completion_target': Real(90300, None, 0, 1, None),
'checkpoint_flush_after': Integer(90600, None, 0, 256, '8kB'),
'checkpoint_segments': Integer(90300, 90500, 1, 2147483647, None),
'checkpoint_timeout': (
Integer(90300, 90600, 30, 3600, 's'),
Integer(90600, None, 30, 86400, 's')
),
'checkpoint_warning': Integer(90300, None, 0, 2147483647, 's'),
'client_connection_check_interval': Integer(140000, None, 0, 2147483647, 'ms'),
'client_encoding': String(90300, None),
'client_min_messages': Enum(90300, None, ('debug5', 'debug4', 'debug3', 'debug2',
'debug1', 'log', 'notice', 'warning', 'error')),
'cluster_name': String(90500, None),
'commit_delay': Integer(90300, None, 0, 100000, None),
'commit_siblings': Integer(90300, None, 0, 1000, None),
'compute_query_id': (
EnumBool(140000, 150000, ('auto',)),
EnumBool(150000, None, ('auto', 'regress'))
),
'config_file': String(90300, None),
'constraint_exclusion': EnumBool(90300, None, ('partition',)),
'cpu_index_tuple_cost': Real(90300, None, 0, 1.79769e+308, None),
'cpu_operator_cost': Real(90300, None, 0, 1.79769e+308, None),
'cpu_tuple_cost': Real(90300, None, 0, 1.79769e+308, None),
'cursor_tuple_fraction': Real(90300, None, 0, 1, None),
'data_directory': String(90300, None),
'data_sync_retry': Bool(90400, None),
'DateStyle': String(90300, None),
'db_user_namespace': Bool(90300, None),
'deadlock_timeout': Integer(90300, None, 1, 2147483647, 'ms'),
'debug_discard_caches': Integer(150000, None, 0, 0, None),
'debug_pretty_print': Bool(90300, None),
'debug_print_parse': Bool(90300, None),
'debug_print_plan': Bool(90300, None),
'debug_print_rewritten': Bool(90300, None),
'default_statistics_target': Integer(90300, None, 1, 10000, None),
'default_table_access_method': String(120000, None),
'default_tablespace': String(90300, None),
'default_text_search_config': String(90300, None),
'default_toast_compression': Enum(140000, None, ('pglz', 'lz4')),
'default_transaction_deferrable': Bool(90300, None),
'default_transaction_isolation': Enum(90300, None, ('serializable', 'repeatable read',
'read committed', 'read uncommitted')),
'default_transaction_read_only': Bool(90300, None),
'default_with_oids': Bool(90300, 120000),
'dynamic_library_path': String(90300, None),
'dynamic_shared_memory_type': (
Enum(90400, 120000, ('posix', 'sysv', 'mmap', 'none')),
Enum(120000, None, ('posix', 'sysv', 'mmap'))
),
'effective_cache_size': Integer(90300, None, 1, 2147483647, '8kB'),
'effective_io_concurrency': Integer(90300, None, 0, 1000, None),
'enable_async_append': Bool(140000, None),
'enable_bitmapscan': Bool(90300, None),
'enable_gathermerge': Bool(100000, None),
'enable_hashagg': Bool(90300, None),
'enable_hashjoin': Bool(90300, None),
'enable_incremental_sort': Bool(130000, None),
'enable_indexonlyscan': Bool(90300, None),
'enable_indexscan': Bool(90300, None),
'enable_material': Bool(90300, None),
'enable_memoize': Bool(150000, None),
'enable_mergejoin': Bool(90300, None),
'enable_nestloop': Bool(90300, None),
'enable_parallel_append': Bool(110000, None),
'enable_parallel_hash': Bool(110000, None),
'enable_partition_pruning': Bool(110000, None),
'enable_partitionwise_aggregate': Bool(110000, None),
'enable_partitionwise_join': Bool(110000, None),
'enable_seqscan': Bool(90300, None),
'enable_sort': Bool(90300, None),
'enable_tidscan': Bool(90300, None),
'escape_string_warning': Bool(90300, None),
'event_source': String(90300, None),
'exit_on_error': Bool(90300, None),
'extension_destdir': String(140000, None),
'external_pid_file': String(90300, None),
'extra_float_digits': Integer(90300, None, -15, 3, None),
'force_parallel_mode': EnumBool(90600, None, ('regress',)),
'from_collapse_limit': Integer(90300, None, 1, 2147483647, None),
'fsync': Bool(90300, None),
'full_page_writes': Bool(90300, None),
'geqo': Bool(90300, None),
'geqo_effort': Integer(90300, None, 1, 10, None),
'geqo_generations': Integer(90300, None, 0, 2147483647, None),
'geqo_pool_size': Integer(90300, None, 0, 2147483647, None),
'geqo_seed': Real(90300, None, 0, 1, None),
'geqo_selection_bias': Real(90300, None, 1.5, 2, None),
'geqo_threshold': Integer(90300, None, 2, 2147483647, None),
'gin_fuzzy_search_limit': Integer(90300, None, 0, 2147483647, None),
'gin_pending_list_limit': Integer(90500, None, 64, 2147483647, 'kB'),
'hash_mem_multiplier': Real(130000, None, 1, 1000, None),
'hba_file': String(90300, None),
'hot_standby': Bool(90300, None),
'hot_standby_feedback': Bool(90300, None),
'huge_pages': EnumBool(90400, None, ('try',)),
'huge_page_size': Integer(140000, None, 0, 2147483647, 'kB'),
'ident_file': String(90300, None),
'idle_in_transaction_session_timeout': Integer(90600, None, 0, 2147483647, 'ms'),
'idle_session_timeout': Integer(140000, None, 0, 2147483647, 'ms'),
'ignore_checksum_failure': Bool(90300, None),
'ignore_invalid_pages': Bool(130000, None),
'ignore_system_indexes': Bool(90300, None),
'IntervalStyle': Enum(90300, None, ('postgres', 'postgres_verbose', 'sql_standard', 'iso_8601')),
'jit': Bool(110000, None),
'jit_above_cost': Real(110000, None, -1, 1.79769e+308, None),
'jit_debugging_support': Bool(110000, None),
'jit_dump_bitcode': Bool(110000, None),
'jit_expressions': Bool(110000, None),
'jit_inline_above_cost': Real(110000, None, -1, 1.79769e+308, None),
'jit_optimize_above_cost': Real(110000, None, -1, 1.79769e+308, None),
'jit_profiling_support': Bool(110000, None),
'jit_provider': String(110000, None),
'jit_tuple_deforming': Bool(110000, None),
'join_collapse_limit': Integer(90300, None, 1, 2147483647, None),
'krb_caseins_users': Bool(90300, None),
'krb_server_keyfile': String(90300, None),
'krb_srvname': String(90300, 90400),
'lc_messages': String(90300, None),
'lc_monetary': String(90300, None),
'lc_numeric': String(90300, None),
'lc_time': String(90300, None),
'listen_addresses': String(90300, None),
'local_preload_libraries': String(90300, None),
'lock_timeout': Integer(90300, None, 0, 2147483647, 'ms'),
'lo_compat_privileges': Bool(90300, None),
'log_autovacuum_min_duration': Integer(90300, None, -1, 2147483647, 'ms'),
'log_checkpoints': Bool(90300, None),
'log_connections': Bool(90300, None),
'log_destination': String(90300, None),
'log_directory': String(90300, None),
'log_disconnections': Bool(90300, None),
'log_duration': Bool(90300, None),
'log_error_verbosity': Enum(90300, None, ('terse', 'default', 'verbose')),
'log_executor_stats': Bool(90300, None),
'log_file_mode': Integer(90300, None, 0, 511, None),
'log_filename': String(90300, None),
'logging_collector': Bool(90300, None),
'log_hostname': Bool(90300, None),
'logical_decoding_work_mem': Integer(130000, None, 64, 2147483647, 'kB'),
'log_line_prefix': String(90300, None),
'log_lock_waits': Bool(90300, None),
'log_min_duration_sample': Integer(130000, None, -1, 2147483647, 'ms'),
'log_min_duration_statement': Integer(90300, None, -1, 2147483647, 'ms'),
'log_min_error_statement': Enum(90300, None, ('debug5', 'debug4', 'debug3', 'debug2', 'debug1', 'info',
'notice', 'warning', 'error', 'log', 'fatal', 'panic')),
'log_min_messages': Enum(90300, None, ('debug5', 'debug4', 'debug3', 'debug2', 'debug1', 'info',
'notice', 'warning', 'error', 'log', 'fatal', 'panic')),
'log_parameter_max_length': Integer(130000, None, -1, 1073741823, 'B'),
'log_parameter_max_length_on_error': Integer(130000, None, -1, 1073741823, 'B'),
'log_parser_stats': Bool(90300, None),
'log_planner_stats': Bool(90300, None),
'log_recovery_conflict_waits': Bool(140000, None),
'log_replication_commands': Bool(90500, None),
'log_rotation_age': Integer(90300, None, 0, 35791394, 'min'),
'log_rotation_size': Integer(90300, None, 0, 2097151, 'kB'),
'log_startup_progress_interval': Integer(150000, None, 0, 2147483647, 'ms'),
'log_statement': Enum(90300, None, ('none', 'ddl', 'mod', 'all')),
'log_statement_sample_rate': Real(130000, None, 0, 1, None),
'log_statement_stats': Bool(90300, None),
'log_temp_files': Integer(90300, None, -1, 2147483647, 'kB'),
'log_timezone': String(90300, None),
'log_transaction_sample_rate': Real(120000, None, 0, 1, None),
'log_truncate_on_rotation': Bool(90300, None),
'maintenance_io_concurrency': Integer(130000, None, 0, 1000, None),
'maintenance_work_mem': Integer(90300, None, 1024, 2147483647, 'kB'),
'max_connections': (
Integer(90300, 90600, 1, 8388607, None),
Integer(90600, None, 1, 262143, None)
),
'max_files_per_process': (
Integer(90300, 130000, 25, 2147483647, None),
Integer(130000, None, 64, 2147483647, None)
),
'max_locks_per_transaction': Integer(90300, None, 10, 2147483647, None),
'max_logical_replication_workers': Integer(100000, None, 0, 262143, None),
'max_parallel_maintenance_workers': Integer(110000, None, 0, 1024, None),
'max_parallel_workers': Integer(100000, None, 0, 1024, None),
'max_parallel_workers_per_gather': Integer(90600, None, 0, 1024, None),
'max_pred_locks_per_page': Integer(100000, None, 0, 2147483647, None),
'max_pred_locks_per_relation': Integer(100000, None, -2147483648, 2147483647, None),
'max_pred_locks_per_transaction': Integer(90300, None, 10, 2147483647, None),
'max_prepared_transactions': (
Integer(90300, 90600, 0, 8388607, None),
Integer(90600, None, 0, 262143, None)
),
'max_replication_slots': (
Integer(90400, 90600, 0, 8388607, None),
Integer(90600, None, 0, 262143, None)
),
'max_slot_wal_keep_size': Integer(130000, None, -1, 2147483647, 'MB'),
'max_stack_depth': Integer(90300, None, 100, 2147483647, 'kB'),
'max_standby_archive_delay': Integer(90300, None, -1, 2147483647, 'ms'),
'max_standby_streaming_delay': Integer(90300, None, -1, 2147483647, 'ms'),
'max_sync_workers_per_subscription': Integer(100000, None, 0, 262143, None),
'max_wal_senders': (
Integer(90300, 90600, 0, 8388607, None),
Integer(90600, None, 0, 262143, None)
),
'max_wal_size': (
Integer(90500, 100000, 2, 2147483647, '16MB'),
Integer(100000, None, 2, 2147483647, 'MB')
),
'max_worker_processes': (
Integer(90400, 90600, 1, 8388607, None),
Integer(90600, None, 0, 262143, None)
),
'min_dynamic_shared_memory': Integer(140000, None, 0, 2147483647, 'MB'),
'min_parallel_index_scan_size': Integer(100000, None, 0, 715827882, '8kB'),
'min_parallel_relation_size': Integer(90600, 100000, 0, 715827882, '8kB'),
'min_parallel_table_scan_size': Integer(100000, None, 0, 715827882, '8kB'),
'min_wal_size': (
Integer(90500, 100000, 2, 2147483647, '16MB'),
Integer(100000, None, 2, 2147483647, 'MB')
),
'old_snapshot_threshold': Integer(90600, None, -1, 86400, 'min'),
'operator_precedence_warning': Bool(90500, 140000),
'parallel_leader_participation': Bool(110000, None),
'parallel_setup_cost': Real(90600, None, 0, 1.79769e+308, None),
'parallel_tuple_cost': Real(90600, None, 0, 1.79769e+308, None),
'password_encryption': (
Bool(90300, 100000),
Enum(100000, None, ('md5', 'scram-sha-256'))
),
'plan_cache_mode': Enum(120000, None, ('auto', 'force_generic_plan', 'force_custom_plan')),
'port': Integer(90300, None, 1, 65535, None),
'post_auth_delay': Integer(90300, None, 0, 2147, 's'),
'pre_auth_delay': Integer(90300, None, 0, 60, 's'),
'quote_all_identifiers': Bool(90300, None),
'random_page_cost': Real(90300, None, 0, 1.79769e+308, None),
'recovery_init_sync_method': Enum(140000, None, ('fsync', 'syncfs')),
'recovery_prefetch': EnumBool(150000, None, ('try',)),
'recursive_worktable_factor': Real(150000, None, 0.001, 1e+06, None),
'remove_temp_files_after_crash': Bool(140000, None),
'replacement_sort_tuples': Integer(90600, 110000, 0, 2147483647, None),
'restart_after_crash': Bool(90300, None),
'row_security': Bool(90500, None),
'search_path': String(90300, None),
'seq_page_cost': Real(90300, None, 0, 1.79769e+308, None),
'session_preload_libraries': String(90400, None),
'session_replication_role': Enum(90300, None, ('origin', 'replica', 'local')),
'shared_buffers': Integer(90300, None, 16, 1073741823, '8kB'),
'shared_memory_type': Enum(120000, None, ('sysv', 'mmap')),
'shared_preload_libraries': String(90300, None),
'sql_inheritance': Bool(90300, 100000),
'ssl': Bool(90300, None),
'ssl_ca_file': String(90300, None),
'ssl_cert_file': String(90300, None),
'ssl_ciphers': String(90300, None),
'ssl_crl_dir': String(140000, None),
'ssl_crl_file': String(90300, None),
'ssl_dh_params_file': String(100000, None),
'ssl_ecdh_curve': String(90400, None),
'ssl_key_file': String(90300, None),
'ssl_max_protocol_version': Enum(120000, None, ('', 'tlsv1', 'tlsv1.1', 'tlsv1.2', 'tlsv1.3')),
'ssl_min_protocol_version': Enum(120000, None, ('tlsv1', 'tlsv1.1', 'tlsv1.2', 'tlsv1.3')),
'ssl_passphrase_command': String(110000, None),
'ssl_passphrase_command_supports_reload': Bool(110000, None),
'ssl_prefer_server_ciphers': Bool(90400, None),
'ssl_renegotiation_limit': Integer(90300, 90500, 0, 2147483647, 'kB'),
'standard_conforming_strings': Bool(90300, None),
'statement_timeout': Integer(90300, None, 0, 2147483647, 'ms'),
'stats_fetch_consistency': Enum(150000, None, ('none', 'cache', 'snapshot')),
'stats_temp_directory': String(90300, 150000),
'superuser_reserved_connections': (
Integer(90300, 90600, 0, 8388607, None),
Integer(90600, None, 0, 262143, None)
),
'synchronize_seqscans': Bool(90300, None),
'synchronous_commit': (
EnumBool(90300, 90600, ('local', 'remote_write')),
EnumBool(90600, None, ('local', 'remote_write', 'remote_apply'))
),
'synchronous_standby_names': String(90300, None),
'syslog_facility': Enum(90300, None, ('local0', 'local1', 'local2', 'local3',
'local4', 'local5', 'local6', 'local7')),
'syslog_ident': String(90300, None),
'syslog_sequence_numbers': Bool(90600, None),
'syslog_split_messages': Bool(90600, None),
'tcp_keepalives_count': Integer(90300, None, 0, 2147483647, None),
'tcp_keepalives_idle': Integer(90300, None, 0, 2147483647, 's'),
'tcp_keepalives_interval': Integer(90300, None, 0, 2147483647, 's'),
'tcp_user_timeout': Integer(120000, None, 0, 2147483647, 'ms'),
'temp_buffers': Integer(90300, None, 100, 1073741823, '8kB'),
'temp_file_limit': Integer(90300, None, -1, 2147483647, 'kB'),
'temp_tablespaces': String(90300, None),
'TimeZone': String(90300, None),
'timezone_abbreviations': String(90300, None),
'trace_notify': Bool(90300, None),
'trace_recovery_messages': Enum(90300, None, ('debug5', 'debug4', 'debug3', 'debug2',
'debug1', 'log', 'notice', 'warning', 'error')),
'trace_sort': Bool(90300, None),
'track_activities': Bool(90300, None),
'track_activity_query_size': (
Integer(90300, 110000, 100, 102400, None),
Integer(110000, 130000, 100, 102400, 'B'),
Integer(130000, None, 100, 1048576, 'B')
),
'track_commit_timestamp': Bool(90500, None),
'track_counts': Bool(90300, None),
'track_functions': Enum(90300, None, ('none', 'pl', 'all')),
'track_io_timing': Bool(90300, None),
'track_wal_io_timing': Bool(140000, None),
'transaction_deferrable': Bool(90300, None),
'transaction_isolation': Enum(90300, None, ('serializable', 'repeatable read',
'read committed', 'read uncommitted')),
'transaction_read_only': Bool(90300, None),
'transform_null_equals': Bool(90300, None),
'unix_socket_directories': String(90300, None),
'unix_socket_group': String(90300, None),
'unix_socket_permissions': Integer(90300, None, 0, 511, None),
'update_process_title': Bool(90300, None),
'vacuum_cleanup_index_scale_factor': Real(110000, 140000, 0, 1e+10, None),
'vacuum_cost_delay': (
Integer(90300, 120000, 0, 100, 'ms'),
Real(120000, None, 0, 100, 'ms')
),
'vacuum_cost_limit': Integer(90300, None, 1, 10000, None),
'vacuum_cost_page_dirty': Integer(90300, None, 0, 10000, None),
'vacuum_cost_page_hit': Integer(90300, None, 0, 10000, None),
'vacuum_cost_page_miss': Integer(90300, None, 0, 10000, None),
'vacuum_defer_cleanup_age': Integer(90300, None, 0, 1000000, None),
'vacuum_failsafe_age': Integer(140000, None, 0, 2100000000, None),
'vacuum_freeze_min_age': Integer(90300, None, 0, 1000000000, None),
'vacuum_freeze_table_age': Integer(90300, None, 0, 2000000000, None),
'vacuum_multixact_failsafe_age': Integer(140000, None, 0, 2100000000, None),
'vacuum_multixact_freeze_min_age': Integer(90300, None, 0, 1000000000, None),
'vacuum_multixact_freeze_table_age': Integer(90300, None, 0, 2000000000, None),
'wal_buffers': Integer(90300, None, -1, 262143, '8kB'),
'wal_compression': (
Bool(90500, 150000),
EnumBool(150000, None, ('pglz', 'lz4', 'zstd'))
),
'wal_consistency_checking': String(100000, None),
'wal_decode_buffer_size': Integer(150000, None, 65536, 1073741823, 'B'),
'wal_init_zero': Bool(120000, None),
'wal_keep_segments': Integer(90300, 130000, 0, 2147483647, None),
'wal_keep_size': Integer(130000, None, 0, 2147483647, 'MB'),
'wal_level': (
Enum(90300, 90400, ('minimal', 'archive', 'hot_standby')),
Enum(90400, 90600, ('minimal', 'archive', 'hot_standby', 'logical')),
Enum(90600, None, ('minimal', 'replica', 'logical'))
),
'wal_log_hints': Bool(90400, None),
'wal_receiver_create_temp_slot': Bool(130000, None),
'wal_receiver_status_interval': Integer(90300, None, 0, 2147483, 's'),
'wal_receiver_timeout': Integer(90300, None, 0, 2147483647, 'ms'),
'wal_recycle': Bool(120000, None),
'wal_retrieve_retry_interval': Integer(90500, None, 1, 2147483647, 'ms'),
'wal_sender_timeout': Integer(90300, None, 0, 2147483647, 'ms'),
'wal_skip_threshold': Integer(130000, None, 0, 2147483647, 'kB'),
'wal_sync_method': Enum(90300, None, ('fsync', 'fdatasync', 'open_sync', 'open_datasync')),
'wal_writer_delay': Integer(90300, None, 1, 10000, 'ms'),
'wal_writer_flush_after': Integer(90600, None, 0, 2147483647, '8kB'),
'work_mem': Integer(90300, None, 64, 2147483647, 'kB'),
'xmlbinary': Enum(90300, None, ('base64', 'hex')),
'xmloption': Enum(90300, None, ('content', 'document')),
'zero_damaged_pages': Bool(90300, None)
})
# key - parameter name
# value - variable length tuple of `_Transformable` objects. Each object in the tuple represents a different
# validation of the GUC across postgres versions. If a GUC validation has never changed over time, then it will
# have a single object in the tuple. For example, `password_encryption` used to be a boolean GUC up to Postgres
# 10, at which point it started being an enum. In that case the value of `password_encryption` would be a tuple
# of 2 `_Transformable` objects (`Bool` and `Enum`, respectively), each one reprensenting a different
# validation rule.
parameters = CaseInsensitiveDict()
recovery_parameters = CaseInsensitiveDict()
recovery_parameters = CaseInsensitiveDict({
'archive_cleanup_command': String(90300, None),
'pause_at_recovery_target': Bool(90300, 90500),
'primary_conninfo': String(90300, None),
'primary_slot_name': String(90400, None),
'promote_trigger_file': String(120000, None),
'recovery_end_command': String(90300, None),
'recovery_min_apply_delay': Integer(90400, None, 0, 2147483647, 'ms'),
'recovery_target': Enum(90400, None, ('immediate', '')),
'recovery_target_action': Enum(90500, None, ('pause', 'promote', 'shutdown')),
'recovery_target_inclusive': Bool(90300, None),
'recovery_target_lsn': String(100000, None),
'recovery_target_name': String(90400, None),
'recovery_target_time': String(90300, None),
'recovery_target_timeline': String(90300, None),
'recovery_target_xid': String(90300, None),
'restore_command': String(90300, None),
'standby_mode': Bool(90300, 120000),
'trigger_file': String(90300, 120000)
})
class ValidatorFactoryNoType(PatroniException):
"""Raised when a validator spec misses a type."""
def _transform_parameter_value(validators: MutableMapping[str, Union[_Transformable, Tuple[_Transformable, ...]]],
version: int, name: str, value: Any) -> Optional[Any]:
name_validators = validators.get(name)
if name_validators:
for validator in (name_validators if isinstance(name_validators, tuple) else (name_validators,)):
class ValidatorFactoryInvalidType(PatroniException):
"""Raised when a validator spec contains an invalid type."""
class ValidatorFactoryInvalidSpec(PatroniException):
"""Raised when a validator spec contains an invalid set of attributes."""
class ValidatorFactory:
"""Factory class used to build Patroni validator objects based on the given specs."""
TYPES: Dict[str, Type[_Transformable]] = {cls.__name__: cls for cls in _Transformable.get_subclasses()}
def __new__(cls, validator: Dict[str, Any]) -> _Transformable:
"""Parse a given Postgres GUC *validator* into the corresponding Patroni validator object.
:param validator: a validator spec for a given parameter. It usually comes from a parsed YAML file.
:returns: the Patroni validator object that corresponds to the specification found in *validator*.
:raises :class:`ValidatorFactoryNoType`: if *validator* contains no ``type`` key.
:raises :class:`ValidatorFactoryInvalidType`: if ``type`` key from *validator* contains an invalid value.
:raises :class:`ValidatorFactoryInvalidSpec`: if *validator* contains an invalid set of attributes for the
given ``type``.
:Example:
If a given validator was defined as follows in the YAML file:
```yaml
- type: String
version_from: 90300
version_till: null
```
Then this method would receive *validator* as:
```python
{
'type': 'String',
'version_from': 90300,
'version_till': None
}
```
And this method would return a :class:`String`:
```python
String(90300, None)
```
"""
validator = deepcopy(validator)
try:
type_ = validator.pop('type')
except KeyError as exc:
raise ValidatorFactoryNoType('Validator contains no type.') from exc
if type_ not in cls.TYPES:
raise ValidatorFactoryInvalidType(f'Unexpected validator type: `{type_}`.')
for key, value in validator.items():
# :func:`_transform_parameter_value` expects :class:`tuple` instead of :class:`list`
if isinstance(value, list):
tmp_value: List[Any] = value
validator[key] = tuple(tmp_value)
try:
return cls.TYPES[type_](**validator)
except Exception as exc:
raise ValidatorFactoryInvalidSpec(
f'Failed to parse `{type_}` validator (`{validator}`): `{str(exc)}`.') from exc
def _get_postgres_guc_validators(config: Dict[str, Any], parameter: str) -> Tuple[_Transformable, ...]:
"""Get all validators of *parameter* from *config*.
Loop over all validators specs of *parameter* and return them parsed as Patroni validators.
:param config: Python object corresponding to an YAML file, with values of either ``parameters`` or
``recovery_parameters`` key.
:param parameter: name of the parameter found under *config* which validators should be parsed and returned.
:rtype: yields any exception that is faced while parsing a validator spec into a Patroni validator object.
"""
validators: List[_Transformable] = []
for validator_spec in config.get(parameter, []):
try:
validator = ValidatorFactory(validator_spec)
validators.append(validator)
except (ValidatorFactoryNoType, ValidatorFactoryInvalidType, ValidatorFactoryInvalidSpec) as exc:
logger.warning('Faced an issue while parsing a validator for parameter `%s`: `%r`', parameter, exc)
return tuple(validators)
class InvalidGucValidatorsFile(PatroniException):
"""Raised when reading or parsing of a YAML file faces an issue."""
def _read_postgres_gucs_validators_file(file: str) -> Dict[str, Any]:
"""Read an YAML file and return the corresponding Python object.
:param file: path to the file to be read. It is expected to be encoded with ``UTF-8``, and to be a YAML document.
:returns: the YAML content parsed into a Python object. If any issue is faced while reading/parsing the file, then
return ``None``.
:raises :class:`InvalidGucValidatorsFile`: if faces an issue while reading or parsing *file*.
"""
try:
with open(file, encoding='UTF-8') as stream:
return yaml.safe_load(stream)
except Exception as exc:
raise InvalidGucValidatorsFile(
f'Unexpected issue while reading parameters file `{file}`: `{str(exc)}`.') from exc
def _load_postgres_gucs_validators() -> None:
"""Load all Postgres GUC validators from YAML files.
Recursively walk through ``available_parameters`` directory and load validators of each found YAML file into
``parameters`` and/or ``recovery_parameters`` variables.
Walk through directories in top-down fashion and for each of them:
* Sort files by name;
* Load validators from YAML files that were found.
Any problem faced while reading or parsing files will be logged as a ``WARNING`` by the child function, and the
corresponding file or validator will be ignored.
By default Patroni only ships the file ``0_postgres.yml``, which contains Community Postgres GUCs validators, but
that behavior can be extended. For example: if a vendor wants to add GUC validators to Patroni for covering a custom
Postgres build, then they can create their custom YAML files under ``available_parameters`` directory.
Each YAML file may contain either or both of these root attributes, here called sections:
* ``parameters``: general GUCs that would be written to ``postgresql.conf``;
* ``recovery_parameters``: recovery related GUCs that would be written to ``recovery.conf`` (Patroni later
writes them to ``postgresql.conf`` if running PG 12 and above).
Then, each of these sections, if specified, may contain one or more attributes with the following structure:
* key: the name of a GUC;
* value: a list of validators. Each item in the list must contain a ``type`` attribute, which must be one among:
* ``Bool``; or
* ``Integer``; or
* ``Real``; or
* ``Enum``; or
* ``EnumBool``; or
* ``String``.
Besides the ``type`` attribute, it should also contain all the required attributes as per the corresponding
class in this module.
.. seealso::
* :class:`Bool`;
* :class:`Integer`;
* :class:`Real`;
* :class:`Enum`;
* :class:`EnumBool`;
* :class:`String`.
:Example:
This is a sample content for an YAML file based on Postgres GUCs, showing each of the supported types and
sections:
```yaml
parameters:
archive_command:
- type: String
version_from: 90300
version_till: null
archive_mode:
- type: Bool
version_from: 90300
version_till: 90500
- type: EnumBool
version_from: 90500
version_till: null
possible_values:
- always
archive_timeout:
- type: Integer
version_from: 90300
version_till: null
min_val: 0
max_val: 1073741823
unit: s
autovacuum_vacuum_cost_delay:
- type: Integer
version_from: 90300
version_till: 120000
min_val: -1
max_val: 100
unit: ms
- type: Real
version_from: 120000
version_till: null
min_val: -1
max_val: 100
unit: ms
client_min_messages:
- type: Enum
version_from: 90300
version_till: null
possible_values:
- debug5
- debug4
- debug3
- debug2
- debug1
- log
- notice
- warning
- error
recovery_parameters:
archive_cleanup_command:
- type: String
version_from: 90300
version_till: null
```
"""
conf_dir = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
'available_parameters',
)
yaml_files: List[str] = []
for root, _, files in os.walk(conf_dir):
for file in sorted(files):
full_path = os.path.join(root, file)
if file.lower().endswith(('.yml', '.yaml')):
yaml_files.append(full_path)
else:
logger.info('Ignored a non-YAML file found under `available_parameters` directory: `%s`.', full_path)
for file in yaml_files:
try:
config: Dict[str, Any] = _read_postgres_gucs_validators_file(file)
except InvalidGucValidatorsFile as exc:
logger.warning(str(exc))
continue
logger.debug(f'Parsing validators from file `{file}`.')
mapping = {
'parameters': parameters,
'recovery_parameters': recovery_parameters,
}
for section in ['parameters', 'recovery_parameters']:
section_var = mapping[section]
config_section = config.get(section, {})
for parameter in config_section.keys():
section_var[parameter] = _get_postgres_guc_validators(config_section, parameter)
_load_postgres_gucs_validators()
def _transform_parameter_value(validators: MutableMapping[str, Tuple[_Transformable, ...]],
version: int, name: str, value: Any,
available_gucs: CaseInsensitiveSet) -> Optional[Any]:
"""Validate *value* of GUC *name* for Postgres *version* using defined *validators* and *available_gucs*.
:param validators: a dictionary of all GUCs across all Postgres versions. Each key is the name of a Postgres GUC,
and the corresponding value is a variable length tuple of :class:`_Transformable`. Each item is a validation
rule for the GUC for a given range of Postgres versions. Should either contain recovery GUCs or general GUCs,
not both.
:param version: Postgres version to validate the GUC against.
:param name: name of the Postgres GUC.
:param value: value of the Postgres GUC.
:param available_gucs: a set of all GUCs available in Postgres *version*. Each item is the name of a Postgres
GUC. Used for a couple purposes:
* Disallow writing GUCs to ``postgresql.conf`` (or ``recovery.conf``) that does not exist in Postgres *version*;
* Avoid ignoring GUC *name* if it does not have a validator in *validators*, but is a valid GUC in Postgres
*version*.
:returns: the return value may be one among:
* *value* transformed to the expected format for GUC *name* in Postgres *version*, if *name* is present in
*available_gucs* and has a validator in *validators* for the corresponding Postgres *version*; or
* The own *value* if *name* is present in *available_gucs* but not in *validators*; or
* ``None`` if *name* is not present in *available_gucs*.
"""
if name in available_gucs:
for validator in validators.get(name, ()) or ():
if version >= validator.version_from and\
(validator.version_till is None or version < validator.version_till):
return validator.transform(name, value)
# Ideally we should have a validator in *validators*. However, if none is available, we will not discard a
# setting that exists in Postgres *version*, but rather allow the value with no validation.
return value
logger.warning('Removing unexpected parameter=%s value=%s from the config', name, value)
def transform_postgresql_parameter_value(version: int, name: str, value: Any) -> Optional[Any]:
if '.' in name:
def transform_postgresql_parameter_value(version: int, name: str, value: Any,
available_gucs: CaseInsensitiveSet) -> Optional[Any]:
"""Validate *value* of GUC *name* for Postgres *version* using ``parameters`` and *available_gucs*.
:param version: Postgres version to validate the GUC against.
:param name: name of the Postgres GUC.
:param value: value of the Postgres GUC.
:param available_gucs: a set of all GUCs available in Postgres *version*. Each item is the name of a Postgres
GUC. Used for a couple purposes:
* Disallow writing GUCs to ``postgresql.conf`` that does not exist in Postgres *version*;
* Avoid ignoring GUC *name* if it does not have a validator in ``parameters``, but is a valid GUC in Postgres
*version*.
:returns: The return value may be one among
* The original *value* if *name* seems to be an extension GUC (contains a period '.'); or
* ``None`` if **name** is a recovery GUC; or
* *value* transformed to the expected format for GUC *name* in Postgres *version* using validators defined in
``parameters``. Can also return ``None``. See :func:`_transform_parameter_value`.
"""
if '.' in name and name not in parameters:
# likely an extension GUC, so just return as it is. Otherwise, if `name` is in `parameters`, it's likely a
# namespaced GUC from a custom Postgres build, so we treat that over the usual validation means.
return value
if name in recovery_parameters:
return None
return _transform_parameter_value(parameters, version, name, value)
return _transform_parameter_value(parameters, version, name, value, available_gucs)
def transform_recovery_parameter_value(version: int, name: str, value: Any) -> Optional[Any]:
return _transform_parameter_value(recovery_parameters, version, name, value)
def transform_recovery_parameter_value(version: int, name: str, value: Any,
available_gucs: CaseInsensitiveSet) -> Optional[Any]:
"""Validate *value* of GUC *name* for Postgres *version* using ``recovery_parameters`` and *available_gucs*.
:param version: Postgres version to validate the recovery GUC against.
:param name: name of the Postgres recovery GUC.
:param value: value of the Postgres recovery GUC.
:param available_gucs: a set of all GUCs available in Postgres *version*. Each item is the name of a Postgres
GUC. Used for a couple purposes:
* Disallow writing GUCs to ``recovery.conf`` (or ``postgresql.conf`` depending on *version*), that does not
exist in Postgres *version*;
* Avoid ignoring recovery GUC *name* if it does not have a validator in ``recovery_parameters``, but is a valid
GUC in Postgres *version*.
:returns: *value* transformed to the expected format for recovery GUC *name* in Postgres *version* using validators
defined in ``recovery_parameters``. It can also return ``None``. See :func:`_transform_parameter_value`.
"""
# Recovery settings are not present in ``postgres --describe-config`` output of Postgres <= 11. In that case we
# just pass down the list of settings defined in Patroni validators so :func:`_transform_parameter_value` will not
# discard the recovery GUCs when running Postgres <= 11.
# NOTE: At the moment this change was done Postgres 11 was almost EOL, and had been likely extensively used with
# Patroni, so we should be able to rely solely on Patroni validators as the source of truth.
return _transform_parameter_value(
recovery_parameters, version, name, value,
available_gucs if version >= 120000 else CaseInsensitiveSet(recovery_parameters.keys()))

5
setup.py
View File

@@ -157,7 +157,10 @@ def setup_package(version):
long_description=read('README.rst'),
classifiers=CLASSIFIERS,
packages=find_packages(exclude=['tests', 'tests.*']),
package_data={MAIN_PACKAGE: ["*.json"]},
package_data={MAIN_PACKAGE: [
"postgresql/available_parameters/*.yml",
"postgresql/available_parameters/*.yaml",
]},
install_requires=install_requires,
extras_require=EXTRAS_REQUIRE,
cmdclass=cmdclass,

11
tests/__init__.py
View File

@@ -3,7 +3,7 @@ import os
import shutil
import unittest
from mock import Mock, patch
from mock import Mock, PropertyMock, patch
import urllib3
@@ -19,6 +19,15 @@ class SleepException(Exception):
pass
mock_available_gucs = PropertyMock(return_value={
'cluster_name', 'constraint_exclusion', 'force_parallel_mode', 'hot_standby', 'listen_addresses', 'max_connections',
'max_locks_per_transaction', 'max_prepared_transactions', 'max_replication_slots', 'max_stack_depth',
'max_wal_senders', 'max_worker_processes', 'port', 'search_path', 'shared_preload_libraries',
'stats_temp_directory', 'synchronous_standby_names', 'track_commit_timestamp', 'unix_socket_directories',
'vacuum_cost_delay', 'vacuum_cost_limit', 'wal_keep_size', 'wal_level', 'wal_log_hints', 'zero_damaged_pages',
})
class MockResponse(object):
def __init__(self, status_code=200):

3
tests/test_bootstrap.py
View File

@@ -9,12 +9,13 @@ from patroni.postgresql.bootstrap import Bootstrap
from patroni.postgresql.cancellable import CancellableSubprocess
from patroni.postgresql.config import ConfigHandler
from . import psycopg_connect, BaseTestPostgresql
from . import psycopg_connect, BaseTestPostgresql, mock_available_gucs
@patch('subprocess.call', Mock(return_value=0))
@patch('patroni.psycopg.connect', psycopg_connect)
@patch('os.rename', Mock())
@patch.object(Postgresql, 'available_gucs', mock_available_gucs)
class TestBootstrap(BaseTestPostgresql):
@patch('patroni.postgresql.CallbackExecutor', Mock())

2
tests/test_patroni.py
View File

@@ -67,6 +67,7 @@ class TestPatroni(unittest.TestCase):
@patch.object(etcd.Client, 'read', etcd_read)
@patch.object(Thread, 'start', Mock())
@patch.object(AbstractEtcdClientWithFailover, '_get_machines_list', Mock(return_value=['http://remotehost:2379']))
@patch.object(Postgresql, '_get_gucs', Mock(return_value={'foo': True, 'bar': True}))
def setUp(self):
self._handlers = logging.getLogger().handlers[:]
RestApiServer._BaseServer__is_shut_down = Mock()
@@ -90,6 +91,7 @@ class TestPatroni(unittest.TestCase):
@patch.object(etcd.Client, 'delete', Mock())
@patch.object(AbstractEtcdClientWithFailover, '_get_machines_list', Mock(return_value=['http://remotehost:2379']))
@patch.object(Thread, 'join', Mock())
@patch.object(Postgresql, '_get_gucs', Mock(return_value={'foo': True, 'bar': True}))
def test_patroni_patroni_main(self):
with patch('subprocess.call', Mock(return_value=1)):
with patch.object(Patroni, 'run', Mock(side_effect=SleepException)):

218
tests/test_postgresql.py
View File

@@ -10,6 +10,7 @@ from mock import Mock, MagicMock, PropertyMock, patch, mock_open
import patroni.psycopg as psycopg
from patroni.async_executor import CriticalTask
from patroni.collections import CaseInsensitiveSet
from patroni.config import GlobalConfig
from patroni.dcs import RemoteMember
from patroni.exceptions import PostgresConnectionException, PatroniException
@@ -17,10 +18,14 @@ from patroni.postgresql import Postgresql, STATE_REJECT, STATE_NO_RESPONSE
from patroni.postgresql.bootstrap import Bootstrap
from patroni.postgresql.callback_executor import CallbackAction
from patroni.postgresql.postmaster import PostmasterProcess
from patroni.postgresql.validator import (ValidatorFactoryNoType, ValidatorFactoryInvalidType,
ValidatorFactoryInvalidSpec, ValidatorFactory, InvalidGucValidatorsFile,
_get_postgres_guc_validators, _read_postgres_gucs_validators_file,
_load_postgres_gucs_validators, Bool, Integer, Real, Enum, EnumBool, String)
from patroni.utils import RetryFailedError
from threading import Thread, current_thread
from . import BaseTestPostgresql, MockCursor, MockPostmaster, psycopg_connect
from . import BaseTestPostgresql, MockCursor, MockPostmaster, psycopg_connect, mock_available_gucs
mtime_ret = {}
@@ -91,6 +96,7 @@ Data page checksum version: 0
@patch('subprocess.call', Mock(return_value=0))
@patch('patroni.psycopg.connect', psycopg_connect)
@patch.object(Postgresql, 'available_gucs', mock_available_gucs)
class TestPostgresql(BaseTestPostgresql):
@patch('subprocess.call', Mock(return_value=0))
@@ -98,6 +104,7 @@ class TestPostgresql(BaseTestPostgresql):
@patch('patroni.postgresql.CallbackExecutor', Mock())
@patch.object(Postgresql, 'get_major_version', Mock(return_value=140000))
@patch.object(Postgresql, 'is_running', Mock(return_value=True))
@patch.object(Postgresql, 'available_gucs', mock_available_gucs)
def setUp(self):
super(TestPostgresql, self).setUp()
self.p.config.write_postgresql_conf()
@@ -739,3 +746,212 @@ class TestPostgresql(BaseTestPostgresql):
@patch.object(Postgresql, '_cluster_info_state_get', Mock(return_value=True))
def test_handle_parameter_change(self):
self.p.handle_parameter_change()
def test_validator_factory(self):
# validator with no type
validator = {
'version_from': 90300,
'version_till': None,
}
with self.assertRaises(ValidatorFactoryNoType) as e:
ValidatorFactory(validator)
self.assertEqual(str(e.exception), 'Validator contains no type.')
# validator with invalid type
validator = {
'type': 'Random',
'version_from': 90300,
'version_till': None,
}
with self.assertRaises(ValidatorFactoryInvalidType) as e:
ValidatorFactory(validator)
self.assertEqual(str(e.exception), f'Unexpected validator type: `{validator["type"]}`.')
# validator with missing attributes
validator = {
'type': 'Integer',
'version_from': 90300,
'min_val': 0,
}
with self.assertRaises(ValidatorFactoryInvalidSpec) as e:
ValidatorFactory(validator)
type_ = validator.pop('type')
self.assertRegex(
str(e.exception),
rf"Failed to parse `{type_}` validator \(`{validator}`\): `(Number\.)?__init__\(\) missing 1 "
"required keyword-only argument: 'max_val'`."
)
# valid validators
# Bool
validator = {
'type': 'Bool',
'version_from': 90300,
'version_till': None,
}
ret = ValidatorFactory(validator)
self.assertIsInstance(ret, Bool)
self.assertEqual(
ret.__dict__,
Bool(version_from=validator['version_from'], version_till=validator['version_till']).__dict__,
)
# Integer
validator = {
'type': 'Integer',
'version_from': 90300,
'version_till': None,
'min_val': 1,
'max_val': 100,
'unit': None,
}
ret = ValidatorFactory(validator)
self.assertIsInstance(ret, Integer)
self.assertEqual(
ret.__dict__,
Integer(version_from=validator['version_from'], version_till=validator['version_till'],
min_val=validator['min_val'], max_val=validator['max_val'], unit=validator['unit']).__dict__,
)
# Real
validator = {
'type': 'Real',
'version_from': 90300,
'version_till': None,
'min_val': 1.0,
'max_val': 100.0,
'unit': None,
}
ret = ValidatorFactory(validator)
self.assertIsInstance(ret, Real)
self.assertEqual(
ret.__dict__,
Real(version_from=validator['version_from'], version_till=validator['version_till'],
min_val=validator['min_val'], max_val=validator['max_val'], unit=validator['unit']).__dict__,
)
# Enum
validator = {
'type': 'Enum',
'version_from': 90300,
'version_till': None,
'possible_values': ('abc', 'def'),
}
ret = ValidatorFactory(validator)
self.assertIsInstance(ret, Enum)
self.assertEqual(
ret.__dict__,
Enum(version_from=validator['version_from'], version_till=validator['version_till'],
possible_values=validator['possible_values']).__dict__,
)
# EnumBool
validator = {
'type': 'EnumBool',
'version_from': 90300,
'version_till': None,
'possible_values': ('abc', 'def'),
}
ret = ValidatorFactory(validator)
self.assertIsInstance(ret, EnumBool)
self.assertEqual(
ret.__dict__,
EnumBool(version_from=validator['version_from'], version_till=validator['version_till'],
possible_values=validator['possible_values']).__dict__,
)
# String
validator = {
'type': 'String',
'version_from': 90300,
'version_till': None,
}
ret = ValidatorFactory(validator)
self.assertIsInstance(ret, String)
self.assertEqual(
ret.__dict__,
String(version_from=validator['version_from'], version_till=validator['version_till']).__dict__,
)
def test__get_postgres_guc_validators(self):
# normal run
parameter = 'my_parameter'
config = {
parameter: [{
'type': 'Bool',
'version_from': 90300,
'version_till': 90500,
}, {
'type': 'EnumBool',
'version_from': 90500,
'version_till': 90600,
'possible_values': [
'always',
],
}]
}
ret = _get_postgres_guc_validators(config, parameter)
self.assertIsInstance(ret, tuple)
self.assertEqual(len(ret), 2)
self.assertIsInstance(ret[0], Bool)
self.assertIsInstance(ret[1], EnumBool)
# log exceptions
del config[parameter][0]['type']
with patch('patroni.postgresql.validator.logger.warning') as mock_logger:
ret = _get_postgres_guc_validators(config, parameter)
self.assertIsInstance(ret, tuple)
self.assertEqual(len(ret), 1)
self.assertIsInstance(ret[0], EnumBool)
mock_logger.assert_called_once()
mock_call = mock_logger.call_args[0]
self.assertEqual(mock_call[0], 'Faced an issue while parsing a validator for parameter `%s`: `%r`')
self.assertEqual(mock_call[1], parameter)
self.assertIsInstance(mock_call[2], ValidatorFactoryNoType)
def test__read_postgres_gucs_validators_file(self):
# raise exception
with self.assertRaises(InvalidGucValidatorsFile) as exc:
_read_postgres_gucs_validators_file('random_file.yaml')
self.assertEqual(
str(exc.exception),
"Unexpected issue while reading parameters file `random_file.yaml`: `[Errno 2] No such file or directory: "
"'random_file.yaml'`."
)
def test__load_postgres_gucs_validators(self):
# log messages
with patch('os.walk', Mock(return_value=iter([('.', [], ['file.txt', 'random.yaml'])]))), \
patch('patroni.postgresql.validator.logger.info') as mock_info, \
patch('patroni.postgresql.validator.logger.warning') as mock_warning:
_load_postgres_gucs_validators()
mock_info.assert_called_once_with('Ignored a non-YAML file found under `available_parameters` directory: '
'`%s`.', os.path.join('.', 'file.txt'))
mock_warning.assert_called_once()
self.assertIn(
"Unexpected issue while reading parameters file `{0}`: `[Errno 2] No such file or "
"directory:".format(os.path.join('.', 'random.yaml')),
mock_warning.call_args[0][0]
)
@patch('subprocess.call', Mock(return_value=0))
@patch('patroni.psycopg.connect', psycopg_connect)
class TestPostgresql2(BaseTestPostgresql):
@patch('subprocess.call', Mock(return_value=0))
@patch('os.rename', Mock())
@patch('patroni.postgresql.CallbackExecutor', Mock())
@patch.object(Postgresql, 'get_major_version', Mock(return_value=140000))
@patch.object(Postgresql, 'is_running', Mock(return_value=True))
def setUp(self):
super(TestPostgresql2, self).setUp()
@patch('subprocess.check_output', Mock(return_value='\n'.join(mock_available_gucs.return_value).encode('utf-8')))
def test_available_gucs(self):
gucs = self.p.available_gucs
self.assertIsInstance(gucs, CaseInsensitiveSet)
self.assertEqual(gucs, mock_available_gucs.return_value)

4
tests/test_sync.py
View File

@@ -7,11 +7,12 @@ from patroni.config import GlobalConfig
from patroni.dcs import Cluster, SyncState
from patroni.postgresql import Postgresql
from . import BaseTestPostgresql, psycopg_connect
from . import BaseTestPostgresql, psycopg_connect, mock_available_gucs
@patch('subprocess.call', Mock(return_value=0))
@patch('patroni.psycopg.connect', psycopg_connect)
@patch.object(Postgresql, 'available_gucs', mock_available_gucs)
class TestSync(BaseTestPostgresql):
@patch('subprocess.call', Mock(return_value=0))
@@ -19,6 +20,7 @@ class TestSync(BaseTestPostgresql):
@patch('patroni.postgresql.CallbackExecutor', Mock())
@patch.object(Postgresql, 'get_major_version', Mock(return_value=140000))
@patch.object(Postgresql, 'is_running', Mock(return_value=True))
@patch.object(Postgresql, 'available_gucs', mock_available_gucs)
def setUp(self):
super(TestSync, self).setUp()
self.p.config.write_postgresql_conf()