diff --git a/patroni.spec b/patroni.spec index 121e6f5d..b5c414a2 100644 --- a/patroni.spec +++ b/patroni.spec @@ -16,7 +16,10 @@ def hiddenimports(): a = Analysis(['patroni/__main__.py'], pathex=[], binaries=None, - datas=None, + datas=[ + ('patroni/postgresql/available_parameters/*.yml', 'patroni/postgresql/available_parameters'), + ('patroni/postgresql/available_parameters/*.yaml', 'patroni/postgresql/available_parameters'), + ], hiddenimports=hiddenimports(), hookspath=[], runtime_hooks=[], diff --git a/patroni/postgresql/__init__.py b/patroni/postgresql/__init__.py index 6d2d18af..b2423c89 100644 --- a/patroni/postgresql/__init__.py +++ b/patroni/postgresql/__init__.py @@ -26,6 +26,7 @@ from .slots import SlotsHandler from .sync import SyncHandler from .. import psycopg from ..async_executor import CriticalTask +from ..collections import CaseInsensitiveSet from ..dcs import Cluster, Leader, Member from ..exceptions import PostgresConnectionException from ..utils import Retry, RetryFailedError, polling_loop, data_directory_is_empty, parse_int @@ -211,6 +212,11 @@ class Postgresql(object): return ("SELECT " + self.TL_LSN + ", {2}").format(self.wal_name, self.lsn_name, extra) + @property + def available_gucs(self) -> CaseInsensitiveSet: + """GUCs available in this Postgres server.""" + return self._get_gucs() + def _version_file_exists(self) -> bool: return not self.data_directory_empty() and os.path.isfile(self._version_file) @@ -1265,3 +1271,13 @@ class Postgresql(object): self.slots_handler.schedule() self.citus_handler.schedule_cache_rebuild() self._sysid = '' + + def _get_gucs(self) -> CaseInsensitiveSet: + """Get all available GUCs based on ``postgres --describe-config`` output. + + :returns: all available GUCs in the local Postgres server. + """ + cmd = [self.pgcommand('postgres'), '--describe-config'] + return CaseInsensitiveSet({ + line.split('\t')[0] for line in subprocess.check_output(cmd).decode('utf-8').strip().split('\n') + }) diff --git a/patroni/postgresql/available_parameters/0_postgres.yml b/patroni/postgresql/available_parameters/0_postgres.yml new file mode 100644 index 00000000..5afd2efb --- /dev/null +++ b/patroni/postgresql/available_parameters/0_postgres.yml @@ -0,0 +1,1710 @@ +parameters: + allow_in_place_tablespaces: + - type: Bool + version_from: 150000 + allow_system_table_mods: + - type: Bool + version_from: 90300 + application_name: + - type: String + version_from: 90300 + archive_command: + - type: String + version_from: 90300 + archive_library: + - type: String + version_from: 150000 + archive_mode: + - type: Bool + version_from: 90300 + version_till: 90500 + - type: EnumBool + version_from: 90500 + possible_values: + - always + archive_timeout: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 1073741823 + unit: s + array_nulls: + - type: Bool + version_from: 90300 + authentication_timeout: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 600 + unit: s + autovacuum: + - type: Bool + version_from: 90300 + autovacuum_analyze_scale_factor: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 100 + autovacuum_analyze_threshold: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + autovacuum_freeze_max_age: + - type: Integer + version_from: 90300 + min_val: 100000 + max_val: 2000000000 + autovacuum_max_workers: + - type: Integer + version_from: 90300 + version_till: 90600 + min_val: 1 + max_val: 8388607 + - type: Integer + version_from: 90600 + min_val: 1 + max_val: 262143 + autovacuum_multixact_freeze_max_age: + - type: Integer + version_from: 90300 + min_val: 10000 + max_val: 2000000000 + autovacuum_naptime: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 2147483 + unit: s + autovacuum_vacuum_cost_delay: + - type: Integer + version_from: 90300 + version_till: 120000 + min_val: -1 + max_val: 100 + unit: ms + - type: Real + version_from: 120000 + min_val: -1 + max_val: 100 + unit: ms + autovacuum_vacuum_cost_limit: + - type: Integer + version_from: 90300 + min_val: -1 + max_val: 10000 + autovacuum_vacuum_insert_scale_factor: + - type: Real + version_from: 130000 + min_val: 0 + max_val: 100 + autovacuum_vacuum_insert_threshold: + - type: Integer + version_from: 130000 + min_val: -1 + max_val: 2147483647 + autovacuum_vacuum_scale_factor: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 100 + autovacuum_vacuum_threshold: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + autovacuum_work_mem: + - type: Integer + version_from: 90400 + min_val: -1 + max_val: 2147483647 + unit: kB + backend_flush_after: + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 256 + unit: 8kB + backslash_quote: + - type: EnumBool + version_from: 90300 + possible_values: + - safe_encoding + backtrace_functions: + - type: String + version_from: 130000 + bgwriter_delay: + - type: Integer + version_from: 90300 + min_val: 10 + max_val: 10000 + unit: ms + bgwriter_flush_after: + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 256 + unit: 8kB + bgwriter_lru_maxpages: + - type: Integer + version_from: 90300 + version_till: 100000 + min_val: 0 + max_val: 1000 + - type: Integer + version_from: 100000 + min_val: 0 + max_val: 1073741823 + bgwriter_lru_multiplier: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 10 + bonjour: + - type: Bool + version_from: 90300 + bonjour_name: + - type: String + version_from: 90300 + bytea_output: + - type: Enum + version_from: 90300 + possible_values: + - escape + - hex + check_function_bodies: + - type: Bool + version_from: 90300 + checkpoint_completion_target: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 1 + checkpoint_flush_after: + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 256 + unit: 8kB + checkpoint_segments: + - type: Integer + version_from: 90300 + version_till: 90500 + min_val: 1 + max_val: 2147483647 + checkpoint_timeout: + - type: Integer + version_from: 90300 + version_till: 90600 + min_val: 30 + max_val: 3600 + unit: s + - type: Integer + version_from: 90600 + min_val: 30 + max_val: 86400 + unit: s + checkpoint_warning: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + unit: s + client_connection_check_interval: + - type: Integer + version_from: 140000 + min_val: 0 + max_val: 2147483647 + unit: ms + client_encoding: + - type: String + version_from: 90300 + client_min_messages: + - type: Enum + version_from: 90300 + possible_values: + - debug5 + - debug4 + - debug3 + - debug2 + - debug1 + - log + - notice + - warning + - error + cluster_name: + - type: String + version_from: 90500 + commit_delay: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 100000 + commit_siblings: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 1000 + compute_query_id: + - type: EnumBool + version_from: 140000 + version_till: 150000 + possible_values: + - auto + - type: EnumBool + version_from: 150000 + possible_values: + - auto + - regress + config_file: + - type: String + version_from: 90300 + constraint_exclusion: + - type: EnumBool + version_from: 90300 + possible_values: + - partition + cpu_index_tuple_cost: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 1.79769e+308 + cpu_operator_cost: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 1.79769e+308 + cpu_tuple_cost: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 1.79769e+308 + cursor_tuple_fraction: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 1 + data_directory: + - type: String + version_from: 90300 + data_sync_retry: + - type: Bool + version_from: 90400 + DateStyle: + - type: String + version_from: 90300 + db_user_namespace: + - type: Bool + version_from: 90300 + deadlock_timeout: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 2147483647 + unit: ms + debug_discard_caches: + - type: Integer + version_from: 150000 + min_val: 0 + max_val: 0 + debug_pretty_print: + - type: Bool + version_from: 90300 + debug_print_parse: + - type: Bool + version_from: 90300 + debug_print_plan: + - type: Bool + version_from: 90300 + debug_print_rewritten: + - type: Bool + version_from: 90300 + default_statistics_target: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 10000 + default_table_access_method: + - type: String + version_from: 120000 + default_tablespace: + - type: String + version_from: 90300 + default_text_search_config: + - type: String + version_from: 90300 + default_toast_compression: + - type: Enum + version_from: 140000 + possible_values: + - pglz + - lz4 + default_transaction_deferrable: + - type: Bool + version_from: 90300 + default_transaction_isolation: + - type: Enum + version_from: 90300 + possible_values: + - serializable + - repeatable read + - read committed + - read uncommitted + default_transaction_read_only: + - type: Bool + version_from: 90300 + default_with_oids: + - type: Bool + version_from: 90300 + version_till: 120000 + dynamic_library_path: + - type: String + version_from: 90300 + dynamic_shared_memory_type: + - type: Enum + version_from: 90400 + version_till: 120000 + possible_values: + - posix + - sysv + - mmap + - none + - type: Enum + version_from: 120000 + possible_values: + - posix + - sysv + - mmap + effective_cache_size: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 2147483647 + unit: 8kB + effective_io_concurrency: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 1000 + enable_async_append: + - type: Bool + version_from: 140000 + enable_bitmapscan: + - type: Bool + version_from: 90300 + enable_gathermerge: + - type: Bool + version_from: 100000 + enable_hashagg: + - type: Bool + version_from: 90300 + enable_hashjoin: + - type: Bool + version_from: 90300 + enable_incremental_sort: + - type: Bool + version_from: 130000 + enable_indexonlyscan: + - type: Bool + version_from: 90300 + enable_indexscan: + - type: Bool + version_from: 90300 + enable_material: + - type: Bool + version_from: 90300 + enable_memoize: + - type: Bool + version_from: 150000 + enable_mergejoin: + - type: Bool + version_from: 90300 + enable_nestloop: + - type: Bool + version_from: 90300 + enable_parallel_append: + - type: Bool + version_from: 110000 + enable_parallel_hash: + - type: Bool + version_from: 110000 + enable_partition_pruning: + - type: Bool + version_from: 110000 + enable_partitionwise_aggregate: + - type: Bool + version_from: 110000 + enable_partitionwise_join: + - type: Bool + version_from: 110000 + enable_seqscan: + - type: Bool + version_from: 90300 + enable_sort: + - type: Bool + version_from: 90300 + enable_tidscan: + - type: Bool + version_from: 90300 + escape_string_warning: + - type: Bool + version_from: 90300 + event_source: + - type: String + version_from: 90300 + exit_on_error: + - type: Bool + version_from: 90300 + extension_destdir: + - type: String + version_from: 140000 + external_pid_file: + - type: String + version_from: 90300 + extra_float_digits: + - type: Integer + version_from: 90300 + min_val: -15 + max_val: 3 + force_parallel_mode: + - type: EnumBool + version_from: 90600 + possible_values: + - regress + from_collapse_limit: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 2147483647 + fsync: + - type: Bool + version_from: 90300 + full_page_writes: + - type: Bool + version_from: 90300 + geqo: + - type: Bool + version_from: 90300 + geqo_effort: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 10 + geqo_generations: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + geqo_pool_size: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + geqo_seed: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 1 + geqo_selection_bias: + - type: Real + version_from: 90300 + min_val: 1.5 + max_val: 2 + geqo_threshold: + - type: Integer + version_from: 90300 + min_val: 2 + max_val: 2147483647 + gin_fuzzy_search_limit: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + gin_pending_list_limit: + - type: Integer + version_from: 90500 + min_val: 64 + max_val: 2147483647 + unit: kB + hash_mem_multiplier: + - type: Real + version_from: 130000 + min_val: 1 + max_val: 1000 + hba_file: + - type: String + version_from: 90300 + hot_standby: + - type: Bool + version_from: 90300 + hot_standby_feedback: + - type: Bool + version_from: 90300 + huge_pages: + - type: EnumBool + version_from: 90400 + possible_values: + - try + huge_page_size: + - type: Integer + version_from: 140000 + min_val: 0 + max_val: 2147483647 + unit: kB + ident_file: + - type: String + version_from: 90300 + idle_in_transaction_session_timeout: + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 2147483647 + unit: ms + idle_session_timeout: + - type: Integer + version_from: 140000 + min_val: 0 + max_val: 2147483647 + unit: ms + ignore_checksum_failure: + - type: Bool + version_from: 90300 + ignore_invalid_pages: + - type: Bool + version_from: 130000 + ignore_system_indexes: + - type: Bool + version_from: 90300 + IntervalStyle: + - type: Enum + version_from: 90300 + possible_values: + - postgres + - postgres_verbose + - sql_standard + - iso_8601 + jit: + - type: Bool + version_from: 110000 + jit_above_cost: + - type: Real + version_from: 110000 + min_val: -1 + max_val: 1.79769e+308 + jit_debugging_support: + - type: Bool + version_from: 110000 + jit_dump_bitcode: + - type: Bool + version_from: 110000 + jit_expressions: + - type: Bool + version_from: 110000 + jit_inline_above_cost: + - type: Real + version_from: 110000 + min_val: -1 + max_val: 1.79769e+308 + jit_optimize_above_cost: + - type: Real + version_from: 110000 + min_val: -1 + max_val: 1.79769e+308 + jit_profiling_support: + - type: Bool + version_from: 110000 + jit_provider: + - type: String + version_from: 110000 + jit_tuple_deforming: + - type: Bool + version_from: 110000 + join_collapse_limit: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 2147483647 + krb_caseins_users: + - type: Bool + version_from: 90300 + krb_server_keyfile: + - type: String + version_from: 90300 + krb_srvname: + - type: String + version_from: 90300 + version_till: 90400 + lc_messages: + - type: String + version_from: 90300 + lc_monetary: + - type: String + version_from: 90300 + lc_numeric: + - type: String + version_from: 90300 + lc_time: + - type: String + version_from: 90300 + listen_addresses: + - type: String + version_from: 90300 + local_preload_libraries: + - type: String + version_from: 90300 + lock_timeout: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + unit: ms + lo_compat_privileges: + - type: Bool + version_from: 90300 + log_autovacuum_min_duration: + - type: Integer + version_from: 90300 + min_val: -1 + max_val: 2147483647 + unit: ms + log_checkpoints: + - type: Bool + version_from: 90300 + log_connections: + - type: Bool + version_from: 90300 + log_destination: + - type: String + version_from: 90300 + log_directory: + - type: String + version_from: 90300 + log_disconnections: + - type: Bool + version_from: 90300 + log_duration: + - type: Bool + version_from: 90300 + log_error_verbosity: + - type: Enum + version_from: 90300 + possible_values: + - terse + - default + - verbose + log_executor_stats: + - type: Bool + version_from: 90300 + log_file_mode: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 511 + log_filename: + - type: String + version_from: 90300 + logging_collector: + - type: Bool + version_from: 90300 + log_hostname: + - type: Bool + version_from: 90300 + logical_decoding_work_mem: + - type: Integer + version_from: 130000 + min_val: 64 + max_val: 2147483647 + unit: kB + log_line_prefix: + - type: String + version_from: 90300 + log_lock_waits: + - type: Bool + version_from: 90300 + log_min_duration_sample: + - type: Integer + version_from: 130000 + min_val: -1 + max_val: 2147483647 + unit: ms + log_min_duration_statement: + - type: Integer + version_from: 90300 + min_val: -1 + max_val: 2147483647 + unit: ms + log_min_error_statement: + - type: Enum + version_from: 90300 + possible_values: + - debug5 + - debug4 + - debug3 + - debug2 + - debug1 + - info + - notice + - warning + - error + - log + - fatal + - panic + log_min_messages: + - type: Enum + version_from: 90300 + possible_values: + - debug5 + - debug4 + - debug3 + - debug2 + - debug1 + - info + - notice + - warning + - error + - log + - fatal + - panic + log_parameter_max_length: + - type: Integer + version_from: 130000 + min_val: -1 + max_val: 1073741823 + unit: B + log_parameter_max_length_on_error: + - type: Integer + version_from: 130000 + min_val: -1 + max_val: 1073741823 + unit: B + log_parser_stats: + - type: Bool + version_from: 90300 + log_planner_stats: + - type: Bool + version_from: 90300 + log_recovery_conflict_waits: + - type: Bool + version_from: 140000 + log_replication_commands: + - type: Bool + version_from: 90500 + log_rotation_age: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 35791394 + unit: min + log_rotation_size: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2097151 + unit: kB + log_startup_progress_interval: + - type: Integer + version_from: 150000 + min_val: 0 + max_val: 2147483647 + unit: ms + log_statement: + - type: Enum + version_from: 90300 + possible_values: + - none + - ddl + - mod + - all + log_statement_sample_rate: + - type: Real + version_from: 130000 + min_val: 0 + max_val: 1 + log_statement_stats: + - type: Bool + version_from: 90300 + log_temp_files: + - type: Integer + version_from: 90300 + min_val: -1 + max_val: 2147483647 + unit: kB + log_timezone: + - type: String + version_from: 90300 + log_transaction_sample_rate: + - type: Real + version_from: 120000 + min_val: 0 + max_val: 1 + log_truncate_on_rotation: + - type: Bool + version_from: 90300 + maintenance_io_concurrency: + - type: Integer + version_from: 130000 + min_val: 0 + max_val: 1000 + maintenance_work_mem: + - type: Integer + version_from: 90300 + min_val: 1024 + max_val: 2147483647 + unit: kB + max_connections: + - type: Integer + version_from: 90300 + version_till: 90600 + min_val: 1 + max_val: 8388607 + - type: Integer + version_from: 90600 + min_val: 1 + max_val: 262143 + max_files_per_process: + - type: Integer + version_from: 90300 + version_till: 130000 + min_val: 25 + max_val: 2147483647 + - type: Integer + version_from: 130000 + min_val: 64 + max_val: 2147483647 + max_locks_per_transaction: + - type: Integer + version_from: 90300 + min_val: 10 + max_val: 2147483647 + max_logical_replication_workers: + - type: Integer + version_from: 100000 + min_val: 0 + max_val: 262143 + max_parallel_maintenance_workers: + - type: Integer + version_from: 110000 + min_val: 0 + max_val: 1024 + max_parallel_workers: + - type: Integer + version_from: 100000 + min_val: 0 + max_val: 1024 + max_parallel_workers_per_gather: + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 1024 + max_pred_locks_per_page: + - type: Integer + version_from: 100000 + min_val: 0 + max_val: 2147483647 + max_pred_locks_per_relation: + - type: Integer + version_from: 100000 + min_val: -2147483648 + max_val: 2147483647 + max_pred_locks_per_transaction: + - type: Integer + version_from: 90300 + min_val: 10 + max_val: 2147483647 + max_prepared_transactions: + - type: Integer + version_from: 90300 + version_till: 90600 + min_val: 0 + max_val: 8388607 + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 262143 + max_replication_slots: + - type: Integer + version_from: 90400 + version_till: 90600 + min_val: 0 + max_val: 8388607 + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 262143 + max_slot_wal_keep_size: + - type: Integer + version_from: 130000 + min_val: -1 + max_val: 2147483647 + unit: MB + max_stack_depth: + - type: Integer + version_from: 90300 + min_val: 100 + max_val: 2147483647 + unit: kB + max_standby_archive_delay: + - type: Integer + version_from: 90300 + min_val: -1 + max_val: 2147483647 + unit: ms + max_standby_streaming_delay: + - type: Integer + version_from: 90300 + min_val: -1 + max_val: 2147483647 + unit: ms + max_sync_workers_per_subscription: + - type: Integer + version_from: 100000 + min_val: 0 + max_val: 262143 + max_wal_senders: + - type: Integer + version_from: 90300 + version_till: 90600 + min_val: 0 + max_val: 8388607 + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 262143 + max_wal_size: + - type: Integer + version_from: 90500 + version_till: 100000 + min_val: 2 + max_val: 2147483647 + unit: 16MB + - type: Integer + version_from: 100000 + min_val: 2 + max_val: 2147483647 + unit: MB + max_worker_processes: + - type: Integer + version_from: 90400 + version_till: 90600 + min_val: 1 + max_val: 8388607 + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 262143 + min_dynamic_shared_memory: + - type: Integer + version_from: 140000 + min_val: 0 + max_val: 2147483647 + unit: MB + min_parallel_index_scan_size: + - type: Integer + version_from: 100000 + min_val: 0 + max_val: 715827882 + unit: 8kB + min_parallel_relation_size: + - type: Integer + version_from: 90600 + version_till: 100000 + min_val: 0 + max_val: 715827882 + unit: 8kB + min_parallel_table_scan_size: + - type: Integer + version_from: 100000 + min_val: 0 + max_val: 715827882 + unit: 8kB + min_wal_size: + - type: Integer + version_from: 90500 + version_till: 100000 + min_val: 2 + max_val: 2147483647 + unit: 16MB + - type: Integer + version_from: 100000 + min_val: 2 + max_val: 2147483647 + unit: MB + old_snapshot_threshold: + - type: Integer + version_from: 90600 + min_val: -1 + max_val: 86400 + unit: min + operator_precedence_warning: + - type: Bool + version_from: 90500 + version_till: 140000 + parallel_leader_participation: + - type: Bool + version_from: 110000 + parallel_setup_cost: + - type: Real + version_from: 90600 + min_val: 0 + max_val: 1.79769e+308 + parallel_tuple_cost: + - type: Real + version_from: 90600 + min_val: 0 + max_val: 1.79769e+308 + password_encryption: + - type: Bool + version_from: 90300 + version_till: 100000 + - type: Enum + version_from: 100000 + possible_values: + - md5 + - scram-sha-256 + plan_cache_mode: + - type: Enum + version_from: 120000 + possible_values: + - auto + - force_generic_plan + - force_custom_plan + port: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 65535 + post_auth_delay: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147 + unit: s + pre_auth_delay: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 60 + unit: s + quote_all_identifiers: + - type: Bool + version_from: 90300 + random_page_cost: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 1.79769e+308 + recovery_init_sync_method: + - type: Enum + version_from: 140000 + possible_values: + - fsync + - syncfs + recovery_prefetch: + - type: EnumBool + version_from: 150000 + possible_values: + - try + recursive_worktable_factor: + - type: Real + version_from: 150000 + min_val: 0.001 + max_val: 1000000.0 + remove_temp_files_after_crash: + - type: Bool + version_from: 140000 + replacement_sort_tuples: + - type: Integer + version_from: 90600 + version_till: 110000 + min_val: 0 + max_val: 2147483647 + restart_after_crash: + - type: Bool + version_from: 90300 + row_security: + - type: Bool + version_from: 90500 + search_path: + - type: String + version_from: 90300 + seq_page_cost: + - type: Real + version_from: 90300 + min_val: 0 + max_val: 1.79769e+308 + session_preload_libraries: + - type: String + version_from: 90400 + session_replication_role: + - type: Enum + version_from: 90300 + possible_values: + - origin + - replica + - local + shared_buffers: + - type: Integer + version_from: 90300 + min_val: 16 + max_val: 1073741823 + unit: 8kB + shared_memory_type: + - type: Enum + version_from: 120000 + possible_values: + - sysv + - mmap + shared_preload_libraries: + - type: String + version_from: 90300 + sql_inheritance: + - type: Bool + version_from: 90300 + version_till: 100000 + ssl: + - type: Bool + version_from: 90300 + ssl_ca_file: + - type: String + version_from: 90300 + ssl_cert_file: + - type: String + version_from: 90300 + ssl_ciphers: + - type: String + version_from: 90300 + ssl_crl_dir: + - type: String + version_from: 140000 + ssl_crl_file: + - type: String + version_from: 90300 + ssl_dh_params_file: + - type: String + version_from: 100000 + ssl_ecdh_curve: + - type: String + version_from: 90400 + ssl_key_file: + - type: String + version_from: 90300 + ssl_max_protocol_version: + - type: Enum + version_from: 120000 + possible_values: + - '' + - tlsv1 + - tlsv1.1 + - tlsv1.2 + - tlsv1.3 + ssl_min_protocol_version: + - type: Enum + version_from: 120000 + possible_values: + - tlsv1 + - tlsv1.1 + - tlsv1.2 + - tlsv1.3 + ssl_passphrase_command: + - type: String + version_from: 110000 + ssl_passphrase_command_supports_reload: + - type: Bool + version_from: 110000 + ssl_prefer_server_ciphers: + - type: Bool + version_from: 90400 + ssl_renegotiation_limit: + - type: Integer + version_from: 90300 + version_till: 90500 + min_val: 0 + max_val: 2147483647 + unit: kB + standard_conforming_strings: + - type: Bool + version_from: 90300 + statement_timeout: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + unit: ms + stats_fetch_consistency: + - type: Enum + version_from: 150000 + possible_values: + - none + - cache + - snapshot + stats_temp_directory: + - type: String + version_from: 90300 + version_till: 150000 + superuser_reserved_connections: + - type: Integer + version_from: 90300 + version_till: 90600 + min_val: 0 + max_val: 8388607 + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 262143 + synchronize_seqscans: + - type: Bool + version_from: 90300 + synchronous_commit: + - type: EnumBool + version_from: 90300 + version_till: 90600 + possible_values: + - local + - remote_write + - type: EnumBool + version_from: 90600 + possible_values: + - local + - remote_write + - remote_apply + synchronous_standby_names: + - type: String + version_from: 90300 + syslog_facility: + - type: Enum + version_from: 90300 + possible_values: + - local0 + - local1 + - local2 + - local3 + - local4 + - local5 + - local6 + - local7 + syslog_ident: + - type: String + version_from: 90300 + syslog_sequence_numbers: + - type: Bool + version_from: 90600 + syslog_split_messages: + - type: Bool + version_from: 90600 + tcp_keepalives_count: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + tcp_keepalives_idle: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + unit: s + tcp_keepalives_interval: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + unit: s + tcp_user_timeout: + - type: Integer + version_from: 120000 + min_val: 0 + max_val: 2147483647 + unit: ms + temp_buffers: + - type: Integer + version_from: 90300 + min_val: 100 + max_val: 1073741823 + unit: 8kB + temp_file_limit: + - type: Integer + version_from: 90300 + min_val: -1 + max_val: 2147483647 + unit: kB + temp_tablespaces: + - type: String + version_from: 90300 + TimeZone: + - type: String + version_from: 90300 + timezone_abbreviations: + - type: String + version_from: 90300 + trace_notify: + - type: Bool + version_from: 90300 + trace_recovery_messages: + - type: Enum + version_from: 90300 + possible_values: + - debug5 + - debug4 + - debug3 + - debug2 + - debug1 + - log + - notice + - warning + - error + trace_sort: + - type: Bool + version_from: 90300 + track_activities: + - type: Bool + version_from: 90300 + track_activity_query_size: + - type: Integer + version_from: 90300 + version_till: 110000 + min_val: 100 + max_val: 102400 + - type: Integer + version_from: 110000 + version_till: 130000 + min_val: 100 + max_val: 102400 + unit: B + - type: Integer + version_from: 130000 + min_val: 100 + max_val: 1048576 + unit: B + track_commit_timestamp: + - type: Bool + version_from: 90500 + track_counts: + - type: Bool + version_from: 90300 + track_functions: + - type: Enum + version_from: 90300 + possible_values: + - none + - pl + - all + track_io_timing: + - type: Bool + version_from: 90300 + track_wal_io_timing: + - type: Bool + version_from: 140000 + transaction_deferrable: + - type: Bool + version_from: 90300 + transaction_isolation: + - type: Enum + version_from: 90300 + possible_values: + - serializable + - repeatable read + - read committed + - read uncommitted + transaction_read_only: + - type: Bool + version_from: 90300 + transform_null_equals: + - type: Bool + version_from: 90300 + unix_socket_directories: + - type: String + version_from: 90300 + unix_socket_group: + - type: String + version_from: 90300 + unix_socket_permissions: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 511 + update_process_title: + - type: Bool + version_from: 90300 + vacuum_cleanup_index_scale_factor: + - type: Real + version_from: 110000 + version_till: 140000 + min_val: 0 + max_val: 10000000000.0 + vacuum_cost_delay: + - type: Integer + version_from: 90300 + version_till: 120000 + min_val: 0 + max_val: 100 + unit: ms + - type: Real + version_from: 120000 + min_val: 0 + max_val: 100 + unit: ms + vacuum_cost_limit: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 10000 + vacuum_cost_page_dirty: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 10000 + vacuum_cost_page_hit: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 10000 + vacuum_cost_page_miss: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 10000 + vacuum_defer_cleanup_age: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 1000000 + vacuum_failsafe_age: + - type: Integer + version_from: 140000 + min_val: 0 + max_val: 2100000000 + vacuum_freeze_min_age: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 1000000000 + vacuum_freeze_table_age: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2000000000 + vacuum_multixact_failsafe_age: + - type: Integer + version_from: 140000 + min_val: 0 + max_val: 2100000000 + vacuum_multixact_freeze_min_age: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 1000000000 + vacuum_multixact_freeze_table_age: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2000000000 + wal_buffers: + - type: Integer + version_from: 90300 + min_val: -1 + max_val: 262143 + unit: 8kB + wal_compression: + - type: Bool + version_from: 90500 + version_till: 150000 + - type: EnumBool + version_from: 150000 + possible_values: + - pglz + - lz4 + - zstd + wal_consistency_checking: + - type: String + version_from: 100000 + wal_decode_buffer_size: + - type: Integer + version_from: 150000 + min_val: 65536 + max_val: 1073741823 + unit: B + wal_init_zero: + - type: Bool + version_from: 120000 + wal_keep_segments: + - type: Integer + version_from: 90300 + version_till: 130000 + min_val: 0 + max_val: 2147483647 + wal_keep_size: + - type: Integer + version_from: 130000 + min_val: 0 + max_val: 2147483647 + unit: MB + wal_level: + - type: Enum + version_from: 90300 + version_till: 90400 + possible_values: + - minimal + - archive + - hot_standby + - type: Enum + version_from: 90400 + version_till: 90600 + possible_values: + - minimal + - archive + - hot_standby + - logical + - type: Enum + version_from: 90600 + possible_values: + - minimal + - replica + - logical + wal_log_hints: + - type: Bool + version_from: 90400 + wal_receiver_create_temp_slot: + - type: Bool + version_from: 130000 + wal_receiver_status_interval: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483 + unit: s + wal_receiver_timeout: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + unit: ms + wal_recycle: + - type: Bool + version_from: 120000 + wal_retrieve_retry_interval: + - type: Integer + version_from: 90500 + min_val: 1 + max_val: 2147483647 + unit: ms + wal_sender_timeout: + - type: Integer + version_from: 90300 + min_val: 0 + max_val: 2147483647 + unit: ms + wal_skip_threshold: + - type: Integer + version_from: 130000 + min_val: 0 + max_val: 2147483647 + unit: kB + wal_sync_method: + - type: Enum + version_from: 90300 + possible_values: + - fsync + - fdatasync + - open_sync + - open_datasync + wal_writer_delay: + - type: Integer + version_from: 90300 + min_val: 1 + max_val: 10000 + unit: ms + wal_writer_flush_after: + - type: Integer + version_from: 90600 + min_val: 0 + max_val: 2147483647 + unit: 8kB + work_mem: + - type: Integer + version_from: 90300 + min_val: 64 + max_val: 2147483647 + unit: kB + xmlbinary: + - type: Enum + version_from: 90300 + possible_values: + - base64 + - hex + xmloption: + - type: Enum + version_from: 90300 + possible_values: + - content + - document + zero_damaged_pages: + - type: Bool + version_from: 90300 +recovery_parameters: + archive_cleanup_command: + - type: String + version_from: 90300 + pause_at_recovery_target: + - type: Bool + version_from: 90300 + version_till: 90500 + primary_conninfo: + - type: String + version_from: 90300 + primary_slot_name: + - type: String + version_from: 90400 + promote_trigger_file: + - type: String + version_from: 120000 + recovery_end_command: + - type: String + version_from: 90300 + recovery_min_apply_delay: + - type: Integer + version_from: 90400 + min_val: 0 + max_val: 2147483647 + unit: ms + recovery_target: + - type: Enum + version_from: 90400 + possible_values: + - immediate + - '' + recovery_target_action: + - type: Enum + version_from: 90500 + possible_values: + - pause + - promote + - shutdown + recovery_target_inclusive: + - type: Bool + version_from: 90300 + recovery_target_lsn: + - type: String + version_from: 100000 + recovery_target_name: + - type: String + version_from: 90400 + recovery_target_time: + - type: String + version_from: 90300 + recovery_target_timeline: + - type: String + version_from: 90300 + recovery_target_xid: + - type: String + version_from: 90300 + restore_command: + - type: String + version_from: 90300 + standby_mode: + - type: Bool + version_from: 90300 + version_till: 120000 + trigger_file: + - type: String + version_from: 90300 + version_till: 120000 + diff --git a/patroni/postgresql/config.py b/patroni/postgresql/config.py index 76a3fb7b..224b48ef 100644 --- a/patroni/postgresql/config.py +++ b/patroni/postgresql/config.py @@ -412,7 +412,8 @@ class ConfigHandler(object): include = self._config.get('custom_conf') or self._postgresql_base_conf_name f.writeline("include '{0}'\n".format(ConfigWriter.escape(include))) for name, value in sorted((configuration).items()): - value = transform_postgresql_parameter_value(self._postgresql.major_version, name, value) + value = transform_postgresql_parameter_value(self._postgresql.major_version, name, value, + self._postgresql.available_gucs) if value is not None and\ (name != 'hba_file' or not self._postgresql.bootstrap.running_custom_bootstrap): f.write_param(name, value) @@ -534,7 +535,8 @@ class ConfigHandler(object): self._passfile_mtime = mtime(self._pgpass) value = self.format_dsn(value) else: - value = transform_recovery_parameter_value(self._postgresql.major_version, name, value) + value = transform_recovery_parameter_value(self._postgresql.major_version, name, value, + self._postgresql.available_gucs) if value is None: continue fd.write_param(name, value) diff --git a/patroni/postgresql/validator.py b/patroni/postgresql/validator.py index 30546444..7da0003a 100644 --- a/patroni/postgresql/validator.py +++ b/patroni/postgresql/validator.py @@ -1,9 +1,13 @@ import abc +from copy import deepcopy import logging +import os +import yaml -from typing import Any, MutableMapping, Optional, Tuple, Union +from typing import Any, Dict, Iterator, List, MutableMapping, Optional, Tuple, Type, Union -from ..collections import CaseInsensitiveDict +from ..collections import CaseInsensitiveDict, CaseInsensitiveSet +from ..exceptions import PatroniException from ..utils import parse_bool, parse_int, parse_real logger = logging.getLogger(__name__) @@ -11,10 +15,20 @@ logger = logging.getLogger(__name__) class _Transformable(abc.ABC): - def __init__(self, version_from: int, version_till: Optional[int]) -> None: + def __init__(self, version_from: int, version_till: Optional[int] = None) -> None: self.__version_from = version_from self.__version_till = version_till + @classmethod + def get_subclasses(cls) -> Iterator[Type['_Transformable']]: + """Recursively get all subclasses of :class:`_Transformable`. + + :yields: each subclass of :class:`_Transformable`. + """ + for subclass in cls.__subclasses__(): + yield from subclass.get_subclasses() + yield subclass + @property def version_from(self) -> int: return self.__version_from @@ -43,8 +57,8 @@ class Bool(_Transformable): class Number(_Transformable): - def __init__(self, version_from: int, version_till: Optional[int], - min_val: Union[int, float], max_val: Union[int, float], unit: Optional[str]) -> None: + def __init__(self, *, version_from: int, version_till: Optional[int] = None, min_val: Union[int, float], + max_val: Union[int, float], unit: Optional[str] = None) -> None: super(Number, self).__init__(version_from, version_till) self.__min_val = min_val self.__max_val = max_val @@ -99,7 +113,8 @@ class Real(Number): class Enum(_Transformable): - def __init__(self, version_from: int, version_till: Optional[int], possible_values: Tuple[str, ...]) -> None: + def __init__(self, *, version_from: int, version_till: Optional[int] = None, + possible_values: Tuple[str, ...]) -> None: super(Enum, self).__init__(version_from, version_till) self.__possible_values = possible_values @@ -128,456 +143,366 @@ class String(_Transformable): # Format: -# key - parameter name -# value - tuple or multiple tuples if something was changing in GUC across postgres versions -parameters = CaseInsensitiveDict({ - 'allow_in_place_tablespaces': Bool(150000, None), - 'allow_system_table_mods': Bool(90300, None), - 'application_name': String(90300, None), - 'archive_command': String(90300, None), - 'archive_library': String(150000, None), - 'archive_mode': ( - Bool(90300, 90500), - EnumBool(90500, None, ('always',)) - ), - 'archive_timeout': Integer(90300, None, 0, 1073741823, 's'), - 'array_nulls': Bool(90300, None), - 'authentication_timeout': Integer(90300, None, 1, 600, 's'), - 'autovacuum': Bool(90300, None), - 'autovacuum_analyze_scale_factor': Real(90300, None, 0, 100, None), - 'autovacuum_analyze_threshold': Integer(90300, None, 0, 2147483647, None), - 'autovacuum_freeze_max_age': Integer(90300, None, 100000, 2000000000, None), - 'autovacuum_max_workers': ( - Integer(90300, 90600, 1, 8388607, None), - Integer(90600, None, 1, 262143, None) - ), - 'autovacuum_multixact_freeze_max_age': Integer(90300, None, 10000, 2000000000, None), - 'autovacuum_naptime': Integer(90300, None, 1, 2147483, 's'), - 'autovacuum_vacuum_cost_delay': ( - Integer(90300, 120000, -1, 100, 'ms'), - Real(120000, None, -1, 100, 'ms') - ), - 'autovacuum_vacuum_cost_limit': Integer(90300, None, -1, 10000, None), - 'autovacuum_vacuum_insert_scale_factor': Real(130000, None, 0, 100, None), - 'autovacuum_vacuum_insert_threshold': Integer(130000, None, -1, 2147483647, None), - 'autovacuum_vacuum_scale_factor': Real(90300, None, 0, 100, None), - 'autovacuum_vacuum_threshold': Integer(90300, None, 0, 2147483647, None), - 'autovacuum_work_mem': Integer(90400, None, -1, 2147483647, 'kB'), - 'backend_flush_after': Integer(90600, None, 0, 256, '8kB'), - 'backslash_quote': EnumBool(90300, None, ('safe_encoding',)), - 'backtrace_functions': String(130000, None), - 'bgwriter_delay': Integer(90300, None, 10, 10000, 'ms'), - 'bgwriter_flush_after': Integer(90600, None, 0, 256, '8kB'), - 'bgwriter_lru_maxpages': ( - Integer(90300, 100000, 0, 1000, None), - Integer(100000, None, 0, 1073741823, None) - ), - 'bgwriter_lru_multiplier': Real(90300, None, 0, 10, None), - 'bonjour': Bool(90300, None), - 'bonjour_name': String(90300, None), - 'bytea_output': Enum(90300, None, ('escape', 'hex')), - 'check_function_bodies': Bool(90300, None), - 'checkpoint_completion_target': Real(90300, None, 0, 1, None), - 'checkpoint_flush_after': Integer(90600, None, 0, 256, '8kB'), - 'checkpoint_segments': Integer(90300, 90500, 1, 2147483647, None), - 'checkpoint_timeout': ( - Integer(90300, 90600, 30, 3600, 's'), - Integer(90600, None, 30, 86400, 's') - ), - 'checkpoint_warning': Integer(90300, None, 0, 2147483647, 's'), - 'client_connection_check_interval': Integer(140000, None, 0, 2147483647, 'ms'), - 'client_encoding': String(90300, None), - 'client_min_messages': Enum(90300, None, ('debug5', 'debug4', 'debug3', 'debug2', - 'debug1', 'log', 'notice', 'warning', 'error')), - 'cluster_name': String(90500, None), - 'commit_delay': Integer(90300, None, 0, 100000, None), - 'commit_siblings': Integer(90300, None, 0, 1000, None), - 'compute_query_id': ( - EnumBool(140000, 150000, ('auto',)), - EnumBool(150000, None, ('auto', 'regress')) - ), - 'config_file': String(90300, None), - 'constraint_exclusion': EnumBool(90300, None, ('partition',)), - 'cpu_index_tuple_cost': Real(90300, None, 0, 1.79769e+308, None), - 'cpu_operator_cost': Real(90300, None, 0, 1.79769e+308, None), - 'cpu_tuple_cost': Real(90300, None, 0, 1.79769e+308, None), - 'cursor_tuple_fraction': Real(90300, None, 0, 1, None), - 'data_directory': String(90300, None), - 'data_sync_retry': Bool(90400, None), - 'DateStyle': String(90300, None), - 'db_user_namespace': Bool(90300, None), - 'deadlock_timeout': Integer(90300, None, 1, 2147483647, 'ms'), - 'debug_discard_caches': Integer(150000, None, 0, 0, None), - 'debug_pretty_print': Bool(90300, None), - 'debug_print_parse': Bool(90300, None), - 'debug_print_plan': Bool(90300, None), - 'debug_print_rewritten': Bool(90300, None), - 'default_statistics_target': Integer(90300, None, 1, 10000, None), - 'default_table_access_method': String(120000, None), - 'default_tablespace': String(90300, None), - 'default_text_search_config': String(90300, None), - 'default_toast_compression': Enum(140000, None, ('pglz', 'lz4')), - 'default_transaction_deferrable': Bool(90300, None), - 'default_transaction_isolation': Enum(90300, None, ('serializable', 'repeatable read', - 'read committed', 'read uncommitted')), - 'default_transaction_read_only': Bool(90300, None), - 'default_with_oids': Bool(90300, 120000), - 'dynamic_library_path': String(90300, None), - 'dynamic_shared_memory_type': ( - Enum(90400, 120000, ('posix', 'sysv', 'mmap', 'none')), - Enum(120000, None, ('posix', 'sysv', 'mmap')) - ), - 'effective_cache_size': Integer(90300, None, 1, 2147483647, '8kB'), - 'effective_io_concurrency': Integer(90300, None, 0, 1000, None), - 'enable_async_append': Bool(140000, None), - 'enable_bitmapscan': Bool(90300, None), - 'enable_gathermerge': Bool(100000, None), - 'enable_hashagg': Bool(90300, None), - 'enable_hashjoin': Bool(90300, None), - 'enable_incremental_sort': Bool(130000, None), - 'enable_indexonlyscan': Bool(90300, None), - 'enable_indexscan': Bool(90300, None), - 'enable_material': Bool(90300, None), - 'enable_memoize': Bool(150000, None), - 'enable_mergejoin': Bool(90300, None), - 'enable_nestloop': Bool(90300, None), - 'enable_parallel_append': Bool(110000, None), - 'enable_parallel_hash': Bool(110000, None), - 'enable_partition_pruning': Bool(110000, None), - 'enable_partitionwise_aggregate': Bool(110000, None), - 'enable_partitionwise_join': Bool(110000, None), - 'enable_seqscan': Bool(90300, None), - 'enable_sort': Bool(90300, None), - 'enable_tidscan': Bool(90300, None), - 'escape_string_warning': Bool(90300, None), - 'event_source': String(90300, None), - 'exit_on_error': Bool(90300, None), - 'extension_destdir': String(140000, None), - 'external_pid_file': String(90300, None), - 'extra_float_digits': Integer(90300, None, -15, 3, None), - 'force_parallel_mode': EnumBool(90600, None, ('regress',)), - 'from_collapse_limit': Integer(90300, None, 1, 2147483647, None), - 'fsync': Bool(90300, None), - 'full_page_writes': Bool(90300, None), - 'geqo': Bool(90300, None), - 'geqo_effort': Integer(90300, None, 1, 10, None), - 'geqo_generations': Integer(90300, None, 0, 2147483647, None), - 'geqo_pool_size': Integer(90300, None, 0, 2147483647, None), - 'geqo_seed': Real(90300, None, 0, 1, None), - 'geqo_selection_bias': Real(90300, None, 1.5, 2, None), - 'geqo_threshold': Integer(90300, None, 2, 2147483647, None), - 'gin_fuzzy_search_limit': Integer(90300, None, 0, 2147483647, None), - 'gin_pending_list_limit': Integer(90500, None, 64, 2147483647, 'kB'), - 'hash_mem_multiplier': Real(130000, None, 1, 1000, None), - 'hba_file': String(90300, None), - 'hot_standby': Bool(90300, None), - 'hot_standby_feedback': Bool(90300, None), - 'huge_pages': EnumBool(90400, None, ('try',)), - 'huge_page_size': Integer(140000, None, 0, 2147483647, 'kB'), - 'ident_file': String(90300, None), - 'idle_in_transaction_session_timeout': Integer(90600, None, 0, 2147483647, 'ms'), - 'idle_session_timeout': Integer(140000, None, 0, 2147483647, 'ms'), - 'ignore_checksum_failure': Bool(90300, None), - 'ignore_invalid_pages': Bool(130000, None), - 'ignore_system_indexes': Bool(90300, None), - 'IntervalStyle': Enum(90300, None, ('postgres', 'postgres_verbose', 'sql_standard', 'iso_8601')), - 'jit': Bool(110000, None), - 'jit_above_cost': Real(110000, None, -1, 1.79769e+308, None), - 'jit_debugging_support': Bool(110000, None), - 'jit_dump_bitcode': Bool(110000, None), - 'jit_expressions': Bool(110000, None), - 'jit_inline_above_cost': Real(110000, None, -1, 1.79769e+308, None), - 'jit_optimize_above_cost': Real(110000, None, -1, 1.79769e+308, None), - 'jit_profiling_support': Bool(110000, None), - 'jit_provider': String(110000, None), - 'jit_tuple_deforming': Bool(110000, None), - 'join_collapse_limit': Integer(90300, None, 1, 2147483647, None), - 'krb_caseins_users': Bool(90300, None), - 'krb_server_keyfile': String(90300, None), - 'krb_srvname': String(90300, 90400), - 'lc_messages': String(90300, None), - 'lc_monetary': String(90300, None), - 'lc_numeric': String(90300, None), - 'lc_time': String(90300, None), - 'listen_addresses': String(90300, None), - 'local_preload_libraries': String(90300, None), - 'lock_timeout': Integer(90300, None, 0, 2147483647, 'ms'), - 'lo_compat_privileges': Bool(90300, None), - 'log_autovacuum_min_duration': Integer(90300, None, -1, 2147483647, 'ms'), - 'log_checkpoints': Bool(90300, None), - 'log_connections': Bool(90300, None), - 'log_destination': String(90300, None), - 'log_directory': String(90300, None), - 'log_disconnections': Bool(90300, None), - 'log_duration': Bool(90300, None), - 'log_error_verbosity': Enum(90300, None, ('terse', 'default', 'verbose')), - 'log_executor_stats': Bool(90300, None), - 'log_file_mode': Integer(90300, None, 0, 511, None), - 'log_filename': String(90300, None), - 'logging_collector': Bool(90300, None), - 'log_hostname': Bool(90300, None), - 'logical_decoding_work_mem': Integer(130000, None, 64, 2147483647, 'kB'), - 'log_line_prefix': String(90300, None), - 'log_lock_waits': Bool(90300, None), - 'log_min_duration_sample': Integer(130000, None, -1, 2147483647, 'ms'), - 'log_min_duration_statement': Integer(90300, None, -1, 2147483647, 'ms'), - 'log_min_error_statement': Enum(90300, None, ('debug5', 'debug4', 'debug3', 'debug2', 'debug1', 'info', - 'notice', 'warning', 'error', 'log', 'fatal', 'panic')), - 'log_min_messages': Enum(90300, None, ('debug5', 'debug4', 'debug3', 'debug2', 'debug1', 'info', - 'notice', 'warning', 'error', 'log', 'fatal', 'panic')), - 'log_parameter_max_length': Integer(130000, None, -1, 1073741823, 'B'), - 'log_parameter_max_length_on_error': Integer(130000, None, -1, 1073741823, 'B'), - 'log_parser_stats': Bool(90300, None), - 'log_planner_stats': Bool(90300, None), - 'log_recovery_conflict_waits': Bool(140000, None), - 'log_replication_commands': Bool(90500, None), - 'log_rotation_age': Integer(90300, None, 0, 35791394, 'min'), - 'log_rotation_size': Integer(90300, None, 0, 2097151, 'kB'), - 'log_startup_progress_interval': Integer(150000, None, 0, 2147483647, 'ms'), - 'log_statement': Enum(90300, None, ('none', 'ddl', 'mod', 'all')), - 'log_statement_sample_rate': Real(130000, None, 0, 1, None), - 'log_statement_stats': Bool(90300, None), - 'log_temp_files': Integer(90300, None, -1, 2147483647, 'kB'), - 'log_timezone': String(90300, None), - 'log_transaction_sample_rate': Real(120000, None, 0, 1, None), - 'log_truncate_on_rotation': Bool(90300, None), - 'maintenance_io_concurrency': Integer(130000, None, 0, 1000, None), - 'maintenance_work_mem': Integer(90300, None, 1024, 2147483647, 'kB'), - 'max_connections': ( - Integer(90300, 90600, 1, 8388607, None), - Integer(90600, None, 1, 262143, None) - ), - 'max_files_per_process': ( - Integer(90300, 130000, 25, 2147483647, None), - Integer(130000, None, 64, 2147483647, None) - ), - 'max_locks_per_transaction': Integer(90300, None, 10, 2147483647, None), - 'max_logical_replication_workers': Integer(100000, None, 0, 262143, None), - 'max_parallel_maintenance_workers': Integer(110000, None, 0, 1024, None), - 'max_parallel_workers': Integer(100000, None, 0, 1024, None), - 'max_parallel_workers_per_gather': Integer(90600, None, 0, 1024, None), - 'max_pred_locks_per_page': Integer(100000, None, 0, 2147483647, None), - 'max_pred_locks_per_relation': Integer(100000, None, -2147483648, 2147483647, None), - 'max_pred_locks_per_transaction': Integer(90300, None, 10, 2147483647, None), - 'max_prepared_transactions': ( - Integer(90300, 90600, 0, 8388607, None), - Integer(90600, None, 0, 262143, None) - ), - 'max_replication_slots': ( - Integer(90400, 90600, 0, 8388607, None), - Integer(90600, None, 0, 262143, None) - ), - 'max_slot_wal_keep_size': Integer(130000, None, -1, 2147483647, 'MB'), - 'max_stack_depth': Integer(90300, None, 100, 2147483647, 'kB'), - 'max_standby_archive_delay': Integer(90300, None, -1, 2147483647, 'ms'), - 'max_standby_streaming_delay': Integer(90300, None, -1, 2147483647, 'ms'), - 'max_sync_workers_per_subscription': Integer(100000, None, 0, 262143, None), - 'max_wal_senders': ( - Integer(90300, 90600, 0, 8388607, None), - Integer(90600, None, 0, 262143, None) - ), - 'max_wal_size': ( - Integer(90500, 100000, 2, 2147483647, '16MB'), - Integer(100000, None, 2, 2147483647, 'MB') - ), - 'max_worker_processes': ( - Integer(90400, 90600, 1, 8388607, None), - Integer(90600, None, 0, 262143, None) - ), - 'min_dynamic_shared_memory': Integer(140000, None, 0, 2147483647, 'MB'), - 'min_parallel_index_scan_size': Integer(100000, None, 0, 715827882, '8kB'), - 'min_parallel_relation_size': Integer(90600, 100000, 0, 715827882, '8kB'), - 'min_parallel_table_scan_size': Integer(100000, None, 0, 715827882, '8kB'), - 'min_wal_size': ( - Integer(90500, 100000, 2, 2147483647, '16MB'), - Integer(100000, None, 2, 2147483647, 'MB') - ), - 'old_snapshot_threshold': Integer(90600, None, -1, 86400, 'min'), - 'operator_precedence_warning': Bool(90500, 140000), - 'parallel_leader_participation': Bool(110000, None), - 'parallel_setup_cost': Real(90600, None, 0, 1.79769e+308, None), - 'parallel_tuple_cost': Real(90600, None, 0, 1.79769e+308, None), - 'password_encryption': ( - Bool(90300, 100000), - Enum(100000, None, ('md5', 'scram-sha-256')) - ), - 'plan_cache_mode': Enum(120000, None, ('auto', 'force_generic_plan', 'force_custom_plan')), - 'port': Integer(90300, None, 1, 65535, None), - 'post_auth_delay': Integer(90300, None, 0, 2147, 's'), - 'pre_auth_delay': Integer(90300, None, 0, 60, 's'), - 'quote_all_identifiers': Bool(90300, None), - 'random_page_cost': Real(90300, None, 0, 1.79769e+308, None), - 'recovery_init_sync_method': Enum(140000, None, ('fsync', 'syncfs')), - 'recovery_prefetch': EnumBool(150000, None, ('try',)), - 'recursive_worktable_factor': Real(150000, None, 0.001, 1e+06, None), - 'remove_temp_files_after_crash': Bool(140000, None), - 'replacement_sort_tuples': Integer(90600, 110000, 0, 2147483647, None), - 'restart_after_crash': Bool(90300, None), - 'row_security': Bool(90500, None), - 'search_path': String(90300, None), - 'seq_page_cost': Real(90300, None, 0, 1.79769e+308, None), - 'session_preload_libraries': String(90400, None), - 'session_replication_role': Enum(90300, None, ('origin', 'replica', 'local')), - 'shared_buffers': Integer(90300, None, 16, 1073741823, '8kB'), - 'shared_memory_type': Enum(120000, None, ('sysv', 'mmap')), - 'shared_preload_libraries': String(90300, None), - 'sql_inheritance': Bool(90300, 100000), - 'ssl': Bool(90300, None), - 'ssl_ca_file': String(90300, None), - 'ssl_cert_file': String(90300, None), - 'ssl_ciphers': String(90300, None), - 'ssl_crl_dir': String(140000, None), - 'ssl_crl_file': String(90300, None), - 'ssl_dh_params_file': String(100000, None), - 'ssl_ecdh_curve': String(90400, None), - 'ssl_key_file': String(90300, None), - 'ssl_max_protocol_version': Enum(120000, None, ('', 'tlsv1', 'tlsv1.1', 'tlsv1.2', 'tlsv1.3')), - 'ssl_min_protocol_version': Enum(120000, None, ('tlsv1', 'tlsv1.1', 'tlsv1.2', 'tlsv1.3')), - 'ssl_passphrase_command': String(110000, None), - 'ssl_passphrase_command_supports_reload': Bool(110000, None), - 'ssl_prefer_server_ciphers': Bool(90400, None), - 'ssl_renegotiation_limit': Integer(90300, 90500, 0, 2147483647, 'kB'), - 'standard_conforming_strings': Bool(90300, None), - 'statement_timeout': Integer(90300, None, 0, 2147483647, 'ms'), - 'stats_fetch_consistency': Enum(150000, None, ('none', 'cache', 'snapshot')), - 'stats_temp_directory': String(90300, 150000), - 'superuser_reserved_connections': ( - Integer(90300, 90600, 0, 8388607, None), - Integer(90600, None, 0, 262143, None) - ), - 'synchronize_seqscans': Bool(90300, None), - 'synchronous_commit': ( - EnumBool(90300, 90600, ('local', 'remote_write')), - EnumBool(90600, None, ('local', 'remote_write', 'remote_apply')) - ), - 'synchronous_standby_names': String(90300, None), - 'syslog_facility': Enum(90300, None, ('local0', 'local1', 'local2', 'local3', - 'local4', 'local5', 'local6', 'local7')), - 'syslog_ident': String(90300, None), - 'syslog_sequence_numbers': Bool(90600, None), - 'syslog_split_messages': Bool(90600, None), - 'tcp_keepalives_count': Integer(90300, None, 0, 2147483647, None), - 'tcp_keepalives_idle': Integer(90300, None, 0, 2147483647, 's'), - 'tcp_keepalives_interval': Integer(90300, None, 0, 2147483647, 's'), - 'tcp_user_timeout': Integer(120000, None, 0, 2147483647, 'ms'), - 'temp_buffers': Integer(90300, None, 100, 1073741823, '8kB'), - 'temp_file_limit': Integer(90300, None, -1, 2147483647, 'kB'), - 'temp_tablespaces': String(90300, None), - 'TimeZone': String(90300, None), - 'timezone_abbreviations': String(90300, None), - 'trace_notify': Bool(90300, None), - 'trace_recovery_messages': Enum(90300, None, ('debug5', 'debug4', 'debug3', 'debug2', - 'debug1', 'log', 'notice', 'warning', 'error')), - 'trace_sort': Bool(90300, None), - 'track_activities': Bool(90300, None), - 'track_activity_query_size': ( - Integer(90300, 110000, 100, 102400, None), - Integer(110000, 130000, 100, 102400, 'B'), - Integer(130000, None, 100, 1048576, 'B') - ), - 'track_commit_timestamp': Bool(90500, None), - 'track_counts': Bool(90300, None), - 'track_functions': Enum(90300, None, ('none', 'pl', 'all')), - 'track_io_timing': Bool(90300, None), - 'track_wal_io_timing': Bool(140000, None), - 'transaction_deferrable': Bool(90300, None), - 'transaction_isolation': Enum(90300, None, ('serializable', 'repeatable read', - 'read committed', 'read uncommitted')), - 'transaction_read_only': Bool(90300, None), - 'transform_null_equals': Bool(90300, None), - 'unix_socket_directories': String(90300, None), - 'unix_socket_group': String(90300, None), - 'unix_socket_permissions': Integer(90300, None, 0, 511, None), - 'update_process_title': Bool(90300, None), - 'vacuum_cleanup_index_scale_factor': Real(110000, 140000, 0, 1e+10, None), - 'vacuum_cost_delay': ( - Integer(90300, 120000, 0, 100, 'ms'), - Real(120000, None, 0, 100, 'ms') - ), - 'vacuum_cost_limit': Integer(90300, None, 1, 10000, None), - 'vacuum_cost_page_dirty': Integer(90300, None, 0, 10000, None), - 'vacuum_cost_page_hit': Integer(90300, None, 0, 10000, None), - 'vacuum_cost_page_miss': Integer(90300, None, 0, 10000, None), - 'vacuum_defer_cleanup_age': Integer(90300, None, 0, 1000000, None), - 'vacuum_failsafe_age': Integer(140000, None, 0, 2100000000, None), - 'vacuum_freeze_min_age': Integer(90300, None, 0, 1000000000, None), - 'vacuum_freeze_table_age': Integer(90300, None, 0, 2000000000, None), - 'vacuum_multixact_failsafe_age': Integer(140000, None, 0, 2100000000, None), - 'vacuum_multixact_freeze_min_age': Integer(90300, None, 0, 1000000000, None), - 'vacuum_multixact_freeze_table_age': Integer(90300, None, 0, 2000000000, None), - 'wal_buffers': Integer(90300, None, -1, 262143, '8kB'), - 'wal_compression': ( - Bool(90500, 150000), - EnumBool(150000, None, ('pglz', 'lz4', 'zstd')) - ), - 'wal_consistency_checking': String(100000, None), - 'wal_decode_buffer_size': Integer(150000, None, 65536, 1073741823, 'B'), - 'wal_init_zero': Bool(120000, None), - 'wal_keep_segments': Integer(90300, 130000, 0, 2147483647, None), - 'wal_keep_size': Integer(130000, None, 0, 2147483647, 'MB'), - 'wal_level': ( - Enum(90300, 90400, ('minimal', 'archive', 'hot_standby')), - Enum(90400, 90600, ('minimal', 'archive', 'hot_standby', 'logical')), - Enum(90600, None, ('minimal', 'replica', 'logical')) - ), - 'wal_log_hints': Bool(90400, None), - 'wal_receiver_create_temp_slot': Bool(130000, None), - 'wal_receiver_status_interval': Integer(90300, None, 0, 2147483, 's'), - 'wal_receiver_timeout': Integer(90300, None, 0, 2147483647, 'ms'), - 'wal_recycle': Bool(120000, None), - 'wal_retrieve_retry_interval': Integer(90500, None, 1, 2147483647, 'ms'), - 'wal_sender_timeout': Integer(90300, None, 0, 2147483647, 'ms'), - 'wal_skip_threshold': Integer(130000, None, 0, 2147483647, 'kB'), - 'wal_sync_method': Enum(90300, None, ('fsync', 'fdatasync', 'open_sync', 'open_datasync')), - 'wal_writer_delay': Integer(90300, None, 1, 10000, 'ms'), - 'wal_writer_flush_after': Integer(90600, None, 0, 2147483647, '8kB'), - 'work_mem': Integer(90300, None, 64, 2147483647, 'kB'), - 'xmlbinary': Enum(90300, None, ('base64', 'hex')), - 'xmloption': Enum(90300, None, ('content', 'document')), - 'zero_damaged_pages': Bool(90300, None) -}) +# key - parameter name +# value - variable length tuple of `_Transformable` objects. Each object in the tuple represents a different +# validation of the GUC across postgres versions. If a GUC validation has never changed over time, then it will +# have a single object in the tuple. For example, `password_encryption` used to be a boolean GUC up to Postgres +# 10, at which point it started being an enum. In that case the value of `password_encryption` would be a tuple +# of 2 `_Transformable` objects (`Bool` and `Enum`, respectively), each one reprensenting a different +# validation rule. +parameters = CaseInsensitiveDict() +recovery_parameters = CaseInsensitiveDict() -recovery_parameters = CaseInsensitiveDict({ - 'archive_cleanup_command': String(90300, None), - 'pause_at_recovery_target': Bool(90300, 90500), - 'primary_conninfo': String(90300, None), - 'primary_slot_name': String(90400, None), - 'promote_trigger_file': String(120000, None), - 'recovery_end_command': String(90300, None), - 'recovery_min_apply_delay': Integer(90400, None, 0, 2147483647, 'ms'), - 'recovery_target': Enum(90400, None, ('immediate', '')), - 'recovery_target_action': Enum(90500, None, ('pause', 'promote', 'shutdown')), - 'recovery_target_inclusive': Bool(90300, None), - 'recovery_target_lsn': String(100000, None), - 'recovery_target_name': String(90400, None), - 'recovery_target_time': String(90300, None), - 'recovery_target_timeline': String(90300, None), - 'recovery_target_xid': String(90300, None), - 'restore_command': String(90300, None), - 'standby_mode': Bool(90300, 120000), - 'trigger_file': String(90300, 120000) -}) +class ValidatorFactoryNoType(PatroniException): + """Raised when a validator spec misses a type.""" -def _transform_parameter_value(validators: MutableMapping[str, Union[_Transformable, Tuple[_Transformable, ...]]], - version: int, name: str, value: Any) -> Optional[Any]: - name_validators = validators.get(name) - if name_validators: - for validator in (name_validators if isinstance(name_validators, tuple) else (name_validators,)): +class ValidatorFactoryInvalidType(PatroniException): + """Raised when a validator spec contains an invalid type.""" + + +class ValidatorFactoryInvalidSpec(PatroniException): + """Raised when a validator spec contains an invalid set of attributes.""" + + +class ValidatorFactory: + """Factory class used to build Patroni validator objects based on the given specs.""" + + TYPES: Dict[str, Type[_Transformable]] = {cls.__name__: cls for cls in _Transformable.get_subclasses()} + + def __new__(cls, validator: Dict[str, Any]) -> _Transformable: + """Parse a given Postgres GUC *validator* into the corresponding Patroni validator object. + + :param validator: a validator spec for a given parameter. It usually comes from a parsed YAML file. + + :returns: the Patroni validator object that corresponds to the specification found in *validator*. + + :raises :class:`ValidatorFactoryNoType`: if *validator* contains no ``type`` key. + :raises :class:`ValidatorFactoryInvalidType`: if ``type`` key from *validator* contains an invalid value. + :raises :class:`ValidatorFactoryInvalidSpec`: if *validator* contains an invalid set of attributes for the + given ``type``. + + :Example: + + If a given validator was defined as follows in the YAML file: + + ```yaml + - type: String + version_from: 90300 + version_till: null + ``` + + Then this method would receive *validator* as: + + ```python + { + 'type': 'String', + 'version_from': 90300, + 'version_till': None + } + ``` + + And this method would return a :class:`String`: + + ```python + String(90300, None) + ``` + """ + validator = deepcopy(validator) + try: + type_ = validator.pop('type') + except KeyError as exc: + raise ValidatorFactoryNoType('Validator contains no type.') from exc + + if type_ not in cls.TYPES: + raise ValidatorFactoryInvalidType(f'Unexpected validator type: `{type_}`.') + + for key, value in validator.items(): + # :func:`_transform_parameter_value` expects :class:`tuple` instead of :class:`list` + if isinstance(value, list): + tmp_value: List[Any] = value + validator[key] = tuple(tmp_value) + + try: + return cls.TYPES[type_](**validator) + except Exception as exc: + raise ValidatorFactoryInvalidSpec( + f'Failed to parse `{type_}` validator (`{validator}`): `{str(exc)}`.') from exc + + +def _get_postgres_guc_validators(config: Dict[str, Any], parameter: str) -> Tuple[_Transformable, ...]: + """Get all validators of *parameter* from *config*. + + Loop over all validators specs of *parameter* and return them parsed as Patroni validators. + + :param config: Python object corresponding to an YAML file, with values of either ``parameters`` or + ``recovery_parameters`` key. + :param parameter: name of the parameter found under *config* which validators should be parsed and returned. + + :rtype: yields any exception that is faced while parsing a validator spec into a Patroni validator object. + """ + validators: List[_Transformable] = [] + for validator_spec in config.get(parameter, []): + try: + validator = ValidatorFactory(validator_spec) + validators.append(validator) + except (ValidatorFactoryNoType, ValidatorFactoryInvalidType, ValidatorFactoryInvalidSpec) as exc: + logger.warning('Faced an issue while parsing a validator for parameter `%s`: `%r`', parameter, exc) + + return tuple(validators) + + +class InvalidGucValidatorsFile(PatroniException): + """Raised when reading or parsing of a YAML file faces an issue.""" + + +def _read_postgres_gucs_validators_file(file: str) -> Dict[str, Any]: + """Read an YAML file and return the corresponding Python object. + + :param file: path to the file to be read. It is expected to be encoded with ``UTF-8``, and to be a YAML document. + + :returns: the YAML content parsed into a Python object. If any issue is faced while reading/parsing the file, then + return ``None``. + + :raises :class:`InvalidGucValidatorsFile`: if faces an issue while reading or parsing *file*. + """ + try: + with open(file, encoding='UTF-8') as stream: + return yaml.safe_load(stream) + except Exception as exc: + raise InvalidGucValidatorsFile( + f'Unexpected issue while reading parameters file `{file}`: `{str(exc)}`.') from exc + + +def _load_postgres_gucs_validators() -> None: + """Load all Postgres GUC validators from YAML files. + + Recursively walk through ``available_parameters`` directory and load validators of each found YAML file into + ``parameters`` and/or ``recovery_parameters`` variables. + + Walk through directories in top-down fashion and for each of them: + * Sort files by name; + * Load validators from YAML files that were found. + + Any problem faced while reading or parsing files will be logged as a ``WARNING`` by the child function, and the + corresponding file or validator will be ignored. + + By default Patroni only ships the file ``0_postgres.yml``, which contains Community Postgres GUCs validators, but + that behavior can be extended. For example: if a vendor wants to add GUC validators to Patroni for covering a custom + Postgres build, then they can create their custom YAML files under ``available_parameters`` directory. + + Each YAML file may contain either or both of these root attributes, here called sections: + * ``parameters``: general GUCs that would be written to ``postgresql.conf``; + * ``recovery_parameters``: recovery related GUCs that would be written to ``recovery.conf`` (Patroni later + writes them to ``postgresql.conf`` if running PG 12 and above). + + Then, each of these sections, if specified, may contain one or more attributes with the following structure: + * key: the name of a GUC; + * value: a list of validators. Each item in the list must contain a ``type`` attribute, which must be one among: + * ``Bool``; or + * ``Integer``; or + * ``Real``; or + * ``Enum``; or + * ``EnumBool``; or + * ``String``. + + Besides the ``type`` attribute, it should also contain all the required attributes as per the corresponding + class in this module. + + .. seealso:: + * :class:`Bool`; + * :class:`Integer`; + * :class:`Real`; + * :class:`Enum`; + * :class:`EnumBool`; + * :class:`String`. + + :Example: + + This is a sample content for an YAML file based on Postgres GUCs, showing each of the supported types and + sections: + + ```yaml + parameters: + archive_command: + - type: String + version_from: 90300 + version_till: null + archive_mode: + - type: Bool + version_from: 90300 + version_till: 90500 + - type: EnumBool + version_from: 90500 + version_till: null + possible_values: + - always + archive_timeout: + - type: Integer + version_from: 90300 + version_till: null + min_val: 0 + max_val: 1073741823 + unit: s + autovacuum_vacuum_cost_delay: + - type: Integer + version_from: 90300 + version_till: 120000 + min_val: -1 + max_val: 100 + unit: ms + - type: Real + version_from: 120000 + version_till: null + min_val: -1 + max_val: 100 + unit: ms + client_min_messages: + - type: Enum + version_from: 90300 + version_till: null + possible_values: + - debug5 + - debug4 + - debug3 + - debug2 + - debug1 + - log + - notice + - warning + - error + recovery_parameters: + archive_cleanup_command: + - type: String + version_from: 90300 + version_till: null + ``` + """ + conf_dir = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'available_parameters', + ) + yaml_files: List[str] = [] + + for root, _, files in os.walk(conf_dir): + for file in sorted(files): + full_path = os.path.join(root, file) + if file.lower().endswith(('.yml', '.yaml')): + yaml_files.append(full_path) + else: + logger.info('Ignored a non-YAML file found under `available_parameters` directory: `%s`.', full_path) + + for file in yaml_files: + try: + config: Dict[str, Any] = _read_postgres_gucs_validators_file(file) + except InvalidGucValidatorsFile as exc: + logger.warning(str(exc)) + continue + + logger.debug(f'Parsing validators from file `{file}`.') + + mapping = { + 'parameters': parameters, + 'recovery_parameters': recovery_parameters, + } + + for section in ['parameters', 'recovery_parameters']: + section_var = mapping[section] + + config_section = config.get(section, {}) + for parameter in config_section.keys(): + section_var[parameter] = _get_postgres_guc_validators(config_section, parameter) + + +_load_postgres_gucs_validators() + + +def _transform_parameter_value(validators: MutableMapping[str, Tuple[_Transformable, ...]], + version: int, name: str, value: Any, + available_gucs: CaseInsensitiveSet) -> Optional[Any]: + """Validate *value* of GUC *name* for Postgres *version* using defined *validators* and *available_gucs*. + + :param validators: a dictionary of all GUCs across all Postgres versions. Each key is the name of a Postgres GUC, + and the corresponding value is a variable length tuple of :class:`_Transformable`. Each item is a validation + rule for the GUC for a given range of Postgres versions. Should either contain recovery GUCs or general GUCs, + not both. + :param version: Postgres version to validate the GUC against. + :param name: name of the Postgres GUC. + :param value: value of the Postgres GUC. + :param available_gucs: a set of all GUCs available in Postgres *version*. Each item is the name of a Postgres + GUC. Used for a couple purposes: + * Disallow writing GUCs to ``postgresql.conf`` (or ``recovery.conf``) that does not exist in Postgres *version*; + * Avoid ignoring GUC *name* if it does not have a validator in *validators*, but is a valid GUC in Postgres + *version*. + + :returns: the return value may be one among: + * *value* transformed to the expected format for GUC *name* in Postgres *version*, if *name* is present in + *available_gucs* and has a validator in *validators* for the corresponding Postgres *version*; or + * The own *value* if *name* is present in *available_gucs* but not in *validators*; or + * ``None`` if *name* is not present in *available_gucs*. + """ + if name in available_gucs: + for validator in validators.get(name, ()) or (): if version >= validator.version_from and\ (validator.version_till is None or version < validator.version_till): return validator.transform(name, value) + # Ideally we should have a validator in *validators*. However, if none is available, we will not discard a + # setting that exists in Postgres *version*, but rather allow the value with no validation. + return value logger.warning('Removing unexpected parameter=%s value=%s from the config', name, value) -def transform_postgresql_parameter_value(version: int, name: str, value: Any) -> Optional[Any]: - if '.' in name: +def transform_postgresql_parameter_value(version: int, name: str, value: Any, + available_gucs: CaseInsensitiveSet) -> Optional[Any]: + """Validate *value* of GUC *name* for Postgres *version* using ``parameters`` and *available_gucs*. + + :param version: Postgres version to validate the GUC against. + :param name: name of the Postgres GUC. + :param value: value of the Postgres GUC. + :param available_gucs: a set of all GUCs available in Postgres *version*. Each item is the name of a Postgres + GUC. Used for a couple purposes: + * Disallow writing GUCs to ``postgresql.conf`` that does not exist in Postgres *version*; + * Avoid ignoring GUC *name* if it does not have a validator in ``parameters``, but is a valid GUC in Postgres + *version*. + + :returns: The return value may be one among + * The original *value* if *name* seems to be an extension GUC (contains a period '.'); or + * ``None`` if **name** is a recovery GUC; or + * *value* transformed to the expected format for GUC *name* in Postgres *version* using validators defined in + ``parameters``. Can also return ``None``. See :func:`_transform_parameter_value`. + """ + if '.' in name and name not in parameters: + # likely an extension GUC, so just return as it is. Otherwise, if `name` is in `parameters`, it's likely a + # namespaced GUC from a custom Postgres build, so we treat that over the usual validation means. return value if name in recovery_parameters: return None - return _transform_parameter_value(parameters, version, name, value) + return _transform_parameter_value(parameters, version, name, value, available_gucs) -def transform_recovery_parameter_value(version: int, name: str, value: Any) -> Optional[Any]: - return _transform_parameter_value(recovery_parameters, version, name, value) +def transform_recovery_parameter_value(version: int, name: str, value: Any, + available_gucs: CaseInsensitiveSet) -> Optional[Any]: + """Validate *value* of GUC *name* for Postgres *version* using ``recovery_parameters`` and *available_gucs*. + + :param version: Postgres version to validate the recovery GUC against. + :param name: name of the Postgres recovery GUC. + :param value: value of the Postgres recovery GUC. + :param available_gucs: a set of all GUCs available in Postgres *version*. Each item is the name of a Postgres + GUC. Used for a couple purposes: + * Disallow writing GUCs to ``recovery.conf`` (or ``postgresql.conf`` depending on *version*), that does not + exist in Postgres *version*; + * Avoid ignoring recovery GUC *name* if it does not have a validator in ``recovery_parameters``, but is a valid + GUC in Postgres *version*. + + :returns: *value* transformed to the expected format for recovery GUC *name* in Postgres *version* using validators + defined in ``recovery_parameters``. It can also return ``None``. See :func:`_transform_parameter_value`. + """ + # Recovery settings are not present in ``postgres --describe-config`` output of Postgres <= 11. In that case we + # just pass down the list of settings defined in Patroni validators so :func:`_transform_parameter_value` will not + # discard the recovery GUCs when running Postgres <= 11. + # NOTE: At the moment this change was done Postgres 11 was almost EOL, and had been likely extensively used with + # Patroni, so we should be able to rely solely on Patroni validators as the source of truth. + return _transform_parameter_value( + recovery_parameters, version, name, value, + available_gucs if version >= 120000 else CaseInsensitiveSet(recovery_parameters.keys())) diff --git a/setup.py b/setup.py index 1c0d3b3b..bff90cd4 100644 --- a/setup.py +++ b/setup.py @@ -157,7 +157,10 @@ def setup_package(version): long_description=read('README.rst'), classifiers=CLASSIFIERS, packages=find_packages(exclude=['tests', 'tests.*']), - package_data={MAIN_PACKAGE: ["*.json"]}, + package_data={MAIN_PACKAGE: [ + "postgresql/available_parameters/*.yml", + "postgresql/available_parameters/*.yaml", + ]}, install_requires=install_requires, extras_require=EXTRAS_REQUIRE, cmdclass=cmdclass, diff --git a/tests/__init__.py b/tests/__init__.py index 838c2636..03598ae1 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -3,7 +3,7 @@ import os import shutil import unittest -from mock import Mock, patch +from mock import Mock, PropertyMock, patch import urllib3 @@ -19,6 +19,15 @@ class SleepException(Exception): pass +mock_available_gucs = PropertyMock(return_value={ + 'cluster_name', 'constraint_exclusion', 'force_parallel_mode', 'hot_standby', 'listen_addresses', 'max_connections', + 'max_locks_per_transaction', 'max_prepared_transactions', 'max_replication_slots', 'max_stack_depth', + 'max_wal_senders', 'max_worker_processes', 'port', 'search_path', 'shared_preload_libraries', + 'stats_temp_directory', 'synchronous_standby_names', 'track_commit_timestamp', 'unix_socket_directories', + 'vacuum_cost_delay', 'vacuum_cost_limit', 'wal_keep_size', 'wal_level', 'wal_log_hints', 'zero_damaged_pages', +}) + + class MockResponse(object): def __init__(self, status_code=200): diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index bdc284e5..4c7c0fcc 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -9,12 +9,13 @@ from patroni.postgresql.bootstrap import Bootstrap from patroni.postgresql.cancellable import CancellableSubprocess from patroni.postgresql.config import ConfigHandler -from . import psycopg_connect, BaseTestPostgresql +from . import psycopg_connect, BaseTestPostgresql, mock_available_gucs @patch('subprocess.call', Mock(return_value=0)) @patch('patroni.psycopg.connect', psycopg_connect) @patch('os.rename', Mock()) +@patch.object(Postgresql, 'available_gucs', mock_available_gucs) class TestBootstrap(BaseTestPostgresql): @patch('patroni.postgresql.CallbackExecutor', Mock()) diff --git a/tests/test_patroni.py b/tests/test_patroni.py index cb40b232..d5c76c8b 100644 --- a/tests/test_patroni.py +++ b/tests/test_patroni.py @@ -67,6 +67,7 @@ class TestPatroni(unittest.TestCase): @patch.object(etcd.Client, 'read', etcd_read) @patch.object(Thread, 'start', Mock()) @patch.object(AbstractEtcdClientWithFailover, '_get_machines_list', Mock(return_value=['http://remotehost:2379'])) + @patch.object(Postgresql, '_get_gucs', Mock(return_value={'foo': True, 'bar': True})) def setUp(self): self._handlers = logging.getLogger().handlers[:] RestApiServer._BaseServer__is_shut_down = Mock() @@ -90,6 +91,7 @@ class TestPatroni(unittest.TestCase): @patch.object(etcd.Client, 'delete', Mock()) @patch.object(AbstractEtcdClientWithFailover, '_get_machines_list', Mock(return_value=['http://remotehost:2379'])) @patch.object(Thread, 'join', Mock()) + @patch.object(Postgresql, '_get_gucs', Mock(return_value={'foo': True, 'bar': True})) def test_patroni_patroni_main(self): with patch('subprocess.call', Mock(return_value=1)): with patch.object(Patroni, 'run', Mock(side_effect=SleepException)): diff --git a/tests/test_postgresql.py b/tests/test_postgresql.py index f7c0c155..c759cde0 100644 --- a/tests/test_postgresql.py +++ b/tests/test_postgresql.py @@ -10,6 +10,7 @@ from mock import Mock, MagicMock, PropertyMock, patch, mock_open import patroni.psycopg as psycopg from patroni.async_executor import CriticalTask +from patroni.collections import CaseInsensitiveSet from patroni.config import GlobalConfig from patroni.dcs import RemoteMember from patroni.exceptions import PostgresConnectionException, PatroniException @@ -17,10 +18,14 @@ from patroni.postgresql import Postgresql, STATE_REJECT, STATE_NO_RESPONSE from patroni.postgresql.bootstrap import Bootstrap from patroni.postgresql.callback_executor import CallbackAction from patroni.postgresql.postmaster import PostmasterProcess +from patroni.postgresql.validator import (ValidatorFactoryNoType, ValidatorFactoryInvalidType, + ValidatorFactoryInvalidSpec, ValidatorFactory, InvalidGucValidatorsFile, + _get_postgres_guc_validators, _read_postgres_gucs_validators_file, + _load_postgres_gucs_validators, Bool, Integer, Real, Enum, EnumBool, String) from patroni.utils import RetryFailedError from threading import Thread, current_thread -from . import BaseTestPostgresql, MockCursor, MockPostmaster, psycopg_connect +from . import BaseTestPostgresql, MockCursor, MockPostmaster, psycopg_connect, mock_available_gucs mtime_ret = {} @@ -91,6 +96,7 @@ Data page checksum version: 0 @patch('subprocess.call', Mock(return_value=0)) @patch('patroni.psycopg.connect', psycopg_connect) +@patch.object(Postgresql, 'available_gucs', mock_available_gucs) class TestPostgresql(BaseTestPostgresql): @patch('subprocess.call', Mock(return_value=0)) @@ -98,6 +104,7 @@ class TestPostgresql(BaseTestPostgresql): @patch('patroni.postgresql.CallbackExecutor', Mock()) @patch.object(Postgresql, 'get_major_version', Mock(return_value=140000)) @patch.object(Postgresql, 'is_running', Mock(return_value=True)) + @patch.object(Postgresql, 'available_gucs', mock_available_gucs) def setUp(self): super(TestPostgresql, self).setUp() self.p.config.write_postgresql_conf() @@ -739,3 +746,212 @@ class TestPostgresql(BaseTestPostgresql): @patch.object(Postgresql, '_cluster_info_state_get', Mock(return_value=True)) def test_handle_parameter_change(self): self.p.handle_parameter_change() + + def test_validator_factory(self): + # validator with no type + validator = { + 'version_from': 90300, + 'version_till': None, + } + with self.assertRaises(ValidatorFactoryNoType) as e: + ValidatorFactory(validator) + self.assertEqual(str(e.exception), 'Validator contains no type.') + + # validator with invalid type + validator = { + 'type': 'Random', + 'version_from': 90300, + 'version_till': None, + } + with self.assertRaises(ValidatorFactoryInvalidType) as e: + ValidatorFactory(validator) + self.assertEqual(str(e.exception), f'Unexpected validator type: `{validator["type"]}`.') + + # validator with missing attributes + validator = { + 'type': 'Integer', + 'version_from': 90300, + 'min_val': 0, + } + with self.assertRaises(ValidatorFactoryInvalidSpec) as e: + ValidatorFactory(validator) + type_ = validator.pop('type') + self.assertRegex( + str(e.exception), + rf"Failed to parse `{type_}` validator \(`{validator}`\): `(Number\.)?__init__\(\) missing 1 " + "required keyword-only argument: 'max_val'`." + ) + + # valid validators + # Bool + validator = { + 'type': 'Bool', + 'version_from': 90300, + 'version_till': None, + } + ret = ValidatorFactory(validator) + self.assertIsInstance(ret, Bool) + self.assertEqual( + ret.__dict__, + Bool(version_from=validator['version_from'], version_till=validator['version_till']).__dict__, + ) + + # Integer + validator = { + 'type': 'Integer', + 'version_from': 90300, + 'version_till': None, + 'min_val': 1, + 'max_val': 100, + 'unit': None, + } + ret = ValidatorFactory(validator) + self.assertIsInstance(ret, Integer) + self.assertEqual( + ret.__dict__, + Integer(version_from=validator['version_from'], version_till=validator['version_till'], + min_val=validator['min_val'], max_val=validator['max_val'], unit=validator['unit']).__dict__, + ) + + # Real + validator = { + 'type': 'Real', + 'version_from': 90300, + 'version_till': None, + 'min_val': 1.0, + 'max_val': 100.0, + 'unit': None, + } + ret = ValidatorFactory(validator) + self.assertIsInstance(ret, Real) + self.assertEqual( + ret.__dict__, + Real(version_from=validator['version_from'], version_till=validator['version_till'], + min_val=validator['min_val'], max_val=validator['max_val'], unit=validator['unit']).__dict__, + ) + + # Enum + validator = { + 'type': 'Enum', + 'version_from': 90300, + 'version_till': None, + 'possible_values': ('abc', 'def'), + } + ret = ValidatorFactory(validator) + self.assertIsInstance(ret, Enum) + self.assertEqual( + ret.__dict__, + Enum(version_from=validator['version_from'], version_till=validator['version_till'], + possible_values=validator['possible_values']).__dict__, + ) + + # EnumBool + validator = { + 'type': 'EnumBool', + 'version_from': 90300, + 'version_till': None, + 'possible_values': ('abc', 'def'), + } + ret = ValidatorFactory(validator) + self.assertIsInstance(ret, EnumBool) + self.assertEqual( + ret.__dict__, + EnumBool(version_from=validator['version_from'], version_till=validator['version_till'], + possible_values=validator['possible_values']).__dict__, + ) + + # String + validator = { + 'type': 'String', + 'version_from': 90300, + 'version_till': None, + } + ret = ValidatorFactory(validator) + self.assertIsInstance(ret, String) + self.assertEqual( + ret.__dict__, + String(version_from=validator['version_from'], version_till=validator['version_till']).__dict__, + ) + + def test__get_postgres_guc_validators(self): + # normal run + parameter = 'my_parameter' + + config = { + parameter: [{ + 'type': 'Bool', + 'version_from': 90300, + 'version_till': 90500, + }, { + 'type': 'EnumBool', + 'version_from': 90500, + 'version_till': 90600, + 'possible_values': [ + 'always', + ], + }] + } + ret = _get_postgres_guc_validators(config, parameter) + self.assertIsInstance(ret, tuple) + self.assertEqual(len(ret), 2) + self.assertIsInstance(ret[0], Bool) + self.assertIsInstance(ret[1], EnumBool) + + # log exceptions + del config[parameter][0]['type'] + + with patch('patroni.postgresql.validator.logger.warning') as mock_logger: + ret = _get_postgres_guc_validators(config, parameter) + self.assertIsInstance(ret, tuple) + self.assertEqual(len(ret), 1) + self.assertIsInstance(ret[0], EnumBool) + + mock_logger.assert_called_once() + mock_call = mock_logger.call_args[0] + self.assertEqual(mock_call[0], 'Faced an issue while parsing a validator for parameter `%s`: `%r`') + self.assertEqual(mock_call[1], parameter) + self.assertIsInstance(mock_call[2], ValidatorFactoryNoType) + + def test__read_postgres_gucs_validators_file(self): + # raise exception + with self.assertRaises(InvalidGucValidatorsFile) as exc: + _read_postgres_gucs_validators_file('random_file.yaml') + self.assertEqual( + str(exc.exception), + "Unexpected issue while reading parameters file `random_file.yaml`: `[Errno 2] No such file or directory: " + "'random_file.yaml'`." + ) + + def test__load_postgres_gucs_validators(self): + # log messages + with patch('os.walk', Mock(return_value=iter([('.', [], ['file.txt', 'random.yaml'])]))), \ + patch('patroni.postgresql.validator.logger.info') as mock_info, \ + patch('patroni.postgresql.validator.logger.warning') as mock_warning: + _load_postgres_gucs_validators() + mock_info.assert_called_once_with('Ignored a non-YAML file found under `available_parameters` directory: ' + '`%s`.', os.path.join('.', 'file.txt')) + mock_warning.assert_called_once() + self.assertIn( + "Unexpected issue while reading parameters file `{0}`: `[Errno 2] No such file or " + "directory:".format(os.path.join('.', 'random.yaml')), + mock_warning.call_args[0][0] + ) + + +@patch('subprocess.call', Mock(return_value=0)) +@patch('patroni.psycopg.connect', psycopg_connect) +class TestPostgresql2(BaseTestPostgresql): + + @patch('subprocess.call', Mock(return_value=0)) + @patch('os.rename', Mock()) + @patch('patroni.postgresql.CallbackExecutor', Mock()) + @patch.object(Postgresql, 'get_major_version', Mock(return_value=140000)) + @patch.object(Postgresql, 'is_running', Mock(return_value=True)) + def setUp(self): + super(TestPostgresql2, self).setUp() + + @patch('subprocess.check_output', Mock(return_value='\n'.join(mock_available_gucs.return_value).encode('utf-8'))) + def test_available_gucs(self): + gucs = self.p.available_gucs + self.assertIsInstance(gucs, CaseInsensitiveSet) + self.assertEqual(gucs, mock_available_gucs.return_value) diff --git a/tests/test_sync.py b/tests/test_sync.py index 0cfba7f8..78a500e3 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -7,11 +7,12 @@ from patroni.config import GlobalConfig from patroni.dcs import Cluster, SyncState from patroni.postgresql import Postgresql -from . import BaseTestPostgresql, psycopg_connect +from . import BaseTestPostgresql, psycopg_connect, mock_available_gucs @patch('subprocess.call', Mock(return_value=0)) @patch('patroni.psycopg.connect', psycopg_connect) +@patch.object(Postgresql, 'available_gucs', mock_available_gucs) class TestSync(BaseTestPostgresql): @patch('subprocess.call', Mock(return_value=0)) @@ -19,6 +20,7 @@ class TestSync(BaseTestPostgresql): @patch('patroni.postgresql.CallbackExecutor', Mock()) @patch.object(Postgresql, 'get_major_version', Mock(return_value=140000)) @patch.object(Postgresql, 'is_running', Mock(return_value=True)) + @patch.object(Postgresql, 'available_gucs', mock_available_gucs) def setUp(self): super(TestSync, self).setUp() self.p.config.write_postgresql_conf()