Add API tests for a stand-alone node. Bugfixes.

Add tests for patroni API.
Fix test failures when an already running etcd is used.

features/patroni_api.feature

@@ -0,0 +1,21 @@
+Feature: patroni api
+	We should check that patroni correctly responds to valid and not-valid API requests.
+
+Scenario: check API requests on a stand-alone server
+	Given I start postgres0
+	And postgres0 is a leader after 10 seconds
+	When I issue a GET request to http://127.0.0.1:8008/
+	Then I receive a response code 200
+	And I receive a response state running
+	And I receive a response role master
+	When I issue a GET request to http://127.0.0.1:8008/replica
+	Then I receive a response code 503
+	When I issue an empty POST request to http://127.0.0.1:8008/reinitialize
+	Then I receive a response code 503
+	And I receive a response text "I am the leader, can not reinitialize"
+	When I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0
+	Then I receive a response code 503
+	And I receive a response text "failover is not possible: cluster does not have members except leader"
+	When I issue an empty POST request to http://127.0.0.1:8008/failover
+	Then I receive a response code 503
+	And I receive a response text "No values given for required parameters leader and member"

features/patroni_api.py

@@ -0,0 +1,83 @@
+from lettuce import world, steps
+import time
+import requests
+
+
+@steps
+class PatroniAPISteps(object):
+
+    def __init__(self, environ):
+        self.env = environ
+        self.response = None
+        self.status_code = None
+
+    # there is no way we can find out if the node has already
+    # started as a leader without checking the DCS. We cannot
+    # just rely on the database availability, since there is
+    # a short gap between the time PostgreSQL becomes available
+    # and Patroni assuming the leader role.
+    @staticmethod
+    def is_a_leader(step, name, time_limit):
+        '''(\w+) is a leader after (\d+) seconds'''
+        max_time = time.time() + int(time_limit)
+        while (world.etcd_ctl.query("leader") != name):
+            time.sleep(1)
+            if time.time() > max_time:
+                assert False, "{0} is not a leader in etcd after {1} seconds".format(name, time_limit)
+
+    @staticmethod
+    def sleep_for_n_seconds(step, value):
+        '''I sleep for (\d+) seconds'''
+        time.sleep(int(value))
+
+    def do_get(self, step, url):
+        '''I issue a GET request to (https?://(?:\w|\.|:|/)+)'''
+        try:
+            r = requests.get(url)
+        except requests.exceptions.RequestException:
+            self.code = None
+            self.response = None
+        else:
+            self.status_code = r.status_code
+            try:
+                self.response = r.json()
+            except ValueError:
+                self.response = r.content
+
+    def do_post_empty(self, step, url):
+        '''I issue an empty POST request to (https?://(?:\w|\.|:|/)+)'''
+        self.do_post(step, url, None)
+
+    def do_post(self, step, url, data):
+        '''I issue a POST request to (https?://(?:\w|\.|:|/)+) with (\s*\w+\s*=\s*\w+\s*,?)+'''
+        post_data = {}
+        if data:
+            post_components = data.split(',')
+            for pc in post_components:
+                if '=' in pc:
+                    k, v = pc.split('=', 2)
+                    post_data[k.strip()] = v.strip()
+        try:
+            r = requests.post(url, json=post_data)
+        except requests.exceptions.RequestException:
+            self.code = None
+            self.response = None
+        else:
+            self.status_code = r.status_code
+            try:
+                self.response = r.json()
+            except ValueError:
+                self.response = r.content
+
+    def check_response(self, step, component, data):
+        '''I receive a response (\w+) (.*)'''
+        if component == 'code':
+            assert self.status_code == int(data), "status code {0} != {1}".format(self.status_code, int(data))
+        elif component == 'text':
+            assert self.response == data.strip('"'), "response {0} does not contain {1}".format(self.response, data)
+        else:
+            assert component in self.response, "{0} is not part of the response".format(component)
+            assert self.response[component] == data, "{0} does not contain {1}".format(component, data)
+
+
+PatroniAPISteps(world)

features/terrain.py

@@ -116,13 +116,15 @@ class PatroniController(object):
 
         with open(patroni_config_name) as f:
             config = yaml.load(f)
-        postgresql = config['postgresql']['parameters']
-        postgresql['logging_collector'] = 'on'
-        postgresql['log_destination'] = 'csvlog'
-        postgresql['log_directory'] = self._output_dir
-        postgresql['log_filename'] = '{0}.log'.format(pg_name)
-        postgresql['log_statement'] = 'all'
-        postgresql['log_min_messages'] = 'debug1'
+        postgresql = config['postgresql']
+        postgresql['name'] = pg_name.encode('utf-8')
+        postgresql_params = postgresql['parameters']
+        postgresql_params['logging_collector'] = 'on'
+        postgresql_params['log_destination'] = 'csvlog'
+        postgresql_params['log_directory'] = self._output_dir
+        postgresql_params['log_filename'] = '{0}.log'.format(pg_name)
+        postgresql_params['log_statement'] = 'all'
+        postgresql_params['log_min_messages'] = 'debug1'
 
         with open(patroni_config_path, 'w') as f:
             yaml.dump(config, f, default_flow_style=False)
@@ -188,6 +190,15 @@ class EtcdController(object):
             time.sleep(1)
         return True
 
+    def query(self, key):
+        """ query etcd for a value of a given key """
+        r = requests.get("http://127.0.0.1:2379/v2/keys/service/batman/{0}".format(key))
+        if r.ok:
+            content = r.json()
+            if content:
+                return content.get('node', {}).get('value', None)
+        return None
+
     def stop_and_remove_work_directory(self):
         """ terminate etcd and wipe out the temp work directory, but only if we actually started it"""
         if self._is_running() and self.handle:
@@ -226,12 +237,14 @@ pctl = PatroniController()
 etcd_ctl = EtcdController()
 # export pctl to manage patroni from scenario files
 world.pctl = pctl
+world.etcd_ctl = etcd_ctl
 
 
 # actions to execute on start/stop of the tests and before running invidual features
 @before.all
 def start_etcd():
     etcd_ctl.start()
+    etcd_ctl.cleanup_service_tree()
 
 
 @after.all