Couple of minor improvements (#1019)

1. Fix race condition on shutdown. It is very annoying when you cancel behave tests but postgres remains running.
2. Dump pg_controldata output to logs when "recovering" stopped postgres. It will help to investigate some annoying issues.

patroni/__init__.py

@@ -85,9 +85,10 @@ class Patroni(object):
         self._received_sighup = True
 
     def sigterm_handler(self, *args):
-        if not self._received_sigterm:
-            self._received_sigterm = True
-            sys.exit()
+        with self._sigterm_lock:
+            if not self._received_sigterm:
+                self._received_sigterm = True
+                sys.exit()
 
     @property
     def noloadbalance(self):
@@ -106,11 +107,16 @@ class Patroni(object):
         elif self.ha.watch(nap_time):
             self.next_run = time.time()
 
+    @property
+    def received_sigterm(self):
+        with self._sigterm_lock:
+            return self._received_sigterm
+
     def run(self):
         self.api.start()
         self.next_run = time.time()
 
-        while not self._received_sigterm:
+        while not self.received_sigterm:
             if self._received_sighup:
                 self._received_sighup = False
                 if self.config.reload_local_configuration():
@@ -128,13 +134,18 @@ class Patroni(object):
             self.schedule_next_run()
 
     def setup_signal_handlers(self):
+        from threading import Lock
+
         self._received_sighup = False
+        self._sigterm_lock = Lock()
         self._received_sigterm = False
         if os.name != 'nt':
             signal.signal(signal.SIGHUP, self.sighup_handler)
         signal.signal(signal.SIGTERM, self.sigterm_handler)
 
     def shutdown(self):
+        with self._sigterm_lock:
+            self._received_sigterm = True
         try:
             self.api.shutdown()
         except Exception:

patroni/ha.py

@@ -305,6 +305,7 @@ class Ha(object):
             timeout = None
 
         data = self.state_handler.controldata()
+        logger.info('pg_controldata:\n%s\n', '\n'.join('  {0}: {1}'.format(k, v) for k, v in data.items()))
         if data.get('Database cluster state') in ('in production', 'shutting down', 'in crash recovery') and \
                 not self._crash_recovery_executed and (self.cluster.is_unlocked() or self.state_handler.can_rewind):
             self._crash_recovery_executed = True