Cancel long-running jobs on Patroni stop (#3232)

Patroni could be doing replica bootstrap and we don't want want pg_basebackup/wal-g/pgBackRest/barman or similar keep running.

Besides that, remove data directory on replica bootstrap failure if configuration allows.

Close #3224

patroni/ha.py

@@ -2298,6 +2298,7 @@ class Ha(object):
                 return 'Unexpected exception raised, please report it as a BUG'
 
     def shutdown(self) -> None:
+        self._async_executor.cancel()
         if self.is_paused():
             logger.info('Leader key is not deleted and Postgresql is not stopped due paused state')
             self.watchdog.disable()

patroni/postgresql/bootstrap.py

@@ -318,6 +318,10 @@ class Bootstrap(object):
                     logger.exception('Error creating replica using method %s', replica_method)
                     ret = 1
 
+                # replica creation method failed, clean up data directory if configuration allows
+                if not method_config.get('keep_data', False) and not self._postgresql.data_directory_empty():
+                    self._postgresql.remove_data_directory()
+
         self._postgresql.set_state('stopped')
         return ret
 
@@ -359,6 +363,9 @@ class Bootstrap(object):
             if bbfailures < maxfailures - 1:
                 logger.warning('Trying again in 5 seconds')
                 time.sleep(5)
+            elif not self._postgresql.data_directory_empty():
+                # pg_basebackup failed, clean up data directory
+                self._postgresql.remove_data_directory()
 
         return ret