diff --git a/patroni/ha.py b/patroni/ha.py
index f12501b2..8a198036 100644
--- a/patroni/ha.py
+++ b/patroni/ha.py
@@ -2298,6 +2298,7 @@ class Ha(object):
                 return 'Unexpected exception raised, please report it as a BUG'
 
     def shutdown(self) -> None:
+        self._async_executor.cancel()
         if self.is_paused():
             logger.info('Leader key is not deleted and Postgresql is not stopped due paused state')
             self.watchdog.disable()
diff --git a/patroni/postgresql/bootstrap.py b/patroni/postgresql/bootstrap.py
index c0606b4e..94fa0ba7 100644
--- a/patroni/postgresql/bootstrap.py
+++ b/patroni/postgresql/bootstrap.py
@@ -318,6 +318,10 @@ class Bootstrap(object):
                     logger.exception('Error creating replica using method %s', replica_method)
                     ret = 1
 
+                # replica creation method failed, clean up data directory if configuration allows
+                if not method_config.get('keep_data', False) and not self._postgresql.data_directory_empty():
+                    self._postgresql.remove_data_directory()
+
         self._postgresql.set_state('stopped')
         return ret
 
@@ -359,6 +363,9 @@ class Bootstrap(object):
             if bbfailures < maxfailures - 1:
                 logger.warning('Trying again in 5 seconds')
                 time.sleep(5)
+            elif not self._postgresql.data_directory_empty():
+                # pg_basebackup failed, clean up data directory
+                self._postgresql.remove_data_directory()
 
         return ret