Merge remote-tracking branch 'upstream/master' into managesieve-proxy

core/admin/mailu/internal/nginx.py

@@ -36,8 +36,7 @@ def check_credentials(user, password, ip, protocol=None, auth_port=None):
     if auth_port in WEBMAIL_PORTS or auth_port == '4190' and password.startswith('token-'):
         if utils.verify_temp_token(user.get_id(), password):
             is_ok = True
-    # All tokens are 32 characters hex lowercase
-    if not is_ok and len(password) == 32:
+    if not is_ok and utils.is_app_token(password):
         for token in user.tokens:
             if (token.check_password(password) and
                 (not token.ip or token.ip == ip)):

core/admin/mailu/internal/views/auth.py

@@ -21,7 +21,8 @@ def nginx_authentication():
         utils.limiter.rate_limit_ip(client_ip)
         return response
     is_from_webmail = headers['Auth-Port'] in ['10143', '10025']
-    if not is_from_webmail and not is_port_25 and utils.limiter.should_rate_limit_ip(client_ip):
+    is_app_token = utils.is_app_token(headers.get('Auth-Pass',''))
+    if not is_from_webmail and not is_port_25 and not is_app_token and utils.limiter.should_rate_limit_ip(client_ip):
         status, code = nginx.get_status(flask.request.headers['Auth-Protocol'], 'ratelimit')
         response = flask.Response()
         response.headers['Auth-Status'] = status
@@ -36,7 +37,7 @@ def nginx_authentication():
     is_valid_user = False
     username = response.headers.get('Auth-User', None)
     if response.headers.get("Auth-User-Exists") == "True":
-        if utils.limiter.should_rate_limit_user(username, client_ip):
+        if not is_app_token and utils.limiter.should_rate_limit_user(username, client_ip):
             # FIXME could be done before handle_authentication()
             status, code = nginx.get_status(flask.request.headers['Auth-Protocol'], 'ratelimit')
             response = flask.Response()

core/admin/mailu/limiter.py

@@ -76,6 +76,7 @@ class LimitWraperFactory(object):
                 return
             self.storage.incr(f'dedup2-{username}-{truncated_password}', limits.parse(app.config['AUTH_RATELIMIT_USER']).GRANULARITY.seconds, True)
             limiter.hit(device_cookie if device_cookie_name == username else username)
+            self.rate_limit_ip(ip, username)
 
     """ Device cookies as described on:
     https://owasp.org/www-community/Slow_Down_Online_Guessing_Attacks_with_Device_Cookies

core/admin/mailu/sso/views/base.py

@@ -42,12 +42,13 @@ def login():
                 destination = app.config['WEB_WEBMAIL']
         device_cookie, device_cookie_username = utils.limiter.parse_device_cookie(flask.request.cookies.get('rate_limit'))
         username = form.email.data
-        if username != device_cookie_username and utils.limiter.should_rate_limit_ip(client_ip):
-            flask.flash('Too many attempts from your IP (rate-limit)', 'error')
-            return flask.render_template('login.html', form=form, fields=fields)
-        if utils.limiter.should_rate_limit_user(username, client_ip, device_cookie, device_cookie_username):
-            flask.flash('Too many attempts for this user (rate-limit)', 'error')
-            return flask.render_template('login.html', form=form, fields=fields)
+        if not utils.is_app_token(form.pw.data):
+            if username != device_cookie_username and utils.limiter.should_rate_limit_ip(client_ip):
+                flask.flash('Too many attempts from your IP (rate-limit)', 'error')
+                return flask.render_template('login.html', form=form, fields=fields)
+            if utils.limiter.should_rate_limit_user(username, client_ip, device_cookie, device_cookie_username):
+                flask.flash('Too many attempts for this user (rate-limit)', 'error')
+                return flask.render_template('login.html', form=form, fields=fields)
         user = models.User.login(username, form.pw.data)
         if user:
             flask.session.regenerate()

core/admin/mailu/utils.py

@@ -15,6 +15,7 @@ import dns.rdataclass
 
 import hmac
 import secrets
+import string
 import time
 
 from multiprocessing import Value
@@ -525,3 +526,9 @@ def formatCSVField(field):
     else:
         data = field.data
     field.data = ", ".join(data)
+
+# All tokens are 32 characters hex lowercase
+def is_app_token(candidate):
+    if len(candidate) == 32 and all(c in string.hexdigits[:-6] for c in candidate):
+        return True
+    return False

core/admin/start.py

@@ -9,7 +9,6 @@ os.system("chown mailu:mailu -R /dkim")
 os.system("find /data | grep -v /fetchmail | xargs -n1 chown mailu:mailu")
 system.drop_privs_to('mailu')
 
-log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "INFO"))
 system.set_env(['SECRET'])
 
 os.system("flask mailu advertise")

core/base/libs/socrate/socrate/system.py

@@ -1,6 +1,8 @@
 import hmac
 import logging as log
 import os
+import sys
+import re
 from pwd import getpwnam
 import socket
 import tenacity
@@ -24,15 +26,72 @@ def _coerce_value(value):
         return False
     return value
 
-def set_env(required_secrets=[]):
+class LogFilter(object):
+    def __init__(self, stream, re_patterns, log_file):
+        self.stream = stream
+        if isinstance(re_patterns, list):
+            self.pattern = re.compile('|'.join([f'(?:{pattern})' for pattern in re_patterns]))
+        elif isinstance(re_patterns, str):
+            self.pattern = re.compile(re_patterns)
+        else:
+            self.pattern = re_patterns
+        self.found = False
+        self.log_file = log_file
+
+    def __getattr__(self, attr_name):
+        return getattr(self.stream, attr_name)
+
+    def write(self, data):
+        if data == '\n' and self.found:
+            self.found = False
+        else:
+            if not self.pattern.search(data):
+                self.stream.write(data)
+                self.stream.flush()
+                if self.log_file:
+                    try:
+                        with open(self.log_file, 'a', encoding='utf-8') as l:
+                            l.write(data)
+                    except:
+                        pass
+            else:
+                # caught bad pattern
+                self.found = True
+
+    def flush(self):
+        self.stream.flush()
+
+def _is_compatible_with_hardened_malloc():
+    with open('/proc/cpuinfo', 'r') as f:
+        lines = f.readlines()
+        for line in lines:
+            # See #2764, we need vmovdqu
+            if line.startswith('flags') and ' avx ' not in line:
+                return False
+            # See #2541
+            if line.startswith('Features') and ' lrcpc ' not in line:
+                return False
+    return True
+
+def set_env(required_secrets=[], log_filters=[], log_file=None):
+    if log_filters:
+        sys.stdout = LogFilter(sys.stdout, log_filters, log_file)
+        sys.stderr = LogFilter(sys.stderr, log_filters, log_file)
+    log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", 'WARNING'))
+
+    if 'LD_PRELOAD' in os.environ and not _is_compatible_with_hardened_malloc():
+        log.warning('Disabling hardened-malloc on this CPU')
+        del os.environ['LD_PRELOAD']
+
     """ This will set all the environment variables and retains only the secrets we need """
-    secret_key = os.environ.get('SECRET_KEY')
-    if not secret_key:
+    if 'SECRET_KEY_FILE' in os.environ:
         try:
             secret_key = open(os.environ.get("SECRET_KEY_FILE"), "r").read().strip()
         except Exception as exc:
             log.error(f"Can't read SECRET_KEY from file: {exc}")
             raise exc
+    else:
+        secret_key = os.environ.get('SECRET_KEY')
     clean_env()
     # derive the keys we need
     for secret in required_secrets:

core/dovecot/start.py

@@ -9,8 +9,7 @@ import sys
 from podop import run_server
 from socrate import system, conf
 
-log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
-system.set_env()
+system.set_env(log_filters=r'waitpid\(\) returned unknown PID \d+$')
 
 def start_podop():
     system.drop_privs_to('mail')
@@ -36,4 +35,4 @@ os.system("chown mail:mail /mail")
 os.system("chown -R mail:mail /var/lib/dovecot /conf")
 
 multiprocessing.Process(target=start_podop).start()
-os.execv("/usr/sbin/dovecot", ["dovecot", "-c", "/etc/dovecot/dovecot.conf", "-F"])
+os.system("dovecot -c /etc/dovecot/dovecot.conf -F")

core/nginx/start.py

@@ -2,6 +2,9 @@
 
 import os
 import subprocess
+from socrate import system
+
+system.set_env(log_filters=r'could not be resolved \(\d\: [^\)]+\) while in resolving client address, client\: [^,]+, server: [^\:]+\:(25,110,143,587,465,993,995)$')
 
 # Check if a stale pid file exists
 if os.path.exists("/var/run/nginx.pid"):

core/postfix/Dockerfile

@@ -7,7 +7,7 @@ ARG VERSION=local
 LABEL version=$VERSION
 
 RUN set -euxo pipefail \
-  ; apk add --no-cache cyrus-sasl-login logrotate postfix postfix-pcre rsyslog
+  ; apk add --no-cache cyrus-sasl-login postfix postfix-pcre logrotate
 
 COPY conf/ /conf/
 COPY start.py /

core/postfix/conf/logrotate.conf

@@ -4,8 +4,4 @@ rotate 52
 nocompress
 extension log
 create 0644 root root
-      postrotate
-          /bin/kill -HUP $(cat /run/rsyslogd.pid)
-          postfix reload
-      endscript
 }

core/postfix/conf/master.cf

@@ -52,7 +52,6 @@ discard   unix  -       -       n       -       -       discard
 lmtp      unix  -       -       n       -       -       lmtp
 anvil     unix  -       -       n       -       1       anvil
 scache    unix  -       -       n       -       1       scache
-postlog   unix-dgram n  -       n       -       1       postlogd
 
 {# Ensure that the rendered file ends with a newline #}
 {{- "\n" }}

core/postfix/conf/rsyslog.conf

@@ -1,43 +0,0 @@
-# rsyslog configuration file
-#
-# For more information see /usr/share/doc/rsyslog-*/rsyslog_conf.html
-# or latest version online at http://www.rsyslog.com/doc/rsyslog_conf.html
-# If you experience problems, see http://www.rsyslog.com/doc/troubleshoot.html
-
-
-#### Global directives ####
-
-# Sets the directory that rsyslog uses for work files.
-$WorkDirectory /var/lib/rsyslog
-
-# Sets default permissions for all log files.
-$FileOwner root
-$FileGroup adm
-$FileCreateMode 0640
-$DirCreateMode 0755
-$Umask 0022
-
-# Reduce repeating messages (default off).
-$RepeatedMsgReduction on
-
-
-#### Modules ####
-
-# Provides support for local system logging (e.g. via logger command).
-module(load="imuxsock")
-
-#### Rules ####
-
-# Discard messages from local test requests
-:msg, contains, "connect from localhost[127.0.0.1]"  ~
-:msg, contains, "connect from localhost[::1]"  ~
-:msg, contains, "haproxy read: short protocol header: QUIT"  ~
-:msg, contains, "discarding EHLO keywords: PIPELINING"  ~
-
-{% if POSTFIX_LOG_FILE %}
-# Log mail logs to file
-mail.*							-{{POSTFIX_LOG_FILE}}
-{% endif %}
-
-# Log mail logs to stdout
-mail.*							-/dev/stdout

core/postfix/start.py

@@ -4,15 +4,18 @@ import os
 import glob
 import shutil
 import multiprocessing
-import logging as log
 import sys
 import re
 
 from podop import run_server
 from socrate import system, conf
 
-log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
-system.set_env()
+system.set_env(log_filters=[
+    r'the Postfix mail system is running\: \d+$',
+    r'(dis)?connect from localhost\[(\:\:1|127\.0\.0\.1)\]( quit=1 commands=1)?$',
+    r'haproxy read\: short protocol header\: QUIT$',
+    r'discarding EHLO keywords\: PIPELINING$',
+    ], log_file=os.environ.get('POSTFIX_LOG_FILE'))
 
 os.system("flock -n /queue/pid/master.pid rm /queue/pid/master.pid")
 
@@ -45,8 +48,6 @@ def is_valid_postconf_line(line):
 
 # Actual startup script
 os.environ['DEFER_ON_TLS_ERROR'] = os.environ['DEFER_ON_TLS_ERROR'] if 'DEFER_ON_TLS_ERROR' in os.environ else 'True'
-os.environ["POSTFIX_LOG_SYSLOG"] = os.environ.get("POSTFIX_LOG_SYSLOG","local")
-os.environ["POSTFIX_LOG_FILE"] = os.environ.get("POSTFIX_LOG_FILE", "")
 
 # Postfix requires IPv6 addresses to be wrapped in square brackets
 if 'RELAYNETS' in os.environ:
@@ -86,11 +87,8 @@ if "RELAYUSER" in os.environ:
     conf.jinja("/conf/sasl_passwd", os.environ, path)
     os.system("postmap {}".format(path))
 
-# Configure and start local rsyslog server
-conf.jinja("/conf/rsyslog.conf", os.environ, "/etc/rsyslog.conf")
-os.system("/usr/sbin/rsyslogd -niNONE &")
 # Configure logrotate and start crond
-if os.environ["POSTFIX_LOG_FILE"] != "":
+if os.environ.get('POSTFIX_LOG_FILE'):
     conf.jinja("/conf/logrotate.conf", os.environ, "/etc/logrotate.d/postfix.conf")
     os.system("/usr/sbin/crond")
     if os.path.exists("/overrides/logrotate.conf"):

core/rspamd/start.py

@@ -9,7 +9,6 @@ import sys
 import time
 from socrate import system,conf
 
-log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
 system.set_env()
 
 # Actual startup script

docs/antispam.rst

@@ -83,8 +83,8 @@ Via the multimap filter it is possible to block emails from a sender domain. See
 
 The following steps have to be taken to configure an additional symbol (rule) that uses the multimap filter to block emails from sender domain.
 
-1. In the overrides folder create a configuration file for the multimap filter. This configuration is included by Rspamd in the main multimap configuration file. This means you do not have to use the "multimap {}" element. Files in the /mailu/overrides/rspamd/ folder are mapped to /etc/rspamd/override.d.
-   Create the file /mailu/overrides/rspamd/multimap.conf with contents:
+1. In the overrides folder create a configuration file for the multimap filter. This configuration is included by Rspamd in the main multimap configuration file. This means you do not have to use the "multimap {}" element. Files in the ``/mailu/overrides/rspamd/`` folder are mapped to ``/overrides``.
+   Create the file ``/mailu/overrides/rspamd/multimap.conf`` with contents:
 
    .. code-block:: bash
 
@@ -93,7 +93,7 @@ The following steps have to be taken to configure an additional symbol (rule) th
     local_bl_domain {
       type = "from";
       filter = "email:domain";
-      map = "/etc/rspamd/override.d/blacklist.inc";
+      map = "/overrides/blacklist.inc";
       score = 14;
       description = "Senders domain part is on the local blacklist";
       group = "local_bl";
@@ -129,8 +129,7 @@ The following steps have to be taken to configure an additional symbol (rule) th
 
    .. code-block:: bash
 
-     docker compose scale antispam=0
-     docker compose scale antispam=1
+     docker compose up antispam --force-recreate -d
 
 4. (Optional) Check if the custom symbol is loaded. To access the Rspamd webgui, log in the Mailu administration web interface with a user that is an administrator and go to Antispam. In Rspamd webgui go to tab Symbols. Change the group drop-down box to local_bl. The following additional rule will be listed.
 

optional/clamav/start.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python3
 
 import os
-import logging as log
+import logging as logger
 import sys
+from socrate import system
 
-log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
-logger=log.getLogger(__name__)
+system.set_env(log_filters=r'SelfCheck: Database status OK\.$')
 
 # Bootstrap the database if clamav is running for the first time
 if not os.path.isfile("/data/main.cvd"):

optional/radicale/Dockerfile

@@ -11,7 +11,7 @@ COPY radicale.conf /
 RUN echo $VERSION >/version
 
 #EXPOSE 5232/tcp
-HEALTHCHECK CMD curl -f -L http://localhost:5232/ || exit 1
+HEALTHCHECK CMD ["/bin/sh", "-c", "ps ax | grep [/]radicale.conf"]
 
 VOLUME ["/data"]
 

optional/unbound/start.py

@@ -5,7 +5,6 @@ import logging as log
 import sys
 from socrate import conf, system
 
-log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
 system.set_env()
 
 conf.jinja("/unbound.conf", os.environ, "/etc/unbound/unbound.conf")

towncrier/newsfragments/2644.misc

@@ -0,0 +1,2 @@
+Filter unwanted logs out.
+Disable hardened-malloc if we detect a processor not supporting the AVX extension set

towncrier/newsfragments/2772.misc

@@ -0,0 +1,2 @@
+Always exempt login attempts that use app-tokens from rate-limits
+Ensure that unsuccessful login attempts against a valid account hit the ip-based rate-limit too

towncrier/newsfragments/2789.bugfix

@@ -0,0 +1 @@
+In front, config.py can be called several times. LD_PRELOAD may have already been removed from ENV

webmails/start.py

@@ -11,7 +11,6 @@ from socrate import conf, system
 
 env = os.environ
 
-logging.basicConfig(stream=sys.stderr, level=env.get("LOG_LEVEL", "WARNING"))
 system.set_env(['ROUNDCUBE','SNUFFLEUPAGUS'])
 
 # jinja context