Merge branch 'master' into fix_ci

core/admin/mailu/configuration.py

@@ -25,6 +25,7 @@ DEFAULT_CONFIG = {
     'DB_PW': None,
     'DB_HOST': 'database',
     'DB_NAME': 'mailu',
+    'DB_APPENDIX': '',
     'SQLITE_DATABASE_FILE': 'data/main.db',
     'SQLALCHEMY_DATABASE_URI': 'sqlite:////data/main.db',
     'SQLALCHEMY_DATABASE_URI_ROUNDCUBE': 'sqlite:////data/roundcube.db',
@@ -102,8 +103,8 @@ class ConfigManager:
 
     DB_TEMPLATES = {
         'sqlite': 'sqlite:////{SQLITE_DATABASE_FILE}',
-        'postgresql': 'postgresql://{DB_USER}:{DB_PW}@{DB_HOST}/{DB_NAME}',
-        'mysql': 'mysql+mysqlconnector://{DB_USER}:{DB_PW}@{DB_HOST}/{DB_NAME}',
+        'postgresql': 'postgresql://{DB_USER}:{DB_PW}@{DB_HOST}/{DB_NAME}{DB_APPENDIX}',
+        'mysql': 'mysql+mysqlconnector://{DB_USER}:{DB_PW}@{DB_HOST}/{DB_NAME}{DB_APPENDIX}',
     }
 
     def __init__(self):

core/admin/mailu/internal/views/rspamd.py

@@ -2,6 +2,7 @@ from mailu import models
 from mailu.internal import internal
 
 import flask
+import idna
 
 def vault_error(*messages, status=404):
     return flask.make_response(flask.jsonify({'errors':messages}), status)
@@ -19,7 +20,7 @@ def rspamd_dkim_key(domain_name):
         if key := domain.dkim_key:
             selectors.append(
                 {
-                    'domain'  : domain.name,
+                    'domain'  : idna.encode(domain.name.lower()).decode('ascii'),
                     'key'     : key.decode('utf8'),
                     'selector': flask.current_app.config.get('DKIM_SELECTOR', 'dkim'),
                 }
@@ -28,7 +29,7 @@ def rspamd_dkim_key(domain_name):
         if key := domain.domain.dkim_key:
             selectors.append(
                 {
-                    'domain'  : domain.name,
+                    'domain'  : idna.encode(domain.name.lower()).decode('ascii'),
                     'key'     : key.decode('utf8'),
                     'selector': flask.current_app.config.get('DKIM_SELECTOR', 'dkim'),
                 }

core/admin/mailu/schemas.py

@@ -1148,7 +1148,7 @@ class TokenSchema(BaseSchema):
         sibling = True
 
     password = PasswordField(required=True, metadata={'model': models.User})
-    hash_password = fields.Boolean(load_only=True, missing=False)
+    hash_password = fields.Boolean(load_only=True, load_default=False)
 
 
 @mapped
@@ -1193,7 +1193,7 @@ class UserSchema(BaseSchema):
     fetches = fields.Nested(FetchSchema, many=True)
 
     password = PasswordField(required=True, metadata={'model': models.User})
-    hash_password = fields.Boolean(load_only=True, missing=False)
+    hash_password = fields.Boolean(load_only=True, load_default=False)
 
 
 @mapped

core/base/Dockerfile

@@ -15,7 +15,7 @@ RUN set -euxo pipefail \
   ; addgroup -Sg ${MAILU_GID} mailu \
   ; adduser -Sg ${MAILU_UID} -G mailu -h /app -g "mailu app" -s /bin/bash mailu \
   ; apk add --no-cache bash ca-certificates curl python3 tzdata \
-  ; ! [[ "$(uname -m)" == x86_64 ]] \
+  ; ! [[ "$(apk --print-arch)" == x86_64 ]] \
     || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc
 
 WORKDIR /app
@@ -52,10 +52,10 @@ ENV \
   SNUFFLEUPAGUS_URL="https://github.com/jvoisin/snuffleupagus/archive/refs/tags/v${SNUFFLEUPAGUS_VERSION}.tar.gz"
 
 RUN set -euxo pipefail \
-  ;  machine="$(uname -m)" \
+  ;  machine="$(apk --print-arch)" \
   ; deps="build-base gcc libffi-dev python3-dev mariadb-dev" \
   ; [[ "${machine}" != x86_64 ]] && \
-      deps="${deps} cargo git libretls-dev mariadb-connector-c-dev postgresql-dev" \
+      deps="${deps} rust cargo git libretls-dev mariadb-connector-c-dev postgresql-dev" \
   ; apk add --virtual .build-deps ${deps} \
   ; [[ "${machine}" == armv7* ]] && \
       mkdir -p /root/.cargo/registry/index && \

core/base/requirements-dev.txt

@@ -46,6 +46,7 @@ watchdog
 
 # core/postfix
 postfix-mta-sts-resolver
+uvloop
 
 # core/oletools
 python-magic
@@ -56,4 +57,3 @@ requests
 
 # optional/radicale
 radicale
-

core/base/requirements-prod.txt

@@ -78,6 +78,7 @@ tabulate==0.9.0
 tenacity==8.2.3
 typing_extensions==4.11.0
 urllib3==2.2.1
+uvloop==0.21.0
 validators==0.28.1
 visitor==0.1.3
 vobject==0.9.7

docs/configuration.rst

@@ -40,6 +40,8 @@ address.
 The ``WILDCARD_SENDERS`` setting is a comma delimited list of user email addresses
 that are allowed to send emails from any existing address (spoofing the sender).
 
+.. _AUTH Ratelimit:
+
 The ``AUTH_RATELIMIT_IP`` (default: 5/hour) holds a security setting for fighting
 attackers that attempt a password spraying attack. The value defines the limit of
 authentication attempts that will be processed on **distinct** non-existing

docs/faq.rst

@@ -438,6 +438,31 @@ Proceed as following for deleting an user:
 
 .. _`github project`: https://github.com/Mailu/Mailu/
 
+
+How to unblock an IP from rate limiter manually?
+````````````````````````````````````````````````
+
+To manually unblock an IP from the rate limiter do the following on your CLI:
+
+.. code-block:: bash
+
+  # list the limited networks (this is not the IP, but only the network part according to AUTH_RATELIMIT_IP_V4_MASK
+  $ docker compose exec redis redis-cli -n 2 --scan --pattern 'LIMITER/auth-ip/*'
+
+  # remove from rate limiter
+  $ IP=8.8.8.8; docker compose exec redis redis-cli -n 2 --scan --pattern "LIMITER/auth-ip/${IP}/*" \
+  | xargs -r docker compose exec -T redis redis-cli -n 2 DEL
+
+Consider using :ref:`AUTH tokens` for your users. Token-based authentication is exempted from rate limits!
+
+Also have a look at the configuration parameters
+``AUTH_RATELIMIT_EXEMPTION`` and ``AUTH_REQUIRE_TOKENS``. More on
+:ref:`Rate limiting<AUTH Ratelimit>` and :ref:`advanced settings<advanced_settings>`.
+
+*Issue reference:* `2856`_.
+
+.. _`2856`: https://github.com/Mailu/Mailu/issues/2856
+
 Changes in .env don't propagate
 ```````````````````````````````
 
@@ -992,7 +1017,26 @@ Admin container fails to connect to external MariaDB database
 `````````````````````````````````````````````````````````````
 
 If the admin container is `unable to connect to an external MariaDB database due to incompatible collation`_, you may need to change the ``SQLALCHEMY_DATABASE_URI`` setting to ensure the right connector is used.
+Alternatively, you may set ``DB_APPENDIX`` accordingly. For example: ``?collation=utf8mb4_unicode_ci`` is appended as is just after the database name in case DB_TYPE and related values are set.
 
 MariaDB has no support for utf8mb4_0900_ai_ci which is the new default since MySQL version 8.0.
 
 .. _`unable to connect to an external MariaDB database due to incompatible collation`: https://github.com/Mailu/Mailu/issues/3449
+
+Why is Rspamd giving me an "Illegal instruction" error ?
+`````````````````````````````````````````````````````````
+
+On Linux amd64 (x84_64), if the antispam container is crashing and gives you an ``Illegal instruction`` error, you may have a CPU that lacks support of the ``SSE4.2`` instruction set.
+The more modern and FOSS ``vectorscan`` library used by rspamd superseeded the now closed source Intel ``hyperscan`` library in Alpine Linux, and since August 2024 it requires the ``SSE4.2`` instruction set to work properly.
+
+Pre-2013 Intel Atom CPUs (Like N2800 or D425), Intel pre-Nehalem architectures and AMD pre-Bulldozer architectures do not support ``SSE4.2``.
+To check if your CPU supports ``SSE4.2`` you can use this one liner command:
+
+``if grep -q sse4_2 /proc/cpuinfo; then echo "CPU is SSE4.2 Capable"; else echo "CPU is NOT SSE4.2 capable"; fi``
+
+A workaround to this issue is to use a x86_32 (or i686) version of rspamd, because the ``vectorscan`` library is only used on 64-bit capable systems.
+Note that this may stop working in the future, as 32-bit software support is being progressively dropped.
+
+*Issue reference:* `3713`_.
+
+.. _`3713`: https://github.com/Mailu/Mailu/issues/3713

docs/setup.rst

@@ -22,6 +22,9 @@ linux/arm64v8 or linux/armv7 hardware, so it
 should run on pretty much any cloud server as long as enough power is
 provided.
 
+On x86_64, check that your processor supports the ``SSE4.2`` instruction set.
+For example, pre-2013 Intel Atom CPUs lacks ``SSE4.2`` support. See :ref:`faq`.
+
 You should also have at least a DNS hostname and a DNS name for receiving
 emails. Some instructions are provided on the matter in the article
 :ref:`dns_setup`.

docs/webadministration.rst

@@ -167,6 +167,8 @@ Click the submit button to apply settings. With the default polling interval, fe
 Make sure ``FETCHMAIL_ENABLED`` is set to ``true`` in ``mailu.env`` to enable fetching and showing fetchmail in the admin interface.
 
 
+.. _AUTH tokens:
+
 Authentication tokens
 ---------------------
 

setup/flavors/compose/docker-compose.yml

@@ -227,12 +227,6 @@ services:
       - clamav
     volumes:
       - "{{ root }}/clamav:/var/lib/clamav"
-    healthcheck:
-      test: ["CMD-SHELL", "kill -0 `cat /tmp/clamd.pid` && kill -0 `cat /tmp/freshclam.pid`"]
-      interval: 10s
-      timeout: 5s
-      retries: 3
-      start_period: 10s
   {% endif %}
 
   {% if webdav_enabled %}

towncrier/newsfragments/3729.misc

@@ -0,0 +1 @@
+Add optional DB_APPENDIX configuration parameter which is appended as is to the postgresql or mysql URI

towncrier/newsfragments/3753.misc

@@ -0,0 +1 @@
+Update roundcube to 1.6.10

towncrier/newsfragments/3758.bugfix

@@ -0,0 +1 @@
+domain name of an IDN domain in the DKIM signature needs to follow RFC6376; puny encoding the domain name when rspamd accesses the vault;

towncrier/newsfragments/3836.misc

@@ -0,0 +1 @@
+Remove the healthcheck section of clamav: use upstream's

towncrier/newsfragments/3851.misc

@@ -0,0 +1 @@
+Upgrade roundcube to 1.6.11. For the record, Mailu is not vulnerable to CVE-2025-49113, thanks to Snuffleupagus (see https://snuffleupagus.readthedocs.io/config.html#unserialize-noclass)