From b894c5ed5164788be2e6df39065d352d3005b228 Mon Sep 17 00:00:00 2001
From: spomata <49432438+spomata@users.noreply.github.com>
Date: Sat, 16 Sep 2023 12:05:10 +0200
Subject: [PATCH 01/41] Fixing Website translation and more untranslated bits

Both webmail and website have been translated as "Correo Web" incorrectly imho.
---
 core/admin/mailu/translations/es/LC_MESSAGES/messages.po | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/core/admin/mailu/translations/es/LC_MESSAGES/messages.po b/core/admin/mailu/translations/es/LC_MESSAGES/messages.po
index c385905e..068712b8 100644
--- a/core/admin/mailu/translations/es/LC_MESSAGES/messages.po
+++ b/core/admin/mailu/translations/es/LC_MESSAGES/messages.po
@@ -53,7 +53,7 @@ msgstr "Configuración del cliente"
 
 #: mailu/sso/templates/sidebar_sso.html:16 mailu/ui/templates/sidebar.html:114
 msgid "Website"
-msgstr "Correo web"
+msgstr "Sitio web"
 
 #: mailu/sso/templates/sidebar_sso.html:22 mailu/ui/templates/sidebar.html:120
 msgid "Help"
@@ -173,7 +173,7 @@ msgstr "Habilitar filtro de spam"
 
 #: mailu/ui/forms.py:103
 msgid "Enable marking spam mails as read"
-msgstr ""
+msgstr "Habilitar marcado de correos spam como leídos"
 
 #: mailu/ui/forms.py:104
 msgid "Spam filter tolerance"
@@ -218,7 +218,7 @@ msgstr "Texto de la respuesta"
 
 #: mailu/ui/forms.py:122
 msgid "Start of vacation"
-msgstr ""
+msgstr "Comienzo de las vacaciones"
 
 #: mailu/ui/forms.py:123
 msgid "End of vacation"
@@ -346,7 +346,7 @@ msgstr "Ocurrió un error en la comunicación con el servidor Docker."
 
 #: mailu/ui/templates/macros.html:129
 msgid "copy to clipboard"
-msgstr ""
+msgstr "copiar en portapapeles"
 
 #: mailu/ui/templates/sidebar.html:15
 msgid "My account"

From 83fd29c597a81c9b8d234b9a78a69f7cf88d08bd Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Wed, 20 Sep 2023 18:00:36 +0200
Subject: [PATCH 02/41] Upgrade snuffleupagus

---
 core/base/Dockerfile              | 2 +-
 towncrier/newsfragments/2950.misc | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 towncrier/newsfragments/2950.misc

diff --git a/core/base/Dockerfile b/core/base/Dockerfile
index 88332d3a..1db10284 100644
--- a/core/base/Dockerfile
+++ b/core/base/Dockerfile
@@ -27,7 +27,7 @@ CMD /bin/bash
 FROM system as build
 
 ARG MAILU_DEPS=prod
-ARG SNUFFLEUPAGUS_VERSION=0.9.0
+ARG SNUFFLEUPAGUS_VERSION=0.10.0
 
 ENV VIRTUAL_ENV=/app/venv
 
diff --git a/towncrier/newsfragments/2950.misc b/towncrier/newsfragments/2950.misc
new file mode 100644
index 00000000..c21a90a8
--- /dev/null
+++ b/towncrier/newsfragments/2950.misc
@@ -0,0 +1 @@
+Upgrade to snuffleupagus 0.10.0

From c137e1a919c98911119192a4ae12471a6e2f0316 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Wed, 20 Sep 2023 18:04:28 +0200
Subject: [PATCH 03/41] Add new rule too

---
 webmails/snuffleupagus.rules | 1 +
 1 file changed, 1 insertion(+)

diff --git a/webmails/snuffleupagus.rules b/webmails/snuffleupagus.rules
index 5e619a8a..b3f69819 100644
--- a/webmails/snuffleupagus.rules
+++ b/webmails/snuffleupagus.rules
@@ -43,6 +43,7 @@ sp.disable_function.function("mail").param("additional_parameters").value_r("\\-
 
 # Since it's now burned, me might as well mitigate it publicly
 sp.disable_function.function("putenv").param("assignment").value_r("LD_").drop()
+sp.disable_function.function("putenv").param("assignment").value("PATH").drop()
 
 # This one was burned in Nov 2019 - https://gist.github.com/LoadLow/90b60bd5535d6c3927bb24d5f9955b80
 sp.disable_function.function("putenv").param("assignment").value_r("GCONV_").drop()

From 8c11655625c7c7c137641f229504110b867b86d8 Mon Sep 17 00:00:00 2001
From: Dimitri Huisman <diman@huisman.xyz>
Date: Wed, 27 Sep 2023 14:05:12 +0000
Subject: [PATCH 04/41] Update admin password for demo server

---
 docs/demo.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/demo.rst b/docs/demo.rst
index 46bedc0e..0619e218 100644
--- a/docs/demo.rst
+++ b/docs/demo.rst
@@ -35,7 +35,7 @@ Connecting to the server
  * Webmail : https://test.mailu.io/webmail/
  * Admin UI : https://test.mailu.io/admin/
  * Admin login : ``admin@test.mailu.io``
- * Admin password : ``letmein``
+ * Admin password : ``Mailu+Demo@test.mailu.io`` (remove + and @test.mailu.io to get the correct password).
  * RESTful API: https://test.mailu.io/api
  * API token: ``Bearer APITokenForMailu``
 

From 9ae6eafb2469c8cedcd8dc5fdf77a8e2664f01df Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Wed, 4 Oct 2023 17:13:56 +0200
Subject: [PATCH 05/41] Remove the version pinning on hardened malloc

---
 core/base/Dockerfile              | 2 +-
 towncrier/newsfragments/2955.misc | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 towncrier/newsfragments/2955.misc

diff --git a/core/base/Dockerfile b/core/base/Dockerfile
index 88332d3a..68febcd6 100644
--- a/core/base/Dockerfile
+++ b/core/base/Dockerfile
@@ -16,7 +16,7 @@ RUN set -euxo pipefail \
   ; adduser -Sg ${MAILU_UID} -G mailu -h /app -g "mailu app" -s /bin/bash mailu \
   ; apk add --no-cache bash ca-certificates curl python3 tzdata \
   ; ! [[ "$(uname -m)" == x86_64 ]] \
-    || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc==11-r0
+    || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc
 
 WORKDIR /app
 
diff --git a/towncrier/newsfragments/2955.misc b/towncrier/newsfragments/2955.misc
new file mode 100644
index 00000000..42d8719c
--- /dev/null
+++ b/towncrier/newsfragments/2955.misc
@@ -0,0 +1 @@
+Remove the version pinning on hardened malloc

From 85cf7e6a7f8884468666f13a1de656d1c0088761 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Thu, 5 Oct 2023 18:55:33 +0200
Subject: [PATCH 06/41] Upgrade to alpine 3.18.4

---
 core/base/Dockerfile                | 2 +-
 towncrier/newsfragments/2934.bugfix | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)
 create mode 100644 towncrier/newsfragments/2934.bugfix

diff --git a/core/base/Dockerfile b/core/base/Dockerfile
index 68febcd6..50d1f49a 100644
--- a/core/base/Dockerfile
+++ b/core/base/Dockerfile
@@ -3,7 +3,7 @@
 # base system image (intermediate)
 # Note when updating the alpine tag, first manually run the workflow .github/workflows/mirror.yml.
 # Just run the workflow with the tag that must be synchronised.
-ARG DISTRO=ghcr.io/mailu/alpine:3.17.2
+ARG DISTRO=ghcr.io/mailu/alpine:3.18.4
 FROM $DISTRO as system
 
 ENV TZ=Etc/UTC LANG=C.UTF-8
diff --git a/towncrier/newsfragments/2934.bugfix b/towncrier/newsfragments/2934.bugfix
new file mode 100644
index 00000000..349bc8e1
--- /dev/null
+++ b/towncrier/newsfragments/2934.bugfix
@@ -0,0 +1,3 @@
+Upgrade to alpine 3.18.4: this will fix a bug whereby musl wasn't retrying using TCP when it received truncated DNS replies from its upstream. In practice, this has been seen in the wild when postfix complains of:
+
+"Host or domain name not found. Name service error for name=outlook-com.olc.protection.outlook.com type=AAAA: Host found but no data record of requested type"

From ee7bb53366ab05792ae336477b4264f5bf577b85 Mon Sep 17 00:00:00 2001
From: Dimitri Huisman <diman@huisman.xyz>
Date: Thu, 5 Oct 2023 18:12:48 +0000
Subject: [PATCH 07/41] Fix extracting of snappymail archive not working.

---
 webmails/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/webmails/Dockerfile b/webmails/Dockerfile
index 2d36c699..0732f6ad 100644
--- a/webmails/Dockerfile
+++ b/webmails/Dockerfile
@@ -60,7 +60,7 @@ RUN set -euxo pipefail \
   ; curl -sLo /dev/shm/snappymail.tgz  ${SNAPPYMAIL_URL} \
   ; curl -sLo /dev/shm/snappymail.tgz.asc  ${SNAPPYMAIL_URL}.asc \
   ; gpg --status-fd 1 --verify /dev/shm/snappymail.tgz.asc \
-  ; tar xzf /dev/shm/snappymail.tgz
+  ; cat /dev/shm/snappymail.tgz | tar xz
 
 # SnappyMail login
 COPY snappymail/login/include.php /var/www/snappymail/

From 096c0be4f704939740508d09dab04e374cd2687e Mon Sep 17 00:00:00 2001
From: Dimitri Huisman <diman@huisman.xyz>
Date: Thu, 5 Oct 2023 18:41:54 +0000
Subject: [PATCH 08/41] Rspamd executable was moved to /usr/bin

---
 core/rspamd/start.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/rspamd/start.py b/core/rspamd/start.py
index d6991253..54b2f192 100755
--- a/core/rspamd/start.py
+++ b/core/rspamd/start.py
@@ -37,4 +37,4 @@ while True:
 os.system("mkdir -m 755 -p /run/rspamd")
 os.system("chown rspamd:rspamd /run/rspamd")
 os.system("find /var/lib/rspamd | grep -v /filter | xargs -n1 chown rspamd:rspamd")
-os.execv("/usr/sbin/rspamd", ["rspamd", "-f", "-u", "rspamd", "-g", "rspamd"])
+os.execv("/usr/bin/rspamd", ["rspamd", "-f", "-u", "rspamd", "-g", "rspamd"])

From 77c48294015bb87865cfe3720f545e4ae3a69021 Mon Sep 17 00:00:00 2001
From: Dimitri Huisman <diman@huisman.xyz>
Date: Fri, 6 Oct 2023 09:48:50 +0000
Subject: [PATCH 09/41] Hardened malloc was not disabled for oletools when an
 CPU with missing flags is used

---
 core/oletools/Dockerfile            | 4 +++-
 core/oletools/start.py              | 8 ++++++++
 towncrier/newsfragments/2959.bugfix | 1 +
 3 files changed, 12 insertions(+), 1 deletion(-)
 create mode 100755 core/oletools/start.py
 create mode 100644 towncrier/newsfragments/2959.bugfix

diff --git a/core/oletools/Dockerfile b/core/oletools/Dockerfile
index 39fd0e18..77d088a0 100644
--- a/core/oletools/Dockerfile
+++ b/core/oletools/Dockerfile
@@ -11,6 +11,8 @@ RUN set -euxo pipefail \
   ; curl -sLo olefy.py https://raw.githubusercontent.com/HeinleinSupport/olefy/f8aac6cc55283886d153e89c8f27fae66b1c24e2/olefy.py \
   ; chmod 755 olefy.py
 
+COPY start.py /
+
 RUN echo $VERSION >/version
 
 HEALTHCHECK --start-period=60s CMD echo PING|nc -q1 127.0.0.1 11343|grep "PONG"
@@ -28,4 +30,4 @@ ENV \
     OLEFY_DEL_TMP="1" \
     OLEFY_DEL_TMP_FAILED="1"
 
-CMD /app/olefy.py
+CMD /start.py
diff --git a/core/oletools/start.py b/core/oletools/start.py
new file mode 100755
index 00000000..58d26cce
--- /dev/null
+++ b/core/oletools/start.py
@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+
+import os
+from socrate import system
+
+system.set_env()
+
+os.execl("/app/olefy.py", "olefy")
\ No newline at end of file
diff --git a/towncrier/newsfragments/2959.bugfix b/towncrier/newsfragments/2959.bugfix
new file mode 100644
index 00000000..eb858956
--- /dev/null
+++ b/towncrier/newsfragments/2959.bugfix
@@ -0,0 +1 @@
+Hardened malloc was not disabled for oletools when an outdated CPU is used.
\ No newline at end of file

From e70db935ecc652692ddd1c051e87d28ad4c2c3c4 Mon Sep 17 00:00:00 2001
From: Dimitri Huisman <diman@huisman.xyz>
Date: Fri, 6 Oct 2023 10:27:31 +0000
Subject: [PATCH 10/41] Hardened malloc also requires AVX2 cpu flag

---
 core/base/libs/socrate/socrate/system.py | 2 +-
 towncrier/newsfragments/2959.bugfix      | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py
index e80863b8..182e09e6 100644
--- a/core/base/libs/socrate/socrate/system.py
+++ b/core/base/libs/socrate/socrate/system.py
@@ -66,7 +66,7 @@ def _is_compatible_with_hardened_malloc():
         lines = f.readlines()
         for line in lines:
             # See #2764, we need vmovdqu
-            if line.startswith('flags') and ' avx ' not in line:
+            if line.startswith('flags') and ' avx ' not in line and ' avx2 ' not in line:
                 return False
             # See #2541
             if line.startswith('Features') and ' lrcpc ' not in line:
diff --git a/towncrier/newsfragments/2959.bugfix b/towncrier/newsfragments/2959.bugfix
index eb858956..8427d876 100644
--- a/towncrier/newsfragments/2959.bugfix
+++ b/towncrier/newsfragments/2959.bugfix
@@ -1 +1,2 @@
-Hardened malloc was not disabled for oletools when an outdated CPU is used.
\ No newline at end of file
+Hardened malloc was not disabled for oletools when CPU misses required flags.
+Updated hardened malloc test that AVX2 is also required now.
\ No newline at end of file

From 5230c287136d2a35565219fe039883514913e234 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Fri, 6 Oct 2023 13:48:09 +0200
Subject: [PATCH 11/41] Fix letsencrypt on master

---
 core/nginx/letsencrypt.py           | 4 ++--
 towncrier/newsfragments/2962.bugfix | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)
 create mode 100644 towncrier/newsfragments/2962.bugfix

diff --git a/core/nginx/letsencrypt.py b/core/nginx/letsencrypt.py
index a14dfa15..edf24430 100755
--- a/core/nginx/letsencrypt.py
+++ b/core/nginx/letsencrypt.py
@@ -47,7 +47,7 @@ time.sleep(5)
 
 class MyRequestHandler(SimpleHTTPRequestHandler):
     def do_GET(self):
-        if self.path == '/testing':
+        if self.path == '/.well-known/acme-challenge/testing':
             self.send_response(204)
         else:
             self.send_response(404)
@@ -55,7 +55,7 @@ class MyRequestHandler(SimpleHTTPRequestHandler):
         self.end_headers()
 
 def serve_one_request():
-    with HTTPServer(("0.0.0.0", 8008), MyRequestHandler) as server:
+    with HTTPServer(("127.0.0.1", 8008), MyRequestHandler) as server:
         server.handle_request()
 
 # Run certbot every day
diff --git a/towncrier/newsfragments/2962.bugfix b/towncrier/newsfragments/2962.bugfix
new file mode 100644
index 00000000..59149a0e
--- /dev/null
+++ b/towncrier/newsfragments/2962.bugfix
@@ -0,0 +1 @@
+Fix letsencrypt on master

From 0379857ab5db055e803176fd682e4a665669fd69 Mon Sep 17 00:00:00 2001
From: Dimitri Huisman <52963853+Diman0@users.noreply.github.com>
Date: Fri, 6 Oct 2023 13:51:05 +0200
Subject: [PATCH 12/41] Update core/base/libs/socrate/socrate/system.py

Only check for avx2 is required

Co-authored-by: Florent Daigniere <nextgens@users.noreply.github.com>
---
 core/base/libs/socrate/socrate/system.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py
index 182e09e6..352954e9 100644
--- a/core/base/libs/socrate/socrate/system.py
+++ b/core/base/libs/socrate/socrate/system.py
@@ -66,7 +66,7 @@ def _is_compatible_with_hardened_malloc():
         lines = f.readlines()
         for line in lines:
             # See #2764, we need vmovdqu
-            if line.startswith('flags') and ' avx ' not in line and ' avx2 ' not in line:
+            if line.startswith('flags') and ' avx2 ' not in line:
                 return False
             # See #2541
             if line.startswith('Features') and ' lrcpc ' not in line:

From 3985d1d0446e39d6722c6ab86d429fd260544b32 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@users.noreply.github.com>
Date: Fri, 6 Oct 2023 14:00:58 +0200
Subject: [PATCH 13/41] clarify

---
 core/base/libs/socrate/socrate/system.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py
index 352954e9..db8944e6 100644
--- a/core/base/libs/socrate/socrate/system.py
+++ b/core/base/libs/socrate/socrate/system.py
@@ -66,6 +66,7 @@ def _is_compatible_with_hardened_malloc():
         lines = f.readlines()
         for line in lines:
             # See #2764, we need vmovdqu
+            # See #2959, we need vpunpckldq
             if line.startswith('flags') and ' avx2 ' not in line:
                 return False
             # See #2541
@@ -80,7 +81,7 @@ def set_env(required_secrets=[], log_filters=[], log_file=None):
     log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", 'WARNING'))
 
     if 'LD_PRELOAD' in os.environ and not _is_compatible_with_hardened_malloc():
-        log.warning('Disabling hardened-malloc on this CPU')
+        log.warning('Disabling hardened-malloc on this CPU: it requires Advanced Vector Extensions.')
         del os.environ['LD_PRELOAD']
 
     """ This will set all the environment variables and retains only the secrets we need """

From dd58d51156383a6e1d7279f1d5c3e5f185e878a3 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Sat, 7 Oct 2023 09:35:53 +0200
Subject: [PATCH 14/41] change the logic as discussed

---
 core/base/Dockerfile                     | 1 -
 core/base/libs/socrate/socrate/system.py | 5 ++---
 docs/compose/.env                        | 5 +++++
 docs/compose/setup.rst                   | 8 ++++++++
 towncrier/newsfragments/2959.bugfix      | 4 ++--
 5 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/core/base/Dockerfile b/core/base/Dockerfile
index 68febcd6..55284b4f 100644
--- a/core/base/Dockerfile
+++ b/core/base/Dockerfile
@@ -79,7 +79,6 @@ COPY --chown=root:root --from=build /app/snuffleupagus.so /usr/lib/php81/modules
 ENV \
   VIRTUAL_ENV=/app/venv \
   PATH="/app/venv/bin:${PATH}" \
-  LD_PRELOAD="/usr/lib/libhardened_malloc.so" \
   ADMIN_ADDRESS="admin" \
   FRONT_ADDRESS="front" \
   SMTP_ADDRESS="smtp" \
diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py
index db8944e6..6cb5bb7c 100644
--- a/core/base/libs/socrate/socrate/system.py
+++ b/core/base/libs/socrate/socrate/system.py
@@ -80,9 +80,8 @@ def set_env(required_secrets=[], log_filters=[], log_file=None):
         sys.stderr = LogFilter(sys.stderr, log_filters, log_file)
     log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", 'WARNING'))
 
-    if 'LD_PRELOAD' in os.environ and not _is_compatible_with_hardened_malloc():
-        log.warning('Disabling hardened-malloc on this CPU: it requires Advanced Vector Extensions.')
-        del os.environ['LD_PRELOAD']
+    if not 'LD_PRELOAD' in os.environ and _is_compatible_with_hardened_malloc():
+        log.warning('Your CPU has Advanced Vector Extensions available, we recommend you enable hardened-malloc by adding LD_PRELOAD=/usr/lib/libhardened_malloc.so to your mailu.env')
 
     """ This will set all the environment variables and retains only the secrets we need """
     if 'SECRET_KEY_FILE' in os.environ:
diff --git a/docs/compose/.env b/docs/compose/.env
index 62e767cf..1d943d2a 100644
--- a/docs/compose/.env
+++ b/docs/compose/.env
@@ -152,3 +152,8 @@ REJECT_UNLISTED_RECIPIENT=
 
 # Log level threshold in start.py (value: CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET)
 LOG_LEVEL=WARNING
+
+# If your CPU supports Advanced Vector Extensions
+# (AVX2 on x86_64, lrcpc on ARM64), you should consider enabling
+# hardened-malloc by uncommenting this
+# LD_PRELOAD=/usr/lib/libhardened_malloc.so
diff --git a/docs/compose/setup.rst b/docs/compose/setup.rst
index f4c9c574..9b0545be 100644
--- a/docs/compose/setup.rst
+++ b/docs/compose/setup.rst
@@ -76,6 +76,14 @@ Review configuration variables
 After downloading the files, open ``mailu.env`` and review the variable settings.
 Make sure to read the comments in the file and instructions from the :ref:`common_cfg` page.
 
+If your CPU supports Advanced Vector Extensions (AVX2 on x86_64, lrcpc on ARM64), you should
+consider enabling hardened-malloc by adding the following to your mailu.env:
+
+.. code-block:: bash
+
+    LD_PRELOAD=/usr/lib/libhardened_malloc.so
+
+
 Finish setting up TLS
 ---------------------
 
diff --git a/towncrier/newsfragments/2959.bugfix b/towncrier/newsfragments/2959.bugfix
index 8427d876..0c7bf28e 100644
--- a/towncrier/newsfragments/2959.bugfix
+++ b/towncrier/newsfragments/2959.bugfix
@@ -1,2 +1,2 @@
-Hardened malloc was not disabled for oletools when CPU misses required flags.
-Updated hardened malloc test that AVX2 is also required now.
\ No newline at end of file
+Update hardened malloc as the original package is not available from alpine anymore.
+The newer version of hardened malloc requires AVX2: Disable it by default and hint in the logs when it should be enabled instead.

From 9e1bf76a0ce82ba943779f5ea8cbe75bed0604f8 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Sat, 7 Oct 2023 10:03:23 +0200
Subject: [PATCH 15/41] Maybe fix olefy

---
 core/oletools/Dockerfile | 6 +++++-
 core/oletools/start.py   | 4 ++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/core/oletools/Dockerfile b/core/oletools/Dockerfile
index 77d088a0..6193888f 100644
--- a/core/oletools/Dockerfile
+++ b/core/oletools/Dockerfile
@@ -6,9 +6,13 @@ FROM base
 ARG VERSION=local
 LABEL version=$VERSION
 
+ARG OLEFY_SCRIPT https://raw.githubusercontent.com/HeinleinSupport/olefy/f8aac6cc55283886d153e89c8f27fae66b1c24e2/olefy.py
+ARG OLEFY_SHA256 1f5aa58b78ca7917350135b4425e5ed4d580c7051aabed1952c6afd12d0345a0
+
 RUN set -euxo pipefail \
   ; apk add --no-cache netcat-openbsd libmagic libffi \
-  ; curl -sLo olefy.py https://raw.githubusercontent.com/HeinleinSupport/olefy/f8aac6cc55283886d153e89c8f27fae66b1c24e2/olefy.py \
+  ; curl -sLo olefy.py $OLEFY_SCRIPT \
+  ; echo "$OLEFY_SHA256 olefy.py" |sha256sum --check \
   ; chmod 755 olefy.py
 
 COPY start.py /
diff --git a/core/oletools/start.py b/core/oletools/start.py
index 58d26cce..b0972908 100755
--- a/core/oletools/start.py
+++ b/core/oletools/start.py
@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 
-import os
 from socrate import system
 
 system.set_env()
 
-os.execl("/app/olefy.py", "olefy")
\ No newline at end of file
+with open('/app/olefy.py') as olefy:
+    exec(olefy.read())

From 92e861d4fa56eafa4f2e1585c3a0c8b89b8e02d0 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Sat, 7 Oct 2023 10:09:04 +0200
Subject: [PATCH 16/41] There is no reason not to enable it ourselves.

---
 core/base/libs/socrate/socrate/system.py | 3 ++-
 docs/compose/.env                        | 2 +-
 docs/compose/setup.rst                   | 3 ++-
 towncrier/newsfragments/2959.bugfix      | 2 +-
 4 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py
index 6cb5bb7c..53cabae6 100644
--- a/core/base/libs/socrate/socrate/system.py
+++ b/core/base/libs/socrate/socrate/system.py
@@ -81,7 +81,8 @@ def set_env(required_secrets=[], log_filters=[], log_file=None):
     log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", 'WARNING'))
 
     if not 'LD_PRELOAD' in os.environ and _is_compatible_with_hardened_malloc():
-        log.warning('Your CPU has Advanced Vector Extensions available, we recommend you enable hardened-malloc by adding LD_PRELOAD=/usr/lib/libhardened_malloc.so to your mailu.env')
+        log.warning('Your CPU has Advanced Vector Extensions available, we recommend you enable hardened-malloc earlier in the boot process by adding LD_PRELOAD=/usr/lib/libhardened_malloc.so to your mailu.env')
+        os.environ['LD_PRELOAD'] = '/usr/lib/libhardened_malloc.so'
 
     """ This will set all the environment variables and retains only the secrets we need """
     if 'SECRET_KEY_FILE' in os.environ:
diff --git a/docs/compose/.env b/docs/compose/.env
index 1d943d2a..5c54815c 100644
--- a/docs/compose/.env
+++ b/docs/compose/.env
@@ -155,5 +155,5 @@ LOG_LEVEL=WARNING
 
 # If your CPU supports Advanced Vector Extensions
 # (AVX2 on x86_64, lrcpc on ARM64), you should consider enabling
-# hardened-malloc by uncommenting this
+# hardened-malloc earlier by uncommenting this
 # LD_PRELOAD=/usr/lib/libhardened_malloc.so
diff --git a/docs/compose/setup.rst b/docs/compose/setup.rst
index 9b0545be..81433ba3 100644
--- a/docs/compose/setup.rst
+++ b/docs/compose/setup.rst
@@ -77,7 +77,8 @@ After downloading the files, open ``mailu.env`` and review the variable settings
 Make sure to read the comments in the file and instructions from the :ref:`common_cfg` page.
 
 If your CPU supports Advanced Vector Extensions (AVX2 on x86_64, lrcpc on ARM64), you should
-consider enabling hardened-malloc by adding the following to your mailu.env:
+consider enabling hardened-malloc earlier in the boot process by adding the following to
+your mailu.env:
 
 .. code-block:: bash
 
diff --git a/towncrier/newsfragments/2959.bugfix b/towncrier/newsfragments/2959.bugfix
index 0c7bf28e..7a60c2a5 100644
--- a/towncrier/newsfragments/2959.bugfix
+++ b/towncrier/newsfragments/2959.bugfix
@@ -1,2 +1,2 @@
 Update hardened malloc as the original package is not available from alpine anymore.
-The newer version of hardened malloc requires AVX2: Disable it by default and hint in the logs when it should be enabled instead.
+The newer version of hardened malloc requires AVX2: Disable it by default at startup and hint in the logs when it should be enabled instead.

From 12e8041ba6136f0421b804439e2e3661be503b07 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Sat, 7 Oct 2023 10:25:12 +0200
Subject: [PATCH 17/41] Doh

---
 core/oletools/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/oletools/Dockerfile b/core/oletools/Dockerfile
index 6193888f..79045175 100644
--- a/core/oletools/Dockerfile
+++ b/core/oletools/Dockerfile
@@ -6,8 +6,8 @@ FROM base
 ARG VERSION=local
 LABEL version=$VERSION
 
-ARG OLEFY_SCRIPT https://raw.githubusercontent.com/HeinleinSupport/olefy/f8aac6cc55283886d153e89c8f27fae66b1c24e2/olefy.py
-ARG OLEFY_SHA256 1f5aa58b78ca7917350135b4425e5ed4d580c7051aabed1952c6afd12d0345a0
+ARG OLEFY_SCRIPT=https://raw.githubusercontent.com/HeinleinSupport/olefy/f8aac6cc55283886d153e89c8f27fae66b1c24e2/olefy.py
+ARG OLEFY_SHA256=1f5aa58b78ca7917350135b4425e5ed4d580c7051aabed1952c6afd12d0345a0
 
 RUN set -euxo pipefail \
   ; apk add --no-cache netcat-openbsd libmagic libffi \

From 037a79206e77ecdc7d295b3d40c94570c770efff Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Sat, 7 Oct 2023 10:55:21 +0200
Subject: [PATCH 18/41] doh2

---
 core/oletools/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/oletools/Dockerfile b/core/oletools/Dockerfile
index 79045175..f636cc6d 100644
--- a/core/oletools/Dockerfile
+++ b/core/oletools/Dockerfile
@@ -12,7 +12,7 @@ ARG OLEFY_SHA256=1f5aa58b78ca7917350135b4425e5ed4d580c7051aabed1952c6afd12d0345a
 RUN set -euxo pipefail \
   ; apk add --no-cache netcat-openbsd libmagic libffi \
   ; curl -sLo olefy.py $OLEFY_SCRIPT \
-  ; echo "$OLEFY_SHA256 olefy.py" |sha256sum --check \
+  ; echo "$OLEFY_SHA256  olefy.py" |sha256sum -c \
   ; chmod 755 olefy.py
 
 COPY start.py /

From a6f57ca3d40df93c4ac985e7ed27765e7e639342 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Sun, 8 Oct 2023 14:58:39 +0200
Subject: [PATCH 19/41] Upgrade snappymail to v2.29.1

---
 towncrier/newsfragments/2959.bugfix | 1 +
 webmails/Dockerfile                 | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/towncrier/newsfragments/2959.bugfix b/towncrier/newsfragments/2959.bugfix
index 7a60c2a5..9da9c300 100644
--- a/towncrier/newsfragments/2959.bugfix
+++ b/towncrier/newsfragments/2959.bugfix
@@ -1,2 +1,3 @@
 Update hardened malloc as the original package is not available from alpine anymore.
 The newer version of hardened malloc requires AVX2: Disable it by default at startup and hint in the logs when it should be enabled instead.
+Upgrade snappymail to v2.29.1
diff --git a/webmails/Dockerfile b/webmails/Dockerfile
index 2d36c699..0c4ec167 100644
--- a/webmails/Dockerfile
+++ b/webmails/Dockerfile
@@ -52,7 +52,7 @@ COPY roundcube/config/config.inc.carddav.php /var/www/roundcube/plugins/carddav/
 
 # snappymail
 
-ENV SNAPPYMAIL_URL https://github.com/the-djmaze/snappymail/releases/download/v2.28.4/snappymail-2.28.4.tar.gz
+ENV SNAPPYMAIL_URL https://github.com/the-djmaze/snappymail/releases/download/v2.29.1/snappymail-2.29.1.tar.gz
 
 RUN set -euxo pipefail \
   ; mkdir  /var/www/snappymail \

From ad5b6fe27edb2da3bdcdbd2bda7e72cc30dc25d4 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Mon, 9 Oct 2023 10:53:45 +0200
Subject: [PATCH 20/41] Upgrade dovecot: fix proxying ipv6 via xclient

---
 core/dovecot/Dockerfile           | 3 ++-
 towncrier/newsfragments/2918.misc | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)
 create mode 100644 towncrier/newsfragments/2918.misc

diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile
index 872e1ecf..4f682ee0 100644
--- a/core/dovecot/Dockerfile
+++ b/core/dovecot/Dockerfile
@@ -7,7 +7,8 @@ ARG VERSION
 LABEL version=$VERSION
 
 RUN set -euxo pipefail \
-  ; apk add --no-cache dovecot dovecot-fts-xapian dovecot-lmtpd dovecot-pigeonhole-plugin dovecot-pop3d dovecot-submissiond rspamd-client xapian-core \
+  ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main 'dovecot<2.4' dovecot-fts-xapian dovecot-lmtpd dovecot-pigeonhole-plugin dovecot-pop3d dovecot-submissiond xapian-core \
+  ; apk add rspamd-client \
   ; mkdir /var/lib/dovecot
 
 COPY conf/ /conf/
diff --git a/towncrier/newsfragments/2918.misc b/towncrier/newsfragments/2918.misc
new file mode 100644
index 00000000..441806c4
--- /dev/null
+++ b/towncrier/newsfragments/2918.misc
@@ -0,0 +1 @@
+Upgrade dovecot to ensure we can proxy ipv6 via XCLIENT.

From 282401e6712a5abedc30e724eb99695fe6edd6ec Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Mon, 9 Oct 2023 11:17:53 +0200
Subject: [PATCH 21/41] Maybe fix CI

---
 webmails/Dockerfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/webmails/Dockerfile b/webmails/Dockerfile
index ef5df6c0..beaffd5d 100644
--- a/webmails/Dockerfile
+++ b/webmails/Dockerfile
@@ -19,6 +19,7 @@ RUN set -euxo pipefail \
     aspell-uk aspell-ru aspell-fr aspell-de aspell-en \
   ; rm /etc/nginx/http.d/default.conf \
   ; rm /etc/php81/php-fpm.d/www.conf \
+  ; mkdir -m 700 /root/.gnupg/ \
   ; gpg --import /tmp/snappymail.asc \
   ; gpg --import /tmp/roundcube.asc \
   ; echo extension=snuffleupagus > /etc/php81/conf.d/snuffleupagus.ini \
@@ -33,7 +34,7 @@ RUN set -euxo pipefail \
   ; cd /var/www \
   ; curl -sLo /dev/shm/roundcube.tgz ${ROUNDCUBE_URL} \
   ; curl -sLo /dev/shm/roundcube.tgz.asc ${ROUNDCUBE_URL}.asc \
-  ; gpg --status-fd 1 --verify /dev/shm/roundcube.tgz.asc \
+  ; gpg --status-fd 1 --verify /dev/shm/roundcube.tgz.asc /dev/shm/roundcube.tgz \
   ; tar xzf /dev/shm/roundcube.tgz \
   ; curl -sL ${CARDDAV_URL} | tar xz \
   ; mv roundcubemail-* roundcube \
@@ -59,7 +60,7 @@ RUN set -euxo pipefail \
   ; cd /var/www/snappymail \
   ; curl -sLo /dev/shm/snappymail.tgz  ${SNAPPYMAIL_URL} \
   ; curl -sLo /dev/shm/snappymail.tgz.asc  ${SNAPPYMAIL_URL}.asc \
-  ; gpg --status-fd 1 --verify /dev/shm/snappymail.tgz.asc \
+  ; gpg --status-fd 1 --verify /dev/shm/snappymail.tgz.asc /dev/shm/snappymail.tgz \
   ; cat /dev/shm/snappymail.tgz | tar xz
 
 # SnappyMail login

From 63a742100941e7b2fb5902babf5186ae0cb6f267 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Mon, 9 Oct 2023 15:43:35 +0200
Subject: [PATCH 22/41] Ensure that the cache is not used

---
 core/dovecot/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile
index 4f682ee0..580b7fbd 100644
--- a/core/dovecot/Dockerfile
+++ b/core/dovecot/Dockerfile
@@ -7,7 +7,7 @@ ARG VERSION
 LABEL version=$VERSION
 
 RUN set -euxo pipefail \
-  ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main 'dovecot<2.4' dovecot-fts-xapian dovecot-lmtpd dovecot-pigeonhole-plugin dovecot-pop3d dovecot-submissiond xapian-core \
+  ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main 'dovecot<2.4' dovecot-fts-xapian dovecot-lmtpd dovecot-pigeonhole-plugin dovecot-pop3d dovecot-submissiond xapian-core  \
   ; apk add rspamd-client \
   ; mkdir /var/lib/dovecot
 

From fce092d4ec2490728efb2f59f216944df8eee144 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Mon, 9 Oct 2023 16:21:50 +0200
Subject: [PATCH 23/41] Ensure we use the edge community repo too

---
 core/dovecot/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile
index 580b7fbd..190b5a65 100644
--- a/core/dovecot/Dockerfile
+++ b/core/dovecot/Dockerfile
@@ -7,7 +7,7 @@ ARG VERSION
 LABEL version=$VERSION
 
 RUN set -euxo pipefail \
-  ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main 'dovecot<2.4' dovecot-fts-xapian dovecot-lmtpd dovecot-pigeonhole-plugin dovecot-pop3d dovecot-submissiond xapian-core  \
+  ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main 'dovecot<2.4' dovecot-fts-xapian dovecot-lmtpd dovecot-pigeonhole-plugin dovecot-pop3d dovecot-submissiond xapian-core \
   ; apk add rspamd-client \
   ; mkdir /var/lib/dovecot
 

From a0eac6ea163cd8dd50e4c9b3238bf2be7d295ebc Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@users.noreply.github.com>
Date: Mon, 9 Oct 2023 16:49:01 +0200
Subject: [PATCH 24/41] Update core/dovecot/Dockerfile

Co-authored-by: Dimitri Huisman <52963853+Diman0@users.noreply.github.com>
---
 core/dovecot/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile
index 190b5a65..5aa925cc 100644
--- a/core/dovecot/Dockerfile
+++ b/core/dovecot/Dockerfile
@@ -8,7 +8,7 @@ LABEL version=$VERSION
 
 RUN set -euxo pipefail \
   ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main 'dovecot<2.4' dovecot-fts-xapian dovecot-lmtpd dovecot-pigeonhole-plugin dovecot-pop3d dovecot-submissiond xapian-core \
-  ; apk add rspamd-client \
+  ; apk add --no-cache rspamd-client \
   ; mkdir /var/lib/dovecot
 
 COPY conf/ /conf/

From 36236848d269584310595cd08ac97d264cb37c16 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Mon, 9 Oct 2023 17:15:28 +0200
Subject: [PATCH 25/41] switch to fts-flatcurve

---
 core/dovecot/Dockerfile             | 3 ++-
 core/dovecot/conf/dovecot.conf      | 5 ++---
 towncrier/newsfragments/2971.bugfix | 1 +
 3 files changed, 5 insertions(+), 4 deletions(-)
 create mode 100644 towncrier/newsfragments/2971.bugfix

diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile
index 5aa925cc..2681bcff 100644
--- a/core/dovecot/Dockerfile
+++ b/core/dovecot/Dockerfile
@@ -7,7 +7,8 @@ ARG VERSION
 LABEL version=$VERSION
 
 RUN set -euxo pipefail \
-  ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main 'dovecot<2.4' dovecot-fts-xapian dovecot-lmtpd dovecot-pigeonhole-plugin dovecot-pop3d dovecot-submissiond xapian-core \
+  ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main 'dovecot<2.4' dovecot-lmtpd dovecot-pigeonhole-plugin dovecot-pop3d dovecot-submissiond \
+  ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing dovecot-fts-flatcurve \
   ; apk add --no-cache rspamd-client \
   ; mkdir /var/lib/dovecot
 
diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf
index e35ab4a1..efa0fd0b 100644
--- a/core/dovecot/conf/dovecot.conf
+++ b/core/dovecot/conf/dovecot.conf
@@ -37,7 +37,7 @@ mail_plugins = $mail_plugins quota quota_clone{{ ' ' }}
   zlib{{ ' ' }}
   {%- endif %}
   {%- if (FULL_TEXT_SEARCH or '').lower() not in ['off', 'false', '0'] -%}
-  fts fts_xapian
+  fts fts_flatcurve
   {%- endif %}
 default_vsz_limit = 2GB
 
@@ -57,8 +57,7 @@ plugin {
   quota_clone_dict = proxy:/tmp/podop.socket:quota
 
   {% if (FULL_TEXT_SEARCH or '').lower() not in ['off', 'false', '0'] %}
-  fts = xapian
-  fts_xapian = partial=2 full=30
+  fts = flatcurve
   fts_autoindex = yes
   fts_enforced = yes
   fts_autoindex_exclude = \Trash
diff --git a/towncrier/newsfragments/2971.bugfix b/towncrier/newsfragments/2971.bugfix
new file mode 100644
index 00000000..c77b034a
--- /dev/null
+++ b/towncrier/newsfragments/2971.bugfix
@@ -0,0 +1 @@
+Switch from fts-xapian to fts-flatcurve. This should address the problem with indexes getting too big and will be the default in dovecot 2.4

From eb44783eb2d123500372f9f6220423d1f871f6c6 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Mon, 9 Oct 2023 17:40:44 +0200
Subject: [PATCH 26/41] we need this in front too

---
 core/nginx/Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/nginx/Dockerfile b/core/nginx/Dockerfile
index f2eaac81..cacb6c99 100644
--- a/core/nginx/Dockerfile
+++ b/core/nginx/Dockerfile
@@ -17,7 +17,8 @@ ARG VERSION
 LABEL version=$VERSION
 
 RUN set -euxo pipefail \
-  ; apk add --no-cache certbot nginx nginx-mod-http-brotli nginx-mod-mail openssl dovecot-lua dovecot-pigeonhole-plugin dovecot-lmtpd dovecot-pop3d dovecot-submissiond
+  ; apk add --no-cache certbot nginx nginx-mod-http-brotli nginx-mod-mail openssl \
+  ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main dovecot-lua dovecot-pigeonhole-plugin 'dovecot-lmtpd<2.4' dovecot-pop3d dovecot-submissiond
 
 COPY conf/ /conf/
 COPY --from=static /static/ /static/

From 39af87dff406d0e33cf08d18ebcff9c751abd8f8 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Mon, 9 Oct 2023 17:38:15 +0200
Subject: [PATCH 27/41] Add language stops

---
 core/dovecot/conf/dovecot.conf  | 2 ++
 docs/configuration.rst          | 7 +++++--
 setup/flavors/compose/mailu.env | 4 +++-
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf
index efa0fd0b..20f7054c 100644
--- a/core/dovecot/conf/dovecot.conf
+++ b/core/dovecot/conf/dovecot.conf
@@ -58,6 +58,8 @@ plugin {
 
   {% if (FULL_TEXT_SEARCH or '').lower() not in ['off', 'false', '0'] %}
   fts = flatcurve
+  fts_languages = {% if FULL_TEXT_SEARCH %}{{ FULL_TEXT_SEARCH.split(",") | join(" ") }}{% else %}en{% endif %}
+  fts_tokenizers = generic email-address
   fts_autoindex = yes
   fts_enforced = yes
   fts_autoindex_exclude = \Trash
diff --git a/docs/configuration.rst b/docs/configuration.rst
index 2f7e27a7..fbf284a4 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -131,12 +131,15 @@ later classify incoming mail based on the custom part.
 The ``DMARC_RUA`` and ``DMARC_RUF`` are DMARC protocol specific values. They hold
 the localpart for DMARC rua and ruf email addresses.
 
-Full-text search is enabled for IMAP is enabled by default. This feature can be disabled
-(e.g. for performance reasons) by setting the optional variable ``FULL_TEXT_SEARCH`` to ``off``.
+The ``FULL_TEXT_SEARCH`` variable (default: 'en') is a comma separated list of
+language codes as defined on `fts_languages`_. This feature can be disabled
+(e.g. for performance reasons) by setting the variable to ``off``.
 
 You can set a global ``DEFAULT_QUOTA`` to be used for mailboxes when the domain has
 no specific quota configured.
 
+.. _`fts_languages`: https://doc.dovecot.org/settings/plugin/fts-plugin/#fts-languages
+
 .. _web_settings:
 
 Web settings
diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env
index 616cd89b..ef95d8f7 100644
--- a/setup/flavors/compose/mailu.env
+++ b/setup/flavors/compose/mailu.env
@@ -110,7 +110,9 @@ COMPRESSION={{ compression }}
 # change compression-level, default: 6 (value: 1-9)
 COMPRESSION_LEVEL={{ compression_level }}
 
-# IMAP full-text search is enabled by default. Set the following variable to off in order to disable the feature.
+# IMAP full-text search is enabled by default.
+# Set the following variable to off in order to disable the feature
+# or a comma separated list of language codes to support
 # FULL_TEXT_SEARCH=off
 
 ###################################

From 3d9a8bc21f1ca02ade3f392818174f9d9bcfe0af Mon Sep 17 00:00:00 2001
From: Dimitri Huisman <diman@huisman.xyz>
Date: Mon, 9 Oct 2023 16:19:20 +0000
Subject: [PATCH 28/41] RESTful API bugfix for domains. Add authentication
 tokens to API

---
 core/admin/mailu/api/v1/__init__.py           |   3 +-
 .../mailu/api/v1/{domains.py => domain.py}    |  16 +-
 core/admin/mailu/api/v1/token.py              | 172 ++++++++++++++++++
 3 files changed, 182 insertions(+), 9 deletions(-)
 rename core/admin/mailu/api/v1/{domains.py => domain.py} (97%)
 create mode 100644 core/admin/mailu/api/v1/token.py

diff --git a/core/admin/mailu/api/v1/__init__.py b/core/admin/mailu/api/v1/__init__.py
index 44b6ec57..9b3e98f8 100644
--- a/core/admin/mailu/api/v1/__init__.py
+++ b/core/admin/mailu/api/v1/__init__.py
@@ -37,7 +37,8 @@ error_fields = api.model('Error', {
     'message': fields.String,
 })
 
-from . import domains
+from . import domain
 from . import alias
 from . import relay
 from . import user
+from . import token
diff --git a/core/admin/mailu/api/v1/domains.py b/core/admin/mailu/api/v1/domain.py
similarity index 97%
rename from core/admin/mailu/api/v1/domains.py
rename to core/admin/mailu/api/v1/domain.py
index 7043da3d..c5f98530 100644
--- a/core/admin/mailu/api/v1/domains.py
+++ b/core/admin/mailu/api/v1/domain.py
@@ -115,13 +115,13 @@ class Domains(Resource):
         if 'comment' in data:
             domain_new.comment = data['comment']
         if 'max_users' in data:
-            domain_new.comment = data['max_users']
+            domain_new.max_users = data['max_users']
         if 'max_aliases' in data:
-            domain_new.comment = data['max_aliases']
+            domain_new.max_aliases = data['max_aliases']
         if 'max_quota_bytes' in data:
-            domain_new.comment = data['max_quota_bytes']
+            domain_new.max_quota_bytes = data['max_quota_bytes']
         if 'signup_enabled' in data:
-            domain_new.comment = data['signup_enabled']
+            domain_new.signup_enabled = data['signup_enabled']
         models.db.session.add(domain_new)
         #apply the changes
         db.session.commit()
@@ -177,13 +177,13 @@ class Domain(Resource):
         if 'comment' in data:
             domain_found.comment = data['comment']
         if 'max_users' in data:
-            domain_found.comment = data['max_users']
+            domain_found.max_users = data['max_users']
         if 'max_aliases' in data:
-            domain_found.comment = data['max_aliases']
+            domain_found.max_aliases = data['max_aliases']
         if 'max_quota_bytes' in data:
-            domain_found.comment = data['max_quota_bytes']
+            domain_found.max_quota_bytes = data['max_quota_bytes']
         if 'signup_enabled' in data:
-            domain_found.comment = data['signup_enabled']
+            domain_found.signup_enabled = data['signup_enabled']
         models.db.session.add(domain_found)
 
         #apply the changes
diff --git a/core/admin/mailu/api/v1/token.py b/core/admin/mailu/api/v1/token.py
new file mode 100644
index 00000000..5f9d8d13
--- /dev/null
+++ b/core/admin/mailu/api/v1/token.py
@@ -0,0 +1,172 @@
+from flask_restx import Resource, fields, marshal
+import validators, datetime
+import flask
+from passlib import pwd
+
+from . import api, response_fields
+from .. import common
+from ... import models
+
+db = models.db
+
+token = api.namespace('token', description='Token operations')
+
+token_user_fields = api.model('TokenGetResponse', {
+    'id': fields.String(description='The record id of the token (unique identifier)', example='1'),
+    'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='user_email'),
+    'comment': fields.String(description='A description for the token. This description is shown on the Authentication tokens page', example='my comment'),
+    'AuthorizedIP': fields.String(description='Comma separated list of white listed IP addresses or networks that may use this token', example='88.77.66.55', attribute='ip'),
+    'Created': fields.String(description='The date when the token was created', example='John.Doe@example.com', attribute='created_at'),
+    'Last edit': fields.String(description='The date when the token was last modifified', example='John.Doe@example.com', attribute='updated_at')
+})
+
+token_user_fields_post = api.model('TokenPost', {
+    'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='user_email'),
+    'comment': fields.String(description='A description for the token. This description is shown on the Authentication tokens page', example='my comment'),
+    'AuthorizedIP': fields.String(description='Comma separated list of white listed IP addresses or networks that may use this token', example='88.77.66.55', attribute='ip')
+})
+
+token_user_fields_post2 = api.model('TokenPost2', {
+    'comment': fields.String(description='A description for the token. This description is shown on the Authentication tokens page', example='my comment'),
+    'AuthorizedIP': fields.String(description='Comma separated list of white listed IP addresses or networks that may use this token', example='88.77.66.55', attribute='ip')
+})
+
+token_user_post_response = api.model('TokenPostResponse', {
+    'id': fields.String(description='The record id of the token (unique identifier)', example='1'),
+    'token': fields.String(description='The created authentication token for the user.', example='2caf6607de5129e4748a2c061aee56f2', attribute='password'),
+    'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='user_email'),
+    'comment': fields.String(description='A description for the token. This description is shown on the Authentication tokens page', example='my comment'),
+    'AuthorizedIP': fields.String(description='Comma separated list of white listed IP addresses or networks that may use this token', example='88.77.66.55', attribute='ip'),
+    'Created': fields.String(description='The date when the token was created', example='John.Doe@example.com', attribute='created_at')
+})
+
+@token.route('')
+class Tokens(Resource):
+    @token.doc('list_tokens')
+    @token.marshal_with(token_user_fields, as_list=True, skip_none=True, mask=None)
+    @token.doc(security='Bearer')
+    @common.api_token_authorization
+    def get(self):
+        """List tokens"""
+        return models.Token.query.all()
+
+    @token.doc('create_token')
+    @token.expect(token_user_fields_post)
+    @token.response(200, 'Success', token_user_post_response)
+    @token.response(400, 'Input validation exception')
+    @token.response(409, 'Duplicate relay', response_fields)
+    @token.doc(security='Bearer')
+    @common.api_token_authorization
+    def post(self):
+        """ Create a new token"""
+        data = api.payload
+        email = data['email']
+        if not validators.email(email):
+            return { 'code': 400, 'message': f'Provided email address {email} is not a valid email address'}, 400
+        user_found = models.User.query.get(email)
+        if not user_found:
+            return {'code': 404, 'message': f'User {email} cannot be found'}, 404
+        tokens = user_found.tokens
+
+        token_new = models.Token(user_email=data['email'])
+        if 'comment' in data:
+            token_new.comment = data['comment']
+        if 'AuthorizedIP' in data:
+            token_new.ip = data['AuthorizedIP'].replace(' ','').split(',')
+        raw_password = pwd.genword(entropy=128, length=32, charset="hex")
+        token_new.set_password(raw_password)
+        models.db.session.add(token_new)
+        #apply the changes
+        db.session.commit()
+        flask.current_app.logger.info(f'token_new.id == {token_new.id}.')
+        response_dict  = {
+            'id' : token_new.id,
+            'token' : raw_password,
+            'email' : token_new.user_email,
+            'comment' : token_new.comment,
+            'AuthorizedIP' : token_new.ip,
+            'Created': str(token_new.created_at),
+            }
+
+        return  response_dict
+
+@token.route('user/<string:email>')
+class Token(Resource):
+    @token.doc('find_tokens_of_user')
+    @token.marshal_with(token_user_fields, as_list=True, skip_none=True, mask=None)
+    @token.doc(security='Bearer')
+    @common.api_token_authorization
+    def get(self, email):
+        "Find tokens of user"
+        if not validators.email(email):
+            return { 'code': 400, 'message': f'Provided email address {email} is not a valid email address'}, 400
+        user_found = models.User.query.get(email)
+        if not user_found:
+            return {'code': 404, 'message': f'User {email} cannot be found'}, 404
+        tokens = user_found.tokens
+        return tokens
+
+    @token.doc('create_token')
+    @token.expect(token_user_fields_post2)
+    @token.response(200, 'Success', token_user_post_response)
+    @token.response(400, 'Input validation exception')
+    @token.response(409, 'Duplicate relay', response_fields)
+    @token.doc(security='Bearer')
+    @common.api_token_authorization
+    def post(self, email):
+        """ Create a new token for user"""
+        data = api.payload
+        if not validators.email(email):
+            return { 'code': 400, 'message': f'Provided email address {email} is not a valid email address'}, 400
+        user_found = models.User.query.get(email)
+        if not user_found:
+            return {'code': 404, 'message': f'User {email} cannot be found'}, 404
+
+        token_new = models.Token(user_email=email)
+        if 'comment' in data:
+            token_new.comment = data['comment']
+        if 'AuthorizedIP' in data:
+            token_new.ip = token_new.ip = data['AuthorizedIP'].replace(' ','').split(',')
+        raw_password = pwd.genword(entropy=128, length=32, charset="hex")
+        token_new.set_password(raw_password)
+        models.db.session.add(token_new)
+        #apply the changes
+        db.session.commit()
+        flask.current_app.logger.info(f'token_new.id == {token_new.id}.')
+        response_dict  = {
+            'id' : token_new.id,
+            'token' : raw_password,
+            'email' : token_new.user_email,
+            'comment' : token_new.comment,
+            'AuthorizedIP' : token_new.ip,
+            'Created': str(token_new.created_at),
+            }
+        return  response_dict
+
+@token.route('/<string:token_id>')
+class Token(Resource):
+    @token.doc('find_token')
+    @token.marshal_with(token_user_fields, as_list=True, skip_none=True, mask=None)
+    @token.doc(security='Bearer')
+    @common.api_token_authorization
+    def get(self, token_id):
+        "Find token"
+        token = models.Token.query.get(token_id)
+        if not token:
+            return { 'code' : 404, 'message' : f'Record cannot be found for id {token_id} or invalid id provided'}, 404
+        return token
+
+    @token.doc('delete_token')
+    @token.response(200, 'Success', response_fields)
+    @token.response(400, 'Input validation exception', response_fields)
+    @token.response(404, 'Token not found', response_fields)
+    @token.doc(security='Bearer')
+    @common.api_token_authorization
+    def delete(self, token_id):
+        """ Delete token """
+        token = models.Token.query.get(token_id)
+        if not token:
+            return { 'code' : 404, 'message' : f'Record cannot be found for id {token_id} or invalid id provided'}, 404
+        db.session.delete(token)
+        db.session.commit()
+        return {'code': 200, 'message': f'Token with id {token_id} has been deleted'}, 200

From fa2fb1369df1ae5de540b957ac274090498e5210 Mon Sep 17 00:00:00 2001
From: Dimitri Huisman <diman@huisman.xyz>
Date: Mon, 9 Oct 2023 16:23:22 +0000
Subject: [PATCH 29/41] Add changelog entry

---
 towncrier/newsfragments/2974.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 towncrier/newsfragments/2974.feature

diff --git a/towncrier/newsfragments/2974.feature b/towncrier/newsfragments/2974.feature
new file mode 100644
index 00000000..5351654e
--- /dev/null
+++ b/towncrier/newsfragments/2974.feature
@@ -0,0 +1 @@
+Enhance RESTful API with functionality for managing authentication tokens of users

From 5056fa01384e92d270c3aebad4fd97d59a90b895 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Mon, 9 Oct 2023 18:32:28 +0200
Subject: [PATCH 30/41] Add fts_filters

---
 core/dovecot/conf/dovecot.conf | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf
index 20f7054c..cd89af29 100644
--- a/core/dovecot/conf/dovecot.conf
+++ b/core/dovecot/conf/dovecot.conf
@@ -63,6 +63,8 @@ plugin {
   fts_autoindex = yes
   fts_enforced = yes
   fts_autoindex_exclude = \Trash
+  fts_filters = normalizer-icu stopwords
+  fts_filters_en = lowercase english-possessive stopwords
   {% endif %}
 
   {% if COMPRESSION in [ 'gz', 'bz2', 'lz4', 'zstd' ] %}

From 6ba8d62572eb295e981007c478f74df4abbbf49f Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Tue, 10 Oct 2023 08:20:46 +0200
Subject: [PATCH 31/41] Optimize french

---
 core/dovecot/conf/dovecot.conf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf
index cd89af29..ea14ae33 100644
--- a/core/dovecot/conf/dovecot.conf
+++ b/core/dovecot/conf/dovecot.conf
@@ -65,6 +65,7 @@ plugin {
   fts_autoindex_exclude = \Trash
   fts_filters = normalizer-icu stopwords
   fts_filters_en = lowercase english-possessive stopwords
+  fts_filters_fr = lowercase contractions stopwords
   {% endif %}
 
   {% if COMPRESSION in [ 'gz', 'bz2', 'lz4', 'zstd' ] %}

From 6962b9bcb5c7195c1e836a495a63bf4951abb2a4 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Tue, 10 Oct 2023 08:40:33 +0200
Subject: [PATCH 32/41] Enable FTS on attachments too

---
 core/dovecot/Dockerfile        |  4 ++--
 core/dovecot/conf/dovecot.conf | 12 ++++++++++++
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile
index 2681bcff..bb7507cd 100644
--- a/core/dovecot/Dockerfile
+++ b/core/dovecot/Dockerfile
@@ -7,8 +7,8 @@ ARG VERSION
 LABEL version=$VERSION
 
 RUN set -euxo pipefail \
-  ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main 'dovecot<2.4' dovecot-lmtpd dovecot-pigeonhole-plugin dovecot-pop3d dovecot-submissiond \
-  ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing dovecot-fts-flatcurve \
+  ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main 'dovecot<2.4' dovecot-lmtpd dovecot-pigeonhole-plugin dovecot-pop3d dovecot-submissiond poppler-utils \
+  ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing dovecot-fts-flatcurve catdoc \
   ; apk add --no-cache rspamd-client \
   ; mkdir /var/lib/dovecot
 
diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf
index ea14ae33..6f25e99a 100644
--- a/core/dovecot/conf/dovecot.conf
+++ b/core/dovecot/conf/dovecot.conf
@@ -66,6 +66,8 @@ plugin {
   fts_filters = normalizer-icu stopwords
   fts_filters_en = lowercase english-possessive stopwords
   fts_filters_fr = lowercase contractions stopwords
+
+  fts_decoder = decode2text
   {% endif %}
 
   {% if COMPRESSION in [ 'gz', 'bz2', 'lz4', 'zstd' ] %}
@@ -77,6 +79,16 @@ plugin {
   {% endif %}
 }
 
+{% if FULL_TEXT_SEARCH %}
+service decode2text {
+  executable = script /usr/libexec/dovecot/decode2text.sh
+  user = nobody
+  unix_listener decode2text {
+    mode = 0666
+  }
+}
+{% endif %}
+
 ###############
 # Authentication
 ###############

From 9ae8715a6afe58b3a57ab063d2cc960630e40b25 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Tue, 10 Oct 2023 08:46:13 +0200
Subject: [PATCH 33/41] Clarify how one should upgrade

---
 towncrier/newsfragments/2971.bugfix | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/towncrier/newsfragments/2971.bugfix b/towncrier/newsfragments/2971.bugfix
index c77b034a..f767da6b 100644
--- a/towncrier/newsfragments/2971.bugfix
+++ b/towncrier/newsfragments/2971.bugfix
@@ -1 +1,9 @@
 Switch from fts-xapian to fts-flatcurve. This should address the problem with indexes getting too big and will be the default in dovecot 2.4
+
+You can dispose of old indexes using a command such as:
+
+find /mailu/mail -type d -name xapian-indexes -prune -exec rm -r {} \+
+
+And re-index using:
+
+docker compose exec imap doveadm fts rescan -A

From 25964b61d59be0705c0c41c437680fd658fee314 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Tue, 10 Oct 2023 09:16:05 +0200
Subject: [PATCH 34/41] Add the working command

---
 towncrier/newsfragments/2971.bugfix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/towncrier/newsfragments/2971.bugfix b/towncrier/newsfragments/2971.bugfix
index f767da6b..02b72758 100644
--- a/towncrier/newsfragments/2971.bugfix
+++ b/towncrier/newsfragments/2971.bugfix
@@ -6,4 +6,4 @@ find /mailu/mail -type d -name xapian-indexes -prune -exec rm -r {} \+
 
 And re-index using:
 
-docker compose exec imap doveadm fts rescan -A
+docker compose exec imap doveadm index -A '*'

From 16e9d152dd9a4de69d9ba155a1859aee43387cc9 Mon Sep 17 00:00:00 2001
From: Dimitri Huisman <diman@huisman.xyz>
Date: Tue, 10 Oct 2023 08:19:36 +0000
Subject: [PATCH 35/41] Forbidden_file_extension.map could not be overridden.

---
 core/rspamd/start.py                | 3 ++-
 towncrier/newsfragments/2937.bugfix | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)
 create mode 100644 towncrier/newsfragments/2937.bugfix

diff --git a/core/rspamd/start.py b/core/rspamd/start.py
index 54b2f192..67574a01 100755
--- a/core/rspamd/start.py
+++ b/core/rspamd/start.py
@@ -16,7 +16,8 @@ env = system.set_env()
 config_files = []
 for rspamd_file in glob.glob("/conf/*"):
     conf.jinja(rspamd_file, env, os.path.join("/etc/rspamd/local.d", os.path.basename(rspamd_file)))
-    config_files.append(os.path.basename(rspamd_file))
+    if rspamd_file != '/conf/forbidden_file_extension.map':
+        config_files.append(os.path.basename(rspamd_file))
 
 for override_file in glob.glob("/overrides/*"):
     if os.path.basename(override_file) not in config_files:
diff --git a/towncrier/newsfragments/2937.bugfix b/towncrier/newsfragments/2937.bugfix
new file mode 100644
index 00000000..7ccd316c
--- /dev/null
+++ b/towncrier/newsfragments/2937.bugfix
@@ -0,0 +1,2 @@
+forbidden_file_extension.map could not be overridden. This file can be overriden to tweak with file extensions are allowed.
+The instructions on https://mailu.io/master/antispam.html#can-i-change-the-list-of-authorized-file-attachments work again.

From 0718de824b5a0780aea64217c90c12d4b185690b Mon Sep 17 00:00:00 2001
From: Dimitri Huisman <diman@huisman.xyz>
Date: Tue, 10 Oct 2023 10:46:52 +0000
Subject: [PATCH 36/41] Forgot to remove debug logging. Use rfc5737 for IP
 addresses in documentation.

---
 core/admin/mailu/api/v1/token.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/core/admin/mailu/api/v1/token.py b/core/admin/mailu/api/v1/token.py
index 5f9d8d13..0f2b5b7a 100644
--- a/core/admin/mailu/api/v1/token.py
+++ b/core/admin/mailu/api/v1/token.py
@@ -15,7 +15,7 @@ token_user_fields = api.model('TokenGetResponse', {
     'id': fields.String(description='The record id of the token (unique identifier)', example='1'),
     'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='user_email'),
     'comment': fields.String(description='A description for the token. This description is shown on the Authentication tokens page', example='my comment'),
-    'AuthorizedIP': fields.String(description='Comma separated list of white listed IP addresses or networks that may use this token', example='88.77.66.55', attribute='ip'),
+    'AuthorizedIP': fields.String(description='Comma separated list of white listed IP addresses or networks that may use this token.', example="['203.0.113.0/24']", attribute='ip'),
     'Created': fields.String(description='The date when the token was created', example='John.Doe@example.com', attribute='created_at'),
     'Last edit': fields.String(description='The date when the token was last modifified', example='John.Doe@example.com', attribute='updated_at')
 })
@@ -23,12 +23,12 @@ token_user_fields = api.model('TokenGetResponse', {
 token_user_fields_post = api.model('TokenPost', {
     'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='user_email'),
     'comment': fields.String(description='A description for the token. This description is shown on the Authentication tokens page', example='my comment'),
-    'AuthorizedIP': fields.String(description='Comma separated list of white listed IP addresses or networks that may use this token', example='88.77.66.55', attribute='ip')
+    'AuthorizedIP': fields.String(description='Comma separated list of white listed IP addresses or networks that may use this token.', example="['203.0.113.0/24']", attribute='ip'),
 })
 
 token_user_fields_post2 = api.model('TokenPost2', {
     'comment': fields.String(description='A description for the token. This description is shown on the Authentication tokens page', example='my comment'),
-    'AuthorizedIP': fields.String(description='Comma separated list of white listed IP addresses or networks that may use this token', example='88.77.66.55', attribute='ip')
+    'AuthorizedIP': fields.String(description='Comma separated list of white listed IP addresses or networks that may use this token.', example="['203.0.113.0/24']", attribute='ip'),
 })
 
 token_user_post_response = api.model('TokenPostResponse', {
@@ -36,7 +36,7 @@ token_user_post_response = api.model('TokenPostResponse', {
     'token': fields.String(description='The created authentication token for the user.', example='2caf6607de5129e4748a2c061aee56f2', attribute='password'),
     'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='user_email'),
     'comment': fields.String(description='A description for the token. This description is shown on the Authentication tokens page', example='my comment'),
-    'AuthorizedIP': fields.String(description='Comma separated list of white listed IP addresses or networks that may use this token', example='88.77.66.55', attribute='ip'),
+    'AuthorizedIP': fields.String(description='Comma separated list of white listed IP addresses or networks that may use this token.', example="['203.0.113.0/24']", attribute='ip'),
     'Created': fields.String(description='The date when the token was created', example='John.Doe@example.com', attribute='created_at')
 })
 
@@ -78,7 +78,6 @@ class Tokens(Resource):
         models.db.session.add(token_new)
         #apply the changes
         db.session.commit()
-        flask.current_app.logger.info(f'token_new.id == {token_new.id}.')
         response_dict  = {
             'id' : token_new.id,
             'token' : raw_password,
@@ -132,7 +131,6 @@ class Token(Resource):
         models.db.session.add(token_new)
         #apply the changes
         db.session.commit()
-        flask.current_app.logger.info(f'token_new.id == {token_new.id}.')
         response_dict  = {
             'id' : token_new.id,
             'token' : raw_password,

From 27e357190880a7863032361c70b23bee0f2ebfb9 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Tue, 10 Oct 2023 15:04:55 +0200
Subject: [PATCH 37/41] doc

---
 README.md      | 2 +-
 docs/index.rst | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 6a202940..29bd55a4 100644
--- a/README.md
+++ b/README.md
@@ -18,7 +18,7 @@ Features
 Main features include:
 
 - **Standard email server**, IMAP and IMAP+, SMTP and Submission with autoconfiguration profiles for clients
-- **Advanced email features**, aliases, domain aliases, custom routing
+- **Advanced email features**, aliases, domain aliases, custom routing, full-text search of email attachments
 - **Web access**, multiple Webmails and administration interface
 - **User features**, aliases, auto-reply, auto-forward, fetched accounts, managesieve
 - **Admin features**, global admins, announcements, per-domain delegation, quotas
diff --git a/docs/index.rst b/docs/index.rst
index 0f16335c..ed281e3a 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -24,7 +24,7 @@ popular groupware.
 Main features include:
 
 - **Standard email server**, IMAP and IMAP+, SMTP and Submission with autoconfiguration profiles for clients
-- **Advanced email features**, aliases, domain aliases, custom routing
+- **Advanced email features**, aliases, domain aliases, custom routing, full-text search of email attachments
 - **Web access**, multiple Webmails and administration interface
 - **User features**, aliases, auto-reply, auto-forward, fetched accounts, managesieve
 - **Admin features**, global admins, announcements, per-domain delegation, quotas

From 954fe40134e1ff450d3ee92f05e55fb560a87a45 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Tue, 10 Oct 2023 15:07:37 +0200
Subject: [PATCH 38/41] Towncrier

---
 towncrier/newsfragments/2971.bugfix | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/towncrier/newsfragments/2971.bugfix b/towncrier/newsfragments/2971.bugfix
index 02b72758..fd981e75 100644
--- a/towncrier/newsfragments/2971.bugfix
+++ b/towncrier/newsfragments/2971.bugfix
@@ -1,9 +1,12 @@
-Switch from fts-xapian to fts-flatcurve. This should address the problem with indexes getting too big and will be the default in dovecot 2.4
+- Switch from fts-xapian to fts-flatcurve. This should address the problem with indexes getting too big and will be the default in dovecot 2.4
+- Enable full-text search of email attachments
 
-You can dispose of old indexes using a command such as:
+If you would like more than english to be supported, please ensure you update your FULL_TEXT_SEARCH configuration variable.
+
+You may also want to dispose of old indexes using a command such as:
 
 find /mailu/mail -type d -name xapian-indexes -prune -exec rm -r {} \+
 
-And re-index using:
+And proactively force a reindexing using:
 
 docker compose exec imap doveadm index -A '*'

From 054fde8ac133ae6f99dafbd5375d9fd2280d0029 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Wed, 11 Oct 2023 14:37:27 +0200
Subject: [PATCH 39/41] Tika v1

---
 core/admin/mailu/configuration.py             |  2 ++
 core/admin/run_dev.sh                         |  1 +
 core/base/Dockerfile                          |  1 +
 core/dovecot/Dockerfile                       |  4 +--
 core/dovecot/conf/dovecot.conf                | 17 ++++-------
 setup/flavors/compose/docker-compose.yml      | 28 +++++++++++++++++++
 setup/flavors/compose/mailu.env               |  4 ++-
 .../templates/steps/compose/02_services.html  |  9 ++++++
 towncrier/newsfragments/2971.bugfix           |  2 +-
 9 files changed, 52 insertions(+), 16 deletions(-)

diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py
index bb7080c9..d324bf8d 100644
--- a/core/admin/mailu/configuration.py
+++ b/core/admin/mailu/configuration.py
@@ -75,6 +75,8 @@ DEFAULT_CONFIG = {
     'API': False,
     'WEB_API': '/api',
     'API_TOKEN': None,
+    'FULL_TEXT_SEARCH': 'en',
+    'FULL_TEXT_SEARCH_ATTACHMENTS': False,
     'LOG_LEVEL': 'INFO',
     'SESSION_KEY_BITS': 128,
     'SESSION_TIMEOUT': 3600,
diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh
index 0f7c6e05..3d1fc771 100755
--- a/core/admin/run_dev.sh
+++ b/core/admin/run_dev.sh
@@ -78,6 +78,7 @@ ENV \
     \
     ADMIN_ADDRESS="127.0.0.1" \
     FRONT_ADDRESS="127.0.0.1" \
+    FTS_ATTACHMENTS_ADDRESS="127.0.0.1" \
     SMTP_ADDRESS="127.0.0.1" \
     IMAP_ADDRESS="127.0.0.1" \
     REDIS_ADDRESS="127.0.0.1" \
diff --git a/core/base/Dockerfile b/core/base/Dockerfile
index e1087488..2f9c1142 100644
--- a/core/base/Dockerfile
+++ b/core/base/Dockerfile
@@ -81,6 +81,7 @@ ENV \
   PATH="/app/venv/bin:${PATH}" \
   ADMIN_ADDRESS="admin" \
   FRONT_ADDRESS="front" \
+  FTS_ATTACHMENTS_ADDRESS="tika" \
   SMTP_ADDRESS="smtp" \
   IMAP_ADDRESS="imap" \
   OLETOOLS_ADDRESS="oletools" \
diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile
index bb7507cd..2681bcff 100644
--- a/core/dovecot/Dockerfile
+++ b/core/dovecot/Dockerfile
@@ -7,8 +7,8 @@ ARG VERSION
 LABEL version=$VERSION
 
 RUN set -euxo pipefail \
-  ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main 'dovecot<2.4' dovecot-lmtpd dovecot-pigeonhole-plugin dovecot-pop3d dovecot-submissiond poppler-utils \
-  ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing dovecot-fts-flatcurve catdoc \
+  ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main 'dovecot<2.4' dovecot-lmtpd dovecot-pigeonhole-plugin dovecot-pop3d dovecot-submissiond \
+  ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing dovecot-fts-flatcurve \
   ; apk add --no-cache rspamd-client \
   ; mkdir /var/lib/dovecot
 
diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf
index 6f25e99a..c5173787 100644
--- a/core/dovecot/conf/dovecot.conf
+++ b/core/dovecot/conf/dovecot.conf
@@ -63,11 +63,14 @@ plugin {
   fts_autoindex = yes
   fts_enforced = yes
   fts_autoindex_exclude = \Trash
+  fts_autoindex_exclude1 = \Junk
   fts_filters = normalizer-icu stopwords
   fts_filters_en = lowercase english-possessive stopwords
   fts_filters_fr = lowercase contractions stopwords
-
-  fts_decoder = decode2text
+  fts_header_excludes = Received DKIM-* ARC-* X-* x-* Comments Delivered-To Return-Path Authentication-Results Message-ID References In-Reply-To Thread-* Accept-Language Content-* MIME-Version
+  {% if FULL_TEXT_SEARCH_ATTACHMENTS %}
+  fts_tika = http://{{ FTS_ATTACHMENTS_ADDRESS }}:9998/tika/
+  {% endif %}
   {% endif %}
 
   {% if COMPRESSION in [ 'gz', 'bz2', 'lz4', 'zstd' ] %}
@@ -79,16 +82,6 @@ plugin {
   {% endif %}
 }
 
-{% if FULL_TEXT_SEARCH %}
-service decode2text {
-  executable = script /usr/libexec/dovecot/decode2text.sh
-  user = nobody
-  unix_listener decode2text {
-    mode = 0666
-  }
-}
-{% endif %}
-
 ###############
 # Authentication
 ###############
diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml
index b266fec0..a81f9f44 100644
--- a/setup/flavors/compose/docker-compose.yml
+++ b/setup/flavors/compose/docker-compose.yml
@@ -98,8 +98,16 @@ services:
     volumes:
       - "{{ root }}/mail:/mail"
       - "{{ root }}/overrides/dovecot:/overrides:ro"
+    networks:
+      - default
+    {% if tika_enabled %}
+      - fts_attachments
+    {% endif %}
     depends_on:
       - front
+    {% if tika_enabled %}
+      - fts_attachments
+    {% endif %}
     {% if resolver_enabled %}
       - resolver
     dns:
@@ -140,6 +148,21 @@ services:
     {% endif %}
 {% endif %}
 
+{% if tika_enabled %}
+  fts_attachments:
+    image: apache/tika:2.9.0.0-full
+    hostname: tika
+    restart: always
+    networks:
+      - fts_attachments
+    depends_on:
+    {% if resolver_enabled %}
+      - resolver
+    dns:
+      - {{ dns }}
+    {% endif %}
+{% endif %}
+
   antispam:
     image: ${DOCKER_ORG:-ghcr.io/mailu}/${DOCKER_PREFIX:-}rspamd:${MAILU_VERSION:-{{ version }}}
     hostname: antispam
@@ -257,3 +280,8 @@ networks:
     driver: bridge
     internal: true
 {% endif %}
+{% if tika_enabled %}
+  fts_attachments:
+    driver: bridge
+    internal: true
+{% endif %}
diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env
index ef95d8f7..9380bab6 100644
--- a/setup/flavors/compose/mailu.env
+++ b/setup/flavors/compose/mailu.env
@@ -113,7 +113,7 @@ COMPRESSION_LEVEL={{ compression_level }}
 # IMAP full-text search is enabled by default.
 # Set the following variable to off in order to disable the feature
 # or a comma separated list of language codes to support
-# FULL_TEXT_SEARCH=off
+FULL_TEXT_SEARCH=en
 
 ###################################
 # Web settings
@@ -188,3 +188,5 @@ DEFAULT_SPAM_THRESHOLD=80
 # This is a mandatory setting for using the RESTful API.
 API_TOKEN={{ api_token }}
 
+# Whether tika should be enabled (scan/OCR email attachements)
+FULL_TEXT_SEARCH_ATTACHMENTS={{ tika_enabled }}
diff --git a/setup/templates/steps/compose/02_services.html b/setup/templates/steps/compose/02_services.html
index 2311e4a3..afa5e726 100644
--- a/setup/templates/steps/compose/02_services.html
+++ b/setup/templates/steps/compose/02_services.html
@@ -64,6 +64,15 @@ the security implications caused by such an increase of attack surface.<p>
   <i>Oletools scans documents in email attachements for malicious macros. It has a much lower memory footprint than a full-fledged anti-virus.</i>
 </div>
 
+<div class="form-check form-check-inline">
+  <label class="form-check-label">
+    <input class="form-check-input" type="checkbox" name="tika_enabled" value="true">
+    Enable Tika
+  </label>
+
+  <i>Tika scans documents in email attachments, process (OCR, keyword extraction) and then index them in a way they can be efficiently searched. This requires significant ressources (RAM, CPU and storage).</i>
+</div>
+
 <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.6.0/jquery.min.js"></script>
 
 <script type="text/javascript" src="{{ url_for('static', filename='render.js') }}"></script>
diff --git a/towncrier/newsfragments/2971.bugfix b/towncrier/newsfragments/2971.bugfix
index fd981e75..55d775bb 100644
--- a/towncrier/newsfragments/2971.bugfix
+++ b/towncrier/newsfragments/2971.bugfix
@@ -1,5 +1,5 @@
 - Switch from fts-xapian to fts-flatcurve. This should address the problem with indexes getting too big and will be the default in dovecot 2.4
-- Enable full-text search of email attachments
+- Enable full-text search of email attachments if configured (via Tika: you'll need to re-run setup)
 
 If you would like more than english to be supported, please ensure you update your FULL_TEXT_SEARCH configuration variable.
 

From 0026b8db241b2eb8404886963b6ec60bdd724a21 Mon Sep 17 00:00:00 2001
From: Dimitri Huisman <diman@huisman.xyz>
Date: Wed, 11 Oct 2023 15:49:52 +0000
Subject: [PATCH 40/41] Enhance RESTful API user retrieval with quota used
 bytes. This is the current size of the user's email box in bytes.

---
 core/admin/mailu/api/v1/user.py      | 1 +
 towncrier/newsfragments/2824.feature | 1 +
 2 files changed, 2 insertions(+)
 create mode 100644 towncrier/newsfragments/2824.feature

diff --git a/core/admin/mailu/api/v1/user.py b/core/admin/mailu/api/v1/user.py
index d9195e3d..441845c2 100644
--- a/core/admin/mailu/api/v1/user.py
+++ b/core/admin/mailu/api/v1/user.py
@@ -14,6 +14,7 @@ user_fields_get = api.model('UserGet', {
     'password': fields.String(description="Hash of the user's password; Example='$bcrypt-sha256$v=2,t=2b,r=12$fmsAdJbYAD1gGQIE5nfJq.$zLkQUEs2XZfTpAEpcix/1k5UTNPm0jO'"),
     'comment': fields.String(description='A description for the user. This description is shown on the Users page', example='my comment'),
     'quota_bytes': fields.Integer(description='The maximum quota for the user’s email box in bytes', example='1000000000'),
+    'quota_bytes_used': fields.Integer(description='The size of the user’s email box in bytes', example='5000000'),
     'global_admin': fields.Boolean(description='Make the user a global administrator'),
     'enabled': fields.Boolean(description='Enable the user. When an user is disabled, the user is unable to login to the Admin GUI or webmail or access his email via IMAP/POP3 or send mail'),
     'change_pw_next_login': fields.Boolean(description='Force the user to change their password at next login'),
diff --git a/towncrier/newsfragments/2824.feature b/towncrier/newsfragments/2824.feature
new file mode 100644
index 00000000..a5c65570
--- /dev/null
+++ b/towncrier/newsfragments/2824.feature
@@ -0,0 +1 @@
+Enhance RESTful API user retrieval with quota used bytes. This is the current size of the user's email box in bytes.

From 0a7797e96b6bdcbcd330d5106b034586c8eea53d Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Thu, 12 Oct 2023 20:42:16 +0200
Subject: [PATCH 41/41] Review

---
 setup/flavors/compose/mailu.env                | 10 +++++-----
 setup/templates/steps/compose/02_services.html |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env
index 9380bab6..cffafe15 100644
--- a/setup/flavors/compose/mailu.env
+++ b/setup/flavors/compose/mailu.env
@@ -49,19 +49,19 @@ DISABLE_STATISTICS={{ disable_statistics or 'False' }}
 # Expose the admin interface (value: true, false)
 ADMIN={{ admin_enabled or 'false' }}
 
-# Choose which webmail to run if any (values: roundcube, snappymail, none)
+# Choose which webmail to run if any (values: roundcube, snappymail, none). To enable this feature, recreate the docker-compose.yml file via setup.
 WEBMAIL={{ webmail_type }}
 
 # Expose the API interface (value: true, false)
 API={{ api_enabled or 'false' }}
 
-# Dav server implementation (value: radicale, none)
+# Dav server implementation (value: radicale, none). To enable this feature, recreate the docker-compose.yml file via setup.
 WEBDAV={{ webdav_enabled or 'none' }}
 
-# Antivirus solution (value: clamav, none)
+# Antivirus solution (value: clamav, none). To enable this feature, recreate the docker-compose.yml file via setup.
 ANTIVIRUS={{ antivirus_enabled or 'none' }}
 
-# Scan Macros solution (value: true, false)
+# Scan Macros solution (value: true, false). To enable this feature, recreate the docker-compose.yml file via setup.
 SCAN_MACROS={{ oletools_enabled or 'false' }}
 
 ###################################
@@ -188,5 +188,5 @@ DEFAULT_SPAM_THRESHOLD=80
 # This is a mandatory setting for using the RESTful API.
 API_TOKEN={{ api_token }}
 
-# Whether tika should be enabled (scan/OCR email attachements)
+# Whether tika should be enabled (scan/OCR email attachements). To enable this feature, recreate the docker-compose.yml file via setup.
 FULL_TEXT_SEARCH_ATTACHMENTS={{ tika_enabled }}
diff --git a/setup/templates/steps/compose/02_services.html b/setup/templates/steps/compose/02_services.html
index afa5e726..701fa82f 100644
--- a/setup/templates/steps/compose/02_services.html
+++ b/setup/templates/steps/compose/02_services.html
@@ -70,7 +70,7 @@ the security implications caused by such an increase of attack surface.<p>
     Enable Tika
   </label>
 
-  <i>Tika scans documents in email attachments, process (OCR, keyword extraction) and then index them in a way they can be efficiently searched. This requires significant ressources (RAM, CPU and storage).</i>
+  <i>Tika enables the functionality for searching through attachments. Tika scans documents in email attachments, process (OCR, keyword extraction) and then index them in a way they can be efficiently searched. This requires significant ressources (RAM, CPU and storage).</i>
 </div>
 
 <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.6.0/jquery.min.js"></script>