Merge branch 'Mailu:master' into master

README.md

@@ -18,7 +18,7 @@ Features
 Main features include:
 
 - **Standard email server**, IMAP and IMAP+, SMTP and Submission with autoconfiguration profiles for clients
-- **Advanced email features**, aliases, domain aliases, custom routing
+- **Advanced email features**, aliases, domain aliases, custom routing, full-text search of email attachments
 - **Web access**, multiple Webmails and administration interface
 - **User features**, aliases, auto-reply, auto-forward, fetched accounts, managesieve
 - **Admin features**, global admins, announcements, per-domain delegation, quotas

core/admin/mailu/api/v1/__init__.py

@@ -37,7 +37,8 @@ error_fields = api.model('Error', {
     'message': fields.String,
 })
 
-from . import domains
+from . import domain
 from . import alias
 from . import relay
 from . import user
+from . import token

core/admin/mailu/api/v1/domain.py

@@ -115,13 +115,13 @@ class Domains(Resource):
         if 'comment' in data:
             domain_new.comment = data['comment']
         if 'max_users' in data:
-            domain_new.comment = data['max_users']
+            domain_new.max_users = data['max_users']
         if 'max_aliases' in data:
-            domain_new.comment = data['max_aliases']
+            domain_new.max_aliases = data['max_aliases']
         if 'max_quota_bytes' in data:
-            domain_new.comment = data['max_quota_bytes']
+            domain_new.max_quota_bytes = data['max_quota_bytes']
         if 'signup_enabled' in data:
-            domain_new.comment = data['signup_enabled']
+            domain_new.signup_enabled = data['signup_enabled']
         models.db.session.add(domain_new)
         #apply the changes
         db.session.commit()
@@ -177,13 +177,13 @@ class Domain(Resource):
         if 'comment' in data:
             domain_found.comment = data['comment']
         if 'max_users' in data:
-            domain_found.comment = data['max_users']
+            domain_found.max_users = data['max_users']
         if 'max_aliases' in data:
-            domain_found.comment = data['max_aliases']
+            domain_found.max_aliases = data['max_aliases']
         if 'max_quota_bytes' in data:
-            domain_found.comment = data['max_quota_bytes']
+            domain_found.max_quota_bytes = data['max_quota_bytes']
         if 'signup_enabled' in data:
-            domain_found.comment = data['signup_enabled']
+            domain_found.signup_enabled = data['signup_enabled']
         models.db.session.add(domain_found)
 
         #apply the changes

core/admin/mailu/api/v1/token.py

@@ -0,0 +1,170 @@
+from flask_restx import Resource, fields, marshal
+import validators, datetime
+import flask
+from passlib import pwd
+
+from . import api, response_fields
+from .. import common
+from ... import models
+
+db = models.db
+
+token = api.namespace('token', description='Token operations')
+
+token_user_fields = api.model('TokenGetResponse', {
+    'id': fields.String(description='The record id of the token (unique identifier)', example='1'),
+    'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='user_email'),
+    'comment': fields.String(description='A description for the token. This description is shown on the Authentication tokens page', example='my comment'),
+    'AuthorizedIP': fields.String(description='Comma separated list of white listed IP addresses or networks that may use this token.', example="['203.0.113.0/24']", attribute='ip'),
+    'Created': fields.String(description='The date when the token was created', example='John.Doe@example.com', attribute='created_at'),
+    'Last edit': fields.String(description='The date when the token was last modifified', example='John.Doe@example.com', attribute='updated_at')
+})
+
+token_user_fields_post = api.model('TokenPost', {
+    'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='user_email'),
+    'comment': fields.String(description='A description for the token. This description is shown on the Authentication tokens page', example='my comment'),
+    'AuthorizedIP': fields.String(description='Comma separated list of white listed IP addresses or networks that may use this token.', example="['203.0.113.0/24']", attribute='ip'),
+})
+
+token_user_fields_post2 = api.model('TokenPost2', {
+    'comment': fields.String(description='A description for the token. This description is shown on the Authentication tokens page', example='my comment'),
+    'AuthorizedIP': fields.String(description='Comma separated list of white listed IP addresses or networks that may use this token.', example="['203.0.113.0/24']", attribute='ip'),
+})
+
+token_user_post_response = api.model('TokenPostResponse', {
+    'id': fields.String(description='The record id of the token (unique identifier)', example='1'),
+    'token': fields.String(description='The created authentication token for the user.', example='2caf6607de5129e4748a2c061aee56f2', attribute='password'),
+    'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='user_email'),
+    'comment': fields.String(description='A description for the token. This description is shown on the Authentication tokens page', example='my comment'),
+    'AuthorizedIP': fields.String(description='Comma separated list of white listed IP addresses or networks that may use this token.', example="['203.0.113.0/24']", attribute='ip'),
+    'Created': fields.String(description='The date when the token was created', example='John.Doe@example.com', attribute='created_at')
+})
+
+@token.route('')
+class Tokens(Resource):
+    @token.doc('list_tokens')
+    @token.marshal_with(token_user_fields, as_list=True, skip_none=True, mask=None)
+    @token.doc(security='Bearer')
+    @common.api_token_authorization
+    def get(self):
+        """List tokens"""
+        return models.Token.query.all()
+
+    @token.doc('create_token')
+    @token.expect(token_user_fields_post)
+    @token.response(200, 'Success', token_user_post_response)
+    @token.response(400, 'Input validation exception')
+    @token.response(409, 'Duplicate relay', response_fields)
+    @token.doc(security='Bearer')
+    @common.api_token_authorization
+    def post(self):
+        """ Create a new token"""
+        data = api.payload
+        email = data['email']
+        if not validators.email(email):
+            return { 'code': 400, 'message': f'Provided email address {email} is not a valid email address'}, 400
+        user_found = models.User.query.get(email)
+        if not user_found:
+            return {'code': 404, 'message': f'User {email} cannot be found'}, 404
+        tokens = user_found.tokens
+
+        token_new = models.Token(user_email=data['email'])
+        if 'comment' in data:
+            token_new.comment = data['comment']
+        if 'AuthorizedIP' in data:
+            token_new.ip = data['AuthorizedIP'].replace(' ','').split(',')
+        raw_password = pwd.genword(entropy=128, length=32, charset="hex")
+        token_new.set_password(raw_password)
+        models.db.session.add(token_new)
+        #apply the changes
+        db.session.commit()
+        response_dict  = {
+            'id' : token_new.id,
+            'token' : raw_password,
+            'email' : token_new.user_email,
+            'comment' : token_new.comment,
+            'AuthorizedIP' : token_new.ip,
+            'Created': str(token_new.created_at),
+            }
+
+        return  response_dict
+
+@token.route('user/<string:email>')
+class Token(Resource):
+    @token.doc('find_tokens_of_user')
+    @token.marshal_with(token_user_fields, as_list=True, skip_none=True, mask=None)
+    @token.doc(security='Bearer')
+    @common.api_token_authorization
+    def get(self, email):
+        "Find tokens of user"
+        if not validators.email(email):
+            return { 'code': 400, 'message': f'Provided email address {email} is not a valid email address'}, 400
+        user_found = models.User.query.get(email)
+        if not user_found:
+            return {'code': 404, 'message': f'User {email} cannot be found'}, 404
+        tokens = user_found.tokens
+        return tokens
+
+    @token.doc('create_token')
+    @token.expect(token_user_fields_post2)
+    @token.response(200, 'Success', token_user_post_response)
+    @token.response(400, 'Input validation exception')
+    @token.response(409, 'Duplicate relay', response_fields)
+    @token.doc(security='Bearer')
+    @common.api_token_authorization
+    def post(self, email):
+        """ Create a new token for user"""
+        data = api.payload
+        if not validators.email(email):
+            return { 'code': 400, 'message': f'Provided email address {email} is not a valid email address'}, 400
+        user_found = models.User.query.get(email)
+        if not user_found:
+            return {'code': 404, 'message': f'User {email} cannot be found'}, 404
+
+        token_new = models.Token(user_email=email)
+        if 'comment' in data:
+            token_new.comment = data['comment']
+        if 'AuthorizedIP' in data:
+            token_new.ip = token_new.ip = data['AuthorizedIP'].replace(' ','').split(',')
+        raw_password = pwd.genword(entropy=128, length=32, charset="hex")
+        token_new.set_password(raw_password)
+        models.db.session.add(token_new)
+        #apply the changes
+        db.session.commit()
+        response_dict  = {
+            'id' : token_new.id,
+            'token' : raw_password,
+            'email' : token_new.user_email,
+            'comment' : token_new.comment,
+            'AuthorizedIP' : token_new.ip,
+            'Created': str(token_new.created_at),
+            }
+        return  response_dict
+
+@token.route('/<string:token_id>')
+class Token(Resource):
+    @token.doc('find_token')
+    @token.marshal_with(token_user_fields, as_list=True, skip_none=True, mask=None)
+    @token.doc(security='Bearer')
+    @common.api_token_authorization
+    def get(self, token_id):
+        "Find token"
+        token = models.Token.query.get(token_id)
+        if not token:
+            return { 'code' : 404, 'message' : f'Record cannot be found for id {token_id} or invalid id provided'}, 404
+        return token
+
+    @token.doc('delete_token')
+    @token.response(200, 'Success', response_fields)
+    @token.response(400, 'Input validation exception', response_fields)
+    @token.response(404, 'Token not found', response_fields)
+    @token.doc(security='Bearer')
+    @common.api_token_authorization
+    def delete(self, token_id):
+        """ Delete token """
+        token = models.Token.query.get(token_id)
+        if not token:
+            return { 'code' : 404, 'message' : f'Record cannot be found for id {token_id} or invalid id provided'}, 404
+        db.session.delete(token)
+        db.session.commit()
+        return {'code': 200, 'message': f'Token with id {token_id} has been deleted'}, 200

core/admin/mailu/api/v1/user.py

@@ -14,6 +14,7 @@ user_fields_get = api.model('UserGet', {
     'password': fields.String(description="Hash of the user's password; Example='$bcrypt-sha256$v=2,t=2b,r=12$fmsAdJbYAD1gGQIE5nfJq.$zLkQUEs2XZfTpAEpcix/1k5UTNPm0jO'"),
     'comment': fields.String(description='A description for the user. This description is shown on the Users page', example='my comment'),
     'quota_bytes': fields.Integer(description='The maximum quota for the user’s email box in bytes', example='1000000000'),
+    'quota_bytes_used': fields.Integer(description='The size of the user’s email box in bytes', example='5000000'),
     'global_admin': fields.Boolean(description='Make the user a global administrator'),
     'enabled': fields.Boolean(description='Enable the user. When an user is disabled, the user is unable to login to the Admin GUI or webmail or access his email via IMAP/POP3 or send mail'),
     'change_pw_next_login': fields.Boolean(description='Force the user to change their password at next login'),

core/admin/mailu/configuration.py

@@ -75,6 +75,8 @@ DEFAULT_CONFIG = {
     'API': False,
     'WEB_API': '/api',
     'API_TOKEN': None,
+    'FULL_TEXT_SEARCH': 'en',
+    'FULL_TEXT_SEARCH_ATTACHMENTS': False,
     'LOG_LEVEL': 'INFO',
     'SESSION_KEY_BITS': 128,
     'SESSION_TIMEOUT': 3600,

core/admin/mailu/translations/es/LC_MESSAGES/messages.po

@@ -53,7 +53,7 @@ msgstr "Configuración del cliente"
 
 #: mailu/sso/templates/sidebar_sso.html:16 mailu/ui/templates/sidebar.html:114
 msgid "Website"
-msgstr "Correo web"
+msgstr "Sitio web"
 
 #: mailu/sso/templates/sidebar_sso.html:22 mailu/ui/templates/sidebar.html:120
 msgid "Help"
@@ -173,7 +173,7 @@ msgstr "Habilitar filtro de spam"
 
 #: mailu/ui/forms.py:103
 msgid "Enable marking spam mails as read"
-msgstr ""
+msgstr "Habilitar marcado de correos spam como leídos"
 
 #: mailu/ui/forms.py:104
 msgid "Spam filter tolerance"
@@ -218,7 +218,7 @@ msgstr "Texto de la respuesta"
 
 #: mailu/ui/forms.py:122
 msgid "Start of vacation"
-msgstr ""
+msgstr "Comienzo de las vacaciones"
 
 #: mailu/ui/forms.py:123
 msgid "End of vacation"
@@ -346,7 +346,7 @@ msgstr "Ocurrió un error en la comunicación con el servidor Docker."
 
 #: mailu/ui/templates/macros.html:129
 msgid "copy to clipboard"
-msgstr ""
+msgstr "copiar en portapapeles"
 
 #: mailu/ui/templates/sidebar.html:15
 msgid "My account"

core/admin/run_dev.sh

@@ -78,6 +78,7 @@ ENV \
     \
     ADMIN_ADDRESS="127.0.0.1" \
     FRONT_ADDRESS="127.0.0.1" \
+    FTS_ATTACHMENTS_ADDRESS="127.0.0.1" \
     SMTP_ADDRESS="127.0.0.1" \
     IMAP_ADDRESS="127.0.0.1" \
     REDIS_ADDRESS="127.0.0.1" \

core/base/Dockerfile

@@ -3,7 +3,7 @@
 # base system image (intermediate)
 # Note when updating the alpine tag, first manually run the workflow .github/workflows/mirror.yml.
 # Just run the workflow with the tag that must be synchronised.
-ARG DISTRO=ghcr.io/mailu/alpine:3.17.2
+ARG DISTRO=ghcr.io/mailu/alpine:3.18.4
 FROM $DISTRO as system
 
 ENV TZ=Etc/UTC LANG=C.UTF-8
@@ -16,7 +16,7 @@ RUN set -euxo pipefail \
   ; adduser -Sg ${MAILU_UID} -G mailu -h /app -g "mailu app" -s /bin/bash mailu \
   ; apk add --no-cache bash ca-certificates curl python3 tzdata \
   ; ! [[ "$(uname -m)" == x86_64 ]] \
-    || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc==11-r0
+    || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc
 
 WORKDIR /app
 
@@ -27,7 +27,7 @@ CMD /bin/bash
 FROM system as build
 
 ARG MAILU_DEPS=prod
-ARG SNUFFLEUPAGUS_VERSION=0.9.0
+ARG SNUFFLEUPAGUS_VERSION=0.10.0
 
 ENV VIRTUAL_ENV=/app/venv
 
@@ -79,9 +79,9 @@ COPY --chown=root:root --from=build /app/snuffleupagus.so /usr/lib/php81/modules
 ENV \
   VIRTUAL_ENV=/app/venv \
   PATH="/app/venv/bin:${PATH}" \
-  LD_PRELOAD="/usr/lib/libhardened_malloc.so" \
   ADMIN_ADDRESS="admin" \
   FRONT_ADDRESS="front" \
+  FTS_ATTACHMENTS_ADDRESS="tika" \
   SMTP_ADDRESS="smtp" \
   IMAP_ADDRESS="imap" \
   OLETOOLS_ADDRESS="oletools" \

core/base/libs/socrate/socrate/system.py

@@ -66,7 +66,8 @@ def _is_compatible_with_hardened_malloc():
         lines = f.readlines()
         for line in lines:
             # See #2764, we need vmovdqu
-            if line.startswith('flags') and ' avx ' not in line:
+            # See #2959, we need vpunpckldq
+            if line.startswith('flags') and ' avx2 ' not in line:
                 return False
             # See #2541
             if line.startswith('Features') and ' lrcpc ' not in line:
@@ -79,9 +80,9 @@ def set_env(required_secrets=[], log_filters=[], log_file=None):
         sys.stderr = LogFilter(sys.stderr, log_filters, log_file)
     log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", 'WARNING'))
 
-    if 'LD_PRELOAD' in os.environ and not _is_compatible_with_hardened_malloc():
-        log.warning('Disabling hardened-malloc on this CPU')
-        del os.environ['LD_PRELOAD']
+    if not 'LD_PRELOAD' in os.environ and _is_compatible_with_hardened_malloc():
+        log.warning('Your CPU has Advanced Vector Extensions available, we recommend you enable hardened-malloc earlier in the boot process by adding LD_PRELOAD=/usr/lib/libhardened_malloc.so to your mailu.env')
+        os.environ['LD_PRELOAD'] = '/usr/lib/libhardened_malloc.so'
 
     """ This will set all the environment variables and retains only the secrets we need """
     if 'SECRET_KEY_FILE' in os.environ:

core/dovecot/Dockerfile

@@ -7,7 +7,9 @@ ARG VERSION
 LABEL version=$VERSION
 
 RUN set -euxo pipefail \
-  ; apk add --no-cache dovecot dovecot-fts-xapian dovecot-lmtpd dovecot-pigeonhole-plugin dovecot-pop3d dovecot-submissiond rspamd-client xapian-core \
+  ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main 'dovecot<2.4' dovecot-lmtpd dovecot-pigeonhole-plugin dovecot-pop3d dovecot-submissiond \
+  ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing dovecot-fts-flatcurve \
+  ; apk add --no-cache rspamd-client \
   ; mkdir /var/lib/dovecot
 
 COPY conf/ /conf/

core/dovecot/conf/dovecot.conf

@@ -37,7 +37,7 @@ mail_plugins = $mail_plugins quota quota_clone{{ ' ' }}
   zlib{{ ' ' }}
   {%- endif %}
   {%- if (FULL_TEXT_SEARCH or '').lower() not in ['off', 'false', '0'] -%}
-  fts fts_xapian
+  fts fts_flatcurve
   {%- endif %}
 default_vsz_limit = 2GB
 
@@ -57,11 +57,20 @@ plugin {
   quota_clone_dict = proxy:/tmp/podop.socket:quota
 
   {% if (FULL_TEXT_SEARCH or '').lower() not in ['off', 'false', '0'] %}
-  fts = xapian
-  fts_xapian = partial=2 full=30
+  fts = flatcurve
+  fts_languages = {% if FULL_TEXT_SEARCH %}{{ FULL_TEXT_SEARCH.split(",") | join(" ") }}{% else %}en{% endif %}
+  fts_tokenizers = generic email-address
   fts_autoindex = yes
   fts_enforced = yes
   fts_autoindex_exclude = \Trash
+  fts_autoindex_exclude1 = \Junk
+  fts_filters = normalizer-icu stopwords
+  fts_filters_en = lowercase english-possessive stopwords
+  fts_filters_fr = lowercase contractions stopwords
+  fts_header_excludes = Received DKIM-* ARC-* X-* x-* Comments Delivered-To Return-Path Authentication-Results Message-ID References In-Reply-To Thread-* Accept-Language Content-* MIME-Version
+  {% if FULL_TEXT_SEARCH_ATTACHMENTS %}
+  fts_tika = http://{{ FTS_ATTACHMENTS_ADDRESS }}:9998/tika/
+  {% endif %}
   {% endif %}
 
   {% if COMPRESSION in [ 'gz', 'bz2', 'lz4', 'zstd' ] %}

core/nginx/Dockerfile

@@ -17,7 +17,8 @@ ARG VERSION
 LABEL version=$VERSION
 
 RUN set -euxo pipefail \
-  ; apk add --no-cache certbot nginx nginx-mod-http-brotli nginx-mod-mail openssl dovecot-lua dovecot-pigeonhole-plugin dovecot-lmtpd dovecot-pop3d dovecot-submissiond
+  ; apk add --no-cache certbot nginx nginx-mod-http-brotli nginx-mod-mail openssl \
+  ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main dovecot-lua dovecot-pigeonhole-plugin 'dovecot-lmtpd<2.4' dovecot-pop3d dovecot-submissiond
 
 COPY conf/ /conf/
 COPY --from=static /static/ /static/

core/nginx/letsencrypt.py

@@ -47,7 +47,7 @@ time.sleep(5)
 
 class MyRequestHandler(SimpleHTTPRequestHandler):
     def do_GET(self):
-        if self.path == '/testing':
+        if self.path == '/.well-known/acme-challenge/testing':
             self.send_response(204)
         else:
             self.send_response(404)
@@ -55,7 +55,7 @@ class MyRequestHandler(SimpleHTTPRequestHandler):
         self.end_headers()
 
 def serve_one_request():
-    with HTTPServer(("0.0.0.0", 8008), MyRequestHandler) as server:
+    with HTTPServer(("127.0.0.1", 8008), MyRequestHandler) as server:
         server.handle_request()
 
 # Run certbot every day

core/oletools/Dockerfile

@@ -6,11 +6,17 @@ FROM base
 ARG VERSION=local
 LABEL version=$VERSION
 
+ARG OLEFY_SCRIPT=https://raw.githubusercontent.com/HeinleinSupport/olefy/f8aac6cc55283886d153e89c8f27fae66b1c24e2/olefy.py
+ARG OLEFY_SHA256=1f5aa58b78ca7917350135b4425e5ed4d580c7051aabed1952c6afd12d0345a0
+
 RUN set -euxo pipefail \
   ; apk add --no-cache netcat-openbsd libmagic libffi \
-  ; curl -sLo olefy.py https://raw.githubusercontent.com/HeinleinSupport/olefy/f8aac6cc55283886d153e89c8f27fae66b1c24e2/olefy.py \
+  ; curl -sLo olefy.py $OLEFY_SCRIPT \
+  ; echo "$OLEFY_SHA256  olefy.py" |sha256sum -c \
   ; chmod 755 olefy.py
 
+COPY start.py /
+
 RUN echo $VERSION >/version
 
 HEALTHCHECK --start-period=60s CMD echo PING|nc -q1 127.0.0.1 11343|grep "PONG"
@@ -28,4 +34,4 @@ ENV \
     OLEFY_DEL_TMP="1" \
     OLEFY_DEL_TMP_FAILED="1"
 
-CMD /app/olefy.py
+CMD /start.py

core/oletools/start.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+
+from socrate import system
+
+system.set_env()
+
+with open('/app/olefy.py') as olefy:
+    exec(olefy.read())

core/rspamd/start.py

@@ -16,7 +16,8 @@ env = system.set_env()
 config_files = []
 for rspamd_file in glob.glob("/conf/*"):
     conf.jinja(rspamd_file, env, os.path.join("/etc/rspamd/local.d", os.path.basename(rspamd_file)))
-    config_files.append(os.path.basename(rspamd_file))
+    if rspamd_file != '/conf/forbidden_file_extension.map':
+        config_files.append(os.path.basename(rspamd_file))
 
 for override_file in glob.glob("/overrides/*"):
     if os.path.basename(override_file) not in config_files:
@@ -37,4 +38,4 @@ while True:
 os.system("mkdir -m 755 -p /run/rspamd")
 os.system("chown rspamd:rspamd /run/rspamd")
 os.system("find /var/lib/rspamd | grep -v /filter | xargs -n1 chown rspamd:rspamd")
-os.execv("/usr/sbin/rspamd", ["rspamd", "-f", "-u", "rspamd", "-g", "rspamd"])
+os.execv("/usr/bin/rspamd", ["rspamd", "-f", "-u", "rspamd", "-g", "rspamd"])

docs/compose/.env

@@ -152,3 +152,8 @@ REJECT_UNLISTED_RECIPIENT=
 
 # Log level threshold in start.py (value: CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET)
 LOG_LEVEL=WARNING
+
+# If your CPU supports Advanced Vector Extensions
+# (AVX2 on x86_64, lrcpc on ARM64), you should consider enabling
+# hardened-malloc earlier by uncommenting this
+# LD_PRELOAD=/usr/lib/libhardened_malloc.so

docs/compose/setup.rst

@@ -76,6 +76,15 @@ Review configuration variables
 After downloading the files, open ``mailu.env`` and review the variable settings.
 Make sure to read the comments in the file and instructions from the :ref:`common_cfg` page.
 
+If your CPU supports Advanced Vector Extensions (AVX2 on x86_64, lrcpc on ARM64), you should
+consider enabling hardened-malloc earlier in the boot process by adding the following to
+your mailu.env:
+
+.. code-block:: bash
+
+    LD_PRELOAD=/usr/lib/libhardened_malloc.so
+
+
 Finish setting up TLS
 ---------------------
 

docs/configuration.rst

@@ -131,12 +131,15 @@ later classify incoming mail based on the custom part.
 The ``DMARC_RUA`` and ``DMARC_RUF`` are DMARC protocol specific values. They hold
 the localpart for DMARC rua and ruf email addresses.
 
-Full-text search is enabled for IMAP is enabled by default. This feature can be disabled
-(e.g. for performance reasons) by setting the optional variable ``FULL_TEXT_SEARCH`` to ``off``.
+The ``FULL_TEXT_SEARCH`` variable (default: 'en') is a comma separated list of
+language codes as defined on `fts_languages`_. This feature can be disabled
+(e.g. for performance reasons) by setting the variable to ``off``.
 
 You can set a global ``DEFAULT_QUOTA`` to be used for mailboxes when the domain has
 no specific quota configured.
 
+.. _`fts_languages`: https://doc.dovecot.org/settings/plugin/fts-plugin/#fts-languages
+
 .. _web_settings:
 
 Web settings

docs/demo.rst

@@ -35,7 +35,7 @@ Connecting to the server
  * Webmail : https://test.mailu.io/webmail/
  * Admin UI : https://test.mailu.io/admin/
  * Admin login : ``admin@test.mailu.io``
- * Admin password : ``letmein``
+ * Admin password : ``Mailu+Demo@test.mailu.io`` (remove + and @test.mailu.io to get the correct password).
  * RESTful API: https://test.mailu.io/api
  * API token: ``Bearer APITokenForMailu``
 

docs/index.rst

@@ -24,7 +24,7 @@ popular groupware.
 Main features include:
 
 - **Standard email server**, IMAP and IMAP+, SMTP and Submission with autoconfiguration profiles for clients
-- **Advanced email features**, aliases, domain aliases, custom routing
+- **Advanced email features**, aliases, domain aliases, custom routing, full-text search of email attachments
 - **Web access**, multiple Webmails and administration interface
 - **User features**, aliases, auto-reply, auto-forward, fetched accounts, managesieve
 - **Admin features**, global admins, announcements, per-domain delegation, quotas

setup/flavors/compose/docker-compose.yml

@@ -98,8 +98,16 @@ services:
     volumes:
       - "{{ root }}/mail:/mail"
       - "{{ root }}/overrides/dovecot:/overrides:ro"
+    networks:
+      - default
+    {% if tika_enabled %}
+      - fts_attachments
+    {% endif %}
     depends_on:
       - front
+    {% if tika_enabled %}
+      - fts_attachments
+    {% endif %}
     {% if resolver_enabled %}
       - resolver
     dns:
@@ -140,6 +148,21 @@ services:
     {% endif %}
 {% endif %}
 
+{% if tika_enabled %}
+  fts_attachments:
+    image: apache/tika:2.9.0.0-full
+    hostname: tika
+    restart: always
+    networks:
+      - fts_attachments
+    depends_on:
+    {% if resolver_enabled %}
+      - resolver
+    dns:
+      - {{ dns }}
+    {% endif %}
+{% endif %}
+
   antispam:
     image: ${DOCKER_ORG:-ghcr.io/mailu}/${DOCKER_PREFIX:-}rspamd:${MAILU_VERSION:-{{ version }}}
     hostname: antispam
@@ -257,3 +280,8 @@ networks:
     driver: bridge
     internal: true
 {% endif %}
+{% if tika_enabled %}
+  fts_attachments:
+    driver: bridge
+    internal: true
+{% endif %}

setup/flavors/compose/mailu.env

@@ -49,19 +49,19 @@ DISABLE_STATISTICS={{ disable_statistics or 'False' }}
 # Expose the admin interface (value: true, false)
 ADMIN={{ admin_enabled or 'false' }}
 
-# Choose which webmail to run if any (values: roundcube, snappymail, none)
+# Choose which webmail to run if any (values: roundcube, snappymail, none). To enable this feature, recreate the docker-compose.yml file via setup.
 WEBMAIL={{ webmail_type }}
 
 # Expose the API interface (value: true, false)
 API={{ api_enabled or 'false' }}
 
-# Dav server implementation (value: radicale, none)
+# Dav server implementation (value: radicale, none). To enable this feature, recreate the docker-compose.yml file via setup.
 WEBDAV={{ webdav_enabled or 'none' }}
 
-# Antivirus solution (value: clamav, none)
+# Antivirus solution (value: clamav, none). To enable this feature, recreate the docker-compose.yml file via setup.
 ANTIVIRUS={{ antivirus_enabled or 'none' }}
 
-# Scan Macros solution (value: true, false)
+# Scan Macros solution (value: true, false). To enable this feature, recreate the docker-compose.yml file via setup.
 SCAN_MACROS={{ oletools_enabled or 'false' }}
 
 ###################################
@@ -110,8 +110,10 @@ COMPRESSION={{ compression }}
 # change compression-level, default: 6 (value: 1-9)
 COMPRESSION_LEVEL={{ compression_level }}
 
-# IMAP full-text search is enabled by default. Set the following variable to off in order to disable the feature.
-# FULL_TEXT_SEARCH=off
+# IMAP full-text search is enabled by default.
+# Set the following variable to off in order to disable the feature
+# or a comma separated list of language codes to support
+FULL_TEXT_SEARCH=en
 
 ###################################
 # Web settings
@@ -186,3 +188,5 @@ DEFAULT_SPAM_THRESHOLD=80
 # This is a mandatory setting for using the RESTful API.
 API_TOKEN={{ api_token }}
 
+# Whether tika should be enabled (scan/OCR email attachements). To enable this feature, recreate the docker-compose.yml file via setup.
+FULL_TEXT_SEARCH_ATTACHMENTS={{ tika_enabled }}

setup/templates/steps/compose/02_services.html

@@ -64,6 +64,15 @@ the security implications caused by such an increase of attack surface.<p>
   <i>Oletools scans documents in email attachements for malicious macros. It has a much lower memory footprint than a full-fledged anti-virus.</i>
 </div>
 
+<div class="form-check form-check-inline">
+  <label class="form-check-label">
+    <input class="form-check-input" type="checkbox" name="tika_enabled" value="true">
+    Enable Tika
+  </label>
+
+  <i>Tika enables the functionality for searching through attachments. Tika scans documents in email attachments, process (OCR, keyword extraction) and then index them in a way they can be efficiently searched. This requires significant ressources (RAM, CPU and storage).</i>
+</div>
+
 <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.6.0/jquery.min.js"></script>
 
 <script type="text/javascript" src="{{ url_for('static', filename='render.js') }}"></script>

towncrier/newsfragments/2824.feature

@@ -0,0 +1 @@
+Enhance RESTful API user retrieval with quota used bytes. This is the current size of the user's email box in bytes.

towncrier/newsfragments/2918.misc

@@ -0,0 +1 @@
+Upgrade dovecot to ensure we can proxy ipv6 via XCLIENT.

towncrier/newsfragments/2934.bugfix

@@ -0,0 +1,3 @@
+Upgrade to alpine 3.18.4: this will fix a bug whereby musl wasn't retrying using TCP when it received truncated DNS replies from its upstream. In practice, this has been seen in the wild when postfix complains of:
+
+"Host or domain name not found. Name service error for name=outlook-com.olc.protection.outlook.com type=AAAA: Host found but no data record of requested type"

towncrier/newsfragments/2937.bugfix

@@ -0,0 +1,2 @@
+forbidden_file_extension.map could not be overridden. This file can be overriden to tweak with file extensions are allowed.
+The instructions on https://mailu.io/master/antispam.html#can-i-change-the-list-of-authorized-file-attachments work again.

towncrier/newsfragments/2950.misc

@@ -0,0 +1 @@
+Upgrade to snuffleupagus 0.10.0

towncrier/newsfragments/2955.misc

@@ -0,0 +1 @@
+Remove the version pinning on hardened malloc

towncrier/newsfragments/2959.bugfix

@@ -0,0 +1,3 @@
+Update hardened malloc as the original package is not available from alpine anymore.
+The newer version of hardened malloc requires AVX2: Disable it by default at startup and hint in the logs when it should be enabled instead.
+Upgrade snappymail to v2.29.1

towncrier/newsfragments/2962.bugfix

@@ -0,0 +1 @@
+Fix letsencrypt on master

towncrier/newsfragments/2971.bugfix

@@ -0,0 +1,12 @@
+- Switch from fts-xapian to fts-flatcurve. This should address the problem with indexes getting too big and will be the default in dovecot 2.4
+- Enable full-text search of email attachments if configured (via Tika: you'll need to re-run setup)
+
+If you would like more than english to be supported, please ensure you update your FULL_TEXT_SEARCH configuration variable.
+
+You may also want to dispose of old indexes using a command such as:
+
+find /mailu/mail -type d -name xapian-indexes -prune -exec rm -r {} \+
+
+And proactively force a reindexing using:
+
+docker compose exec imap doveadm index -A '*'

towncrier/newsfragments/2974.feature

@@ -0,0 +1 @@
+Enhance RESTful API with functionality for managing authentication tokens of users

webmails/Dockerfile

@@ -19,6 +19,7 @@ RUN set -euxo pipefail \
     aspell-uk aspell-ru aspell-fr aspell-de aspell-en \
   ; rm /etc/nginx/http.d/default.conf \
   ; rm /etc/php81/php-fpm.d/www.conf \
+  ; mkdir -m 700 /root/.gnupg/ \
   ; gpg --import /tmp/snappymail.asc \
   ; gpg --import /tmp/roundcube.asc \
   ; echo extension=snuffleupagus > /etc/php81/conf.d/snuffleupagus.ini \
@@ -33,7 +34,7 @@ RUN set -euxo pipefail \
   ; cd /var/www \
   ; curl -sLo /dev/shm/roundcube.tgz ${ROUNDCUBE_URL} \
   ; curl -sLo /dev/shm/roundcube.tgz.asc ${ROUNDCUBE_URL}.asc \
-  ; gpg --status-fd 1 --verify /dev/shm/roundcube.tgz.asc \
+  ; gpg --status-fd 1 --verify /dev/shm/roundcube.tgz.asc /dev/shm/roundcube.tgz \
   ; tar xzf /dev/shm/roundcube.tgz \
   ; curl -sL ${CARDDAV_URL} | tar xz \
   ; mv roundcubemail-* roundcube \
@@ -52,15 +53,15 @@ COPY roundcube/config/config.inc.carddav.php /var/www/roundcube/plugins/carddav/
 
 # snappymail
 
-ENV SNAPPYMAIL_URL https://github.com/the-djmaze/snappymail/releases/download/v2.28.4/snappymail-2.28.4.tar.gz
+ENV SNAPPYMAIL_URL https://github.com/the-djmaze/snappymail/releases/download/v2.29.1/snappymail-2.29.1.tar.gz
 
 RUN set -euxo pipefail \
   ; mkdir  /var/www/snappymail \
   ; cd /var/www/snappymail \
   ; curl -sLo /dev/shm/snappymail.tgz  ${SNAPPYMAIL_URL} \
   ; curl -sLo /dev/shm/snappymail.tgz.asc  ${SNAPPYMAIL_URL}.asc \
-  ; gpg --status-fd 1 --verify /dev/shm/snappymail.tgz.asc \
-  ; tar xzf /dev/shm/snappymail.tgz
+  ; gpg --status-fd 1 --verify /dev/shm/snappymail.tgz.asc /dev/shm/snappymail.tgz \
+  ; cat /dev/shm/snappymail.tgz | tar xz
 
 # SnappyMail login
 COPY snappymail/login/include.php /var/www/snappymail/

webmails/snuffleupagus.rules

@@ -43,6 +43,7 @@ sp.disable_function.function("mail").param("additional_parameters").value_r("\\-
 
 # Since it's now burned, me might as well mitigate it publicly
 sp.disable_function.function("putenv").param("assignment").value_r("LD_").drop()
+sp.disable_function.function("putenv").param("assignment").value("PATH").drop()
 
 # This one was burned in Nov 2019 - https://gist.github.com/LoadLow/90b60bd5535d6c3927bb24d5f9955b80
 sp.disable_function.function("putenv").param("assignment").value_r("GCONV_").drop()