diff --git a/README.md b/README.md index 6a202940..29bd55a4 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Features Main features include: - **Standard email server**, IMAP and IMAP+, SMTP and Submission with autoconfiguration profiles for clients -- **Advanced email features**, aliases, domain aliases, custom routing +- **Advanced email features**, aliases, domain aliases, custom routing, full-text search of email attachments - **Web access**, multiple Webmails and administration interface - **User features**, aliases, auto-reply, auto-forward, fetched accounts, managesieve - **Admin features**, global admins, announcements, per-domain delegation, quotas diff --git a/core/admin/mailu/api/v1/__init__.py b/core/admin/mailu/api/v1/__init__.py index 44b6ec57..9b3e98f8 100644 --- a/core/admin/mailu/api/v1/__init__.py +++ b/core/admin/mailu/api/v1/__init__.py @@ -37,7 +37,8 @@ error_fields = api.model('Error', { 'message': fields.String, }) -from . import domains +from . import domain from . import alias from . import relay from . import user +from . import token diff --git a/core/admin/mailu/api/v1/domains.py b/core/admin/mailu/api/v1/domain.py similarity index 97% rename from core/admin/mailu/api/v1/domains.py rename to core/admin/mailu/api/v1/domain.py index 7043da3d..c5f98530 100644 --- a/core/admin/mailu/api/v1/domains.py +++ b/core/admin/mailu/api/v1/domain.py @@ -115,13 +115,13 @@ class Domains(Resource): if 'comment' in data: domain_new.comment = data['comment'] if 'max_users' in data: - domain_new.comment = data['max_users'] + domain_new.max_users = data['max_users'] if 'max_aliases' in data: - domain_new.comment = data['max_aliases'] + domain_new.max_aliases = data['max_aliases'] if 'max_quota_bytes' in data: - domain_new.comment = data['max_quota_bytes'] + domain_new.max_quota_bytes = data['max_quota_bytes'] if 'signup_enabled' in data: - domain_new.comment = data['signup_enabled'] + domain_new.signup_enabled = data['signup_enabled'] models.db.session.add(domain_new) #apply the changes db.session.commit() @@ -177,13 +177,13 @@ class Domain(Resource): if 'comment' in data: domain_found.comment = data['comment'] if 'max_users' in data: - domain_found.comment = data['max_users'] + domain_found.max_users = data['max_users'] if 'max_aliases' in data: - domain_found.comment = data['max_aliases'] + domain_found.max_aliases = data['max_aliases'] if 'max_quota_bytes' in data: - domain_found.comment = data['max_quota_bytes'] + domain_found.max_quota_bytes = data['max_quota_bytes'] if 'signup_enabled' in data: - domain_found.comment = data['signup_enabled'] + domain_found.signup_enabled = data['signup_enabled'] models.db.session.add(domain_found) #apply the changes diff --git a/core/admin/mailu/api/v1/token.py b/core/admin/mailu/api/v1/token.py new file mode 100644 index 00000000..0f2b5b7a --- /dev/null +++ b/core/admin/mailu/api/v1/token.py @@ -0,0 +1,170 @@ +from flask_restx import Resource, fields, marshal +import validators, datetime +import flask +from passlib import pwd + +from . import api, response_fields +from .. import common +from ... import models + +db = models.db + +token = api.namespace('token', description='Token operations') + +token_user_fields = api.model('TokenGetResponse', { + 'id': fields.String(description='The record id of the token (unique identifier)', example='1'), + 'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='user_email'), + 'comment': fields.String(description='A description for the token. This description is shown on the Authentication tokens page', example='my comment'), + 'AuthorizedIP': fields.String(description='Comma separated list of white listed IP addresses or networks that may use this token.', example="['203.0.113.0/24']", attribute='ip'), + 'Created': fields.String(description='The date when the token was created', example='John.Doe@example.com', attribute='created_at'), + 'Last edit': fields.String(description='The date when the token was last modifified', example='John.Doe@example.com', attribute='updated_at') +}) + +token_user_fields_post = api.model('TokenPost', { + 'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='user_email'), + 'comment': fields.String(description='A description for the token. This description is shown on the Authentication tokens page', example='my comment'), + 'AuthorizedIP': fields.String(description='Comma separated list of white listed IP addresses or networks that may use this token.', example="['203.0.113.0/24']", attribute='ip'), +}) + +token_user_fields_post2 = api.model('TokenPost2', { + 'comment': fields.String(description='A description for the token. This description is shown on the Authentication tokens page', example='my comment'), + 'AuthorizedIP': fields.String(description='Comma separated list of white listed IP addresses or networks that may use this token.', example="['203.0.113.0/24']", attribute='ip'), +}) + +token_user_post_response = api.model('TokenPostResponse', { + 'id': fields.String(description='The record id of the token (unique identifier)', example='1'), + 'token': fields.String(description='The created authentication token for the user.', example='2caf6607de5129e4748a2c061aee56f2', attribute='password'), + 'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='user_email'), + 'comment': fields.String(description='A description for the token. This description is shown on the Authentication tokens page', example='my comment'), + 'AuthorizedIP': fields.String(description='Comma separated list of white listed IP addresses or networks that may use this token.', example="['203.0.113.0/24']", attribute='ip'), + 'Created': fields.String(description='The date when the token was created', example='John.Doe@example.com', attribute='created_at') +}) + +@token.route('') +class Tokens(Resource): + @token.doc('list_tokens') + @token.marshal_with(token_user_fields, as_list=True, skip_none=True, mask=None) + @token.doc(security='Bearer') + @common.api_token_authorization + def get(self): + """List tokens""" + return models.Token.query.all() + + @token.doc('create_token') + @token.expect(token_user_fields_post) + @token.response(200, 'Success', token_user_post_response) + @token.response(400, 'Input validation exception') + @token.response(409, 'Duplicate relay', response_fields) + @token.doc(security='Bearer') + @common.api_token_authorization + def post(self): + """ Create a new token""" + data = api.payload + email = data['email'] + if not validators.email(email): + return { 'code': 400, 'message': f'Provided email address {email} is not a valid email address'}, 400 + user_found = models.User.query.get(email) + if not user_found: + return {'code': 404, 'message': f'User {email} cannot be found'}, 404 + tokens = user_found.tokens + + token_new = models.Token(user_email=data['email']) + if 'comment' in data: + token_new.comment = data['comment'] + if 'AuthorizedIP' in data: + token_new.ip = data['AuthorizedIP'].replace(' ','').split(',') + raw_password = pwd.genword(entropy=128, length=32, charset="hex") + token_new.set_password(raw_password) + models.db.session.add(token_new) + #apply the changes + db.session.commit() + response_dict = { + 'id' : token_new.id, + 'token' : raw_password, + 'email' : token_new.user_email, + 'comment' : token_new.comment, + 'AuthorizedIP' : token_new.ip, + 'Created': str(token_new.created_at), + } + + return response_dict + +@token.route('user/') +class Token(Resource): + @token.doc('find_tokens_of_user') + @token.marshal_with(token_user_fields, as_list=True, skip_none=True, mask=None) + @token.doc(security='Bearer') + @common.api_token_authorization + def get(self, email): + "Find tokens of user" + if not validators.email(email): + return { 'code': 400, 'message': f'Provided email address {email} is not a valid email address'}, 400 + user_found = models.User.query.get(email) + if not user_found: + return {'code': 404, 'message': f'User {email} cannot be found'}, 404 + tokens = user_found.tokens + return tokens + + @token.doc('create_token') + @token.expect(token_user_fields_post2) + @token.response(200, 'Success', token_user_post_response) + @token.response(400, 'Input validation exception') + @token.response(409, 'Duplicate relay', response_fields) + @token.doc(security='Bearer') + @common.api_token_authorization + def post(self, email): + """ Create a new token for user""" + data = api.payload + if not validators.email(email): + return { 'code': 400, 'message': f'Provided email address {email} is not a valid email address'}, 400 + user_found = models.User.query.get(email) + if not user_found: + return {'code': 404, 'message': f'User {email} cannot be found'}, 404 + + token_new = models.Token(user_email=email) + if 'comment' in data: + token_new.comment = data['comment'] + if 'AuthorizedIP' in data: + token_new.ip = token_new.ip = data['AuthorizedIP'].replace(' ','').split(',') + raw_password = pwd.genword(entropy=128, length=32, charset="hex") + token_new.set_password(raw_password) + models.db.session.add(token_new) + #apply the changes + db.session.commit() + response_dict = { + 'id' : token_new.id, + 'token' : raw_password, + 'email' : token_new.user_email, + 'comment' : token_new.comment, + 'AuthorizedIP' : token_new.ip, + 'Created': str(token_new.created_at), + } + return response_dict + +@token.route('/') +class Token(Resource): + @token.doc('find_token') + @token.marshal_with(token_user_fields, as_list=True, skip_none=True, mask=None) + @token.doc(security='Bearer') + @common.api_token_authorization + def get(self, token_id): + "Find token" + token = models.Token.query.get(token_id) + if not token: + return { 'code' : 404, 'message' : f'Record cannot be found for id {token_id} or invalid id provided'}, 404 + return token + + @token.doc('delete_token') + @token.response(200, 'Success', response_fields) + @token.response(400, 'Input validation exception', response_fields) + @token.response(404, 'Token not found', response_fields) + @token.doc(security='Bearer') + @common.api_token_authorization + def delete(self, token_id): + """ Delete token """ + token = models.Token.query.get(token_id) + if not token: + return { 'code' : 404, 'message' : f'Record cannot be found for id {token_id} or invalid id provided'}, 404 + db.session.delete(token) + db.session.commit() + return {'code': 200, 'message': f'Token with id {token_id} has been deleted'}, 200 diff --git a/core/admin/mailu/api/v1/user.py b/core/admin/mailu/api/v1/user.py index d9195e3d..441845c2 100644 --- a/core/admin/mailu/api/v1/user.py +++ b/core/admin/mailu/api/v1/user.py @@ -14,6 +14,7 @@ user_fields_get = api.model('UserGet', { 'password': fields.String(description="Hash of the user's password; Example='$bcrypt-sha256$v=2,t=2b,r=12$fmsAdJbYAD1gGQIE5nfJq.$zLkQUEs2XZfTpAEpcix/1k5UTNPm0jO'"), 'comment': fields.String(description='A description for the user. This description is shown on the Users page', example='my comment'), 'quota_bytes': fields.Integer(description='The maximum quota for the user’s email box in bytes', example='1000000000'), + 'quota_bytes_used': fields.Integer(description='The size of the user’s email box in bytes', example='5000000'), 'global_admin': fields.Boolean(description='Make the user a global administrator'), 'enabled': fields.Boolean(description='Enable the user. When an user is disabled, the user is unable to login to the Admin GUI or webmail or access his email via IMAP/POP3 or send mail'), 'change_pw_next_login': fields.Boolean(description='Force the user to change their password at next login'), diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index bb7080c9..d324bf8d 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -75,6 +75,8 @@ DEFAULT_CONFIG = { 'API': False, 'WEB_API': '/api', 'API_TOKEN': None, + 'FULL_TEXT_SEARCH': 'en', + 'FULL_TEXT_SEARCH_ATTACHMENTS': False, 'LOG_LEVEL': 'INFO', 'SESSION_KEY_BITS': 128, 'SESSION_TIMEOUT': 3600, diff --git a/core/admin/mailu/translations/es/LC_MESSAGES/messages.po b/core/admin/mailu/translations/es/LC_MESSAGES/messages.po index c385905e..068712b8 100644 --- a/core/admin/mailu/translations/es/LC_MESSAGES/messages.po +++ b/core/admin/mailu/translations/es/LC_MESSAGES/messages.po @@ -53,7 +53,7 @@ msgstr "Configuración del cliente" #: mailu/sso/templates/sidebar_sso.html:16 mailu/ui/templates/sidebar.html:114 msgid "Website" -msgstr "Correo web" +msgstr "Sitio web" #: mailu/sso/templates/sidebar_sso.html:22 mailu/ui/templates/sidebar.html:120 msgid "Help" @@ -173,7 +173,7 @@ msgstr "Habilitar filtro de spam" #: mailu/ui/forms.py:103 msgid "Enable marking spam mails as read" -msgstr "" +msgstr "Habilitar marcado de correos spam como leídos" #: mailu/ui/forms.py:104 msgid "Spam filter tolerance" @@ -218,7 +218,7 @@ msgstr "Texto de la respuesta" #: mailu/ui/forms.py:122 msgid "Start of vacation" -msgstr "" +msgstr "Comienzo de las vacaciones" #: mailu/ui/forms.py:123 msgid "End of vacation" @@ -346,7 +346,7 @@ msgstr "Ocurrió un error en la comunicación con el servidor Docker." #: mailu/ui/templates/macros.html:129 msgid "copy to clipboard" -msgstr "" +msgstr "copiar en portapapeles" #: mailu/ui/templates/sidebar.html:15 msgid "My account" diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index 0f7c6e05..3d1fc771 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -78,6 +78,7 @@ ENV \ \ ADMIN_ADDRESS="127.0.0.1" \ FRONT_ADDRESS="127.0.0.1" \ + FTS_ATTACHMENTS_ADDRESS="127.0.0.1" \ SMTP_ADDRESS="127.0.0.1" \ IMAP_ADDRESS="127.0.0.1" \ REDIS_ADDRESS="127.0.0.1" \ diff --git a/core/base/Dockerfile b/core/base/Dockerfile index 88332d3a..2f9c1142 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -3,7 +3,7 @@ # base system image (intermediate) # Note when updating the alpine tag, first manually run the workflow .github/workflows/mirror.yml. # Just run the workflow with the tag that must be synchronised. -ARG DISTRO=ghcr.io/mailu/alpine:3.17.2 +ARG DISTRO=ghcr.io/mailu/alpine:3.18.4 FROM $DISTRO as system ENV TZ=Etc/UTC LANG=C.UTF-8 @@ -16,7 +16,7 @@ RUN set -euxo pipefail \ ; adduser -Sg ${MAILU_UID} -G mailu -h /app -g "mailu app" -s /bin/bash mailu \ ; apk add --no-cache bash ca-certificates curl python3 tzdata \ ; ! [[ "$(uname -m)" == x86_64 ]] \ - || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc==11-r0 + || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc WORKDIR /app @@ -27,7 +27,7 @@ CMD /bin/bash FROM system as build ARG MAILU_DEPS=prod -ARG SNUFFLEUPAGUS_VERSION=0.9.0 +ARG SNUFFLEUPAGUS_VERSION=0.10.0 ENV VIRTUAL_ENV=/app/venv @@ -79,9 +79,9 @@ COPY --chown=root:root --from=build /app/snuffleupagus.so /usr/lib/php81/modules ENV \ VIRTUAL_ENV=/app/venv \ PATH="/app/venv/bin:${PATH}" \ - LD_PRELOAD="/usr/lib/libhardened_malloc.so" \ ADMIN_ADDRESS="admin" \ FRONT_ADDRESS="front" \ + FTS_ATTACHMENTS_ADDRESS="tika" \ SMTP_ADDRESS="smtp" \ IMAP_ADDRESS="imap" \ OLETOOLS_ADDRESS="oletools" \ diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index e80863b8..53cabae6 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -66,7 +66,8 @@ def _is_compatible_with_hardened_malloc(): lines = f.readlines() for line in lines: # See #2764, we need vmovdqu - if line.startswith('flags') and ' avx ' not in line: + # See #2959, we need vpunpckldq + if line.startswith('flags') and ' avx2 ' not in line: return False # See #2541 if line.startswith('Features') and ' lrcpc ' not in line: @@ -79,9 +80,9 @@ def set_env(required_secrets=[], log_filters=[], log_file=None): sys.stderr = LogFilter(sys.stderr, log_filters, log_file) log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", 'WARNING')) - if 'LD_PRELOAD' in os.environ and not _is_compatible_with_hardened_malloc(): - log.warning('Disabling hardened-malloc on this CPU') - del os.environ['LD_PRELOAD'] + if not 'LD_PRELOAD' in os.environ and _is_compatible_with_hardened_malloc(): + log.warning('Your CPU has Advanced Vector Extensions available, we recommend you enable hardened-malloc earlier in the boot process by adding LD_PRELOAD=/usr/lib/libhardened_malloc.so to your mailu.env') + os.environ['LD_PRELOAD'] = '/usr/lib/libhardened_malloc.so' """ This will set all the environment variables and retains only the secrets we need """ if 'SECRET_KEY_FILE' in os.environ: diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index 872e1ecf..2681bcff 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -7,7 +7,9 @@ ARG VERSION LABEL version=$VERSION RUN set -euxo pipefail \ - ; apk add --no-cache dovecot dovecot-fts-xapian dovecot-lmtpd dovecot-pigeonhole-plugin dovecot-pop3d dovecot-submissiond rspamd-client xapian-core \ + ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main 'dovecot<2.4' dovecot-lmtpd dovecot-pigeonhole-plugin dovecot-pop3d dovecot-submissiond \ + ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing dovecot-fts-flatcurve \ + ; apk add --no-cache rspamd-client \ ; mkdir /var/lib/dovecot COPY conf/ /conf/ diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index e35ab4a1..c5173787 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -37,7 +37,7 @@ mail_plugins = $mail_plugins quota quota_clone{{ ' ' }} zlib{{ ' ' }} {%- endif %} {%- if (FULL_TEXT_SEARCH or '').lower() not in ['off', 'false', '0'] -%} - fts fts_xapian + fts fts_flatcurve {%- endif %} default_vsz_limit = 2GB @@ -57,11 +57,20 @@ plugin { quota_clone_dict = proxy:/tmp/podop.socket:quota {% if (FULL_TEXT_SEARCH or '').lower() not in ['off', 'false', '0'] %} - fts = xapian - fts_xapian = partial=2 full=30 + fts = flatcurve + fts_languages = {% if FULL_TEXT_SEARCH %}{{ FULL_TEXT_SEARCH.split(",") | join(" ") }}{% else %}en{% endif %} + fts_tokenizers = generic email-address fts_autoindex = yes fts_enforced = yes fts_autoindex_exclude = \Trash + fts_autoindex_exclude1 = \Junk + fts_filters = normalizer-icu stopwords + fts_filters_en = lowercase english-possessive stopwords + fts_filters_fr = lowercase contractions stopwords + fts_header_excludes = Received DKIM-* ARC-* X-* x-* Comments Delivered-To Return-Path Authentication-Results Message-ID References In-Reply-To Thread-* Accept-Language Content-* MIME-Version + {% if FULL_TEXT_SEARCH_ATTACHMENTS %} + fts_tika = http://{{ FTS_ATTACHMENTS_ADDRESS }}:9998/tika/ + {% endif %} {% endif %} {% if COMPRESSION in [ 'gz', 'bz2', 'lz4', 'zstd' ] %} diff --git a/core/nginx/Dockerfile b/core/nginx/Dockerfile index f2eaac81..cacb6c99 100644 --- a/core/nginx/Dockerfile +++ b/core/nginx/Dockerfile @@ -17,7 +17,8 @@ ARG VERSION LABEL version=$VERSION RUN set -euxo pipefail \ - ; apk add --no-cache certbot nginx nginx-mod-http-brotli nginx-mod-mail openssl dovecot-lua dovecot-pigeonhole-plugin dovecot-lmtpd dovecot-pop3d dovecot-submissiond + ; apk add --no-cache certbot nginx nginx-mod-http-brotli nginx-mod-mail openssl \ + ; apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main dovecot-lua dovecot-pigeonhole-plugin 'dovecot-lmtpd<2.4' dovecot-pop3d dovecot-submissiond COPY conf/ /conf/ COPY --from=static /static/ /static/ diff --git a/core/nginx/letsencrypt.py b/core/nginx/letsencrypt.py index a14dfa15..edf24430 100755 --- a/core/nginx/letsencrypt.py +++ b/core/nginx/letsencrypt.py @@ -47,7 +47,7 @@ time.sleep(5) class MyRequestHandler(SimpleHTTPRequestHandler): def do_GET(self): - if self.path == '/testing': + if self.path == '/.well-known/acme-challenge/testing': self.send_response(204) else: self.send_response(404) @@ -55,7 +55,7 @@ class MyRequestHandler(SimpleHTTPRequestHandler): self.end_headers() def serve_one_request(): - with HTTPServer(("0.0.0.0", 8008), MyRequestHandler) as server: + with HTTPServer(("127.0.0.1", 8008), MyRequestHandler) as server: server.handle_request() # Run certbot every day diff --git a/core/oletools/Dockerfile b/core/oletools/Dockerfile index 39fd0e18..f636cc6d 100644 --- a/core/oletools/Dockerfile +++ b/core/oletools/Dockerfile @@ -6,11 +6,17 @@ FROM base ARG VERSION=local LABEL version=$VERSION +ARG OLEFY_SCRIPT=https://raw.githubusercontent.com/HeinleinSupport/olefy/f8aac6cc55283886d153e89c8f27fae66b1c24e2/olefy.py +ARG OLEFY_SHA256=1f5aa58b78ca7917350135b4425e5ed4d580c7051aabed1952c6afd12d0345a0 + RUN set -euxo pipefail \ ; apk add --no-cache netcat-openbsd libmagic libffi \ - ; curl -sLo olefy.py https://raw.githubusercontent.com/HeinleinSupport/olefy/f8aac6cc55283886d153e89c8f27fae66b1c24e2/olefy.py \ + ; curl -sLo olefy.py $OLEFY_SCRIPT \ + ; echo "$OLEFY_SHA256 olefy.py" |sha256sum -c \ ; chmod 755 olefy.py +COPY start.py / + RUN echo $VERSION >/version HEALTHCHECK --start-period=60s CMD echo PING|nc -q1 127.0.0.1 11343|grep "PONG" @@ -28,4 +34,4 @@ ENV \ OLEFY_DEL_TMP="1" \ OLEFY_DEL_TMP_FAILED="1" -CMD /app/olefy.py +CMD /start.py diff --git a/core/oletools/start.py b/core/oletools/start.py new file mode 100755 index 00000000..b0972908 --- /dev/null +++ b/core/oletools/start.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 + +from socrate import system + +system.set_env() + +with open('/app/olefy.py') as olefy: + exec(olefy.read()) diff --git a/core/rspamd/start.py b/core/rspamd/start.py index d6991253..67574a01 100755 --- a/core/rspamd/start.py +++ b/core/rspamd/start.py @@ -16,7 +16,8 @@ env = system.set_env() config_files = [] for rspamd_file in glob.glob("/conf/*"): conf.jinja(rspamd_file, env, os.path.join("/etc/rspamd/local.d", os.path.basename(rspamd_file))) - config_files.append(os.path.basename(rspamd_file)) + if rspamd_file != '/conf/forbidden_file_extension.map': + config_files.append(os.path.basename(rspamd_file)) for override_file in glob.glob("/overrides/*"): if os.path.basename(override_file) not in config_files: @@ -37,4 +38,4 @@ while True: os.system("mkdir -m 755 -p /run/rspamd") os.system("chown rspamd:rspamd /run/rspamd") os.system("find /var/lib/rspamd | grep -v /filter | xargs -n1 chown rspamd:rspamd") -os.execv("/usr/sbin/rspamd", ["rspamd", "-f", "-u", "rspamd", "-g", "rspamd"]) +os.execv("/usr/bin/rspamd", ["rspamd", "-f", "-u", "rspamd", "-g", "rspamd"]) diff --git a/docs/compose/.env b/docs/compose/.env index 62e767cf..5c54815c 100644 --- a/docs/compose/.env +++ b/docs/compose/.env @@ -152,3 +152,8 @@ REJECT_UNLISTED_RECIPIENT= # Log level threshold in start.py (value: CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET) LOG_LEVEL=WARNING + +# If your CPU supports Advanced Vector Extensions +# (AVX2 on x86_64, lrcpc on ARM64), you should consider enabling +# hardened-malloc earlier by uncommenting this +# LD_PRELOAD=/usr/lib/libhardened_malloc.so diff --git a/docs/compose/setup.rst b/docs/compose/setup.rst index f4c9c574..81433ba3 100644 --- a/docs/compose/setup.rst +++ b/docs/compose/setup.rst @@ -76,6 +76,15 @@ Review configuration variables After downloading the files, open ``mailu.env`` and review the variable settings. Make sure to read the comments in the file and instructions from the :ref:`common_cfg` page. +If your CPU supports Advanced Vector Extensions (AVX2 on x86_64, lrcpc on ARM64), you should +consider enabling hardened-malloc earlier in the boot process by adding the following to +your mailu.env: + +.. code-block:: bash + + LD_PRELOAD=/usr/lib/libhardened_malloc.so + + Finish setting up TLS --------------------- diff --git a/docs/configuration.rst b/docs/configuration.rst index 2f7e27a7..fbf284a4 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -131,12 +131,15 @@ later classify incoming mail based on the custom part. The ``DMARC_RUA`` and ``DMARC_RUF`` are DMARC protocol specific values. They hold the localpart for DMARC rua and ruf email addresses. -Full-text search is enabled for IMAP is enabled by default. This feature can be disabled -(e.g. for performance reasons) by setting the optional variable ``FULL_TEXT_SEARCH`` to ``off``. +The ``FULL_TEXT_SEARCH`` variable (default: 'en') is a comma separated list of +language codes as defined on `fts_languages`_. This feature can be disabled +(e.g. for performance reasons) by setting the variable to ``off``. You can set a global ``DEFAULT_QUOTA`` to be used for mailboxes when the domain has no specific quota configured. +.. _`fts_languages`: https://doc.dovecot.org/settings/plugin/fts-plugin/#fts-languages + .. _web_settings: Web settings diff --git a/docs/demo.rst b/docs/demo.rst index 46bedc0e..0619e218 100644 --- a/docs/demo.rst +++ b/docs/demo.rst @@ -35,7 +35,7 @@ Connecting to the server * Webmail : https://test.mailu.io/webmail/ * Admin UI : https://test.mailu.io/admin/ * Admin login : ``admin@test.mailu.io`` - * Admin password : ``letmein`` + * Admin password : ``Mailu+Demo@test.mailu.io`` (remove + and @test.mailu.io to get the correct password). * RESTful API: https://test.mailu.io/api * API token: ``Bearer APITokenForMailu`` diff --git a/docs/index.rst b/docs/index.rst index 0f16335c..ed281e3a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,7 +24,7 @@ popular groupware. Main features include: - **Standard email server**, IMAP and IMAP+, SMTP and Submission with autoconfiguration profiles for clients -- **Advanced email features**, aliases, domain aliases, custom routing +- **Advanced email features**, aliases, domain aliases, custom routing, full-text search of email attachments - **Web access**, multiple Webmails and administration interface - **User features**, aliases, auto-reply, auto-forward, fetched accounts, managesieve - **Admin features**, global admins, announcements, per-domain delegation, quotas diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml index b266fec0..a81f9f44 100644 --- a/setup/flavors/compose/docker-compose.yml +++ b/setup/flavors/compose/docker-compose.yml @@ -98,8 +98,16 @@ services: volumes: - "{{ root }}/mail:/mail" - "{{ root }}/overrides/dovecot:/overrides:ro" + networks: + - default + {% if tika_enabled %} + - fts_attachments + {% endif %} depends_on: - front + {% if tika_enabled %} + - fts_attachments + {% endif %} {% if resolver_enabled %} - resolver dns: @@ -140,6 +148,21 @@ services: {% endif %} {% endif %} +{% if tika_enabled %} + fts_attachments: + image: apache/tika:2.9.0.0-full + hostname: tika + restart: always + networks: + - fts_attachments + depends_on: + {% if resolver_enabled %} + - resolver + dns: + - {{ dns }} + {% endif %} +{% endif %} + antispam: image: ${DOCKER_ORG:-ghcr.io/mailu}/${DOCKER_PREFIX:-}rspamd:${MAILU_VERSION:-{{ version }}} hostname: antispam @@ -257,3 +280,8 @@ networks: driver: bridge internal: true {% endif %} +{% if tika_enabled %} + fts_attachments: + driver: bridge + internal: true +{% endif %} diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env index 616cd89b..cffafe15 100644 --- a/setup/flavors/compose/mailu.env +++ b/setup/flavors/compose/mailu.env @@ -49,19 +49,19 @@ DISABLE_STATISTICS={{ disable_statistics or 'False' }} # Expose the admin interface (value: true, false) ADMIN={{ admin_enabled or 'false' }} -# Choose which webmail to run if any (values: roundcube, snappymail, none) +# Choose which webmail to run if any (values: roundcube, snappymail, none). To enable this feature, recreate the docker-compose.yml file via setup. WEBMAIL={{ webmail_type }} # Expose the API interface (value: true, false) API={{ api_enabled or 'false' }} -# Dav server implementation (value: radicale, none) +# Dav server implementation (value: radicale, none). To enable this feature, recreate the docker-compose.yml file via setup. WEBDAV={{ webdav_enabled or 'none' }} -# Antivirus solution (value: clamav, none) +# Antivirus solution (value: clamav, none). To enable this feature, recreate the docker-compose.yml file via setup. ANTIVIRUS={{ antivirus_enabled or 'none' }} -# Scan Macros solution (value: true, false) +# Scan Macros solution (value: true, false). To enable this feature, recreate the docker-compose.yml file via setup. SCAN_MACROS={{ oletools_enabled or 'false' }} ################################### @@ -110,8 +110,10 @@ COMPRESSION={{ compression }} # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL={{ compression_level }} -# IMAP full-text search is enabled by default. Set the following variable to off in order to disable the feature. -# FULL_TEXT_SEARCH=off +# IMAP full-text search is enabled by default. +# Set the following variable to off in order to disable the feature +# or a comma separated list of language codes to support +FULL_TEXT_SEARCH=en ################################### # Web settings @@ -186,3 +188,5 @@ DEFAULT_SPAM_THRESHOLD=80 # This is a mandatory setting for using the RESTful API. API_TOKEN={{ api_token }} +# Whether tika should be enabled (scan/OCR email attachements). To enable this feature, recreate the docker-compose.yml file via setup. +FULL_TEXT_SEARCH_ATTACHMENTS={{ tika_enabled }} diff --git a/setup/templates/steps/compose/02_services.html b/setup/templates/steps/compose/02_services.html index 2311e4a3..701fa82f 100644 --- a/setup/templates/steps/compose/02_services.html +++ b/setup/templates/steps/compose/02_services.html @@ -64,6 +64,15 @@ the security implications caused by such an increase of attack surface.

Oletools scans documents in email attachements for malicious macros. It has a much lower memory footprint than a full-fledged anti-virus. +

+ + + Tika enables the functionality for searching through attachments. Tika scans documents in email attachments, process (OCR, keyword extraction) and then index them in a way they can be efficiently searched. This requires significant ressources (RAM, CPU and storage). +
+ diff --git a/towncrier/newsfragments/2824.feature b/towncrier/newsfragments/2824.feature new file mode 100644 index 00000000..a5c65570 --- /dev/null +++ b/towncrier/newsfragments/2824.feature @@ -0,0 +1 @@ +Enhance RESTful API user retrieval with quota used bytes. This is the current size of the user's email box in bytes. diff --git a/towncrier/newsfragments/2918.misc b/towncrier/newsfragments/2918.misc new file mode 100644 index 00000000..441806c4 --- /dev/null +++ b/towncrier/newsfragments/2918.misc @@ -0,0 +1 @@ +Upgrade dovecot to ensure we can proxy ipv6 via XCLIENT. diff --git a/towncrier/newsfragments/2934.bugfix b/towncrier/newsfragments/2934.bugfix new file mode 100644 index 00000000..349bc8e1 --- /dev/null +++ b/towncrier/newsfragments/2934.bugfix @@ -0,0 +1,3 @@ +Upgrade to alpine 3.18.4: this will fix a bug whereby musl wasn't retrying using TCP when it received truncated DNS replies from its upstream. In practice, this has been seen in the wild when postfix complains of: + +"Host or domain name not found. Name service error for name=outlook-com.olc.protection.outlook.com type=AAAA: Host found but no data record of requested type" diff --git a/towncrier/newsfragments/2937.bugfix b/towncrier/newsfragments/2937.bugfix new file mode 100644 index 00000000..7ccd316c --- /dev/null +++ b/towncrier/newsfragments/2937.bugfix @@ -0,0 +1,2 @@ +forbidden_file_extension.map could not be overridden. This file can be overriden to tweak with file extensions are allowed. +The instructions on https://mailu.io/master/antispam.html#can-i-change-the-list-of-authorized-file-attachments work again. diff --git a/towncrier/newsfragments/2950.misc b/towncrier/newsfragments/2950.misc new file mode 100644 index 00000000..c21a90a8 --- /dev/null +++ b/towncrier/newsfragments/2950.misc @@ -0,0 +1 @@ +Upgrade to snuffleupagus 0.10.0 diff --git a/towncrier/newsfragments/2955.misc b/towncrier/newsfragments/2955.misc new file mode 100644 index 00000000..42d8719c --- /dev/null +++ b/towncrier/newsfragments/2955.misc @@ -0,0 +1 @@ +Remove the version pinning on hardened malloc diff --git a/towncrier/newsfragments/2959.bugfix b/towncrier/newsfragments/2959.bugfix new file mode 100644 index 00000000..9da9c300 --- /dev/null +++ b/towncrier/newsfragments/2959.bugfix @@ -0,0 +1,3 @@ +Update hardened malloc as the original package is not available from alpine anymore. +The newer version of hardened malloc requires AVX2: Disable it by default at startup and hint in the logs when it should be enabled instead. +Upgrade snappymail to v2.29.1 diff --git a/towncrier/newsfragments/2962.bugfix b/towncrier/newsfragments/2962.bugfix new file mode 100644 index 00000000..59149a0e --- /dev/null +++ b/towncrier/newsfragments/2962.bugfix @@ -0,0 +1 @@ +Fix letsencrypt on master diff --git a/towncrier/newsfragments/2971.bugfix b/towncrier/newsfragments/2971.bugfix new file mode 100644 index 00000000..55d775bb --- /dev/null +++ b/towncrier/newsfragments/2971.bugfix @@ -0,0 +1,12 @@ +- Switch from fts-xapian to fts-flatcurve. This should address the problem with indexes getting too big and will be the default in dovecot 2.4 +- Enable full-text search of email attachments if configured (via Tika: you'll need to re-run setup) + +If you would like more than english to be supported, please ensure you update your FULL_TEXT_SEARCH configuration variable. + +You may also want to dispose of old indexes using a command such as: + +find /mailu/mail -type d -name xapian-indexes -prune -exec rm -r {} \+ + +And proactively force a reindexing using: + +docker compose exec imap doveadm index -A '*' diff --git a/towncrier/newsfragments/2974.feature b/towncrier/newsfragments/2974.feature new file mode 100644 index 00000000..5351654e --- /dev/null +++ b/towncrier/newsfragments/2974.feature @@ -0,0 +1 @@ +Enhance RESTful API with functionality for managing authentication tokens of users diff --git a/webmails/Dockerfile b/webmails/Dockerfile index 2d36c699..beaffd5d 100644 --- a/webmails/Dockerfile +++ b/webmails/Dockerfile @@ -19,6 +19,7 @@ RUN set -euxo pipefail \ aspell-uk aspell-ru aspell-fr aspell-de aspell-en \ ; rm /etc/nginx/http.d/default.conf \ ; rm /etc/php81/php-fpm.d/www.conf \ + ; mkdir -m 700 /root/.gnupg/ \ ; gpg --import /tmp/snappymail.asc \ ; gpg --import /tmp/roundcube.asc \ ; echo extension=snuffleupagus > /etc/php81/conf.d/snuffleupagus.ini \ @@ -33,7 +34,7 @@ RUN set -euxo pipefail \ ; cd /var/www \ ; curl -sLo /dev/shm/roundcube.tgz ${ROUNDCUBE_URL} \ ; curl -sLo /dev/shm/roundcube.tgz.asc ${ROUNDCUBE_URL}.asc \ - ; gpg --status-fd 1 --verify /dev/shm/roundcube.tgz.asc \ + ; gpg --status-fd 1 --verify /dev/shm/roundcube.tgz.asc /dev/shm/roundcube.tgz \ ; tar xzf /dev/shm/roundcube.tgz \ ; curl -sL ${CARDDAV_URL} | tar xz \ ; mv roundcubemail-* roundcube \ @@ -52,15 +53,15 @@ COPY roundcube/config/config.inc.carddav.php /var/www/roundcube/plugins/carddav/ # snappymail -ENV SNAPPYMAIL_URL https://github.com/the-djmaze/snappymail/releases/download/v2.28.4/snappymail-2.28.4.tar.gz +ENV SNAPPYMAIL_URL https://github.com/the-djmaze/snappymail/releases/download/v2.29.1/snappymail-2.29.1.tar.gz RUN set -euxo pipefail \ ; mkdir /var/www/snappymail \ ; cd /var/www/snappymail \ ; curl -sLo /dev/shm/snappymail.tgz ${SNAPPYMAIL_URL} \ ; curl -sLo /dev/shm/snappymail.tgz.asc ${SNAPPYMAIL_URL}.asc \ - ; gpg --status-fd 1 --verify /dev/shm/snappymail.tgz.asc \ - ; tar xzf /dev/shm/snappymail.tgz + ; gpg --status-fd 1 --verify /dev/shm/snappymail.tgz.asc /dev/shm/snappymail.tgz \ + ; cat /dev/shm/snappymail.tgz | tar xz # SnappyMail login COPY snappymail/login/include.php /var/www/snappymail/ diff --git a/webmails/snuffleupagus.rules b/webmails/snuffleupagus.rules index 5e619a8a..b3f69819 100644 --- a/webmails/snuffleupagus.rules +++ b/webmails/snuffleupagus.rules @@ -43,6 +43,7 @@ sp.disable_function.function("mail").param("additional_parameters").value_r("\\- # Since it's now burned, me might as well mitigate it publicly sp.disable_function.function("putenv").param("assignment").value_r("LD_").drop() +sp.disable_function.function("putenv").param("assignment").value("PATH").drop() # This one was burned in Nov 2019 - https://gist.github.com/LoadLow/90b60bd5535d6c3927bb24d5f9955b80 sp.disable_function.function("putenv").param("assignment").value_r("GCONV_").drop()