From a30093c6da6cfd25baba715147dc6c5dbb65a099 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 18 Jun 2024 17:08:34 +0000 Subject: [PATCH 01/83] Radicale was listening only on localhost:5232. Now listens on 0.0.0.0:5232 (cherry picked from commit 957abbf7335c9d490792f7fc96d83d532e7d5269) --- optional/radicale/radicale.conf | 1 + towncrier/newsfragments/3294.bugfix | 1 + 2 files changed, 2 insertions(+) create mode 100644 towncrier/newsfragments/3294.bugfix diff --git a/optional/radicale/radicale.conf b/optional/radicale/radicale.conf index d9859508..bcea34f8 100644 --- a/optional/radicale/radicale.conf +++ b/optional/radicale/radicale.conf @@ -1,5 +1,6 @@ [server] ssl = False +hosts = 0.0.0.0:5232 [encoding] request = utf-8 diff --git a/towncrier/newsfragments/3294.bugfix b/towncrier/newsfragments/3294.bugfix new file mode 100644 index 00000000..21cceeea --- /dev/null +++ b/towncrier/newsfragments/3294.bugfix @@ -0,0 +1 @@ +Webdav (radicale) was not reachable. This has been resolved. From 2165e51707e8d26c01ed7aac2ae1fbb31c45a438 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman <52963853+Diman0@users.noreply.github.com> Date: Tue, 18 Jun 2024 20:22:02 +0200 Subject: [PATCH 02/83] Add IPv6 listener for SUBNET6 --- optional/radicale/radicale.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optional/radicale/radicale.conf b/optional/radicale/radicale.conf index bcea34f8..7530338e 100644 --- a/optional/radicale/radicale.conf +++ b/optional/radicale/radicale.conf @@ -1,6 +1,6 @@ [server] ssl = False -hosts = 0.0.0.0:5232 +hosts = 0.0.0.0:5232, [::]:5232 [encoding] request = utf-8 From 59bcf25b5be3d370fef5e66baf286cb5abd1c582 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 19 Jun 2024 14:37:26 +0200 Subject: [PATCH 03/83] fix warnings (cherry picked from commit 2b6daf6157a681a44e297a18f798e300245dd59e) --- core/admin/mailu/internal/views/postfix.py | 4 ++-- core/admin/mailu/ui/forms.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/admin/mailu/internal/views/postfix.py b/core/admin/mailu/internal/views/postfix.py index cc8a0950..c2399cf7 100644 --- a/core/admin/mailu/internal/views/postfix.py +++ b/core/admin/mailu/internal/views/postfix.py @@ -14,7 +14,7 @@ def postfix_dane_map(domain_name): @internal.route("/postfix/domain/") def postfix_mailbox_domain(domain_name): - if re.match("^\[.*\]$", domain_name): + if re.match(r'^\[.*\]$', domain_name): return flask.abort(404) domain = models.Domain.query.get(domain_name) or \ models.Alternative.query.get(domain_name) or \ @@ -39,7 +39,7 @@ def postfix_alias_map(alias): @internal.route("/postfix/transport/") def postfix_transport(email): - if email == '*' or re.match("(^|.*@)\[.*\]$", email): + if email == '*' or re.match(r'(^|.*@)\[.*\]$', email): return flask.abort(404) _, domain_name = models.Email.resolve_domain(email) relay = models.Relay.query.get(domain_name) or flask.abort(404) diff --git a/core/admin/mailu/ui/forms.py b/core/admin/mailu/ui/forms.py index b6c45189..798bd923 100644 --- a/core/admin/mailu/ui/forms.py +++ b/core/admin/mailu/ui/forms.py @@ -7,7 +7,7 @@ import flask_wtf import re import ipaddress -LOCALPART_REGEX = "^[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*$" +LOCALPART_REGEX = r'^[a-zA-Z0-9!#$%&\'*+/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&\'*+/=?^_`{|}~-]+)*$' class DestinationField(fields.SelectMultipleField): """ Allow for multiple emails selection from current user choices and From 1bcbf4b6133883b4dcff691847067d4e3132cbdc Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 19 Jun 2024 14:50:09 +0200 Subject: [PATCH 04/83] fix warnings again (cherry picked from commit cfd7e6b6908f3e00e232191dd4797c9b11561125) --- docs/compose/docker-compose.yml | 1 - docs/docker-compose.yml | 1 - setup/docker-compose.yml | 1 - setup/flavors/compose/docker-compose.yml | 2 -- tests/compose/api/docker-compose.yml | 1 - tests/compose/core/docker-compose.yml | 1 - tests/compose/fetchmail/docker-compose.yml | 1 - tests/compose/filters/docker-compose.yml | 1 - tests/compose/webdav/docker-compose.yml | 1 - tests/compose/webmail/docker-compose.yml | 1 - 10 files changed, 11 deletions(-) diff --git a/docs/compose/docker-compose.yml b/docs/compose/docker-compose.yml index 3b964449..4c997c4c 100644 --- a/docs/compose/docker-compose.yml +++ b/docs/compose/docker-compose.yml @@ -1,6 +1,5 @@ # WARNING: this file is being deprecated over the new setup utility, found at https://setup.mailu.io -version: '2' services: diff --git a/docs/docker-compose.yml b/docs/docker-compose.yml index 9c5d2473..eccc14a1 100644 --- a/docs/docker-compose.yml +++ b/docs/docker-compose.yml @@ -1,7 +1,6 @@ # This file is used to test the mailu/docs website # Deployment files can be found on github.com/mailu/infra -version: '3' services: docs: diff --git a/setup/docker-compose.yml b/setup/docker-compose.yml index 9c93fd6f..680dd9ab 100644 --- a/setup/docker-compose.yml +++ b/setup/docker-compose.yml @@ -1,7 +1,6 @@ # This file is used to test the mailu/setup utility # Deployment files can be found on github.com/mailu/infra -version: '3.6' services: redis: diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml index 8cd374cd..3eeba1d4 100644 --- a/setup/flavors/compose/docker-compose.yml +++ b/setup/flavors/compose/docker-compose.yml @@ -3,8 +3,6 @@ # Please read the documentation before attempting any change. # Generated for {{ flavor }} flavor -version: '2.2' - services: # External dependencies diff --git a/tests/compose/api/docker-compose.yml b/tests/compose/api/docker-compose.yml index 384e89cd..03879edb 100644 --- a/tests/compose/api/docker-compose.yml +++ b/tests/compose/api/docker-compose.yml @@ -2,7 +2,6 @@ # Please read the documentation before attempting any change. # Generated for compose flavor -version: '3.6' services: diff --git a/tests/compose/core/docker-compose.yml b/tests/compose/core/docker-compose.yml index 384e89cd..03879edb 100644 --- a/tests/compose/core/docker-compose.yml +++ b/tests/compose/core/docker-compose.yml @@ -2,7 +2,6 @@ # Please read the documentation before attempting any change. # Generated for compose flavor -version: '3.6' services: diff --git a/tests/compose/fetchmail/docker-compose.yml b/tests/compose/fetchmail/docker-compose.yml index 067532fa..edd4140e 100644 --- a/tests/compose/fetchmail/docker-compose.yml +++ b/tests/compose/fetchmail/docker-compose.yml @@ -2,7 +2,6 @@ # Please read the documentation before attempting any change. # Generated for compose flavor -version: '3.6' services: diff --git a/tests/compose/filters/docker-compose.yml b/tests/compose/filters/docker-compose.yml index 5a508d58..2d2562a4 100644 --- a/tests/compose/filters/docker-compose.yml +++ b/tests/compose/filters/docker-compose.yml @@ -2,7 +2,6 @@ # Please read the documentation before attempting any change. # Generated for compose flavor -version: '3.6' services: diff --git a/tests/compose/webdav/docker-compose.yml b/tests/compose/webdav/docker-compose.yml index 1391b68d..78bce9d7 100644 --- a/tests/compose/webdav/docker-compose.yml +++ b/tests/compose/webdav/docker-compose.yml @@ -2,7 +2,6 @@ # Please read the documentation before attempting any change. # Generated for compose flavor -version: '3.6' services: diff --git a/tests/compose/webmail/docker-compose.yml b/tests/compose/webmail/docker-compose.yml index d61621d3..0fdc3e5f 100644 --- a/tests/compose/webmail/docker-compose.yml +++ b/tests/compose/webmail/docker-compose.yml @@ -2,7 +2,6 @@ # Please read the documentation before attempting any change. # Generated for compose flavor -version: '3.6' services: From a4542507b5e47ff6be6660f77c645f44fd7903a3 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 19 Jun 2024 14:52:43 +0200 Subject: [PATCH 05/83] remove another warning ERROR:root:Not sure what to do with in PROXY_PROTOCOL (None) (cherry picked from commit 12c1affc05304a8b058afbceacac441a7462fc94) --- core/nginx/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/nginx/config.py b/core/nginx/config.py index 1f06424b..0044772e 100755 --- a/core/nginx/config.py +++ b/core/nginx/config.py @@ -76,7 +76,7 @@ PROTO_ALL_BUT_HTTP=PROTO_MAIL.copy() PROTO_ALL_BUT_HTTP.extend(['443']) PROTO_ALL=PROTO_ALL_BUT_HTTP.copy() PROTO_ALL.extend(['80']) -for item in args.get('PROXY_PROTOCOL', '').split(','): +for item in args.get('PROXY_PROTOCOL', None).split(','): if item.isdigit(): args[f'PROXY_PROTOCOL_{item}']=True elif item == 'mail': From 600235f393fde56086db88f4545d8d6135a0f18e Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 19 Jun 2024 14:54:57 +0200 Subject: [PATCH 06/83] another one nginx: [warn] the "listen ... http2" directive is deprecated, use the "http2" directive instead in /etc/nginx/nginx.conf:90 (cherry picked from commit e9db8c4509638bfad65c78fc41a09c036b191afa) --- core/nginx/conf/nginx.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index e7d4885e..60c263a9 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -104,9 +104,10 @@ http { # Only enable HTTPS if TLS is enabled with no error {% if TLS_443 and not TLS_ERROR %} - listen 443 ssl http2{% if PROXY_PROTOCOL_443 %} proxy_protocol{% endif %}; + listen 443 ssl{% if PROXY_PROTOCOL_443 %} proxy_protocol{% endif %}; {% if SUBNET6 %} - listen [::]:443 ssl http2{% if PROXY_PROTOCOL_443 %} proxy_protocol{% endif %}; + listen [::]:443 ssl{% if PROXY_PROTOCOL_443 %} proxy_protocol{% endif %}; + http2 on; {% endif %} include /etc/nginx/tls.conf; From 2d2e9c4d8ab5fbd2810524a997614359a7aa0bd3 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 19 Jun 2024 14:58:34 +0200 Subject: [PATCH 07/83] another (cherry picked from commit c0d0dcf71377b639d8c83421c830cd6d69ebec4d) --- core/base/libs/socrate/socrate/system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index 3dcdea3d..fe233a5b 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -33,7 +33,7 @@ class LogFilter(object): def __init__(self, stream, re_patterns): self.stream = stream if isinstance(re_patterns, list): - self.pattern = re.compile('|'.join([f'(?:{pattern})' for pattern in re_patterns])) + self.pattern = re.compile('|'.join([fr'(?:{pattern})' for pattern in re_patterns])) elif isinstance(re_patterns, str): self.pattern = re.compile(re_patterns) else: From de922fd7e41355f7702950bfa6c4ce15efe4abbd Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 19 Jun 2024 15:35:44 +0200 Subject: [PATCH 08/83] better (cherry picked from commit 89ff26660f5bf0b54cdda0d519934f4d96f27be8) --- core/nginx/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/nginx/config.py b/core/nginx/config.py index 0044772e..4a381c2c 100755 --- a/core/nginx/config.py +++ b/core/nginx/config.py @@ -76,7 +76,7 @@ PROTO_ALL_BUT_HTTP=PROTO_MAIL.copy() PROTO_ALL_BUT_HTTP.extend(['443']) PROTO_ALL=PROTO_ALL_BUT_HTTP.copy() PROTO_ALL.extend(['80']) -for item in args.get('PROXY_PROTOCOL', None).split(','): +for item in args.get('PROXY_PROTOCOL', '').split(','): if item.isdigit(): args[f'PROXY_PROTOCOL_{item}']=True elif item == 'mail': @@ -85,6 +85,8 @@ for item in args.get('PROXY_PROTOCOL', None).split(','): for p in PROTO_ALL_BUT_HTTP: args[f'PROXY_PROTOCOL_{p}']=True elif item == 'all': for p in PROTO_ALL: args[f'PROXY_PROTOCOL_{p}']=True + elif item == '': + pass else: log.error(f'Not sure what to do with {item} in PROXY_PROTOCOL ({args.get("PROXY_PROTOCOL")})') From bca4fdffdb97ec2ace87f4188f191782439230c9 Mon Sep 17 00:00:00 2001 From: Dominik Hebeler Date: Tue, 25 Jun 2024 12:48:10 +0200 Subject: [PATCH 09/83] define client_ip variable (cherry picked from commit 34766602a87c6feac48b7716a19fae2e8950ae9b) --- core/admin/mailu/sso/views/base.py | 1 + towncrier/newsfragments/3314.bugfix | 1 + 2 files changed, 2 insertions(+) create mode 100644 towncrier/newsfragments/3314.bugfix diff --git a/core/admin/mailu/sso/views/base.py b/core/admin/mailu/sso/views/base.py index b347d3cb..59c4ef04 100644 --- a/core/admin/mailu/sso/views/base.py +++ b/core/admin/mailu/sso/views/base.py @@ -130,6 +130,7 @@ https://mailu.io/master/configuration.html#header-authentication-using-an-extern def _proxy(): proxy_ip = flask.request.headers.get('X-Forwarded-By', flask.request.remote_addr) ip = ipaddress.ip_address(proxy_ip) + client_ip = flask.request.headers.get('X-Real-IP', flask.request.remote_addr) if not any(ip in cidr for cidr in app.config['PROXY_AUTH_WHITELIST']): flask.current_app.logger.error(f'Login failed by proxy - not on whitelist: from {client_ip} through {flask.request.remote_addr}.') return flask.abort(500, '%s is not on PROXY_AUTH_WHITELIST' % proxy_ip) diff --git a/towncrier/newsfragments/3314.bugfix b/towncrier/newsfragments/3314.bugfix new file mode 100644 index 00000000..87558ee4 --- /dev/null +++ b/towncrier/newsfragments/3314.bugfix @@ -0,0 +1 @@ +Define client_ip variable in _proxy method. Fixes server error when using proxy authentication \ No newline at end of file From c9419503441fa4e99615d81069cf22be3a7875f2 Mon Sep 17 00:00:00 2001 From: Dominik Hebeler Date: Tue, 25 Jun 2024 12:38:51 +0200 Subject: [PATCH 10/83] nginx should recursively set the realip until the first no trusted address is found (cherry picked from commit 28f3b60615b0b2f501ea235c84ac7d2bdf1dc26b) --- core/nginx/conf/nginx.conf | 1 + towncrier/newsfragments/3311.bugfix | 1 + 2 files changed, 2 insertions(+) create mode 100644 towncrier/newsfragments/3311.bugfix diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index 60c263a9..487a5ef5 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -28,6 +28,7 @@ http { {% if REAL_IP_FROM %}{% for from_ip in REAL_IP_FROM.split(',') %} set_real_ip_from {{ from_ip }}; + real_ip_recursive on; {% endfor %}{% endif %} # Header maps diff --git a/towncrier/newsfragments/3311.bugfix b/towncrier/newsfragments/3311.bugfix new file mode 100644 index 00000000..f3066465 --- /dev/null +++ b/towncrier/newsfragments/3311.bugfix @@ -0,0 +1 @@ +Enable nginx setting `real_ip_recursive` to recursively replace real user ip \ No newline at end of file From 33516af243c0d2ececab080c8a7a263134d4cba3 Mon Sep 17 00:00:00 2001 From: Dominik Hebeler Date: Tue, 25 Jun 2024 13:34:17 +0200 Subject: [PATCH 11/83] do not put real_ip_recursive within the for-loop (cherry picked from commit 5be25b58873624b973b4356cfe10e299450ad605) --- core/nginx/conf/nginx.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index 487a5ef5..97a32113 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -26,9 +26,10 @@ http { real_ip_header proxy_protocol; {% endif %} - {% if REAL_IP_FROM %}{% for from_ip in REAL_IP_FROM.split(',') %} - set_real_ip_from {{ from_ip }}; + {% if REAL_IP_FROM %} real_ip_recursive on; + {% for from_ip in REAL_IP_FROM.split(',') %} + set_real_ip_from {{ from_ip }}; {% endfor %}{% endif %} # Header maps From 42b4dd2004c8222216dd7f9cdc01930944105538 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 22 Jun 2024 09:38:49 +0200 Subject: [PATCH 12/83] ports not in PORTS should not be bound (cherry picked from commit b7dcf452671c566a846127d8ca8644efcbe8f368) --- core/nginx/dovecot/proxy.conf | 15 +++------------ towncrier/newsfragments/3307.bugfix | 1 + 2 files changed, 4 insertions(+), 12 deletions(-) create mode 100644 towncrier/newsfragments/3307.bugfix diff --git a/core/nginx/dovecot/proxy.conf b/core/nginx/dovecot/proxy.conf index a6e64719..aec5aa5e 100644 --- a/core/nginx/dovecot/proxy.conf +++ b/core/nginx/dovecot/proxy.conf @@ -96,14 +96,12 @@ protocol imap { } service imap-login { -{%- if PORT_143 %} inet_listener imap { - port = 143 + port = {% if PORT_143 %}143{% else %}0{% endif %} {%- if PROXY_PROTOCOL_143 %} haproxy = yes {% endif %} } -{% endif %} {%- if TLS_993 and PORT_993 %} inet_listener imaps { port = 993 @@ -119,14 +117,12 @@ service imap-login { } service pop3-login { -{%- if PORT_110 %} inet_listener pop3 { - port = 110 + port = {% if PORT_110 %}110{% else %}0{% endif %} {%- if PROXY_PROTOCOL_110 %} haproxy = yes {% endif %} } -{% endif %} {%- if TLS_995 and PORT_995 %} inet_listener pop3s { port = 995 @@ -148,14 +144,9 @@ service lmtp { service submission-login { inet_listener submission { -{%- if PORT_587 %} - port = 587 + port = {% if PORT_587 %}587{% else %}0{% endif %} {%- if PROXY_PROTOCOL_587 %} haproxy = yes -{% endif %} -{%- else %} -# if the section is unset the port is bound anyways - port = 0 {% endif %} } {%- if TLS_465 and PORT_465 %} diff --git a/towncrier/newsfragments/3307.bugfix b/towncrier/newsfragments/3307.bugfix new file mode 100644 index 00000000..3639e430 --- /dev/null +++ b/towncrier/newsfragments/3307.bugfix @@ -0,0 +1 @@ +Ensure that ports that do not feature in PORTS are not bound From fd4ba4bd86436f32589d77e85e39a5284f2ff81d Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 22 Jun 2024 10:04:10 +0200 Subject: [PATCH 13/83] Update autoconfig too (cherry picked from commit b3d49599cf802aff2fbb3e51052c98ae658734f6) --- core/admin/mailu/configuration.py | 1 + core/admin/mailu/models.py | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 167d4577..e568deb9 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -87,6 +87,7 @@ DEFAULT_CONFIG = { 'TLS_PERMISSIVE': True, 'TZ': 'Etc/UTC', 'DEFAULT_SPAM_THRESHOLD': 80, + 'PORTS': '25,80,443,465,993,995,4190', 'PROXY_AUTH_WHITELIST': '', 'PROXY_AUTH_HEADER': 'X-Auth-Email', 'PROXY_AUTH_CREATE': False, diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 7cde30b9..16641d72 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -259,16 +259,16 @@ class Domain(Base): """ return list of auto configuration records (RFC6186) """ hostname = app.config['HOSTNAME'] protocols = [ - ('imap', 143, 20), - ('pop3', 110, 20), - ('submission', 587, 20), + ('imap', 143, 20 if 143 in app.config['PORTS'].split(',') else 0), + ('pop3', 110, 20 if 110 in app.config['PORTS'].split(',') else 0), + ('submission', 587, 20 if 587 in app.config['PORTS'].split(',') else 0), ] if app.config['TLS_FLAVOR'] != 'notls': protocols.extend([ - ('autodiscover', 443, 10), - ('submissions', 465, 10), - ('imaps', 993, 10), - ('pop3s', 995, 10), + ('autodiscover', 443, 10 if 443 in app.config['PORTS'].split(',') else 0), + ('submissions', 465, 10 if 465 in app.config['PORTS'].split(',') else 0), + ('imaps', 993, 10 if 993 in app.config['PORTS'].split(',') else 0), + ('pop3s', 995, 10 if 995 in app.config['PORTS'].split(',') else 0), ]) return [ From 8e280505101ca6a97745e5cd0f1c891d6462b42e Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 22 Jun 2024 10:20:32 +0200 Subject: [PATCH 14/83] Don't let people disable 465 and 993 This is what we use for the other type of autoconfig... and really what everyone should be using. (cherry picked from commit 5fb44bd7190ec2e2d70e06927f641d0b5df44ed8) --- core/admin/mailu/models.py | 14 +++++++------- core/nginx/dovecot/proxy.conf | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 16641d72..60ac5455 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -259,16 +259,16 @@ class Domain(Base): """ return list of auto configuration records (RFC6186) """ hostname = app.config['HOSTNAME'] protocols = [ - ('imap', 143, 20 if 143 in app.config['PORTS'].split(',') else 0), - ('pop3', 110, 20 if 110 in app.config['PORTS'].split(',') else 0), - ('submission', 587, 20 if 587 in app.config['PORTS'].split(',') else 0), + ('imap', 143, 20 if '143' in app.config['PORTS'].split(',') else 0), + ('pop3', 110, 20 if '110' in app.config['PORTS'].split(',') else 0), + ('submission', 587, 20 if '587' in app.config['PORTS'].split(',') else 0), ] if app.config['TLS_FLAVOR'] != 'notls': protocols.extend([ - ('autodiscover', 443, 10 if 443 in app.config['PORTS'].split(',') else 0), - ('submissions', 465, 10 if 465 in app.config['PORTS'].split(',') else 0), - ('imaps', 993, 10 if 993 in app.config['PORTS'].split(',') else 0), - ('pop3s', 995, 10 if 995 in app.config['PORTS'].split(',') else 0), + ('autodiscover', 443, 10 if '443' in app.config['PORTS'].split(',') else 0), + ('submissions', 465, 10), + ('imaps', 993, 10), + ('pop3s', 995, 10 if '995' in app.config['PORTS'].split(',') else 0), ]) return [ diff --git a/core/nginx/dovecot/proxy.conf b/core/nginx/dovecot/proxy.conf index aec5aa5e..8decfc77 100644 --- a/core/nginx/dovecot/proxy.conf +++ b/core/nginx/dovecot/proxy.conf @@ -102,7 +102,7 @@ service imap-login { haproxy = yes {% endif %} } -{%- if TLS_993 and PORT_993 %} +{%- if TLS_993 %} inet_listener imaps { port = 993 ssl = yes @@ -149,7 +149,7 @@ service submission-login { haproxy = yes {% endif %} } -{%- if TLS_465 and PORT_465 %} +{%- if TLS_465 %} inet_listener submissions { port = 465 ssl = yes From dc86008cb4cdb7515a57f8133f30b53c81a4fd7d Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 24 Jun 2024 08:58:39 +0200 Subject: [PATCH 15/83] optimize as per review (cherry picked from commit cdbfa4ee01e87ed02e9e0e5e129a6b19c8901f20) --- core/admin/mailu/models.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 60ac5455..f07dcf69 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -257,18 +257,19 @@ class Domain(Base): @cached_property def dns_autoconfig(self): """ return list of auto configuration records (RFC6186) """ + ports = {port.strip() for port in app.config['PORTS'].split(',')} hostname = app.config['HOSTNAME'] protocols = [ - ('imap', 143, 20 if '143' in app.config['PORTS'].split(',') else 0), - ('pop3', 110, 20 if '110' in app.config['PORTS'].split(',') else 0), - ('submission', 587, 20 if '587' in app.config['PORTS'].split(',') else 0), + ('imap', 143, 20 if '143' in ports else 0), + ('pop3', 110, 20 if '110' in ports else 0), + ('submission', 587, 20 if '587' in ports else 0), ] if app.config['TLS_FLAVOR'] != 'notls': protocols.extend([ - ('autodiscover', 443, 10 if '443' in app.config['PORTS'].split(',') else 0), + ('autodiscover', 443, 10 if '443' in ports else 0), ('submissions', 465, 10), ('imaps', 993, 10), - ('pop3s', 995, 10 if '995' in app.config['PORTS'].split(',') else 0), + ('pop3s', 995, 10 if '995' in ports else 0), ]) return [ From da8106ab66569620666aba5bf3c25f4ce8540ccd Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 24 Jun 2024 09:01:55 +0200 Subject: [PATCH 16/83] Document that 25,465 and 993 can't be disabled (cherry picked from commit b0dbaa6719bb1769bc36a4d88bfd2c015f7dde9b) --- docs/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 06b7627e..b4b687b1 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -280,7 +280,7 @@ The ``TZ`` sets the timezone Mailu will use. The timezone naming convention usua The ``PORTS`` (default: '25,80,443,465,993,995,4190') setting determines which services should be enabled. It is a comma delimited list of ports numbers. -If you need to re-enable IMAP, POP3 and Submission, you can append '110,143,587' to that list. +If you need to re-enable IMAP, POP3 and Submission, you can append '110,143,587' to that list. Please note that ports 25,465 and 993 cannot be disabled. The ``PROXY_PROTOCOL`` (default: unset) setting allows the the front container to receive TCP and HTTP connections with the `PROXY protocol`_ (originally introduced in HAProxy, now also configurable in other proxy servers). From 47b2cd47928d497bbe5cf3b3146fb9c9bf9f672e Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 26 Jun 2024 11:21:29 +0200 Subject: [PATCH 17/83] Update models.py (cherry picked from commit 98b3016bbdb0fd96e5823e140ae8e7facfedfd20) --- core/admin/mailu/models.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index f07dcf69..f73bf0b7 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -257,23 +257,23 @@ class Domain(Base): @cached_property def dns_autoconfig(self): """ return list of auto configuration records (RFC6186) """ - ports = {port.strip() for port in app.config['PORTS'].split(',')} + ports = {int(port.strip()) for port in app.config['PORTS'].split(',')}.union({465, 993}) hostname = app.config['HOSTNAME'] protocols = [ - ('imap', 143, 20 if '143' in ports else 0), - ('pop3', 110, 20 if '110' in ports else 0), - ('submission', 587, 20 if '587' in ports else 0), + ('imap', 143, 20), + ('pop3', 110, 20), + ('submission', 587, 20), ] if app.config['TLS_FLAVOR'] != 'notls': protocols.extend([ - ('autodiscover', 443, 10 if '443' in ports else 0), + ('autodiscover', 443, 10), ('submissions', 465, 10), ('imaps', 993, 10), - ('pop3s', 995, 10 if '995' in ports else 0), + ('pop3s', 995, 10), ]) return [ - f'_{proto}._tcp.{self.name}. 600 IN SRV {prio} 1 {port} {hostname}.' + f'_{proto}._tcp.{self.name}. 600 IN SRV {prio} 1 {port} {hostname}.' if port in ports else f'_{proto}._tcp.{self.name}. 600 IN SRV 0 0 0 .' for proto, port, prio in protocols ]+[f'autoconfig.{self.name}. 600 IN CNAME {hostname}.'] From 9392c626644b0afd4ad3e2ecd7bfe29041c60121 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 1 Jul 2024 17:14:36 +0200 Subject: [PATCH 18/83] Fix smtpd_forbid_unauth_pipelining w XCLIENT (cherry picked from commit 648c98ab0500b25e7eef9bc09c53e3ce05c56e25) --- core/postfix/conf/main.cf | 1 + towncrier/newsfragments/3322.bugfix | 1 + 2 files changed, 2 insertions(+) create mode 100644 towncrier/newsfragments/3322.bugfix diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf index e1d9134d..830031fc 100644 --- a/core/postfix/conf/main.cf +++ b/core/postfix/conf/main.cf @@ -26,6 +26,7 @@ podop = socketmap:unix:/tmp/podop.socket: postscreen_upstream_proxy_protocol = haproxy compatibility_level=3.6 smtpd_forbid_bare_newline=yes +smtpd_forbid_unauth_pipelining=no # Only accept virtual emails mydestination = diff --git a/towncrier/newsfragments/3322.bugfix b/towncrier/newsfragments/3322.bugfix new file mode 100644 index 00000000..db17941c --- /dev/null +++ b/towncrier/newsfragments/3322.bugfix @@ -0,0 +1 @@ +Postfix 3.9 has changed the default for smtpd_forbid_unauth_pipelining. We need it to be allowed for XCLIENT to work. From 745d26acf5b2d418b1622003a6dce18b48c32583 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 28 Jul 2024 22:54:55 +0200 Subject: [PATCH 19/83] Fix email forwarding when set from the UI (cherry picked from commit 9120f731fc610407a11fb72020b21c7046924efa) --- core/admin/mailu/ui/views/users.py | 16 +++++++++------- towncrier/newsfragments/3349.bugfix | 1 + 2 files changed, 10 insertions(+), 7 deletions(-) create mode 100644 towncrier/newsfragments/3349.bugfix diff --git a/core/admin/mailu/ui/views/users.py b/core/admin/mailu/ui/views/users.py index d3816c4a..df0a716d 100644 --- a/core/admin/mailu/ui/views/users.py +++ b/core/admin/mailu/ui/views/users.py @@ -92,13 +92,15 @@ def user_settings(user_email): user = models.User.query.get(user_email_or_current) or flask.abort(404) form = forms.UserSettingsForm(obj=user) utils.formatCSVField(form.forward_destination) + form.forward_enabled.data = bool(flask.request.form.get('forward_enabled', False)) if form.validate_on_submit(): - if form.forward_enabled.data and (form.forward_destination.data in ['', None] or type(form.forward_destination.data) is list): + if form.forward_enabled and (form.forward_destination.data in ['', None] or type(form.forward_destination.data) is list): flask.flash('Destination email address is missing', 'error') - user.forward_enabled = True - return flask.render_template('user/settings.html', form=form, user=user) - if form.forward_enabled.data: - form.forward_destination.data = form.forward_destination.data.replace(" ","").split(",") + return flask.redirect( + flask.url_for('.user_settings', user=user.email)) + if not user.forward_enabled and not flask.request.form.get('forward_destination', None): + form.forward_destination.data = "" + form.forward_destination.data = form.forward_destination.data.replace(" ","").split(",") form.populate_obj(user) models.db.session.commit() form.forward_destination.data = ", ".join(form.forward_destination.data) @@ -107,8 +109,8 @@ def user_settings(user_email): return flask.redirect( flask.url_for('.user_list', domain_name=user.domain.name)) elif form.is_submitted() and not form.validate(): - user.forward_enabled = form.forward_enabled.data - return flask.render_template('user/settings.html', form=form, user=user) + return flask.redirect( + flask.url_for('.user_settings', user=user.email)) return flask.render_template('user/settings.html', form=form, user=user) def _process_password_change(form, user_email): diff --git a/towncrier/newsfragments/3349.bugfix b/towncrier/newsfragments/3349.bugfix new file mode 100644 index 00000000..b1dd1e0a --- /dev/null +++ b/towncrier/newsfragments/3349.bugfix @@ -0,0 +1 @@ +Fix email-forwarding when set from the web interface From 6173f5e434c0897f74ad6f400f9e63e24e7b233c Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 29 Jul 2024 20:33:32 +0200 Subject: [PATCH 20/83] simplify again the logic (cherry picked from commit f003c0e712ba609457695ccfa89f55a4fa544c32) --- core/admin/mailu/ui/views/users.py | 9 ++++----- core/admin/mailu/utils.py | 1 + 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/admin/mailu/ui/views/users.py b/core/admin/mailu/ui/views/users.py index df0a716d..3b9e9485 100644 --- a/core/admin/mailu/ui/views/users.py +++ b/core/admin/mailu/ui/views/users.py @@ -92,14 +92,12 @@ def user_settings(user_email): user = models.User.query.get(user_email_or_current) or flask.abort(404) form = forms.UserSettingsForm(obj=user) utils.formatCSVField(form.forward_destination) - form.forward_enabled.data = bool(flask.request.form.get('forward_enabled', False)) if form.validate_on_submit(): + user.forward_enabled = bool(flask.request.form.get('forward_enabled', False)) if form.forward_enabled and (form.forward_destination.data in ['', None] or type(form.forward_destination.data) is list): flask.flash('Destination email address is missing', 'error') return flask.redirect( - flask.url_for('.user_settings', user=user.email)) - if not user.forward_enabled and not flask.request.form.get('forward_destination', None): - form.forward_destination.data = "" + flask.url_for('.user_settings', user_email=user_email)) form.forward_destination.data = form.forward_destination.data.replace(" ","").split(",") form.populate_obj(user) models.db.session.commit() @@ -109,8 +107,9 @@ def user_settings(user_email): return flask.redirect( flask.url_for('.user_list', domain_name=user.domain.name)) elif form.is_submitted() and not form.validate(): + flask.flash('Error validating the form', 'error') return flask.redirect( - flask.url_for('.user_settings', user=user.email)) + flask.url_for('.user_settings', user_email=user_email)) return flask.render_template('user/settings.html', form=form, user=user) def _process_password_change(form, user_email): diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index 13522d79..2af352db 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -532,6 +532,7 @@ def isBadOrPwned(form): def formatCSVField(field): if not field.data: + field.data = '' return if isinstance(field.data,str): data = field.data.replace(" ","").split(",") From ca6488e2ffa6c82b5aaf216158c0a9ca957e10a1 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Tue, 30 Jul 2024 10:41:17 +0200 Subject: [PATCH 21/83] as per review (cherry picked from commit 0eafff00c14b44e7e6970c6fdcfa4aa54553d97e) --- core/admin/mailu/ui/views/users.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/ui/views/users.py b/core/admin/mailu/ui/views/users.py index 3b9e9485..a430db4b 100644 --- a/core/admin/mailu/ui/views/users.py +++ b/core/admin/mailu/ui/views/users.py @@ -94,7 +94,7 @@ def user_settings(user_email): utils.formatCSVField(form.forward_destination) if form.validate_on_submit(): user.forward_enabled = bool(flask.request.form.get('forward_enabled', False)) - if form.forward_enabled and (form.forward_destination.data in ['', None] or type(form.forward_destination.data) is list): + if form.forward_enabled and not form.forward_destination.data: flask.flash('Destination email address is missing', 'error') return flask.redirect( flask.url_for('.user_settings', user_email=user_email)) From 95edcd8dcb0b2e040c1e1715c42cd78b25a9c885 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Tue, 30 Jul 2024 11:51:25 +0200 Subject: [PATCH 22/83] As per review (cherry picked from commit 11e72e70349c3ab89e8a6c538deb15692a40d7c0) --- core/admin/mailu/ui/views/users.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/ui/views/users.py b/core/admin/mailu/ui/views/users.py index a430db4b..6487c326 100644 --- a/core/admin/mailu/ui/views/users.py +++ b/core/admin/mailu/ui/views/users.py @@ -94,7 +94,7 @@ def user_settings(user_email): utils.formatCSVField(form.forward_destination) if form.validate_on_submit(): user.forward_enabled = bool(flask.request.form.get('forward_enabled', False)) - if form.forward_enabled and not form.forward_destination.data: + if user.forward_enabled and not form.forward_destination.data: flask.flash('Destination email address is missing', 'error') return flask.redirect( flask.url_for('.user_settings', user_email=user_email)) From 6fd984afb1c0a7e9d1f2278ab1ce1dd83c8b90d3 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 2 Aug 2024 21:24:41 +0200 Subject: [PATCH 23/83] Update deps (cherry picked from commit da168322d62f29d7736905d834db85cbcd9ce77b) --- setup/flavors/compose/docker-compose.yml | 2 +- towncrier/newsfragments/3347.bugfix | 2 ++ webmails/Dockerfile | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 towncrier/newsfragments/3347.bugfix diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml index 3eeba1d4..428d6995 100644 --- a/setup/flavors/compose/docker-compose.yml +++ b/setup/flavors/compose/docker-compose.yml @@ -217,7 +217,7 @@ services: # Optional services {% if antivirus_enabled %} antivirus: - image: clamav/clamav-debian:1.2.0-6 + image: clamav/clamav-debian:1.2.3-45 restart: always logging: driver: journald diff --git a/towncrier/newsfragments/3347.bugfix b/towncrier/newsfragments/3347.bugfix new file mode 100644 index 00000000..a5723885 --- /dev/null +++ b/towncrier/newsfragments/3347.bugfix @@ -0,0 +1,2 @@ +Update to a newer clamav 1.2.3-45 +Update to snappymail 2.36.4 diff --git a/webmails/Dockerfile b/webmails/Dockerfile index 950c3f3c..4e33a4da 100644 --- a/webmails/Dockerfile +++ b/webmails/Dockerfile @@ -54,7 +54,7 @@ COPY roundcube/config/config.inc.carddav.php /var/www/roundcube/plugins/carddav/ # snappymail -ENV SNAPPYMAIL_URL https://github.com/the-djmaze/snappymail/releases/download/v2.36.1/snappymail-2.36.1.tar.gz +ENV SNAPPYMAIL_URL https://github.com/the-djmaze/snappymail/releases/download/v2.36.4/snappymail-2.36.4.tar.gz RUN set -euxo pipefail \ ; mkdir /var/www/snappymail \ From dcd843d4babddf74fa0448f8e4a00ef3a988db28 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 4 Aug 2024 10:54:49 +0200 Subject: [PATCH 24/83] Add a new CNAME for old MUAs (cherry picked from commit cfec4c58cc20faf18540c574b00bc76568d62f86) --- core/admin/mailu/models.py | 2 +- towncrier/newsfragments/3347.bugfix | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index f73bf0b7..57c82c6b 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -276,7 +276,7 @@ class Domain(Base): f'_{proto}._tcp.{self.name}. 600 IN SRV {prio} 1 {port} {hostname}.' if port in ports else f'_{proto}._tcp.{self.name}. 600 IN SRV 0 0 0 .' for proto, port, prio in protocols - ]+[f'autoconfig.{self.name}. 600 IN CNAME {hostname}.'] + ]+[f'autoconfig.{self.name}. 600 IN CNAME {hostname}.', f'autodiscover.{self.name}. 600 IN CNAME {hostname}.'] @cached_property def dns_tlsa(self): diff --git a/towncrier/newsfragments/3347.bugfix b/towncrier/newsfragments/3347.bugfix index a5723885..cefcfd3e 100644 --- a/towncrier/newsfragments/3347.bugfix +++ b/towncrier/newsfragments/3347.bugfix @@ -1,2 +1,3 @@ Update to a newer clamav 1.2.3-45 Update to snappymail 2.36.4 +Add a new DNS entry for autodiscover (old MUA autoconfiguration) From 60cb72e9caba25f659db9a86c3b4573c49130864 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 4 Aug 2024 17:57:33 +0200 Subject: [PATCH 25/83] Roundcube 1.6.8 (cherry picked from commit 407024d59a08c1b335970ca7f33ed2831ebce6ff) --- towncrier/newsfragments/3347.bugfix | 1 + webmails/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/towncrier/newsfragments/3347.bugfix b/towncrier/newsfragments/3347.bugfix index cefcfd3e..c06c7515 100644 --- a/towncrier/newsfragments/3347.bugfix +++ b/towncrier/newsfragments/3347.bugfix @@ -1,3 +1,4 @@ Update to a newer clamav 1.2.3-45 Update to snappymail 2.36.4 +Update to roundcube 1.6.8 (CVE-2024-42009, CVE-2024-42008, CVE-2024-42010) Add a new DNS entry for autodiscover (old MUA autoconfiguration) diff --git a/webmails/Dockerfile b/webmails/Dockerfile index 4e33a4da..4e6e6896 100644 --- a/webmails/Dockerfile +++ b/webmails/Dockerfile @@ -28,7 +28,7 @@ RUN set -euxo pipefail \ ; mkdir -p /run/nginx /conf # roundcube -ENV ROUNDCUBE_URL https://github.com/roundcube/roundcubemail/releases/download/1.6.7/roundcubemail-1.6.7-complete.tar.gz +ENV ROUNDCUBE_URL https://github.com/roundcube/roundcubemail/releases/download/1.6.8/roundcubemail-1.6.8-complete.tar.gz ENV CARDDAV_URL https://github.com/mstilkerich/rcmcarddav/releases/download/v5.1.0/carddav-v5.1.0.tar.gz RUN set -euxo pipefail \ From 51559a823f9630052ecdc489a8b94cacb8068926 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 5 Aug 2024 14:13:55 +0200 Subject: [PATCH 26/83] Clarify the doc as per https://github.com/Mailu/Mailu/issues/3359#issuecomment-2268909660 (cherry picked from commit 0b8e565272159629e0b43571b4add65a32fc1af7) --- docs/antispam.rst | 6 +++--- towncrier/newsfragments/3347.bugfix | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/antispam.rst b/docs/antispam.rst index 5042ff0c..91ecab58 100644 --- a/docs/antispam.rst +++ b/docs/antispam.rst @@ -113,7 +113,7 @@ The following steps have to be taken to configure an additional symbol (rule) th * soft reject: temporarily delay message (this is used, for instance, to greylist or rate-limit messages) To move an email message to the Junk (Spam) folder, a score of 15 can be used in combination with the action "add header". - The above example configuration will reject all emails send from domains that are listed in '/etc/rspamd/override.d/blacklist.inc'. + The above example configuration will reject all emails send from domains that are listed in '/overrides/blacklist.inc'. 2. In the Rspamd overrides folder create a map that contains the domains to be blocked. You can use # to add comments. @@ -137,12 +137,12 @@ The following steps have to be taken to configure an additional symbol (rule) th The symbol is only displayed if the symbol has no pre-filter (action= line) configured. Changes made in this screen are not saved to the configuration file. -5. Check if the map is available. In rspamd webgui to to configuration. A map is available with the path: +5. Check if the map is available. In rspamd webgui to to configuration, a map is available with the path: /etc/rspamd/override.d/blacklist.inc Senders domain part is on the local blacklist .. image:: assets/screenshots/RspamdMapBlacklist.png - When clicking on this map, you can live-edit the map via the GUI. Changes are effective immediately. Only changes made to maps in the overrides folder are persistent. Changes made to other maps will be reverted when the Rspamd container is recreated. It is also possible to make direct changes to the map on filesystem. These changes are also effective immediately. + When clicking on this map, you can live-edit the map via the GUI. Please note that only changes made to maps in the ``/overrides`` folder are persistent as changes made interractively though the GUI will be reverted when the Rspamd container is recreated. All changes (whether through the GUI or on the filesystem) are effective immediately. For more information on using the multimap filter see the official `multimap documentation`_ of Rspamd. diff --git a/towncrier/newsfragments/3347.bugfix b/towncrier/newsfragments/3347.bugfix index c06c7515..6668768f 100644 --- a/towncrier/newsfragments/3347.bugfix +++ b/towncrier/newsfragments/3347.bugfix @@ -2,3 +2,4 @@ Update to a newer clamav 1.2.3-45 Update to snappymail 2.36.4 Update to roundcube 1.6.8 (CVE-2024-42009, CVE-2024-42008, CVE-2024-42010) Add a new DNS entry for autodiscover (old MUA autoconfiguration) +Clarify the language in the documentation related to rspamd overrides From eeb5deea80d6cca136ef29a0680c55980cbec239 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 5 Aug 2024 14:49:38 +0200 Subject: [PATCH 27/83] more doc updates (cherry picked from commit b359301cc2db40f98ecf4cb0acc821cdf9a003b7) --- docs/antispam.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/antispam.rst b/docs/antispam.rst index 91ecab58..51dffe22 100644 --- a/docs/antispam.rst +++ b/docs/antispam.rst @@ -137,8 +137,8 @@ The following steps have to be taken to configure an additional symbol (rule) th The symbol is only displayed if the symbol has no pre-filter (action= line) configured. Changes made in this screen are not saved to the configuration file. -5. Check if the map is available. In rspamd webgui to to configuration, a map is available with the path: - /etc/rspamd/override.d/blacklist.inc Senders domain part is on the local blacklist +5. Check if the map is available. In rspamd webgui go to configuration, a map is available with the path: + /overrides/blacklist.inc Senders domain part is on the local blacklist .. image:: assets/screenshots/RspamdMapBlacklist.png From 3d4a9ac29cf10f545bd68772156265ad6bc94d2c Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 8 Aug 2024 09:35:27 +0200 Subject: [PATCH 28/83] Fix #3364 (cherry picked from commit ee243ea735744b296bb90b2c1e6a1fded8915c8d) --- core/nginx/dovecot/login.lua | 13 +++++++++++-- towncrier/newsfragments/3364.bugfix | 1 + 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 towncrier/newsfragments/3364.bugfix diff --git a/core/nginx/dovecot/login.lua b/core/nginx/dovecot/login.lua index d24de149..a93b4b29 100644 --- a/core/nginx/dovecot/login.lua +++ b/core/nginx/dovecot/login.lua @@ -10,15 +10,24 @@ local http_client = dovecot.http.client { max_attempts = 3; } +-- on the other end we use urllib.parse.unquote() +function urlEncode(str) + return str:gsub("[^%w_.-~]", function(c) + return string.format("%%%02X", string.byte(c)) + end) +end + function auth_passdb_lookup(req) local auth_request = http_client:request { url = "http://{{ ADMIN_ADDRESS }}:8080/internal/auth/email"; } auth_request:add_header('Auth-Port', req.local_port) - auth_request:add_header('Auth-User', req.user) + local user = urlEncode(req.user) + auth_request:add_header('Auth-User', user) if req.password ~= nil then - auth_request:add_header('Auth-Pass', req.password) + local password = urlEncode(req.password) + auth_request:add_header('Auth-Pass', password) end auth_request:add_header('Auth-Protocol', req.service) auth_request:add_header('Client-IP', req.remote_ip) diff --git a/towncrier/newsfragments/3364.bugfix b/towncrier/newsfragments/3364.bugfix new file mode 100644 index 00000000..10e56cf3 --- /dev/null +++ b/towncrier/newsfragments/3364.bugfix @@ -0,0 +1 @@ +Fix a bug preventing percent characters from being used in passwords From 77c7d2f6919423cd6de62f64e3482b867d2701c7 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 8 Aug 2024 09:49:21 +0200 Subject: [PATCH 29/83] Ensure we test this (cherry picked from commit 1cbd31c7bbccede04970901edf27e3160844aa09) --- tests/compose/core/00_create_users.sh | 2 +- tests/compose/core/05_connectivity.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/compose/core/00_create_users.sh b/tests/compose/core/00_create_users.sh index 142c3cb3..3832c03b 100755 --- a/tests/compose/core/00_create_users.sh +++ b/tests/compose/core/00_create_users.sh @@ -8,5 +8,5 @@ docker compose -f tests/compose/core/docker-compose.yml exec -T admin flask mail docker compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu admin admin mailu.io 'password' --mode=update || exit 1 docker compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu user user mailu.io 'password' || exit 1 docker compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu user 'user/with/slash' mailu.io 'password' || exit 1 -docker compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu user 'user_UTF8' mailu.io 'password€' || exit 1 +docker compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu user 'user_UTF8' mailu.io 'pass%e9word€' || exit 1 echo "User testing successful!" diff --git a/tests/compose/core/05_connectivity.py b/tests/compose/core/05_connectivity.py index 5cc12069..6ae62487 100755 --- a/tests/compose/core/05_connectivity.py +++ b/tests/compose/core/05_connectivity.py @@ -8,7 +8,7 @@ import managesieve SERVER='localhost' USERNAME='user_UTF8@mailu.io' -PASSWORD='password€' +PASSWORD='pass%e9word€' #https://github.com/python/cpython/issues/73936 #SMTPlib does not support UTF8 passwords. USERNAME_ASCII='user@mailu.io' @@ -139,4 +139,4 @@ if __name__ == '__main__': test_SMTP(SERVER, USERNAME_ASCII, PASSWORD_ASCII) test_managesieve(SERVER, USERNAME, PASSWORD) #https://github.com/python/cpython/issues/73936 -#SMTPlib does not support UTF8 passwords. \ No newline at end of file +#SMTPlib does not support UTF8 passwords. From 00ef3cb950606dbf8dce8b48044028d3f1691fad Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 8 Aug 2024 10:24:43 +0200 Subject: [PATCH 30/83] Remove this insanity since we don't use nginx (cherry picked from commit 148c8f9ede32e649129eaec3eba0e3886db1d0a5) --- core/admin/mailu/internal/nginx.py | 12 +++--------- core/admin/mailu/internal/views/auth.py | 11 ++--------- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py index ebd677d0..d236513d 100644 --- a/core/admin/mailu/internal/nginx.py +++ b/core/admin/mailu/internal/nginx.py @@ -91,20 +91,14 @@ def handle_authentication(headers): # Authenticated user elif method in ['plain', 'login']: is_valid_user = False - # According to RFC2616 section 3.7.1 and PEP 3333, HTTP headers should - # be ASCII and are generally considered ISO8859-1. However when passing - # the password, nginx does not transcode the input UTF string, thus - # we need to manually decode. - raw_user_email = urllib.parse.unquote(headers["Auth-User"]) - raw_password = urllib.parse.unquote(headers["Auth-Pass"]) user_email = 'invalid' password = 'invalid' try: - user_email = raw_user_email.encode("iso8859-1").decode("utf8") - password = raw_password.encode("iso8859-1").decode("utf8") + user_email = urllib.parse.unquote(headers["Auth-User"]) + password = urllib.parse.unquote(headers["Auth-Pass"]) ip = urllib.parse.unquote(headers["Client-Ip"]) except: - app.logger.warn(f'Received undecodable user/password from nginx: {raw_user_email!r}/{raw_password!r}') + app.logger.warn(f'Received undecodable user/password from nginx: {headers["Auth-User"]!r}/{headers["Auth-Pass"]!r}') else: try: user = models.User.query.get(user_email) if '@' in user_email else None diff --git a/core/admin/mailu/internal/views/auth.py b/core/admin/mailu/internal/views/auth.py index 4aa31407..3182188c 100644 --- a/core/admin/mailu/internal/views/auth.py +++ b/core/admin/mailu/internal/views/auth.py @@ -29,7 +29,6 @@ def nginx_authentication(): response.headers['Auth-Status'] = status response.headers['Auth-Error-Code'] = code return response - raw_password = urllib.parse.unquote(headers['Auth-Pass']) if 'Auth-Pass' in headers else '' headers = nginx.handle_authentication(flask.request.headers) response = flask.Response() for key, value in headers.items(): @@ -50,14 +49,8 @@ def nginx_authentication(): if not is_port_25: utils.limiter.exempt_ip_from_ratelimits(client_ip) elif is_valid_user: - password = None - try: - password = raw_password.encode("iso8859-1").decode("utf8") - except: - app.logger.warn(f'Received undecodable password for {username} from nginx: {raw_password!r}') - utils.limiter.rate_limit_user(username, client_ip, password=None) - else: - utils.limiter.rate_limit_user(username, client_ip, password=password) + password = urllib.parse.unquote(headers['Auth-Pass']) if 'Auth-Pass' in headers else '' + utils.limiter.rate_limit_user(username, client_ip, password=password) elif not is_from_webmail: utils.limiter.rate_limit_ip(client_ip, username) return response From a5af42a6efb17cd5729413a4514d68ef0f181c5b Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 8 Aug 2024 10:27:35 +0200 Subject: [PATCH 31/83] Better (cherry picked from commit 38ea029bd9b118e58627055a900ae8357e9e48e5) --- core/admin/mailu/internal/views/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/views/auth.py b/core/admin/mailu/internal/views/auth.py index 3182188c..fed10953 100644 --- a/core/admin/mailu/internal/views/auth.py +++ b/core/admin/mailu/internal/views/auth.py @@ -49,7 +49,7 @@ def nginx_authentication(): if not is_port_25: utils.limiter.exempt_ip_from_ratelimits(client_ip) elif is_valid_user: - password = urllib.parse.unquote(headers['Auth-Pass']) if 'Auth-Pass' in headers else '' + password = urllib.parse.unquote(headers.get('Auth-Pass'], None)) utils.limiter.rate_limit_user(username, client_ip, password=password) elif not is_from_webmail: utils.limiter.rate_limit_ip(client_ip, username) From 1366ee3fc7afc3571f21c56cb34964e7f9bbbcfd Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 8 Aug 2024 10:33:19 +0200 Subject: [PATCH 32/83] doh (cherry picked from commit d7c6528f045e50b939675fcb6730ab4813aeb6d4) --- core/admin/mailu/internal/views/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/views/auth.py b/core/admin/mailu/internal/views/auth.py index fed10953..d163cb80 100644 --- a/core/admin/mailu/internal/views/auth.py +++ b/core/admin/mailu/internal/views/auth.py @@ -49,7 +49,7 @@ def nginx_authentication(): if not is_port_25: utils.limiter.exempt_ip_from_ratelimits(client_ip) elif is_valid_user: - password = urllib.parse.unquote(headers.get('Auth-Pass'], None)) + password = urllib.parse.unquote(headers.get('Auth-Pass', None)) utils.limiter.rate_limit_user(username, client_ip, password=password) elif not is_from_webmail: utils.limiter.rate_limit_ip(client_ip, username) From e8ded2f203ed7b7b3bb449d532c7101c65c4be1b Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 8 Aug 2024 10:50:32 +0200 Subject: [PATCH 33/83] More tests See what happens with a character that will url decode to longer (cherry picked from commit 2526d43a98c899f9742228e31ffb0e2c1a1c115d) --- tests/compose/core/00_create_users.sh | 2 +- tests/compose/core/05_connectivity.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/compose/core/00_create_users.sh b/tests/compose/core/00_create_users.sh index 3832c03b..0bed6baf 100755 --- a/tests/compose/core/00_create_users.sh +++ b/tests/compose/core/00_create_users.sh @@ -8,5 +8,5 @@ docker compose -f tests/compose/core/docker-compose.yml exec -T admin flask mail docker compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu admin admin mailu.io 'password' --mode=update || exit 1 docker compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu user user mailu.io 'password' || exit 1 docker compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu user 'user/with/slash' mailu.io 'password' || exit 1 -docker compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu user 'user_UTF8' mailu.io 'pass%e9word€' || exit 1 +docker compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu user 'user_UTF8' mailu.io 'pa…ss%e9word€' || exit 1 echo "User testing successful!" diff --git a/tests/compose/core/05_connectivity.py b/tests/compose/core/05_connectivity.py index 6ae62487..44bb4553 100755 --- a/tests/compose/core/05_connectivity.py +++ b/tests/compose/core/05_connectivity.py @@ -8,7 +8,7 @@ import managesieve SERVER='localhost' USERNAME='user_UTF8@mailu.io' -PASSWORD='pass%e9word€' +PASSWORD='pa…ss%e9word€' #https://github.com/python/cpython/issues/73936 #SMTPlib does not support UTF8 passwords. USERNAME_ASCII='user@mailu.io' From 14196e5054699a150d64274da2e5aa2fc8b6e5d0 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 8 Aug 2024 11:18:50 +0200 Subject: [PATCH 34/83] Do the same with Client-Ip (cherry picked from commit 98f671dc2e4be4fbc2ed64a5e36263caa217183c) --- core/nginx/dovecot/login.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/nginx/dovecot/login.lua b/core/nginx/dovecot/login.lua index a93b4b29..442833f1 100644 --- a/core/nginx/dovecot/login.lua +++ b/core/nginx/dovecot/login.lua @@ -30,7 +30,8 @@ function auth_passdb_lookup(req) auth_request:add_header('Auth-Pass', password) end auth_request:add_header('Auth-Protocol', req.service) - auth_request:add_header('Client-IP', req.remote_ip) + local client_ip = urlEncode(req.remote_ip) + auth_request:add_header('Client-Ip', client_ip) auth_request:add_header('Client-Port', req.remote_port) auth_request:add_header('Auth-SSL', req.secured) auth_request:add_header('Auth-Method', req.mechanism) From f2c0a147faac0fdd3c442865081f5b0479de3a2b Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 9 Aug 2024 15:29:51 +0200 Subject: [PATCH 35/83] as per review (cherry picked from commit 78c5d34227c7e42b4f7aaef8cd431697726f0485) --- core/admin/mailu/internal/nginx.py | 2 +- core/admin/mailu/internal/views/auth.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py index d236513d..3638645b 100644 --- a/core/admin/mailu/internal/nginx.py +++ b/core/admin/mailu/internal/nginx.py @@ -98,7 +98,7 @@ def handle_authentication(headers): password = urllib.parse.unquote(headers["Auth-Pass"]) ip = urllib.parse.unquote(headers["Client-Ip"]) except: - app.logger.warn(f'Received undecodable user/password from nginx: {headers["Auth-User"]!r}/{headers["Auth-Pass"]!r}') + app.logger.warn(f'Received undecodable user/password from nginx: {headers.get("Auth-User", "")!r}') else: try: user = models.User.query.get(user_email) if '@' in user_email else None diff --git a/core/admin/mailu/internal/views/auth.py b/core/admin/mailu/internal/views/auth.py index d163cb80..c74bcc9e 100644 --- a/core/admin/mailu/internal/views/auth.py +++ b/core/admin/mailu/internal/views/auth.py @@ -49,7 +49,7 @@ def nginx_authentication(): if not is_port_25: utils.limiter.exempt_ip_from_ratelimits(client_ip) elif is_valid_user: - password = urllib.parse.unquote(headers.get('Auth-Pass', None)) + password = urllib.parse.unquote(headers.get('Auth-Pass', '')) utils.limiter.rate_limit_user(username, client_ip, password=password) elif not is_from_webmail: utils.limiter.rate_limit_ip(client_ip, username) From 79a393d6017f5ac357a706ed19d7e09f1f9ac20c Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 9 Aug 2024 15:55:33 +0200 Subject: [PATCH 36/83] s/nginx/front (cherry picked from commit 5cfec650dfcee49b840134bd61292ace8fe3ea15) --- core/admin/mailu/internal/nginx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py index 3638645b..daef8b9e 100644 --- a/core/admin/mailu/internal/nginx.py +++ b/core/admin/mailu/internal/nginx.py @@ -98,7 +98,7 @@ def handle_authentication(headers): password = urllib.parse.unquote(headers["Auth-Pass"]) ip = urllib.parse.unquote(headers["Client-Ip"]) except: - app.logger.warn(f'Received undecodable user/password from nginx: {headers.get("Auth-User", "")!r}') + app.logger.warn(f'Received undecodable user/password from front: {headers.get("Auth-User", "")!r}') else: try: user = models.User.query.get(user_email) if '@' in user_email else None From 79e6957be33d4125334fd21f2a0926b7f0be30cf Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 24 Jun 2024 09:22:19 +0200 Subject: [PATCH 37/83] Update network graph (cherry picked from commit 3e8e8cef8e30a03241d61d3938458aa8744826df) --- docs/mailu-network-diagram.ipynb | 601 ++++++++++++++++++------------- 1 file changed, 349 insertions(+), 252 deletions(-) diff --git a/docs/mailu-network-diagram.ipynb b/docs/mailu-network-diagram.ipynb index 7b4336a5..6d2a87d8 100644 --- a/docs/mailu-network-diagram.ipynb +++ b/docs/mailu-network-diagram.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -14,378 +14,461 @@ "\n", "\n", - "\n", - "\n", + "\n", + "\n", "mailu\n", - "\n", - "Mailu\n", + "\n", + "Mailu\n", "\n", "\n", "internet\n", - "\n", - "Internet\n", + "\n", + "Internet\n", + "\n", + "\n", + "\n", + "proxy\n", + "\n", + "Optional proxy\n", + "\n", + "\n", + "\n", + "internet->proxy\n", + "\n", + "\n", + "25/tcp\n", + "\n", + "\n", + "\n", + "internet->proxy\n", + "\n", + "\n", + "80/tcp\n", + "\n", + "\n", + "\n", + "internet->proxy\n", + "\n", + "\n", + "443/tcp\n", + "\n", + "\n", + "\n", + "internet->proxy\n", + "\n", + "\n", + "465/tcp\n", + "\n", + "\n", + "\n", + "internet->proxy\n", + "\n", + "\n", + "587/tcp\n", + "\n", + "\n", + "\n", + "internet->proxy\n", + "\n", + "\n", + "110/tcp\n", + "\n", + "\n", + "\n", + "internet->proxy\n", + "\n", + "\n", + "995/tcp\n", + "\n", + "\n", + "\n", + "internet->proxy\n", + "\n", + "\n", + "143/tcp\n", + "\n", + "\n", + "\n", + "internet->proxy\n", + "\n", + "\n", + "993/tcp\n", + "\n", + "\n", + "\n", + "internet->proxy\n", + "\n", + "\n", + "4190/tcp\n", "\n", "\n", - "\n", + "\n", "front\n", - "\n", - "Front\n", + "\n", + "Front\n", "\n", - "\n", - "\n", - "internet->front\n", - "\n", - "\n", - "80/tcp\n", - "\n", - "\n", - "\n", - "internet->front\n", - "\n", - "\n", - "443/tcp\n", - "\n", - "\n", - "\n", - "internet->front\n", - "\n", - "\n", - "25/tcp\n", - "\n", - "\n", - "\n", - "internet->front\n", - "\n", - "\n", - "465/tcp\n", - "\n", - "\n", - "\n", - "internet->front\n", - "\n", - "\n", - "587/tcp\n", - "\n", - "\n", - "\n", - "internet->front\n", - "\n", - "\n", - "110/tcp\n", - "\n", - "\n", - "\n", - "internet->front\n", - "\n", - "\n", - "995/tcp\n", - "\n", - "\n", - "\n", - "internet->front\n", - "\n", - "\n", - "143/tcp\n", - "\n", - "\n", - "\n", - "internet->front\n", - "\n", - "\n", - "993/tcp\n", - "\n", - "\n", - "\n", - "internet->front\n", - "\n", - "\n", - "4190/tcp\n", - "\n", - "\n", + "\n", "\n", - "front->front\n", - "\n", - "\n", - "8008/tcp\n", + "proxy->front\n", + "\n", + "\n", + "25/tcp\n", + "\n", + "\n", + "\n", + "proxy->front\n", + "\n", + "\n", + "80/tcp\n", + "\n", + "\n", + "\n", + "proxy->front\n", + "\n", + "\n", + "443/tcp\n", + "\n", + "\n", + "\n", + "proxy->front\n", + "\n", + "\n", + "465/tcp\n", + "\n", + "\n", + "\n", + "proxy->front\n", + "\n", + "\n", + "587/tcp\n", + "\n", + "\n", + "\n", + "proxy->front\n", + "\n", + "\n", + "110/tcp\n", + "\n", + "\n", + "\n", + "proxy->front\n", + "\n", + "\n", + "995/tcp\n", + "\n", + "\n", + "\n", + "proxy->front\n", + "\n", + "\n", + "143/tcp\n", + "\n", + "\n", + "\n", + "proxy->front\n", + "\n", + "\n", + "993/tcp\n", + "\n", + "\n", + "\n", + "proxy->front\n", + "\n", + "\n", + "4190/tcp\n", "\n", "\n", - "\n", + "\n", "front->front\n", - "\n", - "\n", - "8000/tcp\n", + "\n", + "\n", + "8008/tcp\n", + "\n", + "\n", + "\n", + "front->front\n", + "\n", + "\n", + "8000/tcp\n", "\n", "\n", - "\n", + "\n", "admin\n", - "\n", - "Admin\n", + "\n", + "Admin\n", "\n", "\n", - "\n", + "\n", "front->admin\n", - "\n", - "\n", - "8080/tcp\n", + "\n", + "\n", + "8080/tcp\n", "\n", "\n", - "\n", + "\n", "smtp\n", - "\n", - "SMTP\n", + "\n", + "SMTP\n", "\n", "\n", - "\n", + "\n", "front->smtp\n", - "\n", - "\n", - "25/tcp\n", + "\n", + "\n", + "25/tcp\n", "\n", "\n", - "\n", + "\n", "front->smtp\n", - "\n", - "\n", - "10025/tcp\n", + "\n", + "\n", + "10025/tcp\n", "\n", "\n", - "\n", + "\n", "antispam\n", - "\n", - "Antispam\n", + "\n", + "Antispam\n", "\n", "\n", - "\n", + "\n", "front->antispam\n", - "\n", - "\n", - "11334/tcp\n", + "\n", + "\n", + "11334/tcp\n", "\n", "\n", - "\n", + "\n", "imap\n", - "\n", - "IMAP\n", + "\n", + "IMAP\n", "\n", "\n", - "\n", + "\n", "front->imap\n", - "\n", - "\n", - "4190/tcp\n", + "\n", + "\n", + "4190/tcp\n", "\n", "\n", - "\n", + "\n", "front->imap\n", - "\n", - "\n", - "143/tcp\n", + "\n", + "\n", + "143/tcp\n", "\n", "\n", - "\n", + "\n", "front->imap\n", - "\n", - "\n", - "110/tcp\n", + "\n", + "\n", + "110/tcp\n", "\n", "\n", - "\n", + "\n", "webdav\n", - "\n", - "WebDAV\n", + "\n", + "WebDAV\n", "\n", "\n", - "\n", + "\n", "front->webdav\n", - "\n", - "\n", - "5232/tcp\n", + "\n", + "\n", + "5232/tcp\n", "\n", "\n", - "\n", + "\n", "webmail\n", - "\n", - "Webmail\n", + "\n", + "Webmail\n", "\n", "\n", - "\n", + "\n", "front->webmail\n", - "\n", - "\n", - "80/tcp\n", + "\n", + "\n", + "80/tcp\n", "\n", "\n", - "\n", + "\n", "redis\n", - "\n", - "Redis\n", + "\n", + "Redis\n", "\n", "\n", - "\n", + "\n", "admin->redis\n", - "\n", - "\n", - "6379/tcp\n", + "\n", + "\n", + "6379/tcp\n", "\n", "\n", - "\n", + "\n", "admin->imap\n", - "\n", - "\n", - "2525/tcp\n", + "\n", + "\n", + "2525/tcp\n", "\n", "\n", - "\n", + "\n", "smtp->front\n", - "\n", - "\n", - "2525/tcp\n", + "\n", + "\n", + "2525/tcp\n", "\n", "\n", - "\n", + "\n", "smtp->admin\n", - "\n", - "\n", - "8080/tcp\n", + "\n", + "\n", + "8080/tcp\n", "\n", "\n", - "\n", + "\n", "smtp->antispam\n", - "\n", - "\n", - "11332/tcp\n", + "\n", + "\n", + "11332/tcp\n", "\n", "\n", - "\n", + "\n", "antispam->admin\n", - "\n", - "\n", - "80/tcp\n", + "\n", + "\n", + "8080/tcp\n", "\n", "\n", - "\n", + "\n", "antispam->redis\n", - "\n", - "\n", - "6379/tcp\n", + "\n", + "\n", + "6379/tcp\n", "\n", "\n", - "\n", + "\n", "antivirus\n", - "\n", - "Anti-Virus\n", + "\n", + "Anti-Virus\n", "\n", "\n", - "\n", + "\n", "antispam->antivirus\n", - "\n", - "\n", - "3310/tcp\n", + "\n", + "\n", + "3310/tcp\n", "\n", "\n", - "\n", + "\n", "oletools\n", - "\n", - "Oletools\n", + "\n", + "Oletools\n", "\n", "\n", - "\n", + "\n", "antispam->oletools\n", - "\n", - "\n", - "11343/tcp\n", + "\n", + "\n", + "11343/tcp\n", "\n", - "\n", - "\n", - "imap->front\n", - "\n", - "\n", - "25/tcp\n", + "\n", + "\n", + "imap->proxy\n", + "\n", + "\n", + "25/tcp\n", "\n", "\n", - "\n", + "\n", "imap->admin\n", - "\n", - "\n", - "8080/tcp\n", + "\n", + "\n", + "8080/tcp\n", "\n", "\n", - "\n", + "\n", "imap->antispam\n", - "\n", - "\n", - "11334/tcp\n", + "\n", + "\n", + "11334/tcp\n", "\n", "\n", - "\n", + "\n", "fts_attachments\n", - "\n", - "Tika\n", + "\n", + "Tika\n", "\n", "\n", - "\n", + "\n", "imap->fts_attachments\n", - "\n", - "\n", - "9998/tcp\n", + "\n", + "\n", + "9998/tcp\n", + "\n", + "\n", + "\n", + "webmail->proxy\n", + "\n", + "\n", + "443/tcp\n", "\n", "\n", - "\n", + "\n", "webmail->front\n", - "\n", - "\n", - "14190/tcp\n", + "\n", + "\n", + "14190/tcp\n", "\n", "\n", - "\n", + "\n", "webmail->front\n", - "\n", - "\n", - "10025/tcp\n", + "\n", + "\n", + "10025/tcp\n", "\n", "\n", - "\n", + "\n", "webmail->front\n", - "\n", - "\n", - "10143/tcp\n", + "\n", + "\n", + "10143/tcp\n", "\n", "\n", - "\n", + "\n", "fetchmail\n", - "\n", - "Fetchmail\n", + "\n", + "Fetchmail\n", "\n", "\n", - "\n", + "\n", "fetchmail->front\n", - "\n", - "\n", - "25/tcp\n", + "\n", + "\n", + "25/tcp\n", "\n", "\n", - "\n", + "\n", "fetchmail->front\n", - "\n", - "\n", - "2525/tcp\n", + "\n", + "\n", + "2525/tcp\n", "\n", "\n", - "\n", + "\n", "fetchmail->admin\n", - "\n", - "\n", - "8080/tcp\n", + "\n", + "\n", + "8080/tcp\n", "\n", "\n", "\n" ], "text/plain": [ - "" + "" ] }, - "execution_count": 2, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -405,6 +488,7 @@ " \n", " # Components\n", " internet [label = \"Internet\";];\n", + " proxy [label = \"Optional proxy\"; shape = \"polygon\"];\n", " front [label = \"Front\";];\n", " admin [label = \"Admin\";];\n", " smtp [label = \"SMTP\";];\n", @@ -417,18 +501,29 @@ " fetchmail [label = \"Fetchmail\";];\n", " oletools [label = \"Oletools\"];\n", " fts_attachments [label = \"Tika\"];\n", + " # proxy from internet\n", + " internet -> proxy [label = \"25/tcp\";];\n", + " internet -> proxy [label = \"80/tcp\";];\n", + " internet -> proxy [label = \"443/tcp\";];\n", + " internet -> proxy [label = \"465/tcp\";];\n", + " internet -> proxy [label = \"587/tcp\";];\n", + " internet -> proxy [label = \"110/tcp\";];\n", + " internet -> proxy [label = \"995/tcp\";];\n", + " internet -> proxy [label = \"143/tcp\";];\n", + " internet -> proxy [label = \"993/tcp\";];\n", + " internet -> proxy [label = \"4190/tcp\";];\n", " \n", - " # Front from internet\n", - " internet -> front [label = \"80/tcp\";];\n", - " internet -> front [label = \"443/tcp\";];\n", - " internet -> front [label = \"25/tcp\";];\n", - " internet -> front [label = \"465/tcp\";];\n", - " internet -> front [label = \"587/tcp\";];\n", - " internet -> front [label = \"110/tcp\";];\n", - " internet -> front [label = \"995/tcp\";];\n", - " internet -> front [label = \"143/tcp\";];\n", - " internet -> front [label = \"993/tcp\";];\n", - " internet -> front [label = \"4190/tcp\";];\n", + " # Front from proxy\n", + " proxy -> front [label = \"25/tcp\";];\n", + " proxy -> front [label = \"80/tcp\";];\n", + " proxy -> front [label = \"443/tcp\";];\n", + " proxy -> front [label = \"465/tcp\";];\n", + " proxy -> front [label = \"587/tcp\";];\n", + " proxy -> front [label = \"110/tcp\";];\n", + " proxy -> front [label = \"995/tcp\";];\n", + " proxy -> front [label = \"143/tcp\";];\n", + " proxy -> front [label = \"993/tcp\";];\n", + " proxy -> front [label = \"4190/tcp\";];\n", " \n", " front -> front [label = \"8008/tcp\";];\n", " front -> front [label = \"8000/tcp\";];\n", @@ -448,18 +543,20 @@ " \n", " imap -> admin [label = \"8080/tcp\";];\n", " imap -> antispam [label = \"11334/tcp\";];\n", - " imap -> front [label = \"25/tcp\";];\n", + " imap -> proxy [label = \"25/tcp\";];\n", " imap -> fts_attachments [label = \"9998/tcp\";];\n", " \n", " webmail -> front [label = \"14190/tcp\";];\n", " webmail -> front [label = \"10025/tcp\";];\n", " webmail -> front [label = \"10143/tcp\";];\n", + " # carddav\n", + " webmail -> proxy [label = \"443/tcp\";];\n", " \n", " admin -> redis [label = \"6379/tcp\";];\n", " admin -> imap [label = \"2525/tcp\";];\n", " \n", " antispam -> redis [label = \"6379/tcp\";];\n", - " antispam -> admin [label = \"80/tcp\";];\n", + " antispam -> admin [label = \"8080/tcp\";];\n", " antispam -> oletools [label = \"11343/tcp\";];\n", " antispam -> antivirus [label = \"3310/tcp\";];\n", " \n", From b36018e0c10bb2c2af9fb40d0ee883adb278bbc2 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 24 Jun 2024 09:46:06 +0200 Subject: [PATCH 38/83] Handle the case where PROXY_PROTOCOL_25 is set (cherry picked from commit 0ff18c607923eaa364327dffd510d127667bea75) --- core/base/libs/socrate/socrate/system.py | 33 +++++++++++++++++++++++- core/nginx/config.py | 32 ----------------------- docs/mailu-network-diagram.ipynb | 2 +- optional/fetchmail/fetchmail.py | 2 +- 4 files changed, 34 insertions(+), 35 deletions(-) diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index fe233a5b..ec726433 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -108,8 +108,39 @@ def set_env(required_secrets=[], log_filters=[]): } def clean_env(): - """ remove all secret keys """ + """ remove all secret keys, normalize PROXY_PROTOCOL """ [os.environ.pop(key, None) for key in os.environ.keys() if key.endswith("_KEY")] + # Configure PROXY_PROTOCOL + PROTO_MAIL=['25', '110', '995', '143', '993', '587', '465', '4190'] + PROTO_ALL_BUT_HTTP=PROTO_MAIL.copy() + PROTO_ALL_BUT_HTTP.extend(['443']) + PROTO_ALL=PROTO_ALL_BUT_HTTP.copy() + PROTO_ALL.extend(['80']) + for item in os.environ.get('PROXY_PROTOCOL', '').split(','): + if item.isdigit(): + os.environ[f'PROXY_PROTOCOL_{item}']=True + elif item == 'mail': + for p in PROTO_MAIL: os.environ[f'PROXY_PROTOCOL_{p}']=True + elif item == 'all-but-http': + for p in PROTO_ALL_BUT_HTTP: os.environ[f'PROXY_PROTOCOL_{p}']=True + elif item == 'all': + for p in PROTO_ALL: os.environ[f'PROXY_PROTOCOL_{p}']=True + elif item == '': + pass + else: + log.error(f'Not sure what to do with {item} in PROXY_PROTOCOL ({args.get("PROXY_PROTOCOL")})') + + PORTS_REQUIRING_TLS=['443', '465', '993', '995'] + ALL_PORTS='25,80,443,465,993,995,4190' + for item in os.environ.get('PORTS', ALL_PORTS).split(','): + if item in PORTS_REQUIRING_TLS and os.environ['TLS_FLAVOR'] == 'notls': + continue + os.environ[f'PORT_{item}']=True + + if os.environ['TLS_FLAVOR'] != 'notls': + for item in os.environ.get('TLS', ALL_PORTS).split(','): + if item in PORTS_REQUIRING_TLS: + os.environ[f'TLS_{item}']=True def drop_privs_to(username='mailu'): pwnam = getpwnam(username) diff --git a/core/nginx/config.py b/core/nginx/config.py index 4a381c2c..96812dba 100755 --- a/core/nginx/config.py +++ b/core/nginx/config.py @@ -70,38 +70,6 @@ with open("/etc/resolv.conf") as handle: resolver = content[content.index("nameserver") + 1] args["RESOLVER"] = f"[{resolver}]" if ":" in resolver else resolver -# Configure PROXY_PROTOCOL -PROTO_MAIL=['25', '110', '995', '143', '993', '587', '465', '4190'] -PROTO_ALL_BUT_HTTP=PROTO_MAIL.copy() -PROTO_ALL_BUT_HTTP.extend(['443']) -PROTO_ALL=PROTO_ALL_BUT_HTTP.copy() -PROTO_ALL.extend(['80']) -for item in args.get('PROXY_PROTOCOL', '').split(','): - if item.isdigit(): - args[f'PROXY_PROTOCOL_{item}']=True - elif item == 'mail': - for p in PROTO_MAIL: args[f'PROXY_PROTOCOL_{p}']=True - elif item == 'all-but-http': - for p in PROTO_ALL_BUT_HTTP: args[f'PROXY_PROTOCOL_{p}']=True - elif item == 'all': - for p in PROTO_ALL: args[f'PROXY_PROTOCOL_{p}']=True - elif item == '': - pass - else: - log.error(f'Not sure what to do with {item} in PROXY_PROTOCOL ({args.get("PROXY_PROTOCOL")})') - -PORTS_REQUIRING_TLS=['443', '465', '993', '995'] -ALL_PORTS='25,80,443,465,993,995,4190' -for item in args.get('PORTS', ALL_PORTS).split(','): - if item in PORTS_REQUIRING_TLS and args['TLS_FLAVOR'] == 'notls': - continue - args[f'PORT_{item}']=True - -if args['TLS_FLAVOR'] != 'notls': - for item in args.get('TLS', ALL_PORTS).split(','): - if item in PORTS_REQUIRING_TLS: - args[f'TLS_{item}']=True - # TLS configuration cert_name = args.get("TLS_CERT_FILENAME", "cert.pem") keypair_name = args.get("TLS_KEYPAIR_FILENAME", "key.pem") diff --git a/docs/mailu-network-diagram.ipynb b/docs/mailu-network-diagram.ipynb index 6d2a87d8..2ef1457d 100644 --- a/docs/mailu-network-diagram.ipynb +++ b/docs/mailu-network-diagram.ipynb @@ -561,7 +561,7 @@ " antispam -> antivirus [label = \"3310/tcp\";];\n", " \n", " fetchmail -> admin [label = \"8080/tcp\"]\n", - " fetchmail -> front [label = \"25/tcp\"]\n", + " fetchmail -> proxy [label = \"25/tcp\"]\n", " fetchmail -> front [label = \"2525/tcp\"]\n", " #\n", " # those don't need internet:\n", diff --git a/optional/fetchmail/fetchmail.py b/optional/fetchmail/fetchmail.py index a298cf53..410ce4fb 100755 --- a/optional/fetchmail/fetchmail.py +++ b/optional/fetchmail/fetchmail.py @@ -60,7 +60,7 @@ def run(debug): protocol=fetch["protocol"], host=escape_rc_string(fetch["host"]), port=fetch["port"], - smtphost=f'{os.environ["FRONT_ADDRESS"]}' if fetch['scan'] else f'{os.environ["FRONT_ADDRESS"]}/2525', + smtphost=f'{os.environ["HOSTNAMES"].split(",")[0]}' if fetch['scan'] and os.environ.get('PROXY_PROTOCOL_25', False) else f'{os.environ["FRONT_ADDRESS"]}' if fetch['scan'] else f'{os.environ["FRONT_ADDRESS"]}/2525', username=escape_rc_string(fetch["username"]), password=escape_rc_string(fetch["password"]), options=options, From 6be869f036f4bfdc7c7725e646a2602ff02bb865 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 24 Jun 2024 10:06:55 +0200 Subject: [PATCH 39/83] Admin talks to front:2525, not imap (cherry picked from commit b4c8829a66691e67220840cbca4b07158563e62b) --- docs/mailu-network-diagram.ipynb | 392 +++++++++++++++---------------- 1 file changed, 196 insertions(+), 196 deletions(-) diff --git a/docs/mailu-network-diagram.ipynb b/docs/mailu-network-diagram.ipynb index 2ef1457d..2a68e0e4 100644 --- a/docs/mailu-network-diagram.ipynb +++ b/docs/mailu-network-diagram.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 5, + "execution_count": 1, "metadata": {}, "outputs": [ { @@ -14,461 +14,461 @@ "\n", "\n", - "\n", - "\n", + "\n", + "\n", "mailu\n", - "\n", - "Mailu\n", + "\n", + "Mailu\n", "\n", "\n", "internet\n", - "\n", - "Internet\n", + "\n", + "Internet\n", "\n", "\n", "\n", "proxy\n", - "\n", - "Optional proxy\n", + "\n", + "Optional proxy\n", "\n", "\n", "\n", "internet->proxy\n", - "\n", - "\n", - "25/tcp\n", + "\n", + "\n", + "25/tcp\n", "\n", "\n", "\n", "internet->proxy\n", - "\n", - "\n", - "80/tcp\n", + "\n", + "\n", + "80/tcp\n", "\n", "\n", "\n", "internet->proxy\n", - "\n", - "\n", - "443/tcp\n", + "\n", + "\n", + "443/tcp\n", "\n", "\n", "\n", "internet->proxy\n", - "\n", - "\n", - "465/tcp\n", + "\n", + "\n", + "465/tcp\n", "\n", "\n", "\n", "internet->proxy\n", - "\n", - "\n", - "587/tcp\n", + "\n", + "\n", + "587/tcp\n", "\n", "\n", "\n", "internet->proxy\n", - "\n", - "\n", - "110/tcp\n", + "\n", + "\n", + "110/tcp\n", "\n", "\n", "\n", "internet->proxy\n", - "\n", - "\n", - "995/tcp\n", + "\n", + "\n", + "995/tcp\n", "\n", "\n", "\n", "internet->proxy\n", - "\n", - "\n", - "143/tcp\n", + "\n", + "\n", + "143/tcp\n", "\n", "\n", "\n", "internet->proxy\n", - "\n", - "\n", - "993/tcp\n", + "\n", + "\n", + "993/tcp\n", "\n", "\n", "\n", "internet->proxy\n", - "\n", - "\n", - "4190/tcp\n", + "\n", + "\n", + "4190/tcp\n", "\n", "\n", "\n", "front\n", - "\n", - "Front\n", + "\n", + "Front\n", "\n", "\n", "\n", "proxy->front\n", - "\n", - "\n", - "25/tcp\n", + "\n", + "\n", + "25/tcp\n", "\n", "\n", "\n", "proxy->front\n", - "\n", - "\n", - "80/tcp\n", + "\n", + "\n", + "80/tcp\n", "\n", "\n", "\n", "proxy->front\n", - "\n", - "\n", - "443/tcp\n", + "\n", + "\n", + "443/tcp\n", "\n", "\n", "\n", "proxy->front\n", - "\n", - "\n", - "465/tcp\n", + "\n", + "\n", + "465/tcp\n", "\n", "\n", "\n", "proxy->front\n", - "\n", - "\n", - "587/tcp\n", + "\n", + "\n", + "587/tcp\n", "\n", "\n", "\n", "proxy->front\n", - "\n", - "\n", - "110/tcp\n", + "\n", + "\n", + "110/tcp\n", "\n", "\n", "\n", "proxy->front\n", - "\n", - "\n", - "995/tcp\n", + "\n", + "\n", + "995/tcp\n", "\n", "\n", "\n", "proxy->front\n", - "\n", - "\n", - "143/tcp\n", + "\n", + "\n", + "143/tcp\n", "\n", "\n", "\n", "proxy->front\n", - "\n", - "\n", - "993/tcp\n", + "\n", + "\n", + "993/tcp\n", "\n", "\n", "\n", "proxy->front\n", - "\n", - "\n", - "4190/tcp\n", + "\n", + "\n", + "4190/tcp\n", "\n", "\n", "\n", "front->front\n", - "\n", - "\n", - "8008/tcp\n", + "\n", + "\n", + "8008/tcp\n", "\n", "\n", "\n", "front->front\n", - "\n", - "\n", - "8000/tcp\n", + "\n", + "\n", + "8000/tcp\n", "\n", "\n", "\n", "admin\n", - "\n", - "Admin\n", + "\n", + "Admin\n", "\n", "\n", "\n", "front->admin\n", - "\n", - "\n", - "8080/tcp\n", + "\n", + "\n", + "8080/tcp\n", "\n", "\n", "\n", "smtp\n", - "\n", - "SMTP\n", + "\n", + "SMTP\n", "\n", "\n", "\n", "front->smtp\n", - "\n", - "\n", - "25/tcp\n", + "\n", + "\n", + "25/tcp\n", "\n", "\n", "\n", "front->smtp\n", - "\n", - "\n", - "10025/tcp\n", + "\n", + "\n", + "10025/tcp\n", "\n", "\n", "\n", "antispam\n", - "\n", - "Antispam\n", + "\n", + "Antispam\n", "\n", "\n", "\n", "front->antispam\n", - "\n", - "\n", - "11334/tcp\n", + "\n", + "\n", + "11334/tcp\n", "\n", "\n", "\n", "imap\n", - "\n", - "IMAP\n", + "\n", + "IMAP\n", "\n", "\n", "\n", "front->imap\n", - "\n", - "\n", - "4190/tcp\n", + "\n", + "\n", + "4190/tcp\n", "\n", "\n", "\n", "front->imap\n", - "\n", - "\n", - "143/tcp\n", + "\n", + "\n", + "143/tcp\n", "\n", "\n", "\n", "front->imap\n", - "\n", - "\n", - "110/tcp\n", + "\n", + "\n", + "110/tcp\n", "\n", "\n", "\n", "webdav\n", - "\n", - "WebDAV\n", + "\n", + "WebDAV\n", "\n", "\n", "\n", "front->webdav\n", - "\n", - "\n", - "5232/tcp\n", + "\n", + "\n", + "5232/tcp\n", "\n", "\n", "\n", "webmail\n", - "\n", - "Webmail\n", + "\n", + "Webmail\n", "\n", "\n", "\n", "front->webmail\n", - "\n", - "\n", - "80/tcp\n", + "\n", + "\n", + "80/tcp\n", + "\n", + "\n", + "\n", + "admin->front\n", + "\n", + "\n", + "2525/tcp\n", "\n", "\n", "\n", "redis\n", - "\n", - "Redis\n", + "\n", + "Redis\n", "\n", "\n", "\n", "admin->redis\n", - "\n", - "\n", - "6379/tcp\n", - "\n", - "\n", - "\n", - "admin->imap\n", - "\n", - "\n", - "2525/tcp\n", + "\n", + "\n", + "6379/tcp\n", "\n", "\n", "\n", "smtp->front\n", - "\n", - "\n", - "2525/tcp\n", + "\n", + "\n", + "2525/tcp\n", "\n", "\n", "\n", "smtp->admin\n", - "\n", - "\n", - "8080/tcp\n", + "\n", + "\n", + "8080/tcp\n", "\n", "\n", "\n", "smtp->antispam\n", - "\n", - "\n", - "11332/tcp\n", + "\n", + "\n", + "11332/tcp\n", "\n", "\n", "\n", "antispam->admin\n", - "\n", - "\n", - "8080/tcp\n", + "\n", + "\n", + "8080/tcp\n", "\n", "\n", "\n", "antispam->redis\n", - "\n", - "\n", - "6379/tcp\n", + "\n", + "\n", + "6379/tcp\n", "\n", "\n", "\n", "antivirus\n", - "\n", - "Anti-Virus\n", + "\n", + "Anti-Virus\n", "\n", "\n", "\n", "antispam->antivirus\n", - "\n", - "\n", - "3310/tcp\n", + "\n", + "\n", + "3310/tcp\n", "\n", "\n", "\n", "oletools\n", - "\n", - "Oletools\n", + "\n", + "Oletools\n", "\n", "\n", "\n", "antispam->oletools\n", - "\n", - "\n", - "11343/tcp\n", + "\n", + "\n", + "11343/tcp\n", "\n", "\n", "\n", "imap->proxy\n", - "\n", - "\n", - "25/tcp\n", + "\n", + "\n", + "25/tcp\n", "\n", "\n", "\n", "imap->admin\n", - "\n", - "\n", - "8080/tcp\n", + "\n", + "\n", + "8080/tcp\n", "\n", "\n", "\n", "imap->antispam\n", - "\n", - "\n", - "11334/tcp\n", + "\n", + "\n", + "11334/tcp\n", "\n", "\n", "\n", "fts_attachments\n", - "\n", - "Tika\n", + "\n", + "Tika\n", "\n", "\n", "\n", "imap->fts_attachments\n", - "\n", - "\n", - "9998/tcp\n", + "\n", + "\n", + "9998/tcp\n", "\n", "\n", "\n", "webmail->proxy\n", - "\n", - "\n", - "443/tcp\n", + "\n", + "\n", + "443/tcp\n", "\n", "\n", "\n", "webmail->front\n", - "\n", - "\n", - "14190/tcp\n", + "\n", + "\n", + "14190/tcp\n", "\n", "\n", "\n", "webmail->front\n", - "\n", - "\n", - "10025/tcp\n", + "\n", + "\n", + "10025/tcp\n", "\n", "\n", "\n", "webmail->front\n", - "\n", - "\n", - "10143/tcp\n", + "\n", + "\n", + "10143/tcp\n", "\n", "\n", "\n", "fetchmail\n", - "\n", - "Fetchmail\n", + "\n", + "Fetchmail\n", "\n", - "\n", + "\n", "\n", - "fetchmail->front\n", - "\n", - "\n", - "25/tcp\n", + "fetchmail->proxy\n", + "\n", + "\n", + "25/tcp\n", "\n", "\n", "\n", "fetchmail->front\n", - "\n", - "\n", - "2525/tcp\n", + "\n", + "\n", + "2525/tcp\n", "\n", "\n", "\n", "fetchmail->admin\n", - "\n", - "\n", - "8080/tcp\n", + "\n", + "\n", + "8080/tcp\n", "\n", "\n", "\n" ], "text/plain": [ - "" + "" ] }, - "execution_count": 5, + "execution_count": 1, "metadata": {}, "output_type": "execute_result" } @@ -553,7 +553,7 @@ " webmail -> proxy [label = \"443/tcp\";];\n", " \n", " admin -> redis [label = \"6379/tcp\";];\n", - " admin -> imap [label = \"2525/tcp\";];\n", + " admin -> front [label = \"2525/tcp\";];\n", " \n", " antispam -> redis [label = \"6379/tcp\";];\n", " antispam -> admin [label = \"8080/tcp\";];\n", From 7556df2fa7dd46e00144e586e2a2995c4f01aff6 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 24 Jun 2024 10:11:22 +0200 Subject: [PATCH 40/83] Fix CI (cherry picked from commit 616376eb5a56278c2ba5c454eb58aa89c18afa6b) --- core/base/libs/socrate/socrate/system.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index ec726433..6debfb67 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -118,13 +118,13 @@ def clean_env(): PROTO_ALL.extend(['80']) for item in os.environ.get('PROXY_PROTOCOL', '').split(','): if item.isdigit(): - os.environ[f'PROXY_PROTOCOL_{item}']=True + os.environ[f'PROXY_PROTOCOL_{item}']='True' elif item == 'mail': - for p in PROTO_MAIL: os.environ[f'PROXY_PROTOCOL_{p}']=True + for p in PROTO_MAIL: os.environ[f'PROXY_PROTOCOL_{p}']='True' elif item == 'all-but-http': - for p in PROTO_ALL_BUT_HTTP: os.environ[f'PROXY_PROTOCOL_{p}']=True + for p in PROTO_ALL_BUT_HTTP: os.environ[f'PROXY_PROTOCOL_{p}']='True' elif item == 'all': - for p in PROTO_ALL: os.environ[f'PROXY_PROTOCOL_{p}']=True + for p in PROTO_ALL: os.environ[f'PROXY_PROTOCOL_{p}']='True' elif item == '': pass else: @@ -135,12 +135,12 @@ def clean_env(): for item in os.environ.get('PORTS', ALL_PORTS).split(','): if item in PORTS_REQUIRING_TLS and os.environ['TLS_FLAVOR'] == 'notls': continue - os.environ[f'PORT_{item}']=True + os.environ[f'PORT_{item}']='True' if os.environ['TLS_FLAVOR'] != 'notls': for item in os.environ.get('TLS', ALL_PORTS).split(','): if item in PORTS_REQUIRING_TLS: - os.environ[f'TLS_{item}']=True + os.environ[f'TLS_{item}']='True' def drop_privs_to(username='mailu'): pwnam = getpwnam(username) From 6fc97ee2cbab9ed8d83d0a9dcf334e6e3757396f Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 24 Jun 2024 10:21:16 +0200 Subject: [PATCH 41/83] doh2 (cherry picked from commit 1917bf7ee4ded980f66f39483547a16f80e2a107) --- core/base/libs/socrate/socrate/system.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index 6debfb67..8ea770a3 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -133,11 +133,11 @@ def clean_env(): PORTS_REQUIRING_TLS=['443', '465', '993', '995'] ALL_PORTS='25,80,443,465,993,995,4190' for item in os.environ.get('PORTS', ALL_PORTS).split(','): - if item in PORTS_REQUIRING_TLS and os.environ['TLS_FLAVOR'] == 'notls': + if item in PORTS_REQUIRING_TLS and os.environ.get('TLS_FLAVOR','') == 'notls': continue os.environ[f'PORT_{item}']='True' - if os.environ['TLS_FLAVOR'] != 'notls': + if os.environ.get('TLS_FLAVOR', '') != 'notls': for item in os.environ.get('TLS', ALL_PORTS).split(','): if item in PORTS_REQUIRING_TLS: os.environ[f'TLS_{item}']='True' From 1143c4c4f9ab88609e46119ad7386a521d850ac7 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 28 Jun 2024 12:44:29 +0200 Subject: [PATCH 42/83] as requested (cherry picked from commit d3ea0f898a46ae0638589573afa05e419e290a32) --- .gitignore | 1 + docs/Dockerfile | 5 +- docs/contributors/firewalling.rst | 10 + docs/index.rst | 1 + docs/mailu-network-diagram.dot | 92 +++++ docs/mailu-network-diagram.ipynb | 614 ------------------------------ 6 files changed, 107 insertions(+), 616 deletions(-) create mode 100644 docs/contributors/firewalling.rst create mode 100644 docs/mailu-network-diagram.dot delete mode 100644 docs/mailu-network-diagram.ipynb diff --git a/.gitignore b/.gitignore index 84ee07d3..845a97ee 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ pip-selfcheck.json /docs/lib* /docs/bin /docs/include +/docs/contributors/mailu-network-diagram.svg /docs/_build /.env /.venv diff --git a/docs/Dockerfile b/docs/Dockerfile index 25ecc496..08e725fc 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -9,7 +9,7 @@ COPY . /docs RUN set -euxo pipefail \ ; machine="$(uname -m)" \ - ; deps="gcc musl-dev" \ + ; deps="gcc musl-dev graphviz" \ ; [[ "${machine}" != x86_64 ]] && \ deps="${deps} cargo" \ ; apk add --no-cache --virtual .build-deps ${deps} \ @@ -17,7 +17,8 @@ RUN set -euxo pipefail \ mkdir -p /root/.cargo/registry/index && \ git clone --bare https://github.com/rust-lang/crates.io-index.git /root/.cargo/registry/index/github.com-1285ae84e5963aae \ ; pip3 install -r /requirements.txt \ - ; mkdir -p /build/$VERSION \ + ; mkdir -p /build/$VERSION/ \ + ; dot -Tsvg /docs/mailu-network-diagram.dot -o /docs/contributors/mailu-network-diagram.svg \ ; sphinx-build -W /docs /build/$VERSION \ ; apk del .build-deps \ ; rm -rf /root/.cargo diff --git a/docs/contributors/firewalling.rst b/docs/contributors/firewalling.rst new file mode 100644 index 00000000..b1a363d5 --- /dev/null +++ b/docs/contributors/firewalling.rst @@ -0,0 +1,10 @@ +Firewalling +=========== + +Network flows within Mailu +-------------------------- + +The following diagram may prove useful in understanding how the different components interact. + +.. image:: mailu-network-diagram.svg + diff --git a/docs/index.rst b/docs/index.rst index f2cf56f3..77c95df9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -81,3 +81,4 @@ the version of Mailu that you are running. contributors/database contributors/memo contributors/localization + contributors/firewalling diff --git a/docs/mailu-network-diagram.dot b/docs/mailu-network-diagram.dot new file mode 100644 index 00000000..ed7d1001 --- /dev/null +++ b/docs/mailu-network-diagram.dot @@ -0,0 +1,92 @@ +digraph mailu { + label = "Mailu"; + fontname = "arial"; + + node [shape = box; fontname = "arial"; fontsize = 8; style = filled; color = "#d3edea";]; + splines = "compound"; + // node [shape = "box"; fontsize = "10";]; + edge [fontsize = "8";]; + + # Components + internet [label = "Internet";]; + proxy [label = "Optional proxy"; shape = "polygon"]; + front [label = "Front";]; + admin [label = "Admin";]; + smtp [label = "SMTP";]; + redis [label = "Redis";]; + antispam [label = "Antispam";]; + antivirus [label = "Anti-Virus";]; + imap [label = "IMAP";]; + webdav [label = "WebDAV";]; + webmail [label = "Webmail";]; + fetchmail [label = "Fetchmail";]; + oletools [label = "Oletools"]; + fts_attachments [label = "Tika"]; + # proxy from internet + internet -> proxy [label = "25/tcp";]; + internet -> proxy [label = "80/tcp";]; + internet -> proxy [label = "443/tcp";]; + internet -> proxy [label = "465/tcp";]; + internet -> proxy [label = "587/tcp";]; + internet -> proxy [label = "110/tcp";]; + internet -> proxy [label = "995/tcp";]; + internet -> proxy [label = "143/tcp";]; + internet -> proxy [label = "993/tcp";]; + internet -> proxy [label = "4190/tcp";]; + + # Front from proxy + proxy -> front [label = "25/tcp";]; + proxy -> front [label = "80/tcp";]; + proxy -> front [label = "443/tcp";]; + proxy -> front [label = "465/tcp";]; + proxy -> front [label = "587/tcp";]; + proxy -> front [label = "110/tcp";]; + proxy -> front [label = "995/tcp";]; + proxy -> front [label = "143/tcp";]; + proxy -> front [label = "993/tcp";]; + proxy -> front [label = "4190/tcp";]; + + front -> front [label = "8008/tcp";]; + front -> front [label = "8000/tcp";]; + front -> admin [label = "8080/tcp";]; + front -> imap [label = "4190/tcp";]; + front -> imap [label = "143/tcp";]; + front -> imap [label = "110/tcp";]; + front -> smtp [label = "25/tcp";]; + front -> smtp [label = "10025/tcp";]; + front -> webmail [label = "80/tcp";]; + front -> antispam [label = "11334/tcp";]; + front -> webdav [label = "5232/tcp";]; + + smtp -> admin [label = "8080/tcp";]; + smtp -> front [label = "2525/tcp";]; + smtp -> antispam [label = "11332/tcp";]; + + imap -> admin [label = "8080/tcp";]; + imap -> antispam [label = "11334/tcp";]; + imap -> proxy [label = "25/tcp";]; + imap -> fts_attachments [label = "9998/tcp";]; + + webmail -> front [label = "14190/tcp";]; + webmail -> front [label = "10025/tcp";]; + webmail -> front [label = "10143/tcp";]; + # carddav + webmail -> proxy [label = "443/tcp";]; + + admin -> redis [label = "6379/tcp";]; + admin -> front [label = "2525/tcp";]; + + antispam -> redis [label = "6379/tcp";]; + antispam -> admin [label = "8080/tcp";]; + antispam -> oletools [label = "11343/tcp";]; + antispam -> antivirus [label = "3310/tcp";]; + + fetchmail -> admin [label = "8080/tcp"] + fetchmail -> proxy [label = "25/tcp"] + fetchmail -> front [label = "2525/tcp"] + # + # those don't need internet: + # oletools + # fts_attachments + # redis +} diff --git a/docs/mailu-network-diagram.ipynb b/docs/mailu-network-diagram.ipynb deleted file mode 100644 index 2a68e0e4..00000000 --- a/docs/mailu-network-diagram.ipynb +++ /dev/null @@ -1,614 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "mailu\n", - "\n", - "Mailu\n", - "\n", - "\n", - "internet\n", - "\n", - "Internet\n", - "\n", - "\n", - "\n", - "proxy\n", - "\n", - "Optional proxy\n", - "\n", - "\n", - "\n", - "internet->proxy\n", - "\n", - "\n", - "25/tcp\n", - "\n", - "\n", - "\n", - "internet->proxy\n", - "\n", - "\n", - "80/tcp\n", - "\n", - "\n", - "\n", - "internet->proxy\n", - "\n", - "\n", - "443/tcp\n", - "\n", - "\n", - "\n", - "internet->proxy\n", - "\n", - "\n", - "465/tcp\n", - "\n", - "\n", - "\n", - "internet->proxy\n", - "\n", - "\n", - "587/tcp\n", - "\n", - "\n", - "\n", - "internet->proxy\n", - "\n", - "\n", - "110/tcp\n", - "\n", - "\n", - "\n", - "internet->proxy\n", - "\n", - "\n", - "995/tcp\n", - "\n", - "\n", - "\n", - "internet->proxy\n", - "\n", - "\n", - "143/tcp\n", - "\n", - "\n", - "\n", - "internet->proxy\n", - "\n", - "\n", - "993/tcp\n", - "\n", - "\n", - "\n", - "internet->proxy\n", - "\n", - "\n", - "4190/tcp\n", - "\n", - "\n", - "\n", - "front\n", - "\n", - "Front\n", - "\n", - "\n", - "\n", - "proxy->front\n", - "\n", - "\n", - "25/tcp\n", - "\n", - "\n", - "\n", - "proxy->front\n", - "\n", - "\n", - "80/tcp\n", - "\n", - "\n", - "\n", - "proxy->front\n", - "\n", - "\n", - "443/tcp\n", - "\n", - "\n", - "\n", - "proxy->front\n", - "\n", - "\n", - "465/tcp\n", - "\n", - "\n", - "\n", - "proxy->front\n", - "\n", - "\n", - "587/tcp\n", - "\n", - "\n", - "\n", - "proxy->front\n", - "\n", - "\n", - "110/tcp\n", - "\n", - "\n", - "\n", - "proxy->front\n", - "\n", - "\n", - "995/tcp\n", - "\n", - "\n", - "\n", - "proxy->front\n", - "\n", - "\n", - "143/tcp\n", - "\n", - "\n", - "\n", - "proxy->front\n", - "\n", - "\n", - "993/tcp\n", - "\n", - "\n", - "\n", - "proxy->front\n", - "\n", - "\n", - "4190/tcp\n", - "\n", - "\n", - "\n", - "front->front\n", - "\n", - "\n", - "8008/tcp\n", - "\n", - "\n", - "\n", - "front->front\n", - "\n", - "\n", - "8000/tcp\n", - "\n", - "\n", - "\n", - "admin\n", - "\n", - "Admin\n", - "\n", - "\n", - "\n", - "front->admin\n", - "\n", - "\n", - "8080/tcp\n", - "\n", - "\n", - "\n", - "smtp\n", - "\n", - "SMTP\n", - "\n", - "\n", - "\n", - "front->smtp\n", - "\n", - "\n", - "25/tcp\n", - "\n", - "\n", - "\n", - "front->smtp\n", - "\n", - "\n", - "10025/tcp\n", - "\n", - "\n", - "\n", - "antispam\n", - "\n", - "Antispam\n", - "\n", - "\n", - "\n", - "front->antispam\n", - "\n", - "\n", - "11334/tcp\n", - "\n", - "\n", - "\n", - "imap\n", - "\n", - "IMAP\n", - "\n", - "\n", - "\n", - "front->imap\n", - "\n", - "\n", - "4190/tcp\n", - "\n", - "\n", - "\n", - "front->imap\n", - "\n", - "\n", - "143/tcp\n", - "\n", - "\n", - "\n", - "front->imap\n", - "\n", - "\n", - "110/tcp\n", - "\n", - "\n", - "\n", - "webdav\n", - "\n", - "WebDAV\n", - "\n", - "\n", - "\n", - "front->webdav\n", - "\n", - "\n", - "5232/tcp\n", - "\n", - "\n", - "\n", - "webmail\n", - "\n", - "Webmail\n", - "\n", - "\n", - "\n", - "front->webmail\n", - "\n", - "\n", - "80/tcp\n", - "\n", - "\n", - "\n", - "admin->front\n", - "\n", - "\n", - "2525/tcp\n", - "\n", - "\n", - "\n", - "redis\n", - "\n", - "Redis\n", - "\n", - "\n", - "\n", - "admin->redis\n", - "\n", - "\n", - "6379/tcp\n", - "\n", - "\n", - "\n", - "smtp->front\n", - "\n", - "\n", - "2525/tcp\n", - "\n", - "\n", - "\n", - "smtp->admin\n", - "\n", - "\n", - "8080/tcp\n", - "\n", - "\n", - "\n", - "smtp->antispam\n", - "\n", - "\n", - "11332/tcp\n", - "\n", - "\n", - "\n", - "antispam->admin\n", - "\n", - "\n", - "8080/tcp\n", - "\n", - "\n", - "\n", - "antispam->redis\n", - "\n", - "\n", - "6379/tcp\n", - "\n", - "\n", - "\n", - "antivirus\n", - "\n", - "Anti-Virus\n", - "\n", - "\n", - "\n", - "antispam->antivirus\n", - "\n", - "\n", - "3310/tcp\n", - "\n", - "\n", - "\n", - "oletools\n", - "\n", - "Oletools\n", - "\n", - "\n", - "\n", - "antispam->oletools\n", - "\n", - "\n", - "11343/tcp\n", - "\n", - "\n", - "\n", - "imap->proxy\n", - "\n", - "\n", - "25/tcp\n", - "\n", - "\n", - "\n", - "imap->admin\n", - "\n", - "\n", - "8080/tcp\n", - "\n", - "\n", - "\n", - "imap->antispam\n", - "\n", - "\n", - "11334/tcp\n", - "\n", - "\n", - "\n", - "fts_attachments\n", - "\n", - "Tika\n", - "\n", - "\n", - "\n", - "imap->fts_attachments\n", - "\n", - "\n", - "9998/tcp\n", - "\n", - "\n", - "\n", - "webmail->proxy\n", - "\n", - "\n", - "443/tcp\n", - "\n", - "\n", - "\n", - "webmail->front\n", - "\n", - "\n", - "14190/tcp\n", - "\n", - "\n", - "\n", - "webmail->front\n", - "\n", - "\n", - "10025/tcp\n", - "\n", - "\n", - "\n", - "webmail->front\n", - "\n", - "\n", - "10143/tcp\n", - "\n", - "\n", - "\n", - "fetchmail\n", - "\n", - "Fetchmail\n", - "\n", - "\n", - "\n", - "fetchmail->proxy\n", - "\n", - "\n", - "25/tcp\n", - "\n", - "\n", - "\n", - "fetchmail->front\n", - "\n", - "\n", - "2525/tcp\n", - "\n", - "\n", - "\n", - "fetchmail->admin\n", - "\n", - "\n", - "8080/tcp\n", - "\n", - "\n", - "\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import graphviz\n", - "\n", - "a = \"\"\"\n", - "digraph mailu {\n", - " label = \"Mailu\";\n", - " fontname = \"arial\";\n", - " \n", - " node [shape = box; fontname = \"arial\"; fontsize = 8; style = filled; color = \"#d3edea\";];\n", - " splines = \"compound\";\n", - " // node [shape = \"box\"; fontsize = \"10\";];\n", - " edge [fontsize = \"8\";];\n", - " \n", - " # Components\n", - " internet [label = \"Internet\";];\n", - " proxy [label = \"Optional proxy\"; shape = \"polygon\"];\n", - " front [label = \"Front\";];\n", - " admin [label = \"Admin\";];\n", - " smtp [label = \"SMTP\";];\n", - " redis [label = \"Redis\";];\n", - " antispam [label = \"Antispam\";];\n", - " antivirus [label = \"Anti-Virus\";];\n", - " imap [label = \"IMAP\";];\n", - " webdav [label = \"WebDAV\";];\n", - " webmail [label = \"Webmail\";];\n", - " fetchmail [label = \"Fetchmail\";];\n", - " oletools [label = \"Oletools\"];\n", - " fts_attachments [label = \"Tika\"];\n", - " # proxy from internet\n", - " internet -> proxy [label = \"25/tcp\";];\n", - " internet -> proxy [label = \"80/tcp\";];\n", - " internet -> proxy [label = \"443/tcp\";];\n", - " internet -> proxy [label = \"465/tcp\";];\n", - " internet -> proxy [label = \"587/tcp\";];\n", - " internet -> proxy [label = \"110/tcp\";];\n", - " internet -> proxy [label = \"995/tcp\";];\n", - " internet -> proxy [label = \"143/tcp\";];\n", - " internet -> proxy [label = \"993/tcp\";];\n", - " internet -> proxy [label = \"4190/tcp\";];\n", - " \n", - " # Front from proxy\n", - " proxy -> front [label = \"25/tcp\";];\n", - " proxy -> front [label = \"80/tcp\";];\n", - " proxy -> front [label = \"443/tcp\";];\n", - " proxy -> front [label = \"465/tcp\";];\n", - " proxy -> front [label = \"587/tcp\";];\n", - " proxy -> front [label = \"110/tcp\";];\n", - " proxy -> front [label = \"995/tcp\";];\n", - " proxy -> front [label = \"143/tcp\";];\n", - " proxy -> front [label = \"993/tcp\";];\n", - " proxy -> front [label = \"4190/tcp\";];\n", - " \n", - " front -> front [label = \"8008/tcp\";];\n", - " front -> front [label = \"8000/tcp\";];\n", - " front -> admin [label = \"8080/tcp\";];\n", - " front -> imap [label = \"4190/tcp\";];\n", - " front -> imap [label = \"143/tcp\";];\n", - " front -> imap [label = \"110/tcp\";];\n", - " front -> smtp [label = \"25/tcp\";];\n", - " front -> smtp [label = \"10025/tcp\";];\n", - " front -> webmail [label = \"80/tcp\";];\n", - " front -> antispam [label = \"11334/tcp\";];\n", - " front -> webdav [label = \"5232/tcp\";];\n", - " \n", - " smtp -> admin [label = \"8080/tcp\";];\n", - " smtp -> front [label = \"2525/tcp\";];\n", - " smtp -> antispam [label = \"11332/tcp\";];\n", - " \n", - " imap -> admin [label = \"8080/tcp\";];\n", - " imap -> antispam [label = \"11334/tcp\";];\n", - " imap -> proxy [label = \"25/tcp\";];\n", - " imap -> fts_attachments [label = \"9998/tcp\";];\n", - " \n", - " webmail -> front [label = \"14190/tcp\";];\n", - " webmail -> front [label = \"10025/tcp\";];\n", - " webmail -> front [label = \"10143/tcp\";];\n", - " # carddav\n", - " webmail -> proxy [label = \"443/tcp\";];\n", - " \n", - " admin -> redis [label = \"6379/tcp\";];\n", - " admin -> front [label = \"2525/tcp\";];\n", - " \n", - " antispam -> redis [label = \"6379/tcp\";];\n", - " antispam -> admin [label = \"8080/tcp\";];\n", - " antispam -> oletools [label = \"11343/tcp\";];\n", - " antispam -> antivirus [label = \"3310/tcp\";];\n", - " \n", - " fetchmail -> admin [label = \"8080/tcp\"]\n", - " fetchmail -> proxy [label = \"25/tcp\"]\n", - " fetchmail -> front [label = \"2525/tcp\"]\n", - " #\n", - " # those don't need internet:\n", - " # oletools\n", - " # fts_attachments\n", - " # redis\n", - "}\n", - "\"\"\"\n", - "\n", - "dot = graphviz.Source(a)\n", - "dot\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.2" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From 6200d64a50aab419163d53d53c9c3f4008887ae6 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 28 Jun 2024 16:19:07 +0200 Subject: [PATCH 43/83] Make it clickable (cherry picked from commit 95f2525eea5c84dab17eb2d4b68c2812192785b2) --- docs/contributors/firewalling.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/contributors/firewalling.rst b/docs/contributors/firewalling.rst index b1a363d5..01cc2b8c 100644 --- a/docs/contributors/firewalling.rst +++ b/docs/contributors/firewalling.rst @@ -7,4 +7,5 @@ Network flows within Mailu The following diagram may prove useful in understanding how the different components interact. .. image:: mailu-network-diagram.svg + :target: ../_images/mailu-network-diagram.svg From b150c84a1f27ea50a6ad1cceae2de4483d467e7b Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 29 Jun 2024 09:57:40 +0200 Subject: [PATCH 44/83] colors from fastlorenzo (cherry picked from commit 020f0f0b02e629dff2653ca6bc2348472e788d21) --- docs/mailu-network-diagram.dot | 185 ++++++++++++++++++++------------- 1 file changed, 115 insertions(+), 70 deletions(-) diff --git a/docs/mailu-network-diagram.dot b/docs/mailu-network-diagram.dot index ed7d1001..9178b820 100644 --- a/docs/mailu-network-diagram.dot +++ b/docs/mailu-network-diagram.dot @@ -1,89 +1,134 @@ digraph mailu { - label = "Mailu"; + label = "Mailu network flows"; fontname = "arial"; - node [shape = box; fontname = "arial"; fontsize = 8; style = filled; color = "#d3edea";]; + node [shape = record; fontname = "arial"; fontsize = 8; style = filled; color = "#d3edea";]; splines = "compound"; // node [shape = "box"; fontsize = "10";]; - edge [fontsize = "8";]; + edge [fontsize = 8; arrowsize = 0.5;]; # Components - internet [label = "Internet";]; - proxy [label = "Optional proxy"; shape = "polygon"]; - front [label = "Front";]; - admin [label = "Admin";]; - smtp [label = "SMTP";]; - redis [label = "Redis";]; - antispam [label = "Antispam";]; - antivirus [label = "Anti-Virus";]; - imap [label = "IMAP";]; - webdav [label = "WebDAV";]; - webmail [label = "Webmail";]; - fetchmail [label = "Fetchmail";]; - oletools [label = "Oletools"]; - fts_attachments [label = "Tika"]; - # proxy from internet - internet -> proxy [label = "25/tcp";]; - internet -> proxy [label = "80/tcp";]; - internet -> proxy [label = "443/tcp";]; - internet -> proxy [label = "465/tcp";]; - internet -> proxy [label = "587/tcp";]; - internet -> proxy [label = "110/tcp";]; - internet -> proxy [label = "995/tcp";]; - internet -> proxy [label = "143/tcp";]; - internet -> proxy [label = "993/tcp";]; - internet -> proxy [label = "4190/tcp";]; - + internet [label = "Internet"; color = "red";]; + proxy [label = "Proxy (optional)"; color = "darkorange";]; + front [label="Front"; color="dodgerblue";]; + admin [label="Admin"; color="green"; fontcolor="white";]; + smtp [label="SMTP"; color="orchid";]; + redis [label="Redis"; color="turquoise";]; + antispam [label="Antispam"; color="magenta";]; + antivirus [label="Antivirus"; color="purple"; fontcolor="white";]; + imap [label="IMAP"; color="cyan";]; + webdav [label="WebDAV"; color="yellow";]; + webmail [label="Webmail"; color="darkgoldenrod";]; + fetchmail [label="Fetchmail"; color="chocolate";]; + oletools [label="Oletools"; color="limegreen";]; + fts_attachments [label="Tika"; color="sienna";]; + + rankdir=LR; + {rank=min; internet}; + // {rank=3; proxy}; + // {rank=4; front}; + // {rank=same; admin smtp redis antispam antivirus imap}; + {rank=max; fetchmail}; + + # Proxy from internet + + internet -> proxy [ + color="red"; + fontcolor="red"; + label = < + + + + + + + + + + + + + + + + + + + + + +
80/tcp443/tcp
25/tcp465/tcp
587/tcp110/tcp
995/tcp143/tcp
993/tcp4190/tcp
+ >; + ]; + # Front from proxy - proxy -> front [label = "25/tcp";]; - proxy -> front [label = "80/tcp";]; - proxy -> front [label = "443/tcp";]; - proxy -> front [label = "465/tcp";]; - proxy -> front [label = "587/tcp";]; - proxy -> front [label = "110/tcp";]; - proxy -> front [label = "995/tcp";]; - proxy -> front [label = "143/tcp";]; - proxy -> front [label = "993/tcp";]; - proxy -> front [label = "4190/tcp";]; + proxy -> front [ + color="darkorange"; + fontcolor="darkorange"; + label = < + + + + + + + + + + + + + + + + + + + + + +
80/tcp443/tcp
25/tcp465/tcp
587/tcp110/tcp
995/tcp143/tcp
993/tcp4190/tcp
+ >; + ]; - front -> front [label = "8008/tcp";]; - front -> front [label = "8000/tcp";]; - front -> admin [label = "8080/tcp";]; - front -> imap [label = "4190/tcp";]; - front -> imap [label = "143/tcp";]; - front -> imap [label = "110/tcp";]; - front -> smtp [label = "25/tcp";]; - front -> smtp [label = "10025/tcp";]; - front -> webmail [label = "80/tcp";]; - front -> antispam [label = "11334/tcp";]; - front -> webdav [label = "5232/tcp";]; + front -> front [label = "8008/tcp"; color="dodgerblue"; fontcolor="dodgerblue";]; + front -> front [label = "8000/tcp"; color="dodgerblue"; fontcolor="dodgerblue";]; + front -> admin [label = "8080/tcp"; color="dodgerblue"; fontcolor="dodgerblue";]; + front -> imap [label = "4190/tcp"; color="dodgerblue"; fontcolor="dodgerblue";]; + front -> imap [label = "143/tcp"; color="dodgerblue"; fontcolor="dodgerblue";]; + front -> imap [label = "110/tcp"; color="dodgerblue"; fontcolor="dodgerblue";]; + front -> smtp [label = "25/tcp"; color="dodgerblue"; fontcolor="dodgerblue";]; + front -> smtp [label = "10025/tcp"; color="dodgerblue"; fontcolor="dodgerblue";]; + front -> webmail [label = "80/tcp"; color="dodgerblue"; fontcolor="dodgerblue";]; + front -> antispam [label = "11334/tcp"; color="dodgerblue"; fontcolor="dodgerblue";]; + front -> webdav [label = "5232/tcp"; color="dodgerblue"; fontcolor="dodgerblue";]; - smtp -> admin [label = "8080/tcp";]; - smtp -> front [label = "2525/tcp";]; - smtp -> antispam [label = "11332/tcp";]; + smtp -> admin [label = "8080/tcp"; color="orchid"; fontcolor="orchid";]; + smtp -> front [label = "2525/tcp"; color="orchid"; fontcolor="orchid";]; + smtp -> antispam [label = "11332/tcp"; color="orchid"; fontcolor="orchid";]; - imap -> admin [label = "8080/tcp";]; - imap -> antispam [label = "11334/tcp";]; - imap -> proxy [label = "25/tcp";]; - imap -> fts_attachments [label = "9998/tcp";]; + imap -> admin [label = "8080/tcp"; color="cyan"; fontcolor="cyan";]; + imap -> antispam [label = "11334/tcp"; color="cyan"; fontcolor="cyan";]; + imap -> proxy [label = "25/tcp"; color="cyan"; fontcolor="cyan";]; + imap -> fts_attachments [label = "9998/tcp"; color="cyan"; fontcolor="cyan";]; - webmail -> front [label = "14190/tcp";]; - webmail -> front [label = "10025/tcp";]; - webmail -> front [label = "10143/tcp";]; + webmail -> front [label = "14190/tcp"; color="darkgoldenrod"; fontcolor="darkgoldenrod";]; + webmail -> front [label = "10025/tcp"; color="darkgoldenrod"; fontcolor="darkgoldenrod";]; + webmail -> front [label = "10143/tcp"; color="darkgoldenrod"; fontcolor="darkgoldenrod";]; # carddav - webmail -> proxy [label = "443/tcp";]; + webmail -> proxy [label = "443/tcp"; color="darkgoldenrod"; fontcolor="darkgoldenrod";]; - admin -> redis [label = "6379/tcp";]; - admin -> front [label = "2525/tcp";]; + admin -> redis [label = "6379/tcp"; color="green"; fontcolor="green";]; + admin -> front [label = "2525/tcp"; color="green"; fontcolor="green";]; - antispam -> redis [label = "6379/tcp";]; - antispam -> admin [label = "8080/tcp";]; - antispam -> oletools [label = "11343/tcp";]; - antispam -> antivirus [label = "3310/tcp";]; + antispam -> redis [label = "6379/tcp"; color="magenta"; fontcolor="magenta";]; + antispam -> admin [label = "8080/tcp"; color="magenta"; fontcolor="magenta";]; + antispam -> oletools [label = "11343/tcp"; color="magenta"; fontcolor="magenta";]; + antispam -> antivirus [label = "3310/tcp"; color="magenta"; fontcolor="magenta";]; - fetchmail -> admin [label = "8080/tcp"] - fetchmail -> proxy [label = "25/tcp"] - fetchmail -> front [label = "2525/tcp"] + fetchmail -> admin [label = "8080/tcp"; color="chocolate"; fontcolor="chocolate";]; + fetchmail -> proxy [label = "25/tcp"; color="chocolate"; fontcolor="chocolate";]; + fetchmail -> front [label = "2525/tcp"; color="chocolate"; fontcolor="chocolate";]; # # those don't need internet: # oletools From a4ddddb4183796efbd64b48ceab420f419960077 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 29 Jun 2024 09:58:46 +0200 Subject: [PATCH 45/83] missing front->imap[2525] (cherry picked from commit 40bad79127f2f3038c273f47d6e264b70eef65ec) --- docs/mailu-network-diagram.dot | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/mailu-network-diagram.dot b/docs/mailu-network-diagram.dot index 9178b820..eb57b393 100644 --- a/docs/mailu-network-diagram.dot +++ b/docs/mailu-network-diagram.dot @@ -95,6 +95,7 @@ digraph mailu { front -> front [label = "8000/tcp"; color="dodgerblue"; fontcolor="dodgerblue";]; front -> admin [label = "8080/tcp"; color="dodgerblue"; fontcolor="dodgerblue";]; front -> imap [label = "4190/tcp"; color="dodgerblue"; fontcolor="dodgerblue";]; + front -> imap [label = "2525/tcp"; color="dodgerblue"; fontcolor="dodgerblue";]; front -> imap [label = "143/tcp"; color="dodgerblue"; fontcolor="dodgerblue";]; front -> imap [label = "110/tcp"; color="dodgerblue"; fontcolor="dodgerblue";]; front -> smtp [label = "25/tcp"; color="dodgerblue"; fontcolor="dodgerblue";]; From 83e10813cfc729661aefde08a13826da025d4282 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 26 Aug 2024 10:10:54 +0200 Subject: [PATCH 46/83] Update snuffleupagus rules (cherry picked from commit cf1088f1c58ddf3dc79f34acc14a14664c1e04df) --- towncrier/newsfragments/3384.bugfix | 1 + webmails/snuffleupagus.rules | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 towncrier/newsfragments/3384.bugfix diff --git a/towncrier/newsfragments/3384.bugfix b/towncrier/newsfragments/3384.bugfix new file mode 100644 index 00000000..e82c78d8 --- /dev/null +++ b/towncrier/newsfragments/3384.bugfix @@ -0,0 +1 @@ +Ensure that file:// protocol is not allowed in CURL diff --git a/webmails/snuffleupagus.rules b/webmails/snuffleupagus.rules index 4cbe966d..10390d45 100644 --- a/webmails/snuffleupagus.rules +++ b/webmails/snuffleupagus.rules @@ -39,7 +39,9 @@ sp.disable_function.function("chmod").param("permissions").value("438").drop(); sp.disable_function.function("chmod").param("permissions").value("511").drop(); # Prevent various `mail`-related vulnerabilities +# Uncommend the second rule if you're using php8.3+ sp.disable_function.function("mail").param("additional_parameters").value_r("\\-").drop(); +sp.disable_function.function("mail").param("additional_params").value_r("\\-").drop(); # Since it's now burned, me might as well mitigate it publicly sp.disable_function.function("putenv").param("assignment").value_r("LD_").drop() @@ -52,8 +54,7 @@ sp.disable_function.function("putenv").param("assignment").value_r("GCONV_").dro sp.disable_function.function("extract").param("array").value_r("^_").drop() sp.disable_function.function("extract").param("flags").value("0").drop() -# This is also burned: -# ini_set('open_basedir','..');chdir('..');…;chdir('..');ini_set('open_basedir','/');echo(file_get_contents('/etc/passwd')); +# See https://dustri.org/b/ini_set-based-open_basedir-bypass.html # Since we have no way of matching on two parameters at the same time, we're # blocking calls to open_basedir altogether: nobody is using it via ini_set anyway. # Moreover, there are non-public bypasses that are also using this vector ;) @@ -119,6 +120,10 @@ sp.disable_function.function("curl_setopt").param("value").value("2").allow(); sp.disable_function.function("curl_setopt").param("option").value("64").drop().alias("Please don't turn CURLOPT_SSL_VERIFYCLIENT off."); sp.disable_function.function("curl_setopt").param("option").value("81").drop().alias("Please don't turn CURLOPT_SSL_VERIFYHOST off."); +# Ensure that file:// protocol is not allowed in CURL +sp.disable_function.function("curl_setopt").param("value").value_r("file://").drop().alias("file:// protocol is disabled"); +sp.disable_function.function("curl_init").param("url").value_r("file://").drop().alias("file:// protocol is disabled"); + # File upload sp.disable_function.function("move_uploaded_file").param("to").value_r("\\.ph").drop(); sp.disable_function.function("move_uploaded_file").param("to").value_r("\\.ht").drop(); From 938d513488f168d6653ab37a044d7dce89020560 Mon Sep 17 00:00:00 2001 From: ctrl-i <1422608+ctrl-i@users.noreply.github.com> Date: Sun, 1 Sep 2024 09:39:11 +0100 Subject: [PATCH 47/83] Update Dockerfile Due to an incremental recommended update to roundcube (cherry picked from commit 23f4744a1c9981d1d870ce5ac0b7f1f68f248586) --- webmails/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webmails/Dockerfile b/webmails/Dockerfile index 4e6e6896..517cb873 100644 --- a/webmails/Dockerfile +++ b/webmails/Dockerfile @@ -28,7 +28,7 @@ RUN set -euxo pipefail \ ; mkdir -p /run/nginx /conf # roundcube -ENV ROUNDCUBE_URL https://github.com/roundcube/roundcubemail/releases/download/1.6.8/roundcubemail-1.6.8-complete.tar.gz +ENV ROUNDCUBE_URL https://github.com/roundcube/roundcubemail/releases/download/1.6.9/roundcubemail-1.6.9-complete.tar.gz ENV CARDDAV_URL https://github.com/mstilkerich/rcmcarddav/releases/download/v5.1.0/carddav-v5.1.0.tar.gz RUN set -euxo pipefail \ From fc98469760c4f79c474ed114051ca72b5d5c4bc5 Mon Sep 17 00:00:00 2001 From: ctrl-i <1422608+ctrl-i@users.noreply.github.com> Date: Sun, 1 Sep 2024 09:40:45 +0100 Subject: [PATCH 48/83] Create 3389.misc (cherry picked from commit c3545901aaec524771b814b2776b27c47489625d) --- towncrier/newsfragments/3389.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 towncrier/newsfragments/3389.misc diff --git a/towncrier/newsfragments/3389.misc b/towncrier/newsfragments/3389.misc new file mode 100644 index 00000000..0cd5b88f --- /dev/null +++ b/towncrier/newsfragments/3389.misc @@ -0,0 +1 @@ +Update roundcube to 1.6.9 From ec69a8ef9946cd8ce10cfd4d9ec1fbd0eb5bb818 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 5 Sep 2024 21:58:11 +0200 Subject: [PATCH 49/83] fix #3405 (cherry picked from commit ebbfd486f749b13505f5565ea004b5120507ed94) --- docs/maintain.rst | 1 + towncrier/newsfragments/3405.bugfix | 1 + 2 files changed, 2 insertions(+) create mode 100644 towncrier/newsfragments/3405.bugfix diff --git a/docs/maintain.rst b/docs/maintain.rst index eff8f5c4..a2041a86 100644 --- a/docs/maintain.rst +++ b/docs/maintain.rst @@ -47,6 +47,7 @@ In the case of *certbot* you could write a script to be executed as `deploy hook cp /etc/letsencrypt/live/domain.com/privkey.pem /mailu/certs/key.pem || exit 1 cp /etc/letsencrypt/live/domain.com/fullchain.pem /mailu/certs/cert.pem || exit 1 docker exec mailu_front_1 nginx -s reload + docker exec mailu_front_1 doveadm reload And the certbot command you will use in crontab would look something like: diff --git a/towncrier/newsfragments/3405.bugfix b/towncrier/newsfragments/3405.bugfix new file mode 100644 index 00000000..07db93a8 --- /dev/null +++ b/towncrier/newsfragments/3405.bugfix @@ -0,0 +1 @@ +Update the documentation: ensure that users reload dovecot too if they manually configure certificates From f568f1de20ca3463c568cbb7e0fbde072e32620f Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 5 Sep 2024 22:18:12 +0200 Subject: [PATCH 50/83] fix #3401 (cherry picked from commit 4af085253eda5911e448881b821e44c7df051529) --- webmails/snuffleupagus.rules | 1 + 1 file changed, 1 insertion(+) diff --git a/webmails/snuffleupagus.rules b/webmails/snuffleupagus.rules index 10390d45..3d4713f8 100644 --- a/webmails/snuffleupagus.rules +++ b/webmails/snuffleupagus.rules @@ -130,6 +130,7 @@ sp.disable_function.function("move_uploaded_file").param("to").value_r("\\.ht"). # Logging lockdown sp.disable_function.function("ini_set").param("option").value_r("error_log").drop() +sp.disable_function.function("ini_set").param("option").value_r("display_errors").filename_r("/var/www/snappymail/snappymail/v/[0-9]+\.[0-9]+\.[0-9]+/app/libraries/snappymail/shutdown.php").allow(); sp.disable_function.function("ini_set").param("option").value_r("display_errors").drop() sp.auto_cookie_secure.enable(); From 8c6b0ccdb3b1c3cf277030b2d535357ce0ff4f0f Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 5 Sep 2024 22:33:23 +0200 Subject: [PATCH 51/83] fix #3403 (cherry picked from commit e5790a297aa3cfaf0092970ea3716e6cb1eda7f9) --- core/admin/mailu/internal/nginx.py | 3 +++ towncrier/newsfragments/3403.bugfix | 1 + 2 files changed, 4 insertions(+) create mode 100644 towncrier/newsfragments/3403.bugfix diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py index daef8b9e..9cf9cfb0 100644 --- a/core/admin/mailu/internal/nginx.py +++ b/core/admin/mailu/internal/nginx.py @@ -19,6 +19,9 @@ STATUSES = { }), "encryption": ("Must issue a STARTTLS command first", { "smtp": "530 5.7.0" + "submission": "530 5.7.0", + "pop3": "-ERR Authentication canceled.", + "sieve": "ENCRYPT-NEEDED" }), "ratelimit": ("Temporary authentication failure (rate-limit)", { "imap": "LIMIT", diff --git a/towncrier/newsfragments/3403.bugfix b/towncrier/newsfragments/3403.bugfix new file mode 100644 index 00000000..d8169ba1 --- /dev/null +++ b/towncrier/newsfragments/3403.bugfix @@ -0,0 +1 @@ +fix INBOUND_TLS_ENFORCE From a0cb9852ac6c60dec48653525cae767492e862f2 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 5 Sep 2024 22:37:30 +0200 Subject: [PATCH 52/83] doh (cherry picked from commit b426c679494ca5e562679b331e3402a7637491d0) --- core/admin/mailu/internal/nginx.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py index 9cf9cfb0..6c320ae4 100644 --- a/core/admin/mailu/internal/nginx.py +++ b/core/admin/mailu/internal/nginx.py @@ -18,6 +18,7 @@ STATUSES = { "sieve": "AuthFailed" }), "encryption": ("Must issue a STARTTLS command first", { + "imap": "PRIVACYREQUIRED", "smtp": "530 5.7.0" "submission": "530 5.7.0", "pop3": "-ERR Authentication canceled.", From 45c39d9416f7dff18391bda48f82adbd3b4bc93d Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 5 Sep 2024 22:45:25 +0200 Subject: [PATCH 53/83] fix #3379 (cherry picked from commit bae4855bea381a7d6f202386da0c08eea9edb306) --- core/admin/mailu/ui/views/users.py | 2 +- towncrier/newsfragments/3379.bugfix | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 towncrier/newsfragments/3379.bugfix diff --git a/core/admin/mailu/ui/views/users.py b/core/admin/mailu/ui/views/users.py index 6487c326..85f7c577 100644 --- a/core/admin/mailu/ui/views/users.py +++ b/core/admin/mailu/ui/views/users.py @@ -24,7 +24,7 @@ def user_create(domain_name): flask.url_for('.user_list', domain_name=domain.name)) form = forms.UserForm() form.pw.validators = [wtforms.validators.DataRequired()] - form.quota_bytes.default = app.config['DEFAULT_QUOTA'] + form.quota_bytes.default = int(app.config['DEFAULT_QUOTA']) if domain.max_quota_bytes: form.quota_bytes.validators = [ wtforms.validators.NumberRange(max=domain.max_quota_bytes)] diff --git a/towncrier/newsfragments/3379.bugfix b/towncrier/newsfragments/3379.bugfix new file mode 100644 index 00000000..c1842c52 --- /dev/null +++ b/towncrier/newsfragments/3379.bugfix @@ -0,0 +1 @@ +Fix #3379: DEFAULT_QUOTA From 0e9721b7f91ead11733f73837f93d128e948f757 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 5 Sep 2024 22:57:35 +0200 Subject: [PATCH 54/83] Untested code that may fix the following: Exception in thread Thread-2 (forward_text_lines): Traceback (most recent call last): File "/usr/lib/python3.12/threading.py", line 1073, in _bootstrap_inner self.run() File "/usr/lib/python3.12/threading.py", line 1010, in run self._target(*self._args, **self._kwargs) File "/app/venv/lib/python3.12/site-packages/socrate/system.py", line 155, in forward_text_lines current_line = src.readline() ^^^^^^^^^^^^^^ File "", line 322, in decode UnicodeDecodeError: 'utf-8' codec can't decode byte 0xa8 in position 166: invalid start byte This was reported in #3398 (cherry picked from commit 61812ac32a96b06497e2d646a60b0a2f349f1937) --- core/base/libs/socrate/socrate/system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index 8ea770a3..c5046f4c 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -158,7 +158,7 @@ def forward_text_lines(src, dst): # runs a process and passes its standard/error output to the standard/error output of the current python script def run_process_and_forward_output(cmd): - process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8') stdout_thread = threading.Thread(target=forward_text_lines, args=(process.stdout, sys.stdout)) stdout_thread.daemon = True From 9b55970ab91054d3b1c3a429a7d4f80a6d3a2204 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 5 Sep 2024 23:08:41 +0200 Subject: [PATCH 55/83] fix #3272 (cherry picked from commit 0021ba9cb9a52d70ee0e71d369e1ae7df2ab016f) --- towncrier/newsfragments/3272.bugfix | 1 + webmails/nginx-webmail.conf | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 towncrier/newsfragments/3272.bugfix diff --git a/towncrier/newsfragments/3272.bugfix b/towncrier/newsfragments/3272.bugfix new file mode 100644 index 00000000..726e9e78 --- /dev/null +++ b/towncrier/newsfragments/3272.bugfix @@ -0,0 +1 @@ +Increase the size of buffers for webmail diff --git a/webmails/nginx-webmail.conf b/webmails/nginx-webmail.conf index 8772c8c8..d403eea4 100644 --- a/webmails/nginx-webmail.conf +++ b/webmails/nginx-webmail.conf @@ -55,6 +55,16 @@ server { {% else %} fastcgi_param SCRIPT_NAME {{WEB_WEBMAIL}}/$fastcgi_script_name; {% endif %} + + # fastcgi buffers for php-fpm # + fastcgi_buffers 16 32k; + fastcgi_buffer_size 64k; + fastcgi_busy_buffers_size 64k; + + # nginx buffers # + proxy_buffer_size 128k; + proxy_buffers 4 256k; + proxy_busy_buffers_size 256k; } location ~ (^|/)\. { From eb2754c211f018eef918cb54506387bf2cf2626f Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 5 Sep 2024 23:22:55 +0200 Subject: [PATCH 56/83] doh (cherry picked from commit 5ea3840fc47029f6d5cc41eb3ac41a155213806b) --- core/admin/mailu/internal/nginx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py index 6c320ae4..cf24e12f 100644 --- a/core/admin/mailu/internal/nginx.py +++ b/core/admin/mailu/internal/nginx.py @@ -19,7 +19,7 @@ STATUSES = { }), "encryption": ("Must issue a STARTTLS command first", { "imap": "PRIVACYREQUIRED", - "smtp": "530 5.7.0" + "smtp": "530 5.7.0", "submission": "530 5.7.0", "pop3": "-ERR Authentication canceled.", "sieve": "ENCRYPT-NEEDED" From d3b48941963314418546dffb5a25912dd12ddafc Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 6 Sep 2024 09:34:34 +0200 Subject: [PATCH 57/83] towncrier (cherry picked from commit 97e21df84573c24682693095d0b9bb21b7ce9b5b) --- towncrier/newsfragments/3401.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 towncrier/newsfragments/3401.bugfix diff --git a/towncrier/newsfragments/3401.bugfix b/towncrier/newsfragments/3401.bugfix new file mode 100644 index 00000000..5309408d --- /dev/null +++ b/towncrier/newsfragments/3401.bugfix @@ -0,0 +1 @@ +Fix an error that can occur when using snappymail From f5bc2265742ca1d050eb3c98ca291c83a86e1386 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 6 Sep 2024 09:40:26 +0200 Subject: [PATCH 58/83] INBOUND_TLS_ENFORCE makes no sense. (cherry picked from commit 28c28b1c84fb8699fed89fc9fc1519051f32d0d4) --- core/admin/mailu/internal/nginx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py index cf24e12f..b9cbe879 100644 --- a/core/admin/mailu/internal/nginx.py +++ b/core/admin/mailu/internal/nginx.py @@ -72,7 +72,7 @@ def handle_authentication(headers): # Incoming mail, no authentication if method in ['', 'none'] and protocol in ['smtp', 'lmtp']: server, port = get_server(protocol, False) - if app.config["INBOUND_TLS_ENFORCE"]: + if app.config["INBOUND_TLS_ENFORCE"] and protocol == 'smtp': if "Auth-SSL" in headers and headers["Auth-SSL"] == "on": return { "Auth-Status": "OK", From 3473d9db20f0f3f312fa24fbf16440a1bef70fd2 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 6 Sep 2024 13:56:32 +0200 Subject: [PATCH 59/83] fix #2996 (cherry picked from commit 61b444f00f900e7c41a5f7314f6a04be38e87b5e) --- optional/fetchmail/fetchmail.py | 18 +++++++++++++++++- towncrier/newsfragments/2296.bugfix | 1 + 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 towncrier/newsfragments/2296.bugfix diff --git a/optional/fetchmail/fetchmail.py b/optional/fetchmail/fetchmail.py index 410ce4fb..8e26e82e 100755 --- a/optional/fetchmail/fetchmail.py +++ b/optional/fetchmail/fetchmail.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import base64 import time import os from pathlib import Path @@ -32,6 +33,21 @@ poll "{host}" proto {protocol} port {port} {lmtp} """ +def imaputf7encode(s): + """"Encode a string into RFC2060 aka IMAP UTF7""" + s=s.replace('&','&-') + iters=iter(s) + unipart=out='' + for c in s: + if 0x20<=ord(c)<=0x7f : + if unipart!='' : + out+='&'+base64.b64encode(unipart.encode('utf-16-be')).decode('ascii').rstrip('=')+'-' + unipart='' + out+=c + else : unipart+=c + if unipart!='' : + out+='&'+base64.b64encode(unipart.encode('utf-16-be')).decode('ascii').rstrip('=')+'-' + return out def escape_rc_string(arg): return "".join("\\x%2x" % ord(char) for char in arg) @@ -54,7 +70,7 @@ def run(debug): options = "options antispam 501, 504, 550, 553, 554" options += " ssl" if fetch["tls"] else "" options += " keep" if fetch["keep"] else " fetchall" - folders = "folders %s" % ((','.join('"' + item + '"' for item in fetch['folders'])) if fetch['folders'] else '"INBOX"') + folders = "folders %s" % ((','.join('"' + imaputf7encode(item) + '"' for item in fetch['folders'])) if fetch['folders'] else '"INBOX"') fetchmailrc += RC_LINE.format( user_email=escape_rc_string(fetch["user_email"]), protocol=fetch["protocol"], diff --git a/towncrier/newsfragments/2296.bugfix b/towncrier/newsfragments/2296.bugfix new file mode 100644 index 00000000..493238ff --- /dev/null +++ b/towncrier/newsfragments/2296.bugfix @@ -0,0 +1 @@ +Ensure fetchmail can deal with special characters in folder names From e441539f2019228d7aed5c0c2becc8d630b983c1 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 7 Sep 2024 12:57:42 +0200 Subject: [PATCH 60/83] as per review Co-authored-by: Alexander Graf (cherry picked from commit d70e82765f55a9c0f8a55af6226dc50801dbfb7d) --- optional/fetchmail/fetchmail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optional/fetchmail/fetchmail.py b/optional/fetchmail/fetchmail.py index 8e26e82e..5bd47cd7 100755 --- a/optional/fetchmail/fetchmail.py +++ b/optional/fetchmail/fetchmail.py @@ -70,7 +70,7 @@ def run(debug): options = "options antispam 501, 504, 550, 553, 554" options += " ssl" if fetch["tls"] else "" options += " keep" if fetch["keep"] else " fetchall" - folders = "folders %s" % ((','.join('"' + imaputf7encode(item) + '"' for item in fetch['folders'])) if fetch['folders'] else '"INBOX"') + folders = f"folders {",".join(f'"{imaputf7encode(item).replace('"',r"\34")}"' for item in fetch["folders"]) or '"INBOX"'}" fetchmailrc += RC_LINE.format( user_email=escape_rc_string(fetch["user_email"]), protocol=fetch["protocol"], From 32a37946c94463705ab65c7205e8b2b36da954dc Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 7 Sep 2024 19:03:02 +0200 Subject: [PATCH 61/83] as per review (cherry picked from commit 2b9ba9ba3a75b8c1e58bb62ea159f05c1f366f91) --- optional/fetchmail/fetchmail.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/optional/fetchmail/fetchmail.py b/optional/fetchmail/fetchmail.py index 5bd47cd7..96f387dd 100755 --- a/optional/fetchmail/fetchmail.py +++ b/optional/fetchmail/fetchmail.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import base64 +import binascii import time import os from pathlib import Path @@ -34,20 +34,18 @@ poll "{host}" proto {protocol} port {port} """ def imaputf7encode(s): - """"Encode a string into RFC2060 aka IMAP UTF7""" - s=s.replace('&','&-') - iters=iter(s) - unipart=out='' - for c in s: - if 0x20<=ord(c)<=0x7f : - if unipart!='' : - out+='&'+base64.b64encode(unipart.encode('utf-16-be')).decode('ascii').rstrip('=')+'-' - unipart='' - out+=c - else : unipart+=c - if unipart!='' : - out+='&'+base64.b64encode(unipart.encode('utf-16-be')).decode('ascii').rstrip('=')+'-' - return out + """Encode a string into RFC2060 aka IMAP UTF7""" + out = '' + enc = '' + for c in s.replace('&','&-') + 'X': + if '\x20' <= c <= '\x7f': + if enc: + out += f'&{binascii.b2a_base64(enc.encode("utf-16-be")).rstrip(b"\n=").replace(b"/", b",").decode("ascii")}-' + enc = '' + out += c + else: + enc += c + return out[:-1] def escape_rc_string(arg): return "".join("\\x%2x" % ord(char) for char in arg) From 883af43f7a5b5bea1a28a335f7bce055cd1e127c Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Mon, 9 Sep 2024 13:01:12 +0000 Subject: [PATCH 62/83] Fix 3420 - fixed syntax errors in certwatcher.py (cherry picked from commit c4f457e7f965c8a8e4a28bc938e147ef2c8ef558) --- core/nginx/certwatcher.py | 4 ++-- towncrier/newsfragments/3420.bugfix | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 towncrier/newsfragments/3420.bugfix diff --git a/core/nginx/certwatcher.py b/core/nginx/certwatcher.py index 0e83666e..48d128ed 100755 --- a/core/nginx/certwatcher.py +++ b/core/nginx/certwatcher.py @@ -27,9 +27,9 @@ class ChangeHandler(FileSystemEventHandler): if exists("/var/run/nginx.pid"): print("Reloading a running nginx") system("nginx -s reload") - if os.path.exists("/run/dovecot/master.pid"): + if exists("/run/dovecot/master.pid"): print("Reloading a running dovecot") - os.system("doveadm reload") + system("doveadm reload") @staticmethod def reexec_config(): diff --git a/towncrier/newsfragments/3420.bugfix b/towncrier/newsfragments/3420.bugfix new file mode 100644 index 00000000..dbc78435 --- /dev/null +++ b/towncrier/newsfragments/3420.bugfix @@ -0,0 +1 @@ +The reload functionality of nginx/dovecot upon change of the certificates failed with an error. \ No newline at end of file From 8e0809ebfe76b2b1e1c49cd87c5ae5d947b98971 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 9 Sep 2024 19:25:33 +0200 Subject: [PATCH 63/83] Fix #3411 (cherry picked from commit 63769c73cfcb1ca0142cf488de65541b1bd0ad3e) --- core/admin/mailu/models.py | 2 +- towncrier/newsfragments/3411.bugfix | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 towncrier/newsfragments/3411.bugfix diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 57c82c6b..4f878bf7 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -637,7 +637,7 @@ in clear-text regardless of the presence of the cache. set() containing the sessions to keep """ self.password = password if raw else User.get_password_context().hash(password) - if keep_sessions is not True: + if keep_sessions is not True and self.email is not None: utils.MailuSessionExtension.prune_sessions(uid=self.email, keep=keep_sessions) def get_managed_domains(self): diff --git a/towncrier/newsfragments/3411.bugfix b/towncrier/newsfragments/3411.bugfix new file mode 100644 index 00000000..3b1e80ed --- /dev/null +++ b/towncrier/newsfragments/3411.bugfix @@ -0,0 +1 @@ +Ensure we do not nuke all web-sessions when a password is changed using the command line From 0aff4144b25ef013336c4cb3c93aac3119a95f48 Mon Sep 17 00:00:00 2001 From: Pegoku <81113533+Pegoku@users.noreply.github.com> Date: Tue, 10 Sep 2024 15:32:20 +0200 Subject: [PATCH 64/83] fix container name --- docs/maintain.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/maintain.rst b/docs/maintain.rst index a2041a86..fd01a5bf 100644 --- a/docs/maintain.rst +++ b/docs/maintain.rst @@ -46,8 +46,8 @@ In the case of *certbot* you could write a script to be executed as `deploy hook #!/bin/sh cp /etc/letsencrypt/live/domain.com/privkey.pem /mailu/certs/key.pem || exit 1 cp /etc/letsencrypt/live/domain.com/fullchain.pem /mailu/certs/cert.pem || exit 1 - docker exec mailu_front_1 nginx -s reload - docker exec mailu_front_1 doveadm reload + docker exec mailu-front-1 nginx -s reload + docker exec mailu-front-1 doveadm reload And the certbot command you will use in crontab would look something like: From 80427fb77c86842c7699d277b6c4d7250fc5ba17 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 12 Sep 2024 10:39:18 +0200 Subject: [PATCH 65/83] Fix #3450 (cherry picked from commit a875d7fe6a8dc50afbd8a08c99af1c724e85de4f) --- core/admin/start.py | 2 +- core/base/libs/socrate/socrate/system.py | 1 + core/dovecot/conf/dovecot.conf | 18 ++++++++++++++++-- core/nginx/dovecot/proxy.conf | 22 ++++++++++++++++++++++ towncrier/newsfragments/3450.bugfix | 3 +++ 5 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 towncrier/newsfragments/3450.bugfix diff --git a/core/admin/start.py b/core/admin/start.py index 9574bbb7..2fc8fb40 100755 --- a/core/admin/start.py +++ b/core/admin/start.py @@ -64,7 +64,7 @@ test_unsupported() cmdline = [ "gunicorn", - "--threads", f"{os.cpu_count()}", + "--threads", f"{CPU_COUNT}", # If SUBNET6 is defined, gunicorn must listen on IPv6 as well as IPv4 "-b", f"{'[::]' if os.environ.get('SUBNET6') else '0.0.0.0'}:8080", "--logger-class mailu.Logger", diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index c5046f4c..e9892325 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -141,6 +141,7 @@ def clean_env(): for item in os.environ.get('TLS', ALL_PORTS).split(','): if item in PORTS_REQUIRING_TLS: os.environ[f'TLS_{item}']='True' + os.environ['CPU_COUNT'] = str(os.cpu_count()) def drop_privs_to(username='mailu'): pwnam = getpwnam(username) diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index 9c92aca6..8ba0a8f2 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -89,8 +89,7 @@ plugin { service indexer-worker { executable = /bin/nice -n 10 /usr/libexec/dovecot/indexer-worker - # TODO: maybe MAXPROC? I guess it depends on how much RAM is available - process_limit = 1 + process_limit = {{ CPU_COUNT }} } ############### @@ -137,12 +136,22 @@ service imap-login { inet_listener imap { port = 143 } + service_count = 0 + client_limit = 25000 + process_min_avail = {{ CPU_COUNT }} + process_limit = {{ CPU_COUNT }} + vsz_limit = 256M } service pop3-login { inet_listener pop3 { port = 110 } + service_count = 0 + client_limit = 25000 + process_min_avail = {{ CPU_COUNT }} + process_limit = {{ CPU_COUNT }} + vsz_limit = 256M } ############### @@ -166,6 +175,11 @@ service managesieve-login { inet_listener sieve { port = 4190 } + service_count = 0 + client_limit = 25000 + process_min_avail = {{ CPU_COUNT }} + process_limit = {{ CPU_COUNT }} + vsz_limit = 256M } protocol sieve { diff --git a/core/nginx/dovecot/proxy.conf b/core/nginx/dovecot/proxy.conf index 8decfc77..d7fcbb0a 100644 --- a/core/nginx/dovecot/proxy.conf +++ b/core/nginx/dovecot/proxy.conf @@ -87,6 +87,11 @@ service managesieve-login { inet_listener sieve-webmail { port = 14190 } + service_count = 0 + client_limit = 25000 + process_min_avail = {{ CPU_COUNT }} + process_limit = {{ CPU_COUNT }} + vsz_limit = 256M } {% endif %} @@ -114,6 +119,11 @@ service imap-login { inet_listener imap-webmail { port = 10143 } + service_count = 0 + client_limit = 25000 + process_min_avail = {{ CPU_COUNT }} + process_limit = {{ CPU_COUNT }} + vsz_limit = 256M } service pop3-login { @@ -132,6 +142,11 @@ service pop3-login { {% endif %} } {% endif %} + service_count = 0 + client_limit = 25000 + process_min_avail = {{ CPU_COUNT }} + process_limit = {{ CPU_COUNT }} + vsz_limit = 256M } recipient_delimiter = {{ RECIPIENT_DELIMITER }} @@ -161,4 +176,11 @@ service submission-login { inet_listener submission-webmail { port = 10025 } + service_count = 0 + client_limit = 25000 + process_min_avail = {{ CPU_COUNT }} + process_limit = {{ CPU_COUNT }} + vsz_limit = 256M } + +!include_try /overrides/dovecot.conf diff --git a/towncrier/newsfragments/3450.bugfix b/towncrier/newsfragments/3450.bugfix new file mode 100644 index 00000000..ec92e429 --- /dev/null +++ b/towncrier/newsfragments/3450.bugfix @@ -0,0 +1,3 @@ +Ensure we can do more than 100 parallel sessions. +Make use of all CPUs available for indexing +Allow dovecot's config to be overriden in front too From cb3ed1598db263f562033da5255efbae3cb67ade Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 12 Sep 2024 10:58:09 +0200 Subject: [PATCH 66/83] doh (cherry picked from commit c6c4c09f08fdd130105c04d3e529f0fcda956f65) --- core/admin/start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/start.py b/core/admin/start.py index 2fc8fb40..07db30fe 100755 --- a/core/admin/start.py +++ b/core/admin/start.py @@ -64,7 +64,7 @@ test_unsupported() cmdline = [ "gunicorn", - "--threads", f"{CPU_COUNT}", + "--threads", f"{os.environ.get('CPU_COUNT', 1)}", # If SUBNET6 is defined, gunicorn must listen on IPv6 as well as IPv4 "-b", f"{'[::]' if os.environ.get('SUBNET6') else '0.0.0.0'}:8080", "--logger-class mailu.Logger", From 0bdb9e1cea0b80b51040c5bd37f209fc57437b41 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 12 Sep 2024 12:30:04 +0200 Subject: [PATCH 67/83] Apply suggestions from code review Co-authored-by: Alexander Graf (cherry picked from commit cdf8ee8820ec0f797c4c75329bfbf8e14a35916c) --- core/admin/start.py | 2 +- core/base/libs/socrate/socrate/system.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/admin/start.py b/core/admin/start.py index 07db30fe..24f31e07 100755 --- a/core/admin/start.py +++ b/core/admin/start.py @@ -64,7 +64,7 @@ test_unsupported() cmdline = [ "gunicorn", - "--threads", f"{os.environ.get('CPU_COUNT', 1)}", + "--threads", os.environ.get('CPU_COUNT', '1'), # If SUBNET6 is defined, gunicorn must listen on IPv6 as well as IPv4 "-b", f"{'[::]' if os.environ.get('SUBNET6') else '0.0.0.0'}:8080", "--logger-class mailu.Logger", diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index e9892325..f494a7b7 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -141,7 +141,8 @@ def clean_env(): for item in os.environ.get('TLS', ALL_PORTS).split(','): if item in PORTS_REQUIRING_TLS: os.environ[f'TLS_{item}']='True' - os.environ['CPU_COUNT'] = str(os.cpu_count()) + if 'CPU_COUNT' not in os.environ: + os.environ['CPU_COUNT'] = str(os.cpu_count()) def drop_privs_to(username='mailu'): pwnam = getpwnam(username) From c193cf1d60c379a3d8b8ee6c09c7f9e252b49880 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 12 Sep 2024 12:34:31 +0200 Subject: [PATCH 68/83] play it safe (cherry picked from commit f4ab22edbce7e4d2181f5821d1ae83f5ce46bd00) --- core/dovecot/conf/dovecot.conf | 3 ++- towncrier/newsfragments/3450.bugfix | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index 8ba0a8f2..e16ba8a6 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -89,7 +89,8 @@ plugin { service indexer-worker { executable = /bin/nice -n 10 /usr/libexec/dovecot/indexer-worker - process_limit = {{ CPU_COUNT }} + # TODO: maybe MAXPROC? I guess it depends on how much RAM is available + process_limit = 1 } ############### diff --git a/towncrier/newsfragments/3450.bugfix b/towncrier/newsfragments/3450.bugfix index ec92e429..50c75f22 100644 --- a/towncrier/newsfragments/3450.bugfix +++ b/towncrier/newsfragments/3450.bugfix @@ -1,3 +1,2 @@ Ensure we can do more than 100 parallel sessions. -Make use of all CPUs available for indexing Allow dovecot's config to be overriden in front too From e7caca3e085a8a21c670a1bfc848a0015b43801a Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 12 Sep 2024 15:45:58 +0200 Subject: [PATCH 69/83] As per review (cherry picked from commit 12e94b7616c1c5447e4c9beb05bb608478736905) --- core/dovecot/conf/dovecot.conf | 2 +- docs/faq.rst | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index e16ba8a6..03709b2e 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -238,4 +238,4 @@ service anvil { # Extensions ############### -!include_try /overrides/dovecot.conf +!include_try /overrides/dovecot/proxy.conf diff --git a/docs/faq.rst b/docs/faq.rst index 6b3011a1..955752d0 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -256,8 +256,10 @@ correct syntax. The following file names will be taken as override configuration - For both ``postfix.cf`` and ``postfix.master``, you need to put one configuration per line, as they are fed line-by-line to postfix. - ``logrotate.conf`` as ``$ROOT/overrides/postfix/logrotate.conf`` - Replaces the logrotate.conf file used for rotating ``POSTFIX_LOG_FILE``. -- `Dovecot`_ - ``dovecot.conf`` in dovecot sub-directory; -- `Nginx`_ - All ``*.conf`` files in the ``nginx`` sub-directory; +- `Dovecot`_ - ``dovecot.conf`` in dovecot sub-directory. +- `Nginx`_ : + - All ``*.conf`` files in the ``nginx`` sub-directory. + - ``proxy.conf`` in the ``nginx/dovecot`` sub-directory. - `Rspamd`_ - All files in the ``rspamd`` sub-directory. - `Roundcube`_ - All ``*.inc.php`` files in the ``roundcube`` sub directory. From 4ec2e5c5d301a998a79cd65f4a96d4848504f615 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Thu, 12 Sep 2024 14:21:48 +0000 Subject: [PATCH 70/83] Fix rst syntax (cherry picked from commit 14a75a101dca97e4a78f82010ec2cbed954526cf) --- docs/faq.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 955752d0..1111481d 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -258,8 +258,8 @@ correct syntax. The following file names will be taken as override configuration - ``logrotate.conf`` as ``$ROOT/overrides/postfix/logrotate.conf`` - Replaces the logrotate.conf file used for rotating ``POSTFIX_LOG_FILE``. - `Dovecot`_ - ``dovecot.conf`` in dovecot sub-directory. - `Nginx`_ : - - All ``*.conf`` files in the ``nginx`` sub-directory. - - ``proxy.conf`` in the ``nginx/dovecot`` sub-directory. + - All ``*.conf`` files in the ``nginx`` sub-directory. + - ``proxy.conf`` in the ``nginx/dovecot`` sub-directory. - `Rspamd`_ - All files in the ``rspamd`` sub-directory. - `Roundcube`_ - All ``*.inc.php`` files in the ``roundcube`` sub directory. From 4dc2e0b48ae2285291a13797ff42f441d7773389 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 12 Sep 2024 23:16:48 +0200 Subject: [PATCH 71/83] Need sleep. (cherry picked from commit 07f0ec3d0b084e93bafb3585f3c43bbcfe433a39) --- core/dovecot/conf/dovecot.conf | 2 +- core/nginx/dovecot/proxy.conf | 2 +- towncrier/newsfragments/3467.bugfix | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 towncrier/newsfragments/3467.bugfix diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index 03709b2e..e16ba8a6 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -238,4 +238,4 @@ service anvil { # Extensions ############### -!include_try /overrides/dovecot/proxy.conf +!include_try /overrides/dovecot.conf diff --git a/core/nginx/dovecot/proxy.conf b/core/nginx/dovecot/proxy.conf index d7fcbb0a..bd3d1d6a 100644 --- a/core/nginx/dovecot/proxy.conf +++ b/core/nginx/dovecot/proxy.conf @@ -183,4 +183,4 @@ service submission-login { vsz_limit = 256M } -!include_try /overrides/dovecot.conf +!include_try /overrides/dovecot/proxy.conf diff --git a/towncrier/newsfragments/3467.bugfix b/towncrier/newsfragments/3467.bugfix new file mode 100644 index 00000000..fe7cdc8c --- /dev/null +++ b/towncrier/newsfragments/3467.bugfix @@ -0,0 +1 @@ +Fix broken overrides in 2024.06.17 From 6d2daebe68f9c9b0cff791ffb4518e4d8efa1904 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 12 Sep 2024 19:21:18 +0200 Subject: [PATCH 72/83] Maybe fix #3402 (cherry picked from commit 8b34fbe8810239d36134cd9756904112d932e194) --- core/nginx/conf/nginx.conf | 6 ++++++ core/nginx/letsencrypt.py | 33 ++++++++--------------------- towncrier/newsfragments/3402.bugfix | 1 + 3 files changed, 16 insertions(+), 24 deletions(-) create mode 100644 towncrier/newsfragments/3402.bugfix diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index 97a32113..dd2176ca 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -66,6 +66,9 @@ http { listen [::]:80{% if PROXY_PROTOCOL_80 %} proxy_protocol{% endif %}; {% endif %} {% if TLS_FLAVOR in ['letsencrypt', 'mail-letsencrypt'] %} + location ^~ /.well-known/acme-challenge/testing { + return 204; + } location ^~ /.well-known/acme-challenge/ { proxy_pass http://127.0.0.1:8008; } @@ -159,6 +162,9 @@ http { } {% if TLS_FLAVOR in ['letsencrypt', 'mail-letsencrypt'] %} + location ^~ /.well-known/acme-challenge/testing { + return 204; + } location ^~ /.well-known/acme-challenge/ { proxy_pass http://127.0.0.1:8008; } diff --git a/core/nginx/letsencrypt.py b/core/nginx/letsencrypt.py index a8abbee7..8428cd66 100755 --- a/core/nginx/letsencrypt.py +++ b/core/nginx/letsencrypt.py @@ -6,8 +6,6 @@ import requests import sys import subprocess import time -from threading import Thread -from http.server import HTTPServer, SimpleHTTPRequestHandler log.basicConfig(stream=sys.stderr, level="WARNING") hostnames = ','.join(set(host.strip() for host in os.environ['HOSTNAMES'].split(','))) @@ -45,33 +43,20 @@ command2 = [ # Wait for nginx to start time.sleep(5) -class MyRequestHandler(SimpleHTTPRequestHandler): - def do_GET(self): - if self.path == '/.well-known/acme-challenge/testing': - self.send_response(204) - else: - self.send_response(404) - self.send_header('Content-Type', 'text/plain') - self.end_headers() - -def serve_one_request(): - with HTTPServer(("127.0.0.1", 8008), MyRequestHandler) as server: - server.handle_request() - # Run certbot every day while True: while True: hostname = os.environ['HOSTNAMES'].split(',')[0] target = f'http://{hostname}/.well-known/acme-challenge/testing' - thread = Thread(target=serve_one_request) - thread.start() - r = requests.get(target) - if r.status_code != 204: - log.critical(f"Can't reach {target}!, please ensure it's fixed or change the TLS_FLAVOR.") - time.sleep(5) - else: - break - thread.join() + try: + r = requests.get(target) + if r.status_code != 204: + log.critical(f"Can't reach {target}!, please ensure it's fixed or change the TLS_FLAVOR.") + time.sleep(5) + else: + break + except Exception as e: + log.error(f"Exception while fetching {target}!", exc_info = e) subprocess.call(command) subprocess.call(command2) diff --git a/towncrier/newsfragments/3402.bugfix b/towncrier/newsfragments/3402.bugfix new file mode 100644 index 00000000..21ddd2ee --- /dev/null +++ b/towncrier/newsfragments/3402.bugfix @@ -0,0 +1 @@ +Fix a potential problem with SO_REUSEADDR that may prevent admin from starting up From bd7a5bd235305582463e76c3df824c1073a1c2b9 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 12 Sep 2024 22:48:20 +0200 Subject: [PATCH 73/83] Add delay (cherry picked from commit c11815b3a012e16bae259b08b517d1df0886486b) --- core/nginx/letsencrypt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/nginx/letsencrypt.py b/core/nginx/letsencrypt.py index 8428cd66..fec3b7eb 100755 --- a/core/nginx/letsencrypt.py +++ b/core/nginx/letsencrypt.py @@ -57,6 +57,7 @@ while True: break except Exception as e: log.error(f"Exception while fetching {target}!", exc_info = e) + time.sleep(15) subprocess.call(command) subprocess.call(command2) From e47da4ba0dccd3a194294ec1e347f44b9d488f61 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 22 Sep 2024 13:14:33 +0200 Subject: [PATCH 74/83] Fix #3531 (cherry picked from commit 8e4af5ad05812c9c8914582312e885a5b6010f5d) --- core/nginx/letsencrypt.py | 1 + towncrier/newsfragments/3531.bugfix | 1 + 2 files changed, 2 insertions(+) create mode 100644 towncrier/newsfragments/3531.bugfix diff --git a/core/nginx/letsencrypt.py b/core/nginx/letsencrypt.py index a8abbee7..f9aab75f 100755 --- a/core/nginx/letsencrypt.py +++ b/core/nginx/letsencrypt.py @@ -22,6 +22,7 @@ command = [ "--preferred-challenges", "http", "--http-01-port", "8008", "--keep-until-expiring", "--allow-subset-of-names", + "--key-type", "rsa", "--renew-with-new-domains", "--config-dir", "/certs/letsencrypt", "--post-hook", "/config.py" diff --git a/towncrier/newsfragments/3531.bugfix b/towncrier/newsfragments/3531.bugfix new file mode 100644 index 00000000..9deddb45 --- /dev/null +++ b/towncrier/newsfragments/3531.bugfix @@ -0,0 +1 @@ +Ensure we have both RSA and ECDSA certs when using letsencrypt From 450c78f7b0d3a2f2efe1829721df26de7802d555 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Tue, 15 Oct 2024 17:14:22 +0200 Subject: [PATCH 75/83] Fix http2 (cherry picked from commit fb49ef905759fd71ec93650f2adcb427fb0b37da) --- core/nginx/conf/nginx.conf | 2 +- towncrier/newsfragments/3613.bugfix | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 towncrier/newsfragments/3613.bugfix diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index dd2176ca..2dba0130 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -98,6 +98,7 @@ http { set $webdav {{ WEBDAV_ADDRESS }}:5232; {% endif %} client_max_body_size {{ MESSAGE_SIZE_LIMIT|int + 8388608 }}; + http2 on; # Listen on HTTP only in kubernetes or behind reverse proxy {% if TLS_FLAVOR in [ 'mail-letsencrypt', 'notls', 'mail' ] %} @@ -112,7 +113,6 @@ http { listen 443 ssl{% if PROXY_PROTOCOL_443 %} proxy_protocol{% endif %}; {% if SUBNET6 %} listen [::]:443 ssl{% if PROXY_PROTOCOL_443 %} proxy_protocol{% endif %}; - http2 on; {% endif %} include /etc/nginx/tls.conf; diff --git a/towncrier/newsfragments/3613.bugfix b/towncrier/newsfragments/3613.bugfix new file mode 100644 index 00000000..0618c857 --- /dev/null +++ b/towncrier/newsfragments/3613.bugfix @@ -0,0 +1 @@ +HTTP/2 does not require ipv6; in fact it does not require SSL certs either From df3329962c83ff5b0133391b8156745a18714ca4 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 5 Oct 2024 18:59:29 +0200 Subject: [PATCH 76/83] Ensure healthchecks timeout (cherry picked from commit 7aaf7ec17b3d3c736c4fce64e7d547752f9e121e) # Conflicts: # towncrier/newsfragments/3398.misc --- core/admin/Dockerfile | 2 +- core/nginx/Dockerfile | 2 +- core/rspamd/Dockerfile | 2 +- setup/Dockerfile | 2 +- towncrier/newsfragments/3398.misc | 2 ++ webmails/Dockerfile | 2 +- 6 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 towncrier/newsfragments/3398.misc diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index 1e54dec3..c73bbd85 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -23,7 +23,7 @@ RUN set -euxo pipefail \ RUN echo $VERSION >/version #EXPOSE 8080/tcp -HEALTHCHECK CMD curl -skfLo /dev/null http://localhost:8080/ping +HEALTHCHECK CMD curl -m3 -skfLo /dev/null http://localhost:8080/ping VOLUME ["/data","/dkim"] diff --git a/core/nginx/Dockerfile b/core/nginx/Dockerfile index cacb6c99..41992f30 100644 --- a/core/nginx/Dockerfile +++ b/core/nginx/Dockerfile @@ -29,7 +29,7 @@ RUN echo $VERSION >/version EXPOSE 80/tcp 443/tcp 110/tcp 143/tcp 465/tcp 587/tcp 993/tcp 995/tcp 25/tcp 4190/tcp # EXPOSE 10025/tcp 10143/tcp 14190/tcp -HEALTHCHECK --start-period=60s CMD curl -skfLo /dev/null http://127.0.0.1:10204/health && kill -0 `cat /run/dovecot/master.pid` +HEALTHCHECK --start-period=60s CMD curl -m3 -skfLo /dev/null http://127.0.0.1:10204/health && kill -0 `cat /run/dovecot/master.pid` VOLUME ["/certs", "/overrides"] diff --git a/core/rspamd/Dockerfile b/core/rspamd/Dockerfile index 9f61f7d0..16f4a610 100644 --- a/core/rspamd/Dockerfile +++ b/core/rspamd/Dockerfile @@ -17,7 +17,7 @@ COPY start.py / RUN echo $VERSION >/version #EXPOSE 11332/tcp 11334/tcp 11335/tcp -HEALTHCHECK --start-period=350s CMD curl -skfLo /dev/null http://localhost:11334/ +HEALTHCHECK --start-period=350s CMD curl -m3 -skfLo /dev/null http://localhost:11334/ VOLUME ["/var/lib/rspamd"] diff --git a/setup/Dockerfile b/setup/Dockerfile index a410871d..4ddfe683 100644 --- a/setup/Dockerfile +++ b/setup/Dockerfile @@ -15,7 +15,7 @@ COPY main.py ./main.py RUN echo $VERSION >> /version EXPOSE 80/tcp -HEALTHCHECK --start-period=350s CMD curl -skfLo /dev/null http://localhost/ +HEALTHCHECK --start-period=350s CMD curl -m3 -skfLo /dev/null http://localhost/ USER mailu CMD gunicorn -w 4 -b :80 --access-logfile - --error-logfile - --preload main:app diff --git a/towncrier/newsfragments/3398.misc b/towncrier/newsfragments/3398.misc new file mode 100644 index 00000000..895b6f1d --- /dev/null +++ b/towncrier/newsfragments/3398.misc @@ -0,0 +1,2 @@ +Disable HARDENED_MALLOC unless the requirements are met +Ensure the healthchecks timeout diff --git a/webmails/Dockerfile b/webmails/Dockerfile index 517cb873..712c5dfd 100644 --- a/webmails/Dockerfile +++ b/webmails/Dockerfile @@ -94,6 +94,6 @@ VOLUME /overrides CMD /start.py -HEALTHCHECK CMD curl -f -L http://localhost/ping || exit 1 +HEALTHCHECK CMD curl -m3 -f -L http://localhost/ping || exit 1 RUN echo $VERSION >> /version From 5e8fdd4c9e53d357dc5d65eca951690205778a7d Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 17 Oct 2024 15:26:32 +0200 Subject: [PATCH 77/83] Filter logs line based and in binary mode without decoding utf-8 (cherry picked from commit 4a49234afa2e762d178f46b3da8365164e3fb765) --- core/base/libs/socrate/socrate/system.py | 40 +++++++++++------------- core/dovecot/start.py | 4 ++- core/nginx/start.py | 4 ++- core/postfix/start.py | 8 ++--- 4 files changed, 29 insertions(+), 27 deletions(-) diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index f494a7b7..9e78fd80 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -31,31 +31,29 @@ def _coerce_value(value): class LogFilter(object): def __init__(self, stream, re_patterns): - self.stream = stream - if isinstance(re_patterns, list): - self.pattern = re.compile('|'.join([fr'(?:{pattern})' for pattern in re_patterns])) - elif isinstance(re_patterns, str): - self.pattern = re.compile(re_patterns) - else: - self.pattern = re_patterns - self.found = False + self.stream = stream + self.pattern = re.compile(b'|'.join([b''.join([b'(?:', pattern, b')']) for pattern in re_patterns])) + self.buffer = b'' def __getattr__(self, attr_name): return getattr(self.stream, attr_name) def write(self, data): - if data == '\n' and self.found: - self.found = False - else: - if not self.pattern.search(data): - self.stream.write(data) + if type(data) is str: + data = data.encode('utf-8') + self.buffer += data + while b'\n' in self.buffer: + line, cr, rest = self.buffer.partition(b'\n') + if not self.pattern.search(line): + self.stream.buffer.write(line) + self.stream.buffer.write(cr) self.stream.flush() - else: - # caught bad pattern - self.found = True + self.buffer = rest - def flush(self): - self.stream.flush() + def close(self): + if self.buffer and not self.pattern.search(self.buffer): + self.stream.buffer.write(self.buffer) + self.stream.close() def _is_compatible_with_hardened_malloc(): with open('/proc/cpuinfo', 'r') as f: @@ -100,7 +98,7 @@ def set_env(required_secrets=[], log_filters=[]): for secret in required_secrets: os.environ[f'{secret}_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray(secret, 'utf-8'), 'sha256').hexdigest() - os.system('find /run -xdev -type f -name \*.pid -print -delete') + os.system(r'find /run -xdev -type f -name \*.pid -print -delete') return { key: _coerce_value(os.environ.get(key, value)) @@ -160,7 +158,7 @@ def forward_text_lines(src, dst): # runs a process and passes its standard/error output to the standard/error output of the current python script def run_process_and_forward_output(cmd): - process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8') + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout_thread = threading.Thread(target=forward_text_lines, args=(process.stdout, sys.stdout)) stdout_thread.daemon = True @@ -170,4 +168,4 @@ def run_process_and_forward_output(cmd): stderr_thread.daemon = True stderr_thread.start() - process.wait() + return process.wait() diff --git a/core/dovecot/start.py b/core/dovecot/start.py index b162db95..83afa1de 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -7,7 +7,9 @@ import multiprocessing from podop import run_server from socrate import system, conf -system.set_env(log_filters=[r'Error\: SSL context initialization failed, disabling SSL\: Can\'t load SSL certificate \(ssl_cert setting\)\: The certificate is empty$']) +system.set_env(log_filters=[ + rb'Error\: SSL context initialization failed, disabling SSL\: Can\'t load SSL certificate \(ssl_cert setting\)\: The certificate is empty$' +]) def start_podop(): system.drop_privs_to('mail') diff --git a/core/nginx/start.py b/core/nginx/start.py index a50abec2..48f62511 100755 --- a/core/nginx/start.py +++ b/core/nginx/start.py @@ -4,7 +4,9 @@ import os import subprocess from socrate import system -system.set_env(log_filters=r'could not be resolved \(\d\: [^\)]+\) while in resolving client address, client\: [^,]+, server: [^\:]+\:(25|110|143|587|465|993|995)$') +system.set_env(log_filters=[ + rb'could not be resolved \(\d\: [^\)]+\) while in resolving client address, client\: [^,]+, server: [^\:]+\:(25|110|143|587|465|993|995)$' +]) # Check if a stale pid file exists if os.path.exists("/var/run/nginx.pid"): diff --git a/core/postfix/start.py b/core/postfix/start.py index 7daf7c3b..aaa141a8 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -11,10 +11,10 @@ from podop import run_server from socrate import system, conf system.set_env(log_filters=[ - r'(dis)?connect from localhost\[(\:\:1|127\.0\.0\.1)\]( quit=1 commands=1)?$', - r'haproxy read\: short protocol header\: QUIT$', - r'discarding EHLO keywords\: PIPELINING$' - ]) + rb'(dis)?connect from localhost\[(\:\:1|127\.0\.0\.1)\]( quit=1 commands=1)?$', + rb'haproxy read\: short protocol header\: QUIT$', + rb'discarding EHLO keywords\: PIPELINING$' +]) os.system("flock -n /queue/pid/master.pid rm /queue/pid/master.pid") From a910f4caac61b87db811cbac995448df29466db5 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 17 Oct 2024 16:57:54 +0200 Subject: [PATCH 78/83] Add newsfragment (cherry picked from commit 70e28df9c727fbe142533388cf834600236a5254) --- towncrier/newsfragments/3618.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 towncrier/newsfragments/3618.bugfix diff --git a/towncrier/newsfragments/3618.bugfix b/towncrier/newsfragments/3618.bugfix new file mode 100644 index 00000000..fb72e3a2 --- /dev/null +++ b/towncrier/newsfragments/3618.bugfix @@ -0,0 +1 @@ +Filter logs line based and in binary mode without decoding utf-8 From 9dae649f31286bb020c578e414faece47ed64638 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 17 Oct 2024 16:59:55 +0200 Subject: [PATCH 79/83] Flush buffer on flush() and flush explicitely after subprocess has ended (cherry picked from commit f16467b984232ffdf7fd7294bfd9457d66514dd4) --- core/base/libs/socrate/socrate/system.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index 9e78fd80..d92caf0c 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -50,10 +50,11 @@ class LogFilter(object): self.stream.flush() self.buffer = rest - def close(self): + def flush(self): + # write out buffer on flush even if it's not a complete line if self.buffer and not self.pattern.search(self.buffer): self.stream.buffer.write(self.buffer) - self.stream.close() + self.stream.flush() def _is_compatible_with_hardened_malloc(): with open('/proc/cpuinfo', 'r') as f: @@ -168,4 +169,7 @@ def run_process_and_forward_output(cmd): stderr_thread.daemon = True stderr_thread.start() - return process.wait() + rc = process.wait() + sys.stdout.flush() + sys.stderr.flush() + return rc From 5b2456cc8eb93995c62a8ecee1b096bdfc71734f Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 21 Oct 2024 11:05:01 +0200 Subject: [PATCH 80/83] alpine 3.20.3 (cherry picked from commit 152b45a0e93c0b029894693017b321cd970da728) --- core/base/Dockerfile | 2 +- towncrier/newsfragments/3622.misc | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 towncrier/newsfragments/3622.misc diff --git a/core/base/Dockerfile b/core/base/Dockerfile index 39d9f29c..5e3ad092 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -3,7 +3,7 @@ # base system image (intermediate) # Note when updating the alpine tag, first manually run the workflow .github/workflows/mirror.yml. # Just run the workflow with the tag that must be synchronised. -ARG DISTRO=ghcr.io/mailu/alpine:3.20 +ARG DISTRO=ghcr.io/mailu/alpine:3.20.3 FROM $DISTRO as system ENV TZ=Etc/UTC LANG=C.UTF-8 diff --git a/towncrier/newsfragments/3622.misc b/towncrier/newsfragments/3622.misc new file mode 100644 index 00000000..2e749d74 --- /dev/null +++ b/towncrier/newsfragments/3622.misc @@ -0,0 +1 @@ +Upgrade to alpine 3.20.3 From 59400f0f1b5aa18b6c1e861526a1a8a70e14df45 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 16 Nov 2024 22:24:26 +0100 Subject: [PATCH 81/83] Upgrade snappymail to v2.38.2 (cherry picked from commit d4a3d5aeaecd486267f9ef640af34eae9a58bbd3) --- towncrier/newsfragments/3648.bugfix | 1 + webmails/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 towncrier/newsfragments/3648.bugfix diff --git a/towncrier/newsfragments/3648.bugfix b/towncrier/newsfragments/3648.bugfix new file mode 100644 index 00000000..3c412b2a --- /dev/null +++ b/towncrier/newsfragments/3648.bugfix @@ -0,0 +1 @@ +Upgrade snappymail to v2.38.1 ; this is a security fix for GHSA-2rq7-79vp-ffxm (mXSS) diff --git a/webmails/Dockerfile b/webmails/Dockerfile index 712c5dfd..95d4a374 100644 --- a/webmails/Dockerfile +++ b/webmails/Dockerfile @@ -54,7 +54,7 @@ COPY roundcube/config/config.inc.carddav.php /var/www/roundcube/plugins/carddav/ # snappymail -ENV SNAPPYMAIL_URL https://github.com/the-djmaze/snappymail/releases/download/v2.36.4/snappymail-2.36.4.tar.gz +ENV SNAPPYMAIL_URL https://github.com/the-djmaze/snappymail/releases/download/v2.38.2/snappymail-2.38.2.tar.gz RUN set -euxo pipefail \ ; mkdir /var/www/snappymail \ From 37964c8d935e1c01a6432c2b4d32633643af5d6c Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 16 Nov 2024 22:48:04 +0100 Subject: [PATCH 82/83] doh; v2.38.2 (cherry picked from commit 6f3257f3b2463e4d8de968625839d315e77861b8) --- towncrier/newsfragments/3648.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/towncrier/newsfragments/3648.bugfix b/towncrier/newsfragments/3648.bugfix index 3c412b2a..95a98590 100644 --- a/towncrier/newsfragments/3648.bugfix +++ b/towncrier/newsfragments/3648.bugfix @@ -1 +1 @@ -Upgrade snappymail to v2.38.1 ; this is a security fix for GHSA-2rq7-79vp-ffxm (mXSS) +Upgrade snappymail to v2.38.2 ; this is a security fix for GHSA-2rq7-79vp-ffxm (mXSS) From 1805ef4480b1466f3b99fb14798b9226ce3f2b82 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 17 Nov 2024 14:42:23 +0100 Subject: [PATCH 83/83] Don't check empty passwords against HIBP (cherry picked from commit f1c5ac5b14e9654218777cfcf311ee253233f837) --- core/admin/assets/assets/app.js | 3 +++ towncrier/newsfragments/3650.bugfix | 1 + 2 files changed, 4 insertions(+) create mode 100644 towncrier/newsfragments/3650.bugfix diff --git a/core/admin/assets/assets/app.js b/core/admin/assets/assets/app.js index 33f63433..12baec4c 100644 --- a/core/admin/assets/assets/app.js +++ b/core/admin/assets/assets/app.js @@ -21,6 +21,9 @@ function sha1(string) { } function hibpCheck(pwd) { + if (pwd === null || pwd === undefined || pwd.length === 0) { + return; + } // We hash the pwd first sha1(pwd).then(function(hash){ // We send the first 5 chars of the hash to hibp's API diff --git a/towncrier/newsfragments/3650.bugfix b/towncrier/newsfragments/3650.bugfix new file mode 100644 index 00000000..97d9aa8f --- /dev/null +++ b/towncrier/newsfragments/3650.bugfix @@ -0,0 +1 @@ +Don't check empty passwords against HIBP