Merge remote-tracking branch 'origin/master' into feature-switch-snappymail

core/admin/Dockerfile

@@ -1,5 +1,5 @@
 # First stage to build assets
-ARG DISTRO=alpine:3.14.3
+ARG DISTRO=alpine:3.14.4
 ARG ARCH=""
 
 FROM ${ARCH}node:16 as assets
@@ -13,7 +13,7 @@ COPY webpack.config.js ./
 COPY assets ./assets
 RUN set -eu \
  && sed -i 's/#007bff/#55a5d9/' node_modules/admin-lte/build/scss/_bootstrap-variables.scss \
- && for l in ca da de:de_de en:en-gb es:es_es eu fr:fr_fr he hu is it:it_it ja nb_NO:no_nb nl:nl_nl pl pt:pt_pt ru sv:sv_se zh; do \
+ && for l in ca da de:de-DE en:en-GB es:es-ES eu fr:fr-FR he hu is it:it-IT ja nb_NO:no-NB nl:nl-NL pl pt:pt-PT ru sv:sv-SE zh; do \
       cp node_modules/datatables.net-plugins/i18n/${l#*:}.json assets/${l%:*}.json; \
     done \
  && node_modules/.bin/webpack-cli --color
@@ -59,4 +59,4 @@ ENV FLASK_APP mailu
 CMD /start.py
 
 HEALTHCHECK CMD curl -f -L http://localhost/sso/login?next=ui.index || exit 1
-RUN echo $VERSION >> /version
+RUN echo $VERSION >> /version

core/admin/mailu/internal/nginx.py

@@ -93,12 +93,12 @@ def handle_authentication(headers):
             app.logger.warn(f'Received undecodable user/password from nginx: {raw_user_email!r}/{raw_password!r}')
         else:
             try:
-                user = models.User.query.get(user_email)
-                is_valid_user = True
+                user = models.User.query.get(user_email) if '@' in user_email else None
             except sqlalchemy.exc.StatementError as exc:
                 exc = str(exc).split('\n', 1)[0]
                 app.logger.warn(f'Invalid user {user_email!r}: {exc}')
             else:
+                is_valid_user = user is not None
                 ip = urllib.parse.unquote(headers["Client-Ip"])
                 if check_credentials(user, password, ip, protocol, headers["Auth-Port"]):
                     server, port = get_server(headers["Auth-Protocol"], True)

core/admin/mailu/internal/views/auth.py

@@ -12,7 +12,7 @@ def nginx_authentication():
     """
     client_ip = flask.request.headers["Client-Ip"]
     headers = flask.request.headers
-    if headers["Auth-Port"] == '25' and headers['Auth-Method'] == 'plain':
+    if headers["Auth-Port"] == '25' and headers['Auth-Method'] != 'none':
         response = flask.Response()
         response.headers['Auth-Status'] = 'AUTH not supported'
         response.headers['Auth-Error-Code'] = '502 5.5.1'
@@ -32,7 +32,7 @@ def nginx_authentication():
     for key, value in headers.items():
         response.headers[key] = str(value)
     is_valid_user = False
-    if response.headers.get("Auth-User-Exists"):
+    if response.headers.get("Auth-User-Exists") == "True":
         username = response.headers["Auth-User"]
         if utils.limiter.should_rate_limit_user(username, client_ip):
             # FIXME could be done before handle_authentication()

core/admin/mailu/internal/views/postfix.py

@@ -5,6 +5,7 @@ from flask import current_app as app
 import flask
 import idna
 import re
+import sqlalchemy.exc
 import srslib
 
 @internal.route("/postfix/dane/<domain_name>")
@@ -158,18 +159,13 @@ def postfix_sender_rate(sender):
 def postfix_sender_access(sender):
     """ Simply reject any sender that pretends to be from a local domain
     """
-    if not is_void_address(sender):
-        localpart, domain_name = models.Email.resolve_domain(sender)
-        return flask.jsonify("REJECT") if models.Domain.query.get(domain_name) else flask.abort(404)
-    else:
-        return flask.abort(404)
-
-
-def is_void_address(email):
-    '''True if the email is void (null) email address.
-    '''
-    if email.startswith('<') and email.endswith('>'):
-        email = email[1:-1]
-    # Some MTAs use things like '<MAILER-DAEMON>' instead of '<>'; so let's
-    # consider void any such thing.
-    return '@' not in email
+    if '@' in sender:
+        if sender.startswith('<') and sender.endswith('>'):
+            sender = sender[1:-1]
+        try:
+            localpart, domain_name = models.Email.resolve_domain(sender)
+            if models.Domain.query.get(domain_name):
+                return flask.jsonify("REJECT")
+        except sqlalchemy.exc.StatementError:
+            pass
+    return flask.abort(404)

core/admin/mailu/ui/views/domains.py

@@ -2,6 +2,7 @@ from mailu import models
 from mailu.ui import ui, forms, access
 from flask import current_app as app
 
+import validators
 import flask
 import flask_login
 import wtforms_components
@@ -18,18 +19,21 @@ def domain_list():
 def domain_create():
     form = forms.DomainForm()
     if form.validate_on_submit():
-        conflicting_domain = models.Domain.query.get(form.name.data)
-        conflicting_alternative = models.Alternative.query.get(form.name.data)
-        conflicting_relay = models.Relay.query.get(form.name.data)
-        if conflicting_domain or conflicting_alternative or conflicting_relay:
-            flask.flash('Domain %s is already used' % form.name.data, 'error')
+        if validators.domain(form.name.data):
+            conflicting_domain = models.Domain.query.get(form.name.data)
+            conflicting_alternative = models.Alternative.query.get(form.name.data)
+            conflicting_relay = models.Relay.query.get(form.name.data)
+            if conflicting_domain or conflicting_alternative or conflicting_relay:
+                flask.flash('Domain %s is already used' % form.name.data, 'error')
+            else:
+                domain = models.Domain()
+                form.populate_obj(domain)
+                models.db.session.add(domain)
+                models.db.session.commit()
+                flask.flash('Domain %s created' % domain)
+                return flask.redirect(flask.url_for('.domain_list'))
         else:
-            domain = models.Domain()
-            form.populate_obj(domain)
-            models.db.session.add(domain)
-            models.db.session.commit()
-            flask.flash('Domain %s created' % domain)
-            return flask.redirect(flask.url_for('.domain_list'))
+            flask.flash('Domain %s is invalid' % form.name.data, 'error')
     return flask.render_template('domain/create.html', form=form)
 
 

core/dovecot/Dockerfile

@@ -1,4 +1,4 @@
-ARG DISTRO=alpine:3.14.3
+ARG DISTRO=alpine:3.14.4
 
 FROM $DISTRO
 ARG VERSION

core/nginx/Dockerfile

@@ -1,4 +1,4 @@
-ARG DISTRO=alpine:3.14.3
+ARG DISTRO=alpine:3.14.4
 FROM $DISTRO
 ARG VERSION
 

core/nginx/conf/nginx.conf

@@ -17,7 +17,7 @@ http {
     keepalive_timeout 65;
     server_tokens off;
     absolute_redirect off;
-    resolver {{ RESOLVER }} ipv6=off valid=30s;
+    resolver {{ RESOLVER }} valid=30s;
 
     {% if REAL_IP_HEADER %}
     real_ip_header {{ REAL_IP_HEADER }};
@@ -257,7 +257,7 @@ mail {
     server_name {{ HOSTNAMES.split(",")[0] }};
     auth_http http://127.0.0.1:8000/auth/email;
     proxy_pass_error_message on;
-    resolver {{ RESOLVER }} ipv6=off valid=30s;
+    resolver {{ RESOLVER }} valid=30s;
     error_log /dev/stderr info;
 
     {% if TLS and not TLS_ERROR %}

core/none/Dockerfile

@@ -1,6 +1,6 @@
 # This is an idle image to dynamically replace any component if disabled.
 
-ARG DISTRO=alpine:3.14.3
+ARG DISTRO=alpine:3.14.4
 FROM $DISTRO
 
 CMD sleep 1000000d

core/postfix/Dockerfile

@@ -1,4 +1,4 @@
-ARG DISTRO=alpine:3.14.3
+ARG DISTRO=alpine:3.14.4
 
 FROM $DISTRO
 ARG VERSION

core/postfix/conf/main.cf

@@ -80,7 +80,7 @@ virtual_mailbox_maps = ${podop}mailbox
 
 # Mails are transported if required, then forwarded to Dovecot for delivery
 relay_domains = ${podop}transport
-transport_maps = ${podop}transport
+transport_maps = lmdb:/etc/postfix/transport.map, ${podop}transport
 virtual_transport = lmtp:inet:{{ LMTP_ADDRESS }}
 
 # Sender and recipient canonical maps, mostly for SRS

core/postfix/conf/master.cf

@@ -15,6 +15,22 @@ outclean  unix n       -       n       -       0       cleanup
   -o header_checks=pcre:/etc/postfix/outclean_header_filter.cf
   -o nested_header_checks=
 
+# Polite policy
+polite      unix  -       -       n       -       -       smtp
+  -o syslog_name=postfix-polite
+  -o polite_destination_concurrency_limit=3
+  -o polite_destination_rate_delay=0
+  -o polite_destination_recipient_limit=20
+  -o polite_destination_concurrency_failed_cohort_limit=10
+
+# Turtle policy
+turtle      unix  -       -       n       -       -       smtp
+  -o syslog_name=postfix-turtle
+  -o turtle_destination_concurrency_limit=1
+  -o turtle_destination_rate_delay=1
+  -o turtle_destination_recipient_limit=5
+  -o turtle_destination_concurrency_failed_cohort_limit=10
+
 # Internal postfix services
 pickup    unix  n       -       n       60      1       pickup
 cleanup   unix  n       -       n       -       0       cleanup

core/postfix/start.py

@@ -74,9 +74,10 @@ if os.path.exists("/overrides/mta-sts-daemon.yml"):
 else:
     conf.jinja("/conf/mta-sts-daemon.yml", os.environ, "/etc/mta-sts-daemon.yml")
 
-if not os.path.exists("/etc/postfix/tls_policy.map.lmdb"):
-    open("/etc/postfix/tls_policy.map", "a").close()
-    os.system("postmap /etc/postfix/tls_policy.map")
+for policy in ['tls_policy', 'transport']:
+    if not os.path.exists(f'/etc/postfix/{policy}.map.lmdb'):
+        open(f'/etc/postfix/{policy}.map', 'a').close()
+        os.system(f'postmap /etc/postfix/{policy}.map')
 
 if "RELAYUSER" in os.environ:
     path = "/etc/postfix/sasl_passwd"

docs/faq.rst

@@ -476,6 +476,22 @@ Any mail related connection is proxied by nginx. Therefore the SMTP Banner is al
 
 .. _`1368`: https://github.com/Mailu/Mailu/issues/1368
 
+My emails are getting rejected, I am being told to slow down, what can I do?
+````````````````````````````````````````````````````````````````````````````
+
+Some email operators insist that emails are delivered slowly. Mailu maintains two separate queues for such destinations: ``polite`` and ``turtle``. To enable them for some destination you can creating an override at ``overrides/postfix/transport.map`` as follow:
+
+.. code-block:: bash
+
+   yahoo.com   polite:
+   orange.fr   turtle:
+
+Re-starting the smtp container will be required for changes to take effect.
+
+*Issue reference:* `2213`_.
+
+.. _`2213`: https://github.com/Mailu/Mailu/issues/2213
+
 My emails are getting defered, what can I do?
 `````````````````````````````````````````````
 
@@ -488,7 +504,7 @@ If delivery to a specific domain fails because their DANE records are invalid or
    domain.example.com   may
    domain.example.org   encrypt
 
-The syntax and options are as described in `postfix's documentation`_. Re-creating the smtp container will be required for changes to take effect.
+The syntax and options are as described in `postfix's documentation`_. Re-starting the smtp container will be required for changes to take effect.
 
 .. _`postfix's documentation`: http://www.postfix.org/postconf.5.html#smtp_tls_policy_maps
 
@@ -511,7 +527,7 @@ These issues are typically caused by four scenarios:
 #. Certificates expired;
 #. When ``TLS_FLAVOR=letsencrypt``, it might be that the *certbot* script is not capable of
    obtaining the certificates for your domain. See `letsencrypt issues`_
-#. When ``TLS_FLAVOR=certs``, certificates are supposed to be copied to ``/mailu/certs``.
+#. When ``TLS_FLAVOR=cert``, certificates are supposed to be copied to ``/mailu/certs``.
    Using an external ``letsencrypt`` program, it tends to happen people copy the whole
    ``letsencrypt/live`` directory containing symlinks. Symlinks do not resolve inside the
    container and therefore it breaks the TLS implementation.

docs/reverse.rst

@@ -189,7 +189,7 @@ Mailu must also be configured with the information what header is used by the re
 .. code-block:: docker
   
   #mailu.env file
-  REAL_IP_HEADER=X-Real-IP
+  REAL_IP_HEADER=X-Real-Ip
   REAL_IP_FROM=x.x.x.x,y.y.y.y.y
   #x.x.x.x,y.y.y.y.y is the static IP address your reverse proxy uses for connecting to Mailu. 
 

docs/rpi_build.rst

@@ -17,8 +17,8 @@ Adjustments
 ``build_arm.sh`` uses some variables passed as ``build-arg`` to docker-compose:
 
 - ``ALPINE_VER``: version of ALPINE to use
-- ``DISTRO``: is the main distro used. Dockerfiles are set on Alpine 3.10, and
-  build script overrides for ``balenalib/rpi-alpine:3.10``
+- ``DISTRO``: is the main distro used. Dockerfiles are set on Alpine 3.14, and
+  build script overrides for ``balenalib/rpi-alpine:3.14``
 - ``QEMU``: Used by webmails dockerfiles. It will add ``qemu-arm-static`` only
   if ``QEMU`` is set to ``arm``
 - ``ARCH``: Architecture to use for ``admin``, and ``webmails`` as their images

optional/clamav/Dockerfile

@@ -1,4 +1,4 @@
-ARG DISTRO=alpine:3.14.3
+ARG DISTRO=alpine:3.14.4
 FROM $DISTRO
 ARG VERSION
 

optional/fetchmail/Dockerfile

@@ -1,4 +1,4 @@
-ARG DISTRO=alpine:3.14.3
+ARG DISTRO=alpine:3.14.4
 FROM $DISTRO
 ARG VERSION
 

optional/radicale/Dockerfile

@@ -1,4 +1,4 @@
-ARG DISTRO=alpine:3.14.3
+ARG DISTRO=alpine:3.14.4
 FROM $DISTRO
 ARG VERSION
 

optional/unbound/Dockerfile

@@ -1,4 +1,4 @@
-ARG DISTRO=alpine:3.14.3
+ARG DISTRO=alpine:3.14.4
 FROM $DISTRO
 ARG VERSION
 

setup/Dockerfile

@@ -1,4 +1,4 @@
-ARG DISTRO=alpine:3.14.3
+ARG DISTRO=alpine:3.14.4
 FROM $DISTRO
 ARG VERSION
 ENV TZ Etc/UTC

setup/templates/steps/compose/03_expose.html

@@ -31,7 +31,7 @@ avoid generic all-interfaces addresses like <code>0.0.0.0</code> or <code>::</co
 </div>
 
 <div class="form-group" id="ipv6" style="display: none">
-  <p><span class="label label-danger">Read this:</span> Docker currently does not expose the IPv6 ports properly, as it does not interface with <code>ip6tables</code>. Be sure to read our <a href="https://mailu.io/{{ version }}/faq.html#how-to-make-ipv6-work">FAQ section</a> and be <b>very careful</b> if you still wish to enable this!</p>
+  <p><span class="label label-danger">Read this:</span> Docker currently does not expose the IPv6 ports properly, as it does not interface with <code>ip6tables</code>. Read <a href="https://mailu.io/{{ version }}/faq.html#how-to-make-ipv6-work">FAQ section</a> and be <b>very careful</b>. We do <b>NOT</b> recommend that you enable this!</p>
   <label>IPv6 listen address</label>
 <!--   Validates IPv6 address -->
   <input class="form-control" type="text" name="bind6" value="::1"

towncrier/newsfragments/2210.bugfix

@@ -0,0 +1 @@
+Add input validation for domain creation

towncrier/newsfragments/2213.feature

@@ -0,0 +1 @@
+Create a polite and turtle delivery queue to accommodate destinations that expect emails to be sent slowly

towncrier/newsfragments/2260.bugfix

@@ -0,0 +1 @@
+Fix a bug where rspamd may trigger HFILTER_HOSTNAME_UNKNOWN if part of the delivery chain was using ipv6

towncrier/newsfragments/2281.bugfix

@@ -0,0 +1 @@
+Update to Alpine Linux 3.14.4 which contains a security fix for openssl. 

towncrier/newsfragments/2284.bugfix

@@ -0,0 +1 @@
+Fixed AUTH_RATELIMIT_IP not working on imap/pop3/smtp.

webmails/snappymail/defaults/application.ini

@@ -19,5 +19,5 @@ custom_logout_link='/sso/logout'
 enable = On
 allow_sync = On
 
-[plugins]
+[defaults]
 contacts_autosave = On