scottk/Mailu
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/Mailu.git synced 2025-11-03 19:47:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into fix-sender-checks

Browse Source
This commit is contained in:
kaiyou
2018-10-20 10:18:36 +02:00
committed by GitHub
parent e784556330 4a5c0a6d21
commit 1fcaef7c7e
36 changed files with 864 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
core/admin/Dockerfile
View File

@@ -7,7 +7,7 @@ COPY requirements-prod.txt requirements.txt
RUN apk add --no-cache openssl \
&& apk add --no-cache --virtual build-dep openssl-dev libffi-dev python-dev build-base \
&& pip install -r requirements.txt \
&& apk del build-dep
&& apk del --no-cache build-dep
COPY mailu ./mailu
COPY migrations ./migrations

12
core/admin/mailu/__init__.py
View File

@@ -12,7 +12,7 @@ import docker
import socket
import uuid
from werkzeug.contrib import fixers
from werkzeug.contrib import fixers, profiler
# Create application
app = flask.Flask(__name__)
@@ -57,12 +57,15 @@ default_config = {
'RECAPTCHA_PUBLIC_KEY': '',
'RECAPTCHA_PRIVATE_KEY': '',
# Advanced settings
'PASSWORD_SCHEME': 'SHA512-CRYPT',
'PASSWORD_SCHEME': 'BLF-CRYPT',
# Host settings
'HOST_IMAP': 'imap',
'HOST_POP3': 'imap',
'HOST_SMTP': 'smtp',
'HOST_WEBMAIL': 'webmail',
'HOST_FRONT': 'front',
'HOST_AUTHSMTP': os.environ.get('HOST_SMTP', 'smtp'),
'POD_ADDRESS_RANGE': None
}
# Load configuration from the environment if available
@@ -80,6 +83,10 @@ if app.config.get("DEBUG"):
import flask_debugtoolbar
toolbar = flask_debugtoolbar.DebugToolbarExtension(app)
# Profiler
if app.config.get("DEBUG"):
app.wsgi_app = profiler.ProfilerMiddleware(app.wsgi_app, restrictions=[30])
# Manager commnad
manager = flask_script.Manager(app)
manager.add_command('db', flask_migrate.MigrateCommand)
@@ -129,4 +136,5 @@ class PrefixMiddleware(object):
environ['SCRIPT_NAME'] = prefix
return self.app(environ, start_response)
app.wsgi_app = PrefixMiddleware(fixers.ProxyFix(app.wsgi_app))

14
core/admin/mailu/internal/views/dovecot.py
View File

@@ -1,14 +1,24 @@
from mailu import db, models
from mailu import db, models, app
from mailu.internal import internal
import flask
import socket
@internal.route("/dovecot/passdb/<user_email>")
def dovecot_passdb_dict(user_email):
user = models.User.query.get(user_email) or flask.abort(404)
allow_nets = []
allow_nets.append(
app.config.get("POD_ADDRESS_RANGE") or
socket.gethostbyname(app.config["HOST_FRONT"])
)
allow_nets.append(socket.gethostbyname(app.config["HOST_WEBMAIL"]))
print(allow_nets)
return flask.jsonify({
"password": user.password,
"password": None,
"nopassword": "Y",
"allow_nets": ",".join(allow_nets)
})

12
core/admin/mailu/models.py
View File

@@ -288,7 +288,9 @@ class User(Base, Email):
def get_id(self):
return self.email
scheme_dict = {'SHA512-CRYPT': "sha512_crypt",
scheme_dict = {'PBKDF2': "pbkdf2_sha512",
'BLF-CRYPT': "bcrypt",
'SHA512-CRYPT': "sha512_crypt",
'SHA256-CRYPT': "sha256_crypt",
'MD5-CRYPT': "md5_crypt",
'CRYPT': "des_crypt"}
@@ -298,8 +300,14 @@ class User(Base, Email):
)
def check_password(self, password):
context = User.pw_context
reference = re.match('({[^}]+})?(.*)', self.password).group(2)
return User.pw_context.verify(password, reference)
result = context.verify(password, reference)
if result and context.identify(reference) != context.default_scheme():
self.set_password(password)
db.session.add(self)
db.session.commit()
return result
def set_password(self, password, hash_scheme=app.config['PASSWORD_SCHEME'], raw=False):
"""Set password for user with specified encryption scheme

1
core/admin/requirements-prod.txt
View File

@@ -1,6 +1,7 @@
alembic==0.9.9
asn1crypto==0.24.0
Babel==2.5.3
bcrypt==3.1.4
blinker==1.4
certifi==2018.4.16
cffi==1.11.5

1
core/admin/requirements.txt
View File

@@ -17,3 +17,4 @@ tabulate
PyYAML
PyOpenSSL
dnspython
bcrypt

2
core/dovecot/Dockerfile
View File

@@ -2,7 +2,7 @@ FROM alpine:3.8
RUN apk add --no-cache \
dovecot dovecot-pigeonhole-plugin dovecot-fts-lucene rspamd-client \
python3 py3-pip \
bash python3 py3-pip \
&& pip3 install --upgrade pip \
&& pip3 install jinja2 podop tenacity

4
core/dovecot/conf/bin/ham Executable file
View File

@@ -0,0 +1,4 @@
#!/bin/bash
tee >(rspamc -h antispam:11334 -P mailu learn_ham /dev/stdin) \
| rspamc -h antispam:11334 -P mailu -f 13 fuzzy_add /dev/stdin

3
core/dovecot/conf/bin/mailtrain
View File

@@ -1,3 +0,0 @@
#!/bin/sh
rspamc -h antispam:11334 -P mailu "learn_$1" /dev/stdin <&0

4
core/dovecot/conf/bin/spam Executable file
View File

@@ -0,0 +1,4 @@
#!/bin/bash
tee >(rspamc -h antispam:11334 -P mailu learn_spam /dev/stdin) \
>(rspamc -h antispam:11334 -P mailu -f 11 fuzzy_add /dev/stdin)

3
core/dovecot/conf/dovecot.conf
View File

@@ -136,7 +136,8 @@ service managesieve {
}
plugin {
sieve = dict:proxy:/tmp/podop.socket:sieve
sieve = file:~/sieve;active=~/.dovecot.sieve
sieve_before = dict:proxy:/tmp/podop.socket:sieve
sieve_plugins = sieve_imapsieve sieve_extprograms
sieve_extensions = +spamtest +spamtestplus +editheader
sieve_global_extensions = +vnd.dovecot.execute

2
core/dovecot/conf/report-ham.sieve
View File

@@ -8,4 +8,4 @@ if string "${mailbox}" "Trash" {
stop;
}
execute :pipe "mailtrain" "ham";
execute :pipe "ham";

2
core/dovecot/conf/report-spam.sieve
View File

@@ -1,3 +1,3 @@
require "vnd.dovecot.execute";
execute :pipe "mailtrain" "spam";
execute :pipe "spam";

2
core/dovecot/start.py
View File

@@ -36,5 +36,5 @@ for dovecot_file in glob.glob("/conf/*.conf"):
# Run Podop, then postfix
multiprocessing.Process(target=start_podop).start()
os.system("chown -R mail:mail /mail /var/lib/dovecot")
os.system("chown -R mail:mail /mail /var/lib/dovecot /conf")
os.execv("/usr/sbin/dovecot", ["dovecot", "-c", "/etc/dovecot/dovecot.conf", "-F"])

2
core/nginx/conf/nginx.conf
View File

@@ -91,8 +91,10 @@ http {
{% endif %}
location {{ WEB_WEBMAIL }} {
{% if WEB_WEBMAIL != '/' %}
rewrite ^({{ WEB_WEBMAIL }})$ $1/ permanent;
rewrite ^{{ WEB_WEBMAIL }}/(.*) /$1 break;
{% endif %}
include /etc/nginx/proxy.conf;
client_max_body_size {{ MESSAGE_SIZE_LIMIT|int + 8388608 }};
proxy_pass http://$webmail;

2
core/postfix/conf/main.cf
View File

@@ -32,7 +32,7 @@ relayhost = {{ RELAYHOST }}
recipient_delimiter = {{ RECIPIENT_DELIMITER }}
# Only the front server is allowed to perform xclient
smtpd_authorized_xclient_hosts={{ FRONT_ADDRESS }}
smtpd_authorized_xclient_hosts={{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}
###############
# TLS

10
docs/compose/.env
View File

@@ -120,12 +120,18 @@ WEBSITE=https://mailu.io
# Advanced settings
###################################
# Log driver for front service. Possible values:
# json-file (default)
# journald (On systemd platforms, useful for Fail2Ban integration)
# syslog (Non systemd platforms, Fail2Ban integration. Disables `docker-compose log` for front!)
LOG_DRIVER=json-file
# Docker-compose project name, this will prepended to containers names.
COMPOSE_PROJECT_NAME=mailu
# Default password scheme used for newly created accounts and changed passwords
# (value: SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT)
PASSWORD_SCHEME=SHA512-CRYPT
# (value: PBKDF2, BLF-CRYPT, SHA512-CRYPT, SHA256-CRYPT)
PASSWORD_SCHEME=PBKDF2
# Header to take the real ip from
REAL_IP_HEADER=

2
docs/compose/docker-compose.yml
View File

@@ -6,6 +6,8 @@ services:
image: mailu/nginx:$VERSION
restart: always
env_file: .env
logging:
driver: $LOG_DRIVER
ports:
- "$BIND_ADDRESS4:80:80"
- "$BIND_ADDRESS4:443:443"

55
docs/compose/setup.rst
View File

@@ -26,36 +26,61 @@ for the ``VERSION_TAG`` branch, use:
wget https://mailu.io/VERSION_TAG/_downloads/docker-compose.yml
wget https://mailu.io/VERSION_TAG/_downloads/.env
Then open the ``.env`` file to setup the mail server. Modify the ``ROOT`` setting
to match your setup directory if different from ``/mailu``.
Important configuration variables
---------------------------------
Modify the ``VERSION`` configuration in the ``.env`` file to reflect the version you picked.
Open the ``.env`` file and review the following variable settings:
Set the common configuration values
-----------------------------------
- Change ``ROOT`` if you have your setup directory in a different location then ``/mailu``.
- Check ``VERSION`` to reflect the version you picked. (``master`` or ``1.5``).
Open the ``.env`` file and set configuration settings after reading the configuration
documentation. Some settings are specific to the Docker Compose setup.
Make sure to read the comments in the file and instructions from the :ref:`common_cfg` section.
Modify ``BIND_ADDRESS4`` to match the public IP address assigned to your server.
This address should be configured on one of the network interfaces of the server.
If the address is not configured directly (NAT) on any of the network interfaces or if
you would simply like the server to listen on all interfaces, use ``0.0.0.0``.
Modify ``BIND_ADDRESS6`` to match the public IPv6 address assigned to your server.
The behavior is identical to ``BIND_ADDRESS4``.
TLS certificates
````````````````
Set the ``TLS_FLAVOR`` to one of the following
values:
- ``cert`` is the default and requires certificates to be setup manually;
- ``letsencrypt`` will use the Letsencrypt! CA to generate automatic ceriticates;
- ``letsencrypt`` will use the *Letsencrypt!* CA to generate automatic ceriticates;
- ``mail`` is similar to ``cert`` except that TLS will only be served for
emails (IMAP and SMTP), not HTTP (use it behind reverse proxies);
- ``mail-letsencrypt`` is similar to ``letsencrypt`` except that TLS will only be served for
emails (IMAP and SMTP), not HTTP (use it behind reverse proxies);
- ``notls`` will disable TLS, this is not recommended except for testing.
.. note::
When using *Letsencrypt!* you have to make sure that the DNS ``A`` and ``AAAA`` records for the
all hostnames mentioned in the ``HOSTNAMES`` variable match with the ip adresses of you server.
Or else certificate generation will fail! See also: :ref:`dns_setup`.
Bind address
````````````
Modify ``BIND_ADDRESS4`` and ``BIND_ADDRESS6`` to match the public IP addresses assigned to your server. For IPv6 you will need the ``<global>`` scope address.
You can find those addresses by running the following:
.. code-block:: bash
[root@mailu ~]$ ifconfig eth0
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 125.189.138.127 netmask 255.255.255.0 broadcast 5.189.138.255
inet6 fd21:aab2:717c:cc5a::1 prefixlen 64 scopeid 0x0<global>
inet6 fe2f:2a73:43a8:7a1b::1 prefixlen 64 scopeid 0x20<link>
ether 00:50:56:3c:b2:23 txqueuelen 1000 (Ethernet)
RX packets 174866612 bytes 127773819607 (118.9 GiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 19905110 bytes 2191519656 (2.0 GiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
If the address is not configured directly (NAT) on any of the network interfaces or if
you would simply like the server to listen on all interfaces, use ``0.0.0.0`` and ``::``. Note that running is this mode is not supported and can lead to `issues`_.
.. _issues: https://github.com/Mailu/Mailu/issues/641
Enable optional features
------------------------

10
docs/configuration.rst
View File

@@ -1,12 +1,20 @@
Mailu configuration settings
============================
.. _common_cfg:
Common configuration
--------------------
The ``SECRET_KEY`` **must** be changed for every setup and set to a 16 bytes
randomly generated value. It is intended to secure authentication cookies
among other critical uses.
among other critical uses. This can be generated with a utility such as *pwgen*,
which can be installed on most Linux systems:
.. code-block:: bash
apt-get install pwgen
pwgen 16 1
The ``DOMAIN`` holds the main e-mail domain for the server. This email domain
is used for bounce emails, for generating the postmaster email and other

66
docs/contributors/environment.rst
View File

@@ -5,39 +5,51 @@ Docker containers
-----------------
The development environment is quite similar to the production one. You should always use
the ``master`` version when developing. Simply add a build directive to the images
you are working on in the ``docker-compose.yml``:
the ``master`` version when developing.
.. code-block:: yaml
Building images
```````````````
webdav:
build: ./optional/radicale
image: mailu/$WEBDAV:$VERSION
restart: always
env_file: .env
volumes:
- "$ROOT/dav:/data"
admin:
build: ./core/admin
image: mailu/admin:$VERSION
restart: always
env_file: .env
volumes:
- "$ROOT/data:/data"
- "$ROOT/dkim:/dkim"
- /var/run/docker.sock:/var/run/docker.sock:ro
depends_on:
- redis
The build these containers.
We supply a separate ``test/build.yml`` file for
convenience. To build all Mailu containers:
.. code-block:: bash
docker-compose build admin webdav
docker-compose -f tests/build.yml build
Then you can simply start the stack as normal, newly-built images will be used.
The ``build.yml`` file has two variables:
#. ``$DOCKER_ORG``: First part of the image tag. Defaults to *mailu* and needs to be changed
only when pushing to your own Docker hub account.
#. ``$VERSION``: Last part of the image tag. Defaults to *local* to differentiate from pulled
images.
To re-build only specific containers at a later time.
.. code-block:: bash
docker-compose -f tests/build.yml build admin webdav
If you have to push the images to Docker Hub for testing in Docker Swarm or a remote
host, you have to define ``DOCKER_ORG`` (usually your Docker user-name) and login to
the hub.
.. code-block:: bash
docker login
Username: Foo
Password: Bar
export DOCKER_ORG="Foo"
export VERSION="feat-extra-app"
docker-compose -f tests/build.yml build
docker-compose -f tests/build.yml push
Running containers
``````````````````
To run the newly created images: ``cd`` to your project directory. Edit ``.env`` to set
``VERSION`` to the same value as used during the build, which defaults to ``local``.
After that you can run:
.. code-block:: bash

2
docs/dns.rst
View File

@@ -1,3 +1,5 @@
.. _dns_setup:
Setting up your DNS
===================

7
docs/setup.rst
View File

@@ -32,7 +32,7 @@ user. Make sure you complete the requirements for the flavor you chose.
You should also have at least a DNS hostname and a DNS name for receiving
emails. Some instructions are provided on the matter in the article
[Setup your DNS](dns).
:ref:`dns_setup`.
.. _`MFAshby's fork`: https://github.com/MFAshby/Mailu
@@ -68,10 +68,9 @@ Make sure that you test properly before going live!
- Try to receive an email from an external service
- Check the logs (``docker-compose logs -f servicenamehere``) to look for
warnings or errors
- Use an open relay checker like `mailradar`_
- Use an open relay checker like `mxtoolbox`_
to ensure you're not contributing to the spam problem on the internet.
All tests there should result in "Relay denied".
- If using DMARC, be sure to check the reports you get to verify that legitimate
email is getting through and forgeries are being properly blocked.
.. _mailradar: http://www.mailradar.com/openrelay/
.. _mxtoolbox: https://mxtoolbox.com/diagnostic.aspx

252
docs/swarm/master/README.md Normal file
View File

@@ -0,0 +1,252 @@
# Install Mailu on a docker swarm
## Prequisites
### Swarm
In order to deploy Mailu on a swarm, you will first need to initialize the swarm:
The main command will be:
```bash
docker swarm init --advertise-addr <IP_ADDR>
```
See https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/
If you want to add other managers or workers, please use:
```bash
docker swarm join --token xxxxx
```
See https://docs.docker.com/engine/swarm/join-nodes/
You have now a working swarm, and you can check its status with:
```bash
core@coreos-01 ~/git/Mailu/docs/swarm/1.5 $ docker node ls
ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS ENGINE VERSION
xhgeekkrlttpmtgmapt5hyxrb black-pearl Ready Active 18.06.0-ce
sczlqjgfhehsfdjhfhhph1nvb * coreos-01 Ready Active Leader 18.03.1-ce
mzrm9nbdggsfz4sgq6dhs5i6n flying-dutchman Ready Active 18.06.0-ce
```
### Volume definition
For data persistance (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm.
Hereafter we will assume that "Mailu Data" is available on every node at "$ROOT/certs:/certs" (GlusterFS and nfs shares have been successfully used).
On this example, we are using:
- the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service.
- the default ingress mode.
### Allow authentification with the mesh routing
In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE.
Let's create the mailu_default network:
```bash
core@coreos-01 ~ $ docker network create -d overlay --attachable mailu_default
core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet
"Subnet": "10.0.1.0/24",
```
In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24
In fact, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it is sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network). The issue is that this endpoint is created while the stack is created, I did'nt figure a way to determine this IP before the stack creation...
### Limitation with the ingress mode
With the default ingress mode, the front(s) container(s) will see origin IP(s) all being 10.255.0.x (which is the ingress-endpoint, can be found by inspecting the ingress network)
This issue is known and discussed here:
https://github.com/moby/moby/issues/25526
A workaround (using network host mode and global deployment) is discussed here:
https://github.com/moby/moby/issues/25526#issuecomment-336363408
### Don't create an open relay !
As a side effect of this ingress mode "feature", make sure that the ingress subnet is not in your RELAYHOST, otherwise you would create an smtp open relay :-(
## Scalability
- smtp and imap are scalable
- front and webmail are scalable (pending POD_ADDRESS_RANGE is used), although the let's encrypt magic might not like it (race condidtion ? or risk to be banned by let's encrypt server if too many front containers attemps to renew the certs at the same time)
- redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file)
## Variable substitution and docker-compose.yml
The docker stack deploy command doesn't support variable substitution in the .yml file itself.
As a consequence, we cannot simply use ``` docker stack deploy -c docker.compose.yml mailu ```
Instead, we will use the following work-around:
``` echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu ```
We need also to:
- add a deploy section for every service
- modify the way the ports are defined for the front service
- add the POD_ADDRESS_RANGE definition for imap, smtp and antispam services
## Docker compose
An example of docker-compose-stack.yml file is available here:
```yaml
version: '3.2'
services:
front:
image: mailu/nginx:$VERSION
restart: always
env_file: .env
ports:
- target: 80
published: 80
- target: 443
published: 443
- target: 110
published: 110
- target: 143
published: 143
- target: 993
published: 993
- target: 995
published: 995
- target: 25
published: 25
- target: 465
published: 465
- target: 587
published: 587
volumes:
- "$ROOT/certs:/certs"
deploy:
replicas: 2
redis:
image: redis:alpine
restart: always
volumes:
- "$ROOT/redis:/data"
deploy:
replicas: 1
imap:
image: mailu/dovecot:$VERSION
restart: always
env_file: .env
environment:
- POD_ADDRESS_RANGE=10.0.1.0/24
volumes:
- "$ROOT/mail:/mail"
- "$ROOT/overrides:/overrides"
depends_on:
- front
deploy:
replicas: 2
smtp:
image: mailu/postfix:$VERSION
restart: always
env_file: .env
environment:
- POD_ADDRESS_RANGE=10.0.1.0/24
volumes:
- "$ROOT/overrides:/overrides"
depends_on:
- front
deploy:
replicas: 2
antispam:
image: mailu/rspamd:$VERSION
restart: always
env_file: .env
environment:
- POD_ADDRESS_RANGE=10.0.1.0/24
volumes:
- "$ROOT/filter:/var/lib/rspamd"
- "$ROOT/dkim:/dkim"
- "$ROOT/overrides/rspamd:/etc/rspamd/override.d"
depends_on:
- front
deploy:
replicas: 1
antivirus:
image: mailu/none:$VERSION
restart: always
env_file: .env
volumes:
- "$ROOT/filter:/data"
deploy:
replicas: 1
webdav:
image: mailu/none:$VERSION
restart: always
env_file: .env
volumes:
- "$ROOT/dav:/data"
deploy:
replicas: 1
admin:
image: mailu/admin:$VERSION
restart: always
env_file: .env
volumes:
- "$ROOT/data:/data"
- "$ROOT/dkim:/dkim"
- /var/run/docker.sock:/var/run/docker.sock:ro
depends_on:
- redis
deploy:
replicas: 1
webmail:
image: mailu/roundcube:$VERSION
restart: always
env_file: .env
volumes:
- "$ROOT/webmail:/data"
depends_on:
- imap
deploy:
replicas: 2
fetchmail:
image: mailu/fetchmail:$VERSION
restart: always
env_file: .env
volumes:
deploy:
replicas: 1
networks:
default:
external:
name: mailu_default
```
## Deploy Mailu on the docker swarm
Run the following command:
```bash
echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu
```
See how the services are being deployed:
```bash
core@coreos-01 ~ $ docker service ls
ID NAME MODE REPLICAS IMAGE PORTS
ywnsetmtkb1l mailu_antivirus replicated 1/1 mailu/none:master
pqokiaz0q128 mailu_fetchmail replicated 1/1 mailu/fetchmail:master
```
check a specific service:
```bash
core@coreos-01 ~ $ docker service ps mailu_fetchmail
ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS
tbu8ppgsdffj mailu_fetchmail.1 mailu/fetchmail:master coreos-01 Running Running 11 days ago
```
You might also have a look on the logs:
```bash
core@coreos-01 ~ $ docker service logs -f mailu_fetchmail
```
## Remove the stack
Run the follwoing command:
```bash
core@coreos-01 ~ $ docker stack rm mailu
```

357
docs/swarm/master/README_nfs_example.md Normal file
View File

@@ -0,0 +1,357 @@
# Install Mailu on a docker swarm
## Prequisites
### Swarm
In order to deploy Mailu on a swarm, you will first need to initialize the swarm:
The main command will be:
```bash
docker swarm init --advertise-addr <IP_ADDR>
```
See https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/
If you want to add other managers or workers, please use:
```bash
docker swarm join --token xxxxx
```
See https://docs.docker.com/engine/swarm/join-nodes/
You have now a working swarm, and you can check its status with:
```bash
core@coreos-01 ~/git/Mailu/docs/swarm/1.5 $ docker node ls
ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS ENGINE VERSION
xhgeekkrlttpmtgmapt5hyxrb black-pearl Ready Active 18.06.0-ce
sczlqjgfhehsfdjhfhhph1nvb * coreos-01 Ready Active Leader 18.03.1-ce
mzrm9nbdggsfz4sgq6dhs5i6n flying-dutchman Ready Active 18.06.0-ce
```
### Volume definition
For data persistance (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm.
Hereafter we will use a NFS share:
```bash
core@coreos-01 ~ $ showmount -e 192.168.0.30
Export list for 192.168.0.30:
/mnt/Pool1/pv 192.168.0.0
```
on the nfs server, I am using the following /etc/exports
```bash
$more /etc/exports
/mnt/Pool1/pv -alldirs -mapall=root -network 192.168.0.0 -mask 255.255.255.0
```
on the nfs server, I created the Mailu directory (in fact I copied a working Mailu set-up)
```bash
$mkdir /mnt/Pool1/pv/mailu
```
On your manager node, mount the nfs share to check that the share is available:
```bash
core@coreos-01 ~ $ sudo mount -t nfs 192.168.0.30:/mnt/Pool1/pv/mailu /mnt/local/
```
If this is ok, you can umount it:
```bash
core@coreos-01 ~ $ sudo umount /mnt/local/
```
## Networking mode
On this example, we are using:
- the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service.
- the default ingress mode.
### Allow authentification with the mesh routing
In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE.
Let's create the mailu_default network:
```bash
core@coreos-01 ~ $ docker network create -d overlay --attachable mailu_default
core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet
"Subnet": "10.0.1.0/24",
```
In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24
In fact, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it is sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network). The issue is that this endpoint is created while the stack is created, I did'nt figure a way to determine this IP before the stack creation...
### Limitation with the ingress mode
With the default ingress mode, the front(s) container(s) will see origin IP(s) all being 10.255.0.x (which is the ingress-endpoint, can be found by inspecting the ingress network)
This issue is known and discussed here:
https://github.com/moby/moby/issues/25526
A workaround (using network host mode and global deployment) is discussed here:
https://github.com/moby/moby/issues/25526#issuecomment-336363408
### Don't create an open relay !
As a side effect of this ingress mode "feature", make sure that the ingress subnet is not in your RELAYHOST, otherwise you would create an smtp open relay :-(
## Scalability
- smtp and imap are scalable
- front and webmail are scalable (pending POD_ADDRESS_RANGE is used), although the let's encrypt magic might not like it (race condidtion ? or risk to be banned by let's encrypt server if too many front containers attemps to renew the certs at the same time)
- redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file)
## Variable substitution and docker-compose.yml
The docker stack deploy command doesn't support variable substitution in the .yml file itself. As a consequence, we need to use the following work-around:
``` echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu ```
We need also to:
- change the way we define the volumes (nfs share in our case)
- add a deploy section for every service
- the way the ports are defined for the front service
## Docker compose
An example of docker-compose-stack.yml file is available here:
```yaml
version: '3.2'
services:
front:
image: mailu/nginx:$VERSION
restart: always
env_file: .env
ports:
- target: 80
published: 80
- target: 443
published: 443
- target: 110
published: 110
- target: 143
published: 143
- target: 993
published: 993
- target: 995
published: 995
- target: 25
published: 25
- target: 465
published: 465
- target: 587
published: 587
volumes:
# - "$ROOT/certs:/certs"
- type: volume
source: mailu_certs
target: /certs
deploy:
replicas: 2
redis:
image: redis:alpine
restart: always
volumes:
# - "$ROOT/redis:/data"
- type: volume
source: mailu_redis
target: /data
deploy:
replicas: 1
imap:
image: mailu/dovecot:$VERSION
restart: always
env_file: .env
environment:
- POD_ADDRESS_RANGE=10.0.1.0/24
volumes:
# - "$ROOT/mail:/mail"
- type: volume
source: mailu_mail
target: /mail
# - "$ROOT/overrides:/overrides"
- type: volume
source: mailu_overrides
target: /overrides
depends_on:
- front
deploy:
replicas: 2
smtp:
image: mailu/postfix:$VERSION
restart: always
env_file: .env
environment:
- POD_ADDRESS_RANGE=10.0.1.0/24
volumes:
# - "$ROOT/overrides:/overrides"
- type: volume
source: mailu_overrides
target: /overrides
depends_on:
- front
deploy:
replicas: 2
antispam:
image: mailu/rspamd:$VERSION
restart: always
env_file: .env
environment:
- POD_ADDRESS_RANGE=10.0.1.0/24
depends_on:
- front
volumes:
# - "$ROOT/filter:/var/lib/rspamd"
- type: volume
source: mailu_filter
target: /var/lib/rspamd
# - "$ROOT/dkim:/dkim"
- type: volume
source: mailu_dkim
target: /dkim
# - "$ROOT/overrides/rspamd:/etc/rspamd/override.d"
- type: volume
source: mailu_overrides_rspamd
target: /etc/rspamd/override.d
deploy:
replicas: 1
antivirus:
image: mailu/none:$VERSION
restart: always
env_file: .env
volumes:
# - "$ROOT/filter:/data"
- type: volume
source: mailu_filter
target: /data
deploy:
replicas: 1
webdav:
image: mailu/none:$VERSION
restart: always
env_file: .env
volumes:
# - "$ROOT/dav:/data"
- type: volume
source: mailu_dav
target: /data
deploy:
replicas: 1
admin:
image: mailu/admin:$VERSION
restart: always
env_file: .env
volumes:
# - "$ROOT/data:/data"
- type: volume
source: mailu_data
target: /data
# - "$ROOT/dkim:/dkim"
- type: volume
source: mailu_dkim
target: /dkim
- /var/run/docker.sock:/var/run/docker.sock:ro
depends_on:
- redis
deploy:
replicas: 1
webmail:
image: mailu/roundcube:$VERSION
restart: always
env_file: .env
volumes:
# - "$ROOT/webmail:/data"
- type: volume
source: mailu_data
target: /data
depends_on:
- imap
deploy:
replicas: 2
fetchmail:
image: mailu/fetchmail:$VERSION
restart: always
env_file: .env
volumes:
deploy:
replicas: 1
networks:
default:
external:
name: mailu_default
volumes:
mailu_filter:
driver_opts:
type: "nfs"
o: "addr=192.168.0.30,soft,rw"
device: ":/mnt/Pool1/pv/mailu/filter"
mailu_dkim:
driver_opts:
type: "nfs"
o: "addr=192.168.0.30,soft,rw"
device: ":/mnt/Pool1/pv/mailu/dkim"
mailu_overrides_rspamd:
driver_opts:
type: "nfs"
o: "addr=192.168.0.30,soft,rw"
device: ":/mnt/Pool1/pv/mailu/overrides/rspamd"
mailu_data:
driver_opts:
type: "nfs"
o: "addr=192.168.0.30,soft,rw"
device: ":/mnt/Pool1/pv/mailu/data"
mailu_mail:
driver_opts:
type: "nfs"
o: "addr=192.168.0.30,soft,rw"
device: ":/mnt/Pool1/pv/mailu/mail"
mailu_overrides:
driver_opts:
type: "nfs"
o: "addr=192.168.0.30,soft,rw"
device: ":/mnt/Pool1/pv/mailu/overrides"
mailu_dav:
driver_opts:
type: "nfs"
o: "addr=192.168.0.30,soft,rw"
device: ":/mnt/Pool1/pv/mailu/dav"
mailu_certs:
driver_opts:
type: "nfs"
o: "addr=192.168.0.30,soft,rw"
device: ":/mnt/Pool1/pv/mailu/certs"
mailu_redis:
driver_opts:
type: "nfs"
o: "addr=192.168.0.30,soft,rw"
device: ":/mnt/Pool1/pv/mailu/redis"
```
## Deploy Mailu on the docker swarm
Run the following command:
```bash
echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu
```
See how the services are being deployed:
```bash
core@coreos-01 ~ $ docker service ls
ID NAME MODE REPLICAS IMAGE PORTS
ywnsetmtkb1l mailu_antivirus replicated 1/1 mailu/none:master
pqokiaz0q128 mailu_fetchmail replicated 1/1 mailu/fetchmail:master
```
check a specific service:
```bash
core@coreos-01 ~ $ docker service ps mailu_fetchmail
ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS
tbu8ppgsdffj mailu_fetchmail.1 mailu/fetchmail:master coreos-01 Running Running 11 days ago
```
## Remove the stack
Run the follwoing command:
```bash
core@coreos-01 ~ $ docker stack rm mailu
```

7
services/rspamd/Dockerfile
View File

@@ -1,6 +1,6 @@
FROM alpine:3.8
RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy ca-certificates py-pip \
RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy rspamd-fuzzy ca-certificates py-pip \
&& pip install --upgrade pip \
&& pip install tenacity
@@ -9,10 +9,7 @@ RUN mkdir /run/rspamd
COPY conf/ /conf
COPY start.py /start.py
# Temporary fix to remove references to rspamd-fuzzy for now
RUN sed -i '/fuzzy/,$d' /etc/rspamd/rspamd.conf
EXPOSE 11332/tcp 11334/tcp
EXPOSE 11332/tcp 11334/tcp 11335/tcp
VOLUME ["/var/lib/rspamd"]

34
services/rspamd/conf/fuzzy_check.conf Normal file
View File

@@ -0,0 +1,34 @@
rule "local" {
# Fuzzy storage server list
servers = "localhost:11335";
# Default symbol for unknown flags
symbol = "LOCAL_FUZZY_UNKNOWN";
# Additional mime types to store/check
mime_types = ["application/*"];
# Hash weight threshold for all maps
max_score = 20.0;
# Whether we can learn this storage
read_only = no;
# Ignore unknown flags
skip_unknown = yes;
# Hash generation algorithm
algorithm = "mumhash";
# Map flags to symbols
fuzzy_map = {
LOCAL_FUZZY_DENIED {
# Local threshold
max_score = 20.0;
# Flag to match
flag = 11;
}
LOCAL_FUZZY_PROB {
max_score = 10.0;
flag = 12;
}
LOCAL_FUZZY_WHITE {
max_score = 2.0;
flag = 13;
}
}
}

19
services/rspamd/conf/metrics.conf Normal file
View File

@@ -0,0 +1,19 @@
group "fuzzy" {
max_score = 12.0;
symbol "LOCAL_FUZZY_UNKNOWN" {
weight = 5.0;
description = "Generic fuzzy hash match";
}
symbol "LOCAL_FUZZY_DENIED" {
weight = 12.0;
description = "Denied fuzzy hash";
}
symbol "LOCAL_FUZZY_PROB" {
weight = 5.0;
description = "Probable fuzzy hash";
}
symbol "LOCAL_FUZZY_WHITE" {
weight = -2.1;
description = "Whitelisted fuzzy hash";
}
}

3
services/rspamd/conf/worker-controller.inc
View File

@@ -1,3 +1,4 @@
type = "controller";
bind_socket = "*:11334";
password = "mailu";
secure_ip = "{{ FRONT_ADDRESS }}";
secure_ip = "{% if POD_ADDRESS_RANGE %}{{ POD_ADDRESS_RANGE }}{% else %}{{ FRONT_ADDRESS }}{% endif %}";

6
services/rspamd/conf/worker-fuzzy.inc Normal file
View File

@@ -0,0 +1,6 @@
type = "fuzzy";
bind_socket = "*:11335";
count = 1;
backend = "redis";
expire = 90d;
allow_update = ["127.0.0.1"];

1
services/rspamd/conf/worker-normal.inc
View File

@@ -1 +1,2 @@
type = "normal";
enabled = false;

26
tests/build.yml
View File

@@ -3,54 +3,54 @@ version: '3'
services:
front:
image: $DOCKER_ORG/nginx:$VERSION
image: ${DOCKER_ORG:-mailu}/nginx:${VERSION:-local}
build: ../core/nginx
imap:
image: $DOCKER_ORG/dovecot:$VERSION
image: ${DOCKER_ORG:-mailu}/dovecot:${VERSION:-local}
build: ../core/dovecot
smtp:
image: $DOCKER_ORG/postfix:$VERSION
image: ${DOCKER_ORG:-mailu}/postfix:${VERSION:-local}
build: ../core/postfix
antispam:
image: $DOCKER_ORG/rspamd:$VERSION
image: ${DOCKER_ORG:-mailu}/rspamd:${VERSION:-local}
build: ../services/rspamd
antivirus:
image: $DOCKER_ORG/clamav:$VERSION
image: ${DOCKER_ORG:-mailu}/clamav:${VERSION:-local}
build: ../optional/clamav
webdav:
image: $DOCKER_ORG/radicale:$VERSION
image: ${DOCKER_ORG:-mailu}/radicale:${VERSION:-local}
build: ../optional/radicale
admin:
image: $DOCKER_ORG/admin:$VERSION
image: ${DOCKER_ORG:-mailu}/admin:${VERSION:-local}
build: ../core/admin
roundcube:
image: $DOCKER_ORG/roundcube:$VERSION
image: ${DOCKER_ORG:-mailu}/roundcube:${VERSION:-local}
build: ../webmails/roundcube
rainloop:
image: $DOCKER_ORG/rainloop:$VERSION
image: ${DOCKER_ORG:-mailu}/rainloop:${VERSION:-local}
build: ../webmails/rainloop
fetchmail:
image: $DOCKER_ORG/fetchmail:$VERSION
image: ${DOCKER_ORG:-mailu}/fetchmail:${VERSION:-local}
build: ../services/fetchmail
none:
image: $DOCKER_ORG/none:$VERSION
image: ${DOCKER_ORG:-mailu}/none:${VERSION:-local}
build: ../core/none
docs:
image: $DOCKER_ORG/docs:$VERSION
image: ${DOCKER_ORG:-mailu}/docs:${VERSION:-local}
build: ../docs
setup:
image: $DOCKER_ORG/setup:$VERSION
image: ${DOCKER_ORG:-mailu}/setup:${VERSION:-local}
build: ../setup

6
tests/compose/core.env
View File

@@ -120,6 +120,12 @@ WEBSITE=https://mailu.io
# Advanced settings
###################################
# Log driver for front service. Possible values:
# json-file (default)
# journald (On systemd platforms, useful for Fail2Ban integration)
# syslog (Non systemd platforms, Fail2Ban integration. Disables `docker-compose log` for front!)
LOG_DRIVER=json-file
# Docker-compose project name, this will prepended to containers names.
#COMPOSE_PROJECT_NAME=mailu

2
tests/compose/run.yml
View File

@@ -6,6 +6,8 @@ services:
image: $DOCKER_ORG/nginx:$VERSION
restart: 'no'
env_file: $PWD/.env
logging:
driver: $LOG_DRIVER
ports:
- "$BIND_ADDRESS4:80:80"
- "$BIND_ADDRESS4:443:443"

13
webmails/rainloop/Dockerfile
View File

@@ -1,20 +1,21 @@
FROM php:7.2-apache
RUN apt-get update && apt-get install -y \
unzip python3 python3-jinja2
ENV RAINLOOP_URL https://github.com/RainLoop/rainloop-webmail/releases/download/v1.12.1/rainloop-community-1.12.1.zip
RUN rm -rf /var/www/html/ \
RUN apt-get update && apt-get install -y \
unzip python3 python3-jinja2 \
&& rm -rf /var/www/html/ \
&& mkdir /var/www/html \
&& cd /var/www/html \
&& curl -L -O ${RAINLOOP_URL} \
&& unzip *.zip \
&& unzip -q *.zip \
&& rm -f *.zip \
&& rm -rf data/ \
&& find . -type d -exec chmod 755 {} \; \
&& find . -type f -exec chmod 644 {} \; \
&& chown -R www-data: *
&& chown -R www-data: * \
&& apt-get purge -y unzip \
&& rm -rf /var/lib/apt/lists
COPY include.php /var/www/html/include.php
COPY php.ini /usr/local/etc/php/conf.d/rainloop.ini

15
webmails/roundcube/Dockerfile
View File

@@ -1,14 +1,12 @@
FROM php:7.2-apache
RUN apt-get update && apt-get install -y \
zlib1g-dev \
&& docker-php-ext-install zip
ENV ROUNDCUBE_URL https://github.com/roundcube/roundcubemail/releases/download/1.3.7/roundcubemail-1.3.7-complete.tar.gz
RUN echo date.timezone=UTC > /usr/local/etc/php/conf.d/timezone.ini
RUN rm -rf /var/www/html/ \
RUN apt-get update && apt-get install -y \
zlib1g-dev \
&& docker-php-ext-install zip \
&& echo date.timezone=UTC > /usr/local/etc/php/conf.d/timezone.ini \
&& rm -rf /var/www/html/ \
&& cd /var/www \
&& curl -L -O ${ROUNDCUBE_URL} \
&& tar -xf *.tar.gz \
@@ -17,7 +15,8 @@ RUN rm -rf /var/www/html/ \
&& cd html \
&& rm -rf CHANGELOG INSTALL LICENSE README.md UPGRADING composer.json-dist installer \
&& sed -i 's,mod_php5.c,mod_php7.c,g' .htaccess \
&& chown -R www-data: logs temp
&& chown -R www-data: logs temp \
&& rm -rf /var/lib/apt/lists
COPY php.ini /usr/local/etc/php/conf.d/roundcube.ini