Merge #2529

2529: Improve fetchmail r=mergify[bot] a=nextgens

## What type of PR?

enhancement

## What does this PR do?

Improve fetchmail:
- allow delivery via LMTP (faster, bypassing the filters)
- allow several folders to be retrieved
- run fetchmail as non-root
- tweak the compose file to ensure we have all the dependencies

### Related issue(s)
- closes #1231 
- closes #2246 
- closes #711

## Prerequisites
Before we can consider review and merge, please make sure the following list is done and checked.
If an entry in not applicable, you can check it or remove it from the list.

- [ ] In case of feature or enhancement: documentation updated accordingly
- [x] Unless it's docs or a minor change: add [changelog](https://mailu.io/master/contributors/workflow.html#changelog) entry file.

Co-authored-by: Florent Daigniere <nextgens@freenetproject.org>
Co-authored-by: Florent Daigniere <nextgens@users.noreply.github.com>

core/admin/mailu/internal/views/fetch.py

@@ -12,10 +12,12 @@ def fetch_list():
             "id": fetch.id,
             "tls": fetch.tls,
             "keep": fetch.keep,
+            "scan": fetch.scan,
             "user_email": fetch.user_email,
             "protocol": fetch.protocol,
             "host": fetch.host,
             "port": fetch.port,
+            "folders": fetch.folders,
             "username": fetch.username,
             "password": fetch.password
         } for fetch in models.Fetch.query.all()

core/admin/mailu/models.py

@@ -771,6 +771,8 @@ class Fetch(Base):
     username = db.Column(db.String(255), nullable=False)
     password = db.Column(db.String(255), nullable=False)
     keep = db.Column(db.Boolean, nullable=False, default=False)
+    scan = db.Column(db.Boolean, nullable=False, default=False)
+    folders = db.Column(CommaSeparatedList, nullable=True, default=list)
     last_check = db.Column(db.DateTime, nullable=True)
     error = db.Column(db.String(1023), nullable=True)
 

core/admin/mailu/ui/forms.py

@@ -41,6 +41,15 @@ class MultipleEmailAddressesVerify(object):
         if not pattern.match(field.data.replace(" ", "")):
             raise validators.ValidationError(self.message)
 
+class MultipleFoldersVerify(object):
+    def __init__(self,message=_('Invalid list of folders.')):
+        self.message = message
+
+    def __call__(self, form, field):
+        pattern = re.compile(r'^\w+(\s*,\s*\w+)*$')
+        if not pattern.match(field.data.replace(" ", "")):
+            raise validators.ValidationError(self.message)
+
 class ConfirmationForm(flask_wtf.FlaskForm):
     submit = fields.SubmitField(_('Confirm'))
 
@@ -164,11 +173,13 @@ class FetchForm(flask_wtf.FlaskForm):
         ('imap', 'IMAP'), ('pop3', 'POP3')
     ])
     host = fields.StringField(_('Hostname or IP'), [validators.DataRequired()])
-    port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)])
-    tls = fields.BooleanField(_('Enable TLS'))
+    port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)], default=993)
+    tls = fields.BooleanField(_('Enable TLS'), default=True)
     username = fields.StringField(_('Username'), [validators.DataRequired()])
     password = fields.PasswordField(_('Password'))
     keep = fields.BooleanField(_('Keep emails on the server'))
+    scan = fields.BooleanField(_('Rescan emails locally'))
+    folders = fields.StringField(_('Folders to fetch on the server'), [validators.Optional(), MultipleFoldersVerify()], default='INBOX,Junk')
     submit = fields.SubmitField(_('Submit'))
 
 

core/admin/mailu/ui/templates/fetch/create.html

@@ -24,6 +24,8 @@
 
   {%- call macros.card(title="Settings") %}
   {{ macros.form_field(form.keep) }}
+  {{ macros.form_field(form.scan) }}
+  {{ macros.form_field(form.folders) }}
   {%- endcall %}
 
   {{ macros.form_field(form.submit) }}

core/admin/mailu/ui/templates/fetch/list.html

@@ -20,6 +20,8 @@
     <th>{% trans %}Endpoint{% endtrans %}</th>
     <th>{% trans %}Username{% endtrans %}</th>
     <th>{% trans %}Keep emails{% endtrans %}</th>
+    <th>{% trans %}Rescan emails{% endtrans %}</th>
+    <th>{% trans %}Folders{% endtrans %}</th>
     <th>{% trans %}Last check{% endtrans %}</th>
     <th>{% trans %}Status{% endtrans %}</th>
     <th>{% trans %}Created{% endtrans %}</th>
@@ -36,6 +38,8 @@
     <td>{{ fetch.protocol }}{{ 's' if fetch.tls else '' }}://{{ fetch.host }}:{{ fetch.port }}</td>
     <td>{{ fetch.username }}</td>
     <td data-sort="{{ fetch.keep }}">{% if fetch.keep %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %}</td>
+    <td data-sort="{{ fetch.scan }}">{% if fetch.scan %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %}</td>
+    <td>{{ fetch.folders.data | join(',') }}</td>
     <td>{{ fetch.last_check | format_datetime or '-' }}</td>
     <td>{{ fetch.error or '-' }}</td>
     <td data-sort="{{ fetch.created_at or '0000-00-00' }}">{{ fetch.created_at | format_date }}</td>

core/admin/mailu/ui/views/fetches.py

@@ -1,4 +1,4 @@
-from mailu import models
+from mailu import models, utils
 from mailu.ui import ui, forms, access
 from flask import current_app as app
 
@@ -28,9 +28,12 @@ def fetch_create(user_email):
     user = models.User.query.get(user_email) or flask.abort(404)
     form = forms.FetchForm()
     form.password.validators = [wtforms.validators.DataRequired()]
+    utils.formatCSVField(form.folders)
     if form.validate_on_submit():
         fetch = models.Fetch(user=user)
         form.populate_obj(fetch)
+        if form.folders.data:
+            fetch.folders = form.folders.data.replace(' ','').split(',')
         models.db.session.add(fetch)
         models.db.session.commit()
         flask.flash('Fetch configuration created')
@@ -46,10 +49,13 @@ def fetch_edit(fetch_id):
         flask.abort(404)
     fetch = models.Fetch.query.get(fetch_id) or flask.abort(404)
     form = forms.FetchForm(obj=fetch)
+    utils.formatCSVField(form.folders)
     if form.validate_on_submit():
         if not form.password.data:
             form.password.data = fetch.password
         form.populate_obj(fetch)
+        if form.folders.data:
+            fetch.folders = form.folders.data.replace(' ','').split(',')
         models.db.session.commit()
         flask.flash('Fetch configuration updated')
         return flask.redirect(

core/admin/mailu/ui/views/users.py

@@ -100,11 +100,7 @@ def user_settings(user_email):
     user_email_or_current = user_email or flask_login.current_user.email
     user = models.User.query.get(user_email_or_current) or flask.abort(404)
     form = forms.UserSettingsForm(obj=user)
-    if isinstance(form.forward_destination.data,str):
-        data = form.forward_destination.data.replace(" ","").split(",")
-    else:
-        data = form.forward_destination.data
-    form.forward_destination.data = ", ".join(data)
+    utils.formatCSVField(form.forward_destination)
     if form.validate_on_submit():
         form.forward_destination.data = form.forward_destination.data.replace(" ","").split(",")
         form.populate_obj(user)

core/admin/mailu/utils.py

@@ -518,3 +518,10 @@ def isBadOrPwned(form):
     if breaches > 0:
         return f"This password appears in {breaches} data breaches! It is not unique; please change it."
     return None
+
+def formatCSVField(field):
+    if isinstance(field.data,str):
+        data = field.data.replace(" ","").split(",")
+    else:
+        data = field.data
+    field.data = ", ".join(data)

core/admin/migrations/versions/f4f0f89e0047_.py

@@ -0,0 +1,25 @@
+""" Add fetch.scan and fetch.folders
+
+Revision ID: f4f0f89e0047
+Revises: 8f9ea78776f4
+Create Date: 2022-11-13 16:29:01.246509
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'f4f0f89e0047'
+down_revision = '8f9ea78776f4'
+
+from alembic import op
+import sqlalchemy as sa
+import mailu
+
+def upgrade():
+    with op.batch_alter_table('fetch') as batch:
+        batch.add_column(sa.Column('scan', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()))
+        batch.add_column(sa.Column('folders', mailu.models.CommaSeparatedList(), nullable=True))
+
+def downgrade():
+    with op.batch_alter_table('fetch') as batch:
+        batch.drop_column('fetch', 'folders')
+        batch.drop_column('fetch', 'scan')

docs/webadministration.rst

@@ -157,7 +157,11 @@ You can add a fetched account by clicking on the `Add an account` button on the
 
 * Keep emails on the server. When ticked, retains the email message in the email account after retrieving it.
 
-Click the submit button to apply settings. With the default polling interval, fetchmail will start polling the email account after 10 minutes.
+* Scan emails. When ticked, all the fetched emails will go through the local filters (rspamd, clamav, ...).
+
+* Folders. A comma separated list of folders to fetch from the server. This is optional, by default only the INBOX will be pulled.
+
+Click the submit button to apply settings. With the default polling interval, fetchmail will start polling the email account after ``FETCHMAIL_DELAY``.
 
 
 Authentication tokens

optional/fetchmail/fetchmail.py

@@ -2,11 +2,14 @@
 
 import time
 import os
+from pathlib import Path
+from pwd import getpwnam
 import tempfile
 import shlex
 import subprocess
 import re
 import requests
+from socrate import system
 import sys
 import traceback
 
@@ -14,6 +17,7 @@ import traceback
 FETCHMAIL = """
 fetchmail -N \
     --idfile /data/fetchids --uidl \
+    --pidfile /dev/shm/fetchmail.pid \
     --sslcertck --sslcertpath /etc/ssl/certs \
     -f {}
 """
@@ -24,7 +28,9 @@ poll "{host}" proto {protocol}  port {port}
     user "{username}" password "{password}"
     is "{user_email}"
     smtphost "{smtphost}"
+    {folders}
     {options}
+    {lmtp}
 """
 
 
@@ -48,26 +54,37 @@ def fetchmail(fetchmailrc):
 
 def run(debug):
     try:
-        fetches = requests.get("http://" + os.environ.get("HOST_ADMIN", "admin") + "/internal/fetch").json()
-        smtphost, smtpport = extract_host_port(os.environ.get("HOST_SMTP", "smtp"), None)
+        os.environ["SMTP_ADDRESS"] = system.get_host_address_from_environment("SMTP", "smtp")
+        os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
+        fetches = requests.get(f"http://{os.environ['ADMIN_ADDRESS']}/internal/fetch").json()
+        smtphost, smtpport = extract_host_port(os.environ["SMTP_ADDRESS"], None)
         if smtpport is None:
             smtphostport = smtphost
         else:
             smtphostport = "%s/%d" % (smtphost, smtpport)
+        os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525")
+        lmtphost, lmtpport = extract_host_port(os.environ["LMTP_ADDRESS"], None)
+        if lmtpport is None:
+            lmtphostport = lmtphost
+        else:
+            lmtphostport = "%s/%d" % (lmtphost, lmtpport)
         for fetch in fetches:
             fetchmailrc = ""
             options = "options antispam 501, 504, 550, 553, 554"
             options += " ssl" if fetch["tls"] else ""
             options += " keep" if fetch["keep"] else " fetchall"
+            folders = "folders %s" % ((','.join('"' + item + '"' for item in fetch['folders'])) if fetch['folders'] else '"INBOX"')
             fetchmailrc += RC_LINE.format(
                 user_email=escape_rc_string(fetch["user_email"]),
                 protocol=fetch["protocol"],
                 host=escape_rc_string(fetch["host"]),
                 port=fetch["port"],
-                smtphost=smtphostport,
+                smtphost=smtphostport if fetch['scan'] else lmtphostport,
                 username=escape_rc_string(fetch["username"]),
                 password=escape_rc_string(fetch["password"]),
-                options=options
+                options=options,
+                folders=folders,
+                lmtp='' if fetch['scan'] else 'lmtp',
             )
             if debug:
                 print(fetchmailrc)
@@ -86,14 +103,21 @@ def run(debug):
                         user_info in error_message):
                     print(error_message)
             finally:
-                requests.post("http://" + os.environ.get("HOST_ADMIN", "admin") + "/internal/fetch/{}".format(fetch["id"]),
-                    json=error_message.split("\n")[0]
+                requests.post("http://{}/internal/fetch/{}".format(os.environ['ADMIN_ADDRESS'],fetch['id']),
+                    json=error_message.split('\n')[0]
                 )
     except Exception:
         traceback.print_exc()
 
 
 if __name__ == "__main__":
+    id_fetchmail = getpwnam('fetchmail')
+    Path('/data/fetchids').touch()
+    os.chown("/data/fetchids", id_fetchmail.pw_uid, id_fetchmail.pw_gid)
+    os.chown("/data/", id_fetchmail.pw_uid, id_fetchmail.pw_gid)
+    os.chmod("/data/fetchids", 0o700)
+    os.setgid(id_fetchmail.pw_gid)
+    os.setuid(id_fetchmail.pw_uid)
     while True:
         delay = int(os.environ.get("FETCHMAIL_DELAY", 60))
         print("Sleeping for {} seconds".format(delay))

setup/flavors/compose/docker-compose.yml

@@ -157,8 +157,11 @@ services:
     env_file: {{ env }}
     volumes:
       - "{{ root }}/data/fetchmail:/data"
-    {% if resolver_enabled %}
     depends_on:
+      - admin
+      - smtp
+      - imap
+    {% if resolver_enabled %}
       - resolver
     dns:
       - {{ dns }}

towncrier/newsfragments/1231.bugfix

@@ -0,0 +1 @@
+Add an option so that emails fetched with fetchmail don't go through the filters (closes #1231)

towncrier/newsfragments/2246.bugfix

@@ -0,0 +1 @@
+Fetchmail: Missing support for '*_ADDRESS' env vars

towncrier/newsfragments/711.feature

@@ -0,0 +1 @@
+Allow other folders to be synced by fetchmail