From a875d7fe6a8dc50afbd8a08c99af1c724e85de4f Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 12 Sep 2024 10:39:18 +0200 Subject: [PATCH 1/6] Fix #3450 --- core/admin/start.py | 2 +- core/base/libs/socrate/socrate/system.py | 1 + core/dovecot/conf/dovecot.conf | 18 ++++++++++++++++-- core/nginx/dovecot/proxy.conf | 22 ++++++++++++++++++++++ towncrier/newsfragments/3450.bugfix | 3 +++ 5 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 towncrier/newsfragments/3450.bugfix diff --git a/core/admin/start.py b/core/admin/start.py index 9574bbb7..2fc8fb40 100755 --- a/core/admin/start.py +++ b/core/admin/start.py @@ -64,7 +64,7 @@ test_unsupported() cmdline = [ "gunicorn", - "--threads", f"{os.cpu_count()}", + "--threads", f"{CPU_COUNT}", # If SUBNET6 is defined, gunicorn must listen on IPv6 as well as IPv4 "-b", f"{'[::]' if os.environ.get('SUBNET6') else '0.0.0.0'}:8080", "--logger-class mailu.Logger", diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index c5046f4c..e9892325 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -141,6 +141,7 @@ def clean_env(): for item in os.environ.get('TLS', ALL_PORTS).split(','): if item in PORTS_REQUIRING_TLS: os.environ[f'TLS_{item}']='True' + os.environ['CPU_COUNT'] = str(os.cpu_count()) def drop_privs_to(username='mailu'): pwnam = getpwnam(username) diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index 9c92aca6..8ba0a8f2 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -89,8 +89,7 @@ plugin { service indexer-worker { executable = /bin/nice -n 10 /usr/libexec/dovecot/indexer-worker - # TODO: maybe MAXPROC? I guess it depends on how much RAM is available - process_limit = 1 + process_limit = {{ CPU_COUNT }} } ############### @@ -137,12 +136,22 @@ service imap-login { inet_listener imap { port = 143 } + service_count = 0 + client_limit = 25000 + process_min_avail = {{ CPU_COUNT }} + process_limit = {{ CPU_COUNT }} + vsz_limit = 256M } service pop3-login { inet_listener pop3 { port = 110 } + service_count = 0 + client_limit = 25000 + process_min_avail = {{ CPU_COUNT }} + process_limit = {{ CPU_COUNT }} + vsz_limit = 256M } ############### @@ -166,6 +175,11 @@ service managesieve-login { inet_listener sieve { port = 4190 } + service_count = 0 + client_limit = 25000 + process_min_avail = {{ CPU_COUNT }} + process_limit = {{ CPU_COUNT }} + vsz_limit = 256M } protocol sieve { diff --git a/core/nginx/dovecot/proxy.conf b/core/nginx/dovecot/proxy.conf index 8decfc77..d7fcbb0a 100644 --- a/core/nginx/dovecot/proxy.conf +++ b/core/nginx/dovecot/proxy.conf @@ -87,6 +87,11 @@ service managesieve-login { inet_listener sieve-webmail { port = 14190 } + service_count = 0 + client_limit = 25000 + process_min_avail = {{ CPU_COUNT }} + process_limit = {{ CPU_COUNT }} + vsz_limit = 256M } {% endif %} @@ -114,6 +119,11 @@ service imap-login { inet_listener imap-webmail { port = 10143 } + service_count = 0 + client_limit = 25000 + process_min_avail = {{ CPU_COUNT }} + process_limit = {{ CPU_COUNT }} + vsz_limit = 256M } service pop3-login { @@ -132,6 +142,11 @@ service pop3-login { {% endif %} } {% endif %} + service_count = 0 + client_limit = 25000 + process_min_avail = {{ CPU_COUNT }} + process_limit = {{ CPU_COUNT }} + vsz_limit = 256M } recipient_delimiter = {{ RECIPIENT_DELIMITER }} @@ -161,4 +176,11 @@ service submission-login { inet_listener submission-webmail { port = 10025 } + service_count = 0 + client_limit = 25000 + process_min_avail = {{ CPU_COUNT }} + process_limit = {{ CPU_COUNT }} + vsz_limit = 256M } + +!include_try /overrides/dovecot.conf diff --git a/towncrier/newsfragments/3450.bugfix b/towncrier/newsfragments/3450.bugfix new file mode 100644 index 00000000..ec92e429 --- /dev/null +++ b/towncrier/newsfragments/3450.bugfix @@ -0,0 +1,3 @@ +Ensure we can do more than 100 parallel sessions. +Make use of all CPUs available for indexing +Allow dovecot's config to be overriden in front too From c6c4c09f08fdd130105c04d3e529f0fcda956f65 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 12 Sep 2024 10:58:09 +0200 Subject: [PATCH 2/6] doh --- core/admin/start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/start.py b/core/admin/start.py index 2fc8fb40..07db30fe 100755 --- a/core/admin/start.py +++ b/core/admin/start.py @@ -64,7 +64,7 @@ test_unsupported() cmdline = [ "gunicorn", - "--threads", f"{CPU_COUNT}", + "--threads", f"{os.environ.get('CPU_COUNT', 1)}", # If SUBNET6 is defined, gunicorn must listen on IPv6 as well as IPv4 "-b", f"{'[::]' if os.environ.get('SUBNET6') else '0.0.0.0'}:8080", "--logger-class mailu.Logger", From cdf8ee8820ec0f797c4c75329bfbf8e14a35916c Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 12 Sep 2024 12:30:04 +0200 Subject: [PATCH 3/6] Apply suggestions from code review Co-authored-by: Alexander Graf --- core/admin/start.py | 2 +- core/base/libs/socrate/socrate/system.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/admin/start.py b/core/admin/start.py index 07db30fe..24f31e07 100755 --- a/core/admin/start.py +++ b/core/admin/start.py @@ -64,7 +64,7 @@ test_unsupported() cmdline = [ "gunicorn", - "--threads", f"{os.environ.get('CPU_COUNT', 1)}", + "--threads", os.environ.get('CPU_COUNT', '1'), # If SUBNET6 is defined, gunicorn must listen on IPv6 as well as IPv4 "-b", f"{'[::]' if os.environ.get('SUBNET6') else '0.0.0.0'}:8080", "--logger-class mailu.Logger", diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index e9892325..f494a7b7 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -141,7 +141,8 @@ def clean_env(): for item in os.environ.get('TLS', ALL_PORTS).split(','): if item in PORTS_REQUIRING_TLS: os.environ[f'TLS_{item}']='True' - os.environ['CPU_COUNT'] = str(os.cpu_count()) + if 'CPU_COUNT' not in os.environ: + os.environ['CPU_COUNT'] = str(os.cpu_count()) def drop_privs_to(username='mailu'): pwnam = getpwnam(username) From f4ab22edbce7e4d2181f5821d1ae83f5ce46bd00 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 12 Sep 2024 12:34:31 +0200 Subject: [PATCH 4/6] play it safe --- core/dovecot/conf/dovecot.conf | 3 ++- towncrier/newsfragments/3450.bugfix | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index 8ba0a8f2..e16ba8a6 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -89,7 +89,8 @@ plugin { service indexer-worker { executable = /bin/nice -n 10 /usr/libexec/dovecot/indexer-worker - process_limit = {{ CPU_COUNT }} + # TODO: maybe MAXPROC? I guess it depends on how much RAM is available + process_limit = 1 } ############### diff --git a/towncrier/newsfragments/3450.bugfix b/towncrier/newsfragments/3450.bugfix index ec92e429..50c75f22 100644 --- a/towncrier/newsfragments/3450.bugfix +++ b/towncrier/newsfragments/3450.bugfix @@ -1,3 +1,2 @@ Ensure we can do more than 100 parallel sessions. -Make use of all CPUs available for indexing Allow dovecot's config to be overriden in front too From 12e94b7616c1c5447e4c9beb05bb608478736905 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 12 Sep 2024 15:45:58 +0200 Subject: [PATCH 5/6] As per review --- core/dovecot/conf/dovecot.conf | 2 +- docs/faq.rst | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index e16ba8a6..03709b2e 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -238,4 +238,4 @@ service anvil { # Extensions ############### -!include_try /overrides/dovecot.conf +!include_try /overrides/dovecot/proxy.conf diff --git a/docs/faq.rst b/docs/faq.rst index 6b3011a1..955752d0 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -256,8 +256,10 @@ correct syntax. The following file names will be taken as override configuration - For both ``postfix.cf`` and ``postfix.master``, you need to put one configuration per line, as they are fed line-by-line to postfix. - ``logrotate.conf`` as ``$ROOT/overrides/postfix/logrotate.conf`` - Replaces the logrotate.conf file used for rotating ``POSTFIX_LOG_FILE``. -- `Dovecot`_ - ``dovecot.conf`` in dovecot sub-directory; -- `Nginx`_ - All ``*.conf`` files in the ``nginx`` sub-directory; +- `Dovecot`_ - ``dovecot.conf`` in dovecot sub-directory. +- `Nginx`_ : + - All ``*.conf`` files in the ``nginx`` sub-directory. + - ``proxy.conf`` in the ``nginx/dovecot`` sub-directory. - `Rspamd`_ - All files in the ``rspamd`` sub-directory. - `Roundcube`_ - All ``*.inc.php`` files in the ``roundcube`` sub directory. From 14a75a101dca97e4a78f82010ec2cbed954526cf Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Thu, 12 Sep 2024 14:21:48 +0000 Subject: [PATCH 6/6] Fix rst syntax --- docs/faq.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 955752d0..1111481d 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -258,8 +258,8 @@ correct syntax. The following file names will be taken as override configuration - ``logrotate.conf`` as ``$ROOT/overrides/postfix/logrotate.conf`` - Replaces the logrotate.conf file used for rotating ``POSTFIX_LOG_FILE``. - `Dovecot`_ - ``dovecot.conf`` in dovecot sub-directory. - `Nginx`_ : - - All ``*.conf`` files in the ``nginx`` sub-directory. - - ``proxy.conf`` in the ``nginx/dovecot`` sub-directory. + - All ``*.conf`` files in the ``nginx`` sub-directory. + - ``proxy.conf`` in the ``nginx/dovecot`` sub-directory. - `Rspamd`_ - All files in the ``rspamd`` sub-directory. - `Roundcube`_ - All ``*.inc.php`` files in the ``roundcube`` sub directory.