From 319949831b593d5eb2162dd3495ac6416b00b71e Mon Sep 17 00:00:00 2001 From: Arjan H Date: Sun, 14 Feb 2021 16:17:11 +0100 Subject: [PATCH 01/12] Fix query buildup when combining tables --- gui/acme.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gui/acme.go b/gui/acme.go index bf5ddb6..6f587d3 100644 --- a/gui/acme.go +++ b/gui/acme.go @@ -297,7 +297,7 @@ func GetOrder(w http.ResponseWriter, r *http.Request, id int) (OrderShow, error) if query != "" { query = query + " UNION " } - query = "SELECT id, identifierValue, registrationID, status, expires FROM authz2 WHERE id IN (SELECT authzID FROM orderToAuthz2 WHERE orderID=?)" + query = query + "SELECT id, identifierValue, registrationID, status, expires FROM authz2 WHERE id IN (SELECT authzID FROM orderToAuthz2 WHERE orderID=?)" } var rows *sql.Rows if tableExists(db, "authz") && tableExists(db, "authz2") { @@ -514,7 +514,7 @@ func GetAuth(w http.ResponseWriter, r *http.Request, id string) (AuthShow, error if query != "" { query = query + " UNION " } - query = "SELECT id, identifierValue, registrationID, status, expires, validationError, validationRecord FROM authz2 WHERE id IN (SELECT authzID FROM orderToAuthz2 WHERE id=?)" + query = query + "SELECT id, identifierValue, registrationID, status, expires, validationError, validationRecord FROM authz2 WHERE id IN (SELECT authzID FROM orderToAuthz2 WHERE id=?)" } if tableExists(db, "authz") && tableExists(db, "authz2") { rows, err = db.Query(query, id, id) From 75f88838356d694349bb1902c2440561db8aece3 Mon Sep 17 00:00:00 2001 From: Arjan H Date: Tue, 23 Feb 2021 20:25:00 +0100 Subject: [PATCH 02/12] Minor tweaks --- gui/dashboard.go | 3 ++- gui/setup.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/gui/dashboard.go b/gui/dashboard.go index 210c3b1..0dbb465 100644 --- a/gui/dashboard.go +++ b/gui/dashboard.go @@ -3,13 +3,14 @@ package main import ( "database/sql" "fmt" - "github.com/dustin/go-humanize" "log" "net/http" "regexp" "strconv" "strings" "time" + + "github.com/dustin/go-humanize" ) // Activity is a message to be shown on the dashboard, with timestamp and css class diff --git a/gui/setup.sh b/gui/setup.sh index 472a9e4..44cd504 100755 --- a/gui/setup.sh +++ b/gui/setup.sh @@ -8,7 +8,7 @@ set -e if [ ! -e bin/labca ]; then go mod download - go build -o bin/labca main.go acme.go certificate.go dashboard.go + go build -o bin/labca fi bin/labca From c0002a8a8d34c44d75b10f0e682e62eeb340ff32 Mon Sep 17 00:00:00 2001 From: Arjan H Date: Wed, 24 Feb 2021 08:08:29 +0100 Subject: [PATCH 03/12] Stick to release tags even if there are newer commits on master --- install | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/install b/install index ac4ff14..991b644 100755 --- a/install +++ b/install @@ -51,6 +51,7 @@ source "$dn/utils.sh" &>/dev/null || true cmdlineFqdn="" cmdlineBranch="" +fullCmdline="" # # Helper functions for informing the user and logging to file @@ -195,6 +196,16 @@ clone_or_pull() { fi } +# Checkout the latest release tag +checkout_release() { + local branch="$1" + if [ "$branch" == "" ] || [ "$branch" == "master" ]; then + cd "$cloneDir" + TAG=$(git describe --tags $(git rev-list --tags --max-count=1)) + sudo -u labca -H git reset --hard $TAG &>>$installLog + fi +} + # Restart the script if it was updated itself restart_if_updated() { local curChecksum="$1" @@ -206,7 +217,7 @@ restart_if_updated() { if [ "$curChecksum" != "$newChecksum" ]; then msg_info "Restarting updated version of install script" echo - exec $cloneDir/install + exec $cloneDir/install $fullCmdline exit $? fi fi @@ -229,6 +240,7 @@ prompt_and_export() { # Parse the command line options, if any parse_cmdline() { + fullCmdline="$@" local parsed=$(getopt --options=n:,b: --longoptions=name:,fqdn:,branch: --name "$0" -- "$@" 2>>$installLog) || msg_fatal "Could not process commandline parameters" eval set -- "$parsed" while true; do @@ -709,6 +721,7 @@ main() { parse_cmdline "$@" clone_or_pull "$cloneDir" "$labcaUrl" "$cmdlineBranch" + checkout_release "$cmdlineBranch" restart_if_updated "$checksum" get_fqdn From 933c0dadb5055abc24de02d1ea5678c725945ba6 Mon Sep 17 00:00:00 2001 From: Arjan H Date: Sat, 27 Feb 2021 10:02:46 +0100 Subject: [PATCH 04/12] Make config substitutions more robust #15 --- gui/apply-boulder | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/gui/apply-boulder b/gui/apply-boulder index 49045d7..46bbb1a 100755 --- a/gui/apply-boulder +++ b/gui/apply-boulder @@ -16,11 +16,13 @@ sed -i -e "s/\"directoryCAAIdentity\": \".*\"/\"directoryCAAIdentity\": \"$PKI_D [ -e ../test/hostname-policy.yaml ] && cp ../test/hostname-policy.yaml ./ || true [ -e ../boulder/test/hostname-policy.yaml ] && cp ../boulder/test/hostname-policy.yaml ./ || true [ -e hostname-policy.json ] && rm hostname-policy.json || true -if [ "$PKI_DOMAIN_MODE" == "lockdown" ]; then +cat hostname-policy.yaml | tr '\n' '\r' | sed -e "s/Lockdown:.*//" | tr '\r' '\n' > hostname-policy.yaml.bak && mv hostname-policy.yaml.bak hostname-policy.yaml +cat hostname-policy.yaml | tr '\n' '\r' | sed -e "s/Whitelist:.*//" | tr '\r' '\n' > hostname-policy.yaml.bak && mv hostname-policy.yaml.bak hostname-policy.yaml +if [ "$PKI_DOMAIN_MODE" == "lockdown" ] && [ "$PKI_LOCKDOWN_DOMAINS" != "" ]; then echo "Lockdown:" >> hostname-policy.yaml echo " - \"$PKI_LOCKDOWN_DOMAINS\"" >> hostname-policy.yaml fi -if [ "$PKI_DOMAIN_MODE" == "whitelist" ]; then +if [ "$PKI_DOMAIN_MODE" == "whitelist" ] && [ "$PKI_LOCKDOWN_DOMAINS" != "" ]; then echo "Whitelist:" >> hostname-policy.yaml echo " - \"$PKI_LOCKDOWN_DOMAINS\"" >> hostname-policy.yaml fi @@ -28,8 +30,12 @@ if [ "$PKI_DOMAIN_MODE" == "lockdown" ] || [ "$PKI_DOMAIN_MODE" == "whitelist" ] sed -i -e "s/^\(.*\)\(\"n_subject_common_name_included\"\)/\1\2,\n\1\"e_dnsname_not_valid_tld\"/" config/ca-a.json sed -i -e "s/^\(.*\)\(\"n_subject_common_name_included\"\)/\1\2,\n\1\"e_dnsname_not_valid_tld\"/" config/ca-b.json - sed -i -e "s/\( registrationOverrides:\)/ $PKI_LOCKDOWN_DOMAINS: 10000\n\1/" rate-limit-policies.yml - echo " $PKI_LOCKDOWN_DOMAINS: 10000" >> rate-limit-policies.yml + REPLACEMENT="" + if [ "$PKI_LOCKDOWN_DOMAINS" != "" ]; then + REPLACEMENT=" $PKI_LOCKDOWN_DOMAINS: 10000\n" + fi + cat rate-limit-policies.yml | tr '\n' '\r' | sed -e "s/\(must-staple.le.wtf: 10000\).*\( registrationOverrides:\)/\1\n$REPLACEMENT\2/" | tr '\r' '\n' > rate-limit-policies.yml.bak && mv rate-limit-policies.yml.bak rate-limit-policies.yml + cat rate-limit-policies.yml | tr '\n' '\r' | sed -e "s/\(certificatesPerFQDNSet:.*must-staple.le.wtf: 10000\).*/\1\n$REPLACEMENT/" | tr '\r' '\n' > rate-limit-policies.yml.bak && mv rate-limit-policies.yml.bak rate-limit-policies.yml fi if [ "$PKI_EXTENDED_TIMEOUT" == "1" ]; then From e66c533e8e8b999a81cd556954c4295928a603fd Mon Sep 17 00:00:00 2001 From: Arjan H Date: Sat, 27 Feb 2021 10:03:40 +0100 Subject: [PATCH 05/12] Make installer more robust #15 --- install | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/install b/install index 991b644..c44ed4e 100755 --- a/install +++ b/install @@ -264,6 +264,21 @@ parse_cmdline() { done } +# Utility method to check if value looks like a host + domain +has_domain() { + local dom="$1" + + if [[ "$dom" =~ ^\..*$ ]]; then + false + elif [[ "$dom" =~ ^.*\.$ ]]; then + false + elif [[ "$dom" =~ ^.*\..*$ ]]; then + true + else + false + fi +} + # Determine the remote address of this machine from (in order): commandline parameter, # existing configuration or full hostname. get_fqdn() { @@ -271,12 +286,23 @@ get_fqdn() { local cfgFqdn=$(grep fqdn $cfgFile 2>/dev/null | grep -v LABCA_FQDN | cut -d ":" -f 2- | tr -d " \",") LABCA_FQDN=${cfgFqdn:-$(hostname -f)} - if [ "$cfgFqdn" == "" ]; then + while [ "$cfgFqdn" == "" ]; do if [ "$cmdlineFqdn" != "" ]; then export LABCA_FQDN="$cmdlineFqdn" else prompt_and_export LABCA_FQDN "$LABCA_FQDN" "FQDN (Fully Qualified Domain Name) for this PKI host (users will use this in their browsers and clients)?" fi + + if has_domain $LABCA_FQDN; then + cfgFqdn="ok" + else + msg_err "FQDN must include a hostname AND a domain!" + cmdlineFqdn="" + fi + done + + if ! has_domain $LABCA_FQDN; then + msg_fatal "FQDN must include a hostname AND a domain!" fi msg_ok "Determine web address" From 6856989e4d40353f65baa21fd1d20703acb13616 Mon Sep 17 00:00:00 2001 From: Arjan H Date: Sun, 28 Feb 2021 19:16:27 +0100 Subject: [PATCH 06/12] Hold off on starting boulder until setup wizzard is completed #15 --- commander | 1 + entrypoint.patch | 8 +++++++- gui/apply-boulder | 2 ++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/commander b/commander index 68b94fc..2fac604 100755 --- a/commander +++ b/commander @@ -55,6 +55,7 @@ case $txt in chown -R www-data:www-data * url=$(grep 'DEFAULT_DIRECTORY_URL =' /home/labca/acme_tiny.py | sed -e 's/.*=[ ]*//' | sed -e 's/\"//g') wait_server $url + sleep 10 /home/labca/labca/renew ln -sf /home/labca/labca/cron_d /etc/cron.d/labca ln -sf /home/labca/labca/logrotate_d /etc/logrotate.d/labca diff --git a/entrypoint.patch b/entrypoint.patch index a42e1d0..aca8e21 100644 --- a/entrypoint.patch +++ b/entrypoint.patch @@ -2,10 +2,16 @@ diff --git a/test/entrypoint.sh b/test/entrypoint.sh index 5ca9929..f18e1d8 100755 --- a/test/entrypoint.sh +++ b/test/entrypoint.sh -@@ -36,6 +36,12 @@ wait_tcp_port boulder-mysql 3306 +@@ -36,6 +36,18 @@ wait_tcp_port boulder-mysql 3306 # create the database MYSQL_CONTAINER=1 $DIR/create_db.sh ++fl=$(pwd)/labca/setup_complete ++while [ ! -f $fl ]; do ++ echo "Waiting for $fl to appear..." ++ sleep 30 ++done ++ +#softhsm2-util --show-slots +softhsm2-util --init-token --slot 0 --label "intermediate signing key (rsa)" --pin 1234 --so-pin 5678 | /bin/true +[ -e labca/test-ca.p8 ] && softhsm2-util --import labca/test-ca.p8 --id 333333 --force --token "intermediate signing key (rsa)" --pin 1234 --so-pin 5678 --label 'intermediate_key' diff --git a/gui/apply-boulder b/gui/apply-boulder index 46bbb1a..5ddd1f5 100755 --- a/gui/apply-boulder +++ b/gui/apply-boulder @@ -95,3 +95,5 @@ openssl rsa -in $PKI_ROOT_CERT_BASE.key -pubout > test-root.pubkey.pem 2>/dev/nu openssl pkcs8 -topk8 -inform PEM -outform PEM -nocrypt -in test-root.key -out test-root.p8 chown -R `ls -l PKI.md | cut -d" " -f 3,4 | sed 's/ /:/g'` . + +[ -f setup_complete ] || touch setup_complete From b2a0738816a63290f77373727dacd0b11d828bf0 Mon Sep 17 00:00:00 2001 From: Arjan H Date: Mon, 1 Mar 2021 19:09:52 +0100 Subject: [PATCH 07/12] Fix config substitutions for whitelist mode (#15) --- gui/apply-boulder | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/gui/apply-boulder b/gui/apply-boulder index 5ddd1f5..985b5bf 100755 --- a/gui/apply-boulder +++ b/gui/apply-boulder @@ -22,18 +22,21 @@ if [ "$PKI_DOMAIN_MODE" == "lockdown" ] && [ "$PKI_LOCKDOWN_DOMAINS" != "" ]; th echo "Lockdown:" >> hostname-policy.yaml echo " - \"$PKI_LOCKDOWN_DOMAINS\"" >> hostname-policy.yaml fi -if [ "$PKI_DOMAIN_MODE" == "whitelist" ] && [ "$PKI_LOCKDOWN_DOMAINS" != "" ]; then +if [ "$PKI_DOMAIN_MODE" == "whitelist" ] && [ "$PKI_WHITELIST_DOMAINS" != "" ]; then echo "Whitelist:" >> hostname-policy.yaml - echo " - \"$PKI_LOCKDOWN_DOMAINS\"" >> hostname-policy.yaml + echo " - \"$PKI_WHITELIST_DOMAINS\"" >> hostname-policy.yaml fi if [ "$PKI_DOMAIN_MODE" == "lockdown" ] || [ "$PKI_DOMAIN_MODE" == "whitelist" ]; then sed -i -e "s/^\(.*\)\(\"n_subject_common_name_included\"\)/\1\2,\n\1\"e_dnsname_not_valid_tld\"/" config/ca-a.json sed -i -e "s/^\(.*\)\(\"n_subject_common_name_included\"\)/\1\2,\n\1\"e_dnsname_not_valid_tld\"/" config/ca-b.json REPLACEMENT="" - if [ "$PKI_LOCKDOWN_DOMAINS" != "" ]; then + if [ "$PKI_DOMAIN_MODE" == "lockdown" ] && [ "$PKI_LOCKDOWN_DOMAINS" != "" ]; then REPLACEMENT=" $PKI_LOCKDOWN_DOMAINS: 10000\n" fi + if [ "$PKI_DOMAIN_MODE" == "whitelist" ] && [ "$PKI_WHITELIST_DOMAINS" != "" ]; then + REPLACEMENT=" $PKI_WHITELIST_DOMAINS: 10000\n" + fi cat rate-limit-policies.yml | tr '\n' '\r' | sed -e "s/\(must-staple.le.wtf: 10000\).*\( registrationOverrides:\)/\1\n$REPLACEMENT\2/" | tr '\r' '\n' > rate-limit-policies.yml.bak && mv rate-limit-policies.yml.bak rate-limit-policies.yml cat rate-limit-policies.yml | tr '\n' '\r' | sed -e "s/\(certificatesPerFQDNSet:.*must-staple.le.wtf: 10000\).*/\1\n$REPLACEMENT/" | tr '\r' '\n' > rate-limit-policies.yml.bak && mv rate-limit-policies.yml.bak rate-limit-policies.yml fi From d9c78c53766ba96b7fc4ca5aefd8d6b8efb5f920 Mon Sep 17 00:00:00 2001 From: Arjan H Date: Mon, 1 Mar 2021 19:59:10 +0100 Subject: [PATCH 08/12] Tweak description string of whitelist mode --- gui/templates/views/manage.tmpl | 2 +- gui/templates/views/setup.tmpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gui/templates/views/manage.tmpl b/gui/templates/views/manage.tmpl index 03631cb..f1772b5 100644 --- a/gui/templates/views/manage.tmpl +++ b/gui/templates/views/manage.tmpl @@ -168,7 +168,7 @@
- Next to all official domains, also allow this domain:
+ Next to all official domains, also allow this domain (whitelist):

diff --git a/gui/templates/views/setup.tmpl b/gui/templates/views/setup.tmpl index c72c2e8..f4537d0 100644 --- a/gui/templates/views/setup.tmpl +++ b/gui/templates/views/setup.tmpl @@ -31,7 +31,7 @@ {{ . }}
{{ end }} - Next to all official domains, also allow this domain:
+ Next to all official domains, also allow this domain (whitelist):

{{ with .Errors.WhitelistDomains }} {{ . }}
From 81073c76a60680395a4b29abb61c65ebb4b9d2aa Mon Sep 17 00:00:00 2001 From: Arjan H Date: Mon, 1 Mar 2021 20:18:34 +0100 Subject: [PATCH 09/12] Wait longer for server to be ready before requesting first certificate --- commander | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/commander b/commander index 2fac604..4d9b8e4 100755 --- a/commander +++ b/commander @@ -18,11 +18,11 @@ function wait_server() { local status=0 local cnt=0 - while [ $cnt -lt 20 ] && [ "$status" != "200" ]; do + while [ $cnt -lt 40 ] && [ "$status" != "200" ]; do status=$(curl -o /dev/null -sSL --head --write-out '%{http_code}\n' $url) let cnt=$cnt+1 - if [ $cnt -lt 10 ] && [ "$status" != "200" ]; then - sleep 3 + if [ "$status" != "200" ]; then + sleep 5 fi done } From 0a77e0e2b38904b8fcf93ec0eb79bfabd4bbc0ad Mon Sep 17 00:00:00 2001 From: Arjan H Date: Mon, 1 Mar 2021 20:29:03 +0100 Subject: [PATCH 10/12] Do not wait for boulder startup when setup is not yet complete --- install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install b/install index c44ed4e..cc5aa6c 100755 --- a/install +++ b/install @@ -702,7 +702,7 @@ startup() { wait_up $PS_MYSQL &>>$installLog wait_up $PS_LABCA &>>$installLog - wait_up $PS_BOULDER $PS_BOULDER_COUNT &>>$installLog + [ -f "$boulderLabCADir/setup_complete" ] && wait_up $PS_BOULDER $PS_BOULDER_COUNT &>>$installLog || /bin/true msg_ok "$msg" } From 4b151c753ef1ad7b4fda64f7f8ae49ab6efedf94 Mon Sep 17 00:00:00 2001 From: Arjan H Date: Tue, 2 Mar 2021 19:42:09 +0100 Subject: [PATCH 11/12] Lower frequency of looking for file changes of hostname and rate-limit policies --- install | 3 +++ reloader_reloader.patch | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 reloader_reloader.patch diff --git a/install b/install index cc5aa6c..58abf08 100755 --- a/install +++ b/install @@ -528,6 +528,9 @@ config_boulder() { sudo -u labca -H patch -p1 < $cloneDir/ra_ra.patch &>>$installLog cp ra/ra.go "$boulderLabCADir/.backup/" + sudo -u labca -H patch -p1 < $cloneDir/reloader_reloader.patch &>>$installLog + cp reloader/reloader.go "$boulderLabCADir/.backup/" + sudo -u labca -H patch -p1 < $cloneDir/mail_mailer.patch &>>$installLog cp mail/mailer.go "$boulderLabCADir/.backup/" diff --git a/reloader_reloader.patch b/reloader_reloader.patch new file mode 100644 index 0000000..cfe0623 --- /dev/null +++ b/reloader_reloader.patch @@ -0,0 +1,28 @@ +diff --git a/reloader/reloader.go b/reloader/reloader.go +index d885af63..ab71babf 100644 +--- a/reloader/reloader.go ++++ b/reloader/reloader.go +@@ -9,7 +9,7 @@ import ( + + // Wrap time.Tick so we can override it in tests. + var makeTicker = func() (func(), <-chan time.Time) { +- t := time.NewTicker(1 * time.Second) ++ t := time.NewTicker(30 * time.Second) + return t.Stop, t.C + } + +@@ -55,8 +55,12 @@ func New(filename string, dataCallback func([]byte) error, errorCallback func(er + case <-tickChan: + currentFileInfo, err := os.Stat(filename) + if err != nil { +- errorCallback(err) +- continue ++ time.Sleep(10 * time.Second) ++ currentFileInfo, err = os.Stat(filename) ++ if err != nil { ++ errorCallback(err) ++ continue ++ } + } + if !currentFileInfo.ModTime().After(fileInfo.ModTime()) { + continue From e0fb69edfb7c29478ade2f08fc83aee62f4a7cb6 Mon Sep 17 00:00:00 2001 From: Arjan H Date: Tue, 2 Mar 2021 19:53:35 +0100 Subject: [PATCH 12/12] Prevent warning when updating LabCA --- install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install b/install index 58abf08..5e45ac8 100755 --- a/install +++ b/install @@ -625,7 +625,7 @@ config_boulder() { export PKI_LOCKDOWN_DOMAINS=$(grep lockdown $adminDir/data/config.json | grep -v domain_mode | sed -e 's/.*:[ ]*//' | sed -e 's/\",//g' | sed -e 's/\"//g') export PKI_WHITELIST_DOMAINS=$(grep whitelist $adminDir/data/config.json | grep -v domain_mode | sed -e 's/.*:[ ]*//' | sed -e 's/\",//g' | sed -e 's/\"//g') - enabled=$(grep "email\": {" config.json -A1 | grep enable | head -1 | perl -p0e 's/.*?:\s+(.*)/\1/' | sed -e 's/\",//g' | sed -e 's/\"//g') + enabled=$(grep "email\": {" $adminDir/data/config.json -A1 | grep enable | head -1 | perl -p0e 's/.*?:\s+(.*)/\1/' | sed -e 's/\",//g' | sed -e 's/\"//g') if [ "$enabled" == "true," ]; then export PKI_EMAIL_SERVER=$(grep server $adminDir/data/config.json | head -1 | perl -p0e 's/.*?:\s+(.*)/\1/' | sed -e 's/\",//g' | sed -e 's/\"//g') export PKI_EMAIL_PORT=$(grep port $adminDir/data/config.json | head -1 | perl -p0e 's/.*?:\s+(.*)/\1/' | sed -e 's/\",//g' | sed -e 's/\"//g')