From 169b1470781dde0bbe23f1a5091e2302a56ad041 Mon Sep 17 00:00:00 2001 From: Arjan H Date: Fri, 15 Apr 2022 11:12:12 +0200 Subject: [PATCH] Extract code patching to separate script --- gui/apply-boulder | 2 +- install | 54 ++++++----------------------- mail-tester.go | 2 +- patch.sh | 41 ++++++++++++++++++++++ patches/errors_errors.patch | 17 +++++++++ patches/ratelimit_rate-limits.patch | 37 ++++++++++++++++++++ 6 files changed, 107 insertions(+), 46 deletions(-) create mode 100755 patch.sh create mode 100644 patches/errors_errors.patch create mode 100644 patches/ratelimit_rate-limits.patch diff --git a/gui/apply-boulder b/gui/apply-boulder index fe93d02..01d445f 100755 --- a/gui/apply-boulder +++ b/gui/apply-boulder @@ -60,7 +60,7 @@ if [ "$PKI_DOMAIN_MODE" == "lockdown" ] || [ "$PKI_DOMAIN_MODE" == "whitelist" ] done fi cat rate-limit-policies.yml | tr '\n' '\r' | sed -e "s/\(must-staple.le.wtf: 10000\).*\( registrationOverrides:\)/\1\n$REPLACEMENT\2/" | tr '\r' '\n' > rate-limit-policies.yml.bak && mv rate-limit-policies.yml.bak rate-limit-policies.yml - cat rate-limit-policies.yml | tr '\n' '\r' | sed -e "s/\(certificatesPerFQDNSet:.*must-staple.le.wtf: 10000\).*/\1\n$REPLACEMENT/" | tr '\r' '\n' > rate-limit-policies.yml.bak && mv rate-limit-policies.yml.bak rate-limit-policies.yml + cat rate-limit-policies.yml | tr '\n' '\r' | sed -e "s|\(certificatesPerFQDNSet:.*must-staple.le.wtf: 10000\).*|\1\n${REPLACEMENT}rateLimitsURL: http://$PKI_FQDN/rate-limits\n|" | tr '\r' '\n' > rate-limit-policies.yml.bak && mv rate-limit-policies.yml.bak rate-limit-policies.yml fi if [ "$PKI_EXTENDED_TIMEOUT" == "1" ]; then diff --git a/install b/install index 7b6c024..1e69df2 100755 --- a/install +++ b/install @@ -543,51 +543,29 @@ config_boulder() { msg_info "$msg" cd "$boulderDir" - if [ "$flag_skip_redis" == true ]; then - sudo -u labca -H patch -p1 < $cloneDir/patches/docker-compose-redis.patch &>>$installLog - fi - sudo -u labca -H patch -p1 < $cloneDir/patches/docker-compose.patch &>>$installLog + $cloneDir/patch.sh "sudo -u labca -H" &>>$installLog + cp docker-compose.yml "$boulderLabCADir/.backup/" - - sudo -u labca -H patch -p1 < $cloneDir/patches/cmd_shell.patch &>>$installLog cp cmd/shell.go "$boulderLabCADir/.backup/" - - sudo -u labca -H patch -p1 < $cloneDir/patches/core_interfaces.patch &>>$installLog cp core/interfaces.go "$boulderLabCADir/.backup/" - - sudo -u labca -H patch -p1 < $cloneDir/patches/policy_pa.patch &>>$installLog cp policy/pa.go "$boulderLabCADir/.backup/" - - sudo -u labca -H patch -p1 < $cloneDir/patches/ra_ra.patch &>>$installLog cp ra/ra.go "$boulderLabCADir/.backup/" - - sudo -u labca -H patch -p1 < $cloneDir/patches/reloader_reloader.patch &>>$installLog cp reloader/reloader.go "$boulderLabCADir/.backup/" - - sudo -u labca -H patch -p1 < $cloneDir/patches/mail_mailer.patch &>>$installLog cp mail/mailer.go "$boulderLabCADir/.backup/" - - sudo -u labca -H patch -p1 < $cloneDir/patches/expiration-mailer_main.patch &>>$installLog cp cmd/expiration-mailer/main.go "$boulderLabCADir/.backup/" - - sudo -u labca -H patch -p1 < $cloneDir/patches/notify-mailer_main.patch &>>$installLog cp cmd/notify-mailer/main.go "$boulderLabCADir/.backup/" - - sudo -u labca -H patch -p1 < $cloneDir/patches/contact-auditor_main.patch &>>$installLog cp cmd/contact-auditor/main.go "$boulderLabCADir/.backup/" - - sudo -u labca -H patch -p1 < $cloneDir/patches/bad-key-revoker_main.patch &>>$installLog cp cmd/bad-key-revoker/main.go "$boulderLabCADir/.backup/" - - sudo -u labca -H patch -p1 < $cloneDir/patches/cert-checker_main.patch &>>$installLog cp cmd/cert-checker/main.go "$boulderLabCADir/.backup/" - - sudo -u labca -H patch -p1 < $cloneDir/patches/log-validator_main.patch &>>$installLog cp cmd/log-validator/main.go "$boulderLabCADir/.backup/" + cp cmd/boulder/main.go "$boulderLabCADir/.backup/" + cp ratelimit/rate-limits.go "$boulderLabCADir/.backup/" + cp errors/errors.go "$boulderLabCADir/.backup/" + cp log/log.go "$boulderLabCADir/.backup/" + cp sa/_db/migrations/20210223140000_CombinedSchema.sql "$boulderLabCADir/.backup/" sudo -u labca -H patch -p1 -o "$boulderLabCADir/entrypoint.sh" < $cloneDir/patches/entrypoint.patch &>>$installLog - sudo -u labca -H patch -p1 -o "$boulderLabCADir/startservers.py" < $cloneDir/patches/startservers.patch &>>$installLog - sudo -u labca -H patch -p1 < $cloneDir/patches/startservers.patch &>>$installLog + cp test/startservers.py "$boulderLabCADir/startservers.py" &>>$installLog sudo -u labca -H patch -p1 -o "$boulderLabCADir/config/ca-a.json" < $cloneDir/patches/test_config_ca_a.patch &>>$installLog sudo -u labca -H patch -p1 -o "$boulderLabCADir/config/ca-b.json" < $cloneDir/patches/test_config_ca_b.patch &>>$installLog @@ -601,19 +579,6 @@ config_boulder() { sudo -u labca -H patch -p1 -o "$boulderLabCADir/config/rocsp-tool.json" < $cloneDir/patches/config_rocsp-tool.patch &>>$installLog sudo -u labca -H patch -p1 -o "$boulderLabCADir/config/orphan-finder.json" < $cloneDir/patches/config_orphan-finder.patch &>>$installLog - sed -i -e "s|https://letsencrypt.org/docs/rate-limits/|http://$LABCA_FQDN/rate-limits|" errors/errors.go &>>$installLog - cp errors/errors.go "$boulderLabCADir/.backup/" - - sed -i -e "s/\"150405/\"060102150405/" log/log.go &>>$installLog - cp log/log.go "$boulderLabCADir/.backup/" - - mkdir -p "cmd/mail-tester" - cp $cloneDir/mail-tester.go cmd/mail-tester/main.go - perl -i -p0e "s/(\n\t\"github.com\/letsencrypt\/boulder\/cmd\")/\t_ \"github.com\/letsencrypt\/boulder\/cmd\/mail-tester\"\n\t\1/igs" cmd/boulder/main.go &>>$installLog - - sudo -u labca -H patch -p1 < $cloneDir/patches/db_migrations.patch &>>$installLog - cp sa/_db/migrations/20210223140000_CombinedSchema.sql "$boulderLabCADir/.backup/" - mkdir -p $baseDir/backup [ -z "$(docker ps | grep boulder_bmysql_1)" ] || docker exec -i boulder_bmysql_1 mysqldump boulder_sa_integration >$baseDir/backup/dbdata-${runId}.sql @@ -690,7 +655,8 @@ config_boulder() { export PKI_ROOT_CERT_BASE="$adminDir/data/root-ca" export PKI_INT_CERT_BASE="$adminDir/data/issuer/ca-int" export PKI_DNS=$(grep dns $adminDir/data/config.json | perl -p0e 's/.*?:\s+(.*)/\1/' | sed -e 's/\",//g' | sed -e 's/\"//g') - export PKI_DOMAIN=$(grep fqdn $adminDir/data/config.json | sed -e 's/.*:[ ]*//' | sed -e 's/\",//g' | sed -e 's/\"//g' | perl -p0e 's/.*?\.//') + export PKI_FQDN=$(grep fqdn $adminDir/data/config.json | sed -e 's/.*:[ ]*//' | sed -e 's/\",//g' | sed -e 's/\"//g') + export PKI_DOMAIN=$(echo $PKI_FQDN | perl -p0e 's/.*?\.//') export PKI_DOMAIN_MODE=$(grep domain_mode $adminDir/data/config.json | sed -e 's/.*:[ ]*//' | sed -e 's/\",//g' | sed -e 's/\"//g') export PKI_LOCKDOWN_DOMAINS=$(grep lockdown $adminDir/data/config.json | grep -v domain_mode | sed -e 's/.*:[ ]*//' | sed -e 's/\",//g' | sed -e 's/\"//g') export PKI_WHITELIST_DOMAINS=$(grep whitelist $adminDir/data/config.json | grep -v domain_mode | sed -e 's/.*:[ ]*//' | sed -e 's/\",//g' | sed -e 's/\"//g') diff --git a/mail-tester.go b/mail-tester.go index 7e23c18..15024b7 100644 --- a/mail-tester.go +++ b/mail-tester.go @@ -55,7 +55,7 @@ type config struct { Features map[string]bool } - Syslog cmd.SyslogConfig + Syslog cmd.SyslogConfig Beeline cmd.BeelineConfig Common struct { diff --git a/patch.sh b/patch.sh new file mode 100755 index 0000000..fab3369 --- /dev/null +++ b/patch.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +set -e + +flag_skip_redis=true +cloneDir=$(dirname $0) + +# For legacy mode, when called from the install script... +SUDO="$1" + + +if [ "$flag_skip_redis" == true ]; then + $SUDO patch -p1 < $cloneDir/patches/docker-compose-redis.patch +fi +$SUDO patch -p1 < $cloneDir/patches/docker-compose.patch + +$SUDO patch -p1 < $cloneDir/patches/cmd_shell.patch +$SUDO patch -p1 < $cloneDir/patches/core_interfaces.patch +$SUDO patch -p1 < $cloneDir/patches/policy_pa.patch +$SUDO patch -p1 < $cloneDir/patches/ra_ra.patch +$SUDO patch -p1 < $cloneDir/patches/reloader_reloader.patch +$SUDO patch -p1 < $cloneDir/patches/mail_mailer.patch +$SUDO patch -p1 < $cloneDir/patches/expiration-mailer_main.patch +$SUDO patch -p1 < $cloneDir/patches/notify-mailer_main.patch +$SUDO patch -p1 < $cloneDir/patches/contact-auditor_main.patch +$SUDO patch -p1 < $cloneDir/patches/bad-key-revoker_main.patch +$SUDO patch -p1 < $cloneDir/patches/cert-checker_main.patch +$SUDO patch -p1 < $cloneDir/patches/log-validator_main.patch +$SUDO patch -p1 < $cloneDir/patches/startservers.patch +$SUDO patch -p1 < $cloneDir/patches/errors_errors.patch +$SUDO patch -p1 < $cloneDir/patches/ratelimit_rate-limits.patch + +sed -i -e "s/berrors.RateLimitError(/berrors.RateLimitError(ra.rlPolicies.RateLimitsURL(), /g" ra/ra.go + +sed -i -e "s/\"150405/\"060102150405/" log/log.go + +mkdir -p "cmd/mail-tester" +cp $cloneDir/mail-tester.go cmd/mail-tester/main.go +perl -i -p0e "s/(\n\t\"github.com\/letsencrypt\/boulder\/cmd\")/\t_ \"github.com\/letsencrypt\/boulder\/cmd\/mail-tester\"\n\1/igs" cmd/boulder/main.go + +$SUDO patch -p1 < $cloneDir/patches/db_migrations.patch diff --git a/patches/errors_errors.patch b/patches/errors_errors.patch new file mode 100644 index 0000000..34f151e --- /dev/null +++ b/patches/errors_errors.patch @@ -0,0 +1,17 @@ +diff --git a/errors/errors.go b/errors/errors.go +index 3ca9988a6..4137fe7a2 100644 +--- a/errors/errors.go ++++ b/errors/errors.go +@@ -94,10 +94,10 @@ func NotFoundError(msg string, args ...interface{}) error { + return New(NotFound, msg, args...) + } + +-func RateLimitError(msg string, args ...interface{}) error { ++func RateLimitError(errURL string, msg string, args ...interface{}) error { + return &BoulderError{ + Type: RateLimit, +- Detail: fmt.Sprintf(msg+": see https://letsencrypt.org/docs/rate-limits/", args...), ++ Detail: fmt.Sprintf(msg+": see "+errURL, args...), + } + } + diff --git a/patches/ratelimit_rate-limits.patch b/patches/ratelimit_rate-limits.patch new file mode 100644 index 0000000..7ea086d --- /dev/null +++ b/patches/ratelimit_rate-limits.patch @@ -0,0 +1,37 @@ +diff --git a/ratelimit/rate-limits.go b/ratelimit/rate-limits.go +index c199b1141..474d8f740 100644 +--- a/ratelimit/rate-limits.go ++++ b/ratelimit/rate-limits.go +@@ -22,6 +22,7 @@ type Limits interface { + PendingOrdersPerAccount() RateLimitPolicy + NewOrdersPerAccount() RateLimitPolicy + LoadPolicies(contents []byte) error ++ RateLimitsURL() string + } + + // limitsImpl is an unexported implementation of the Limits interface. It acts +@@ -114,6 +115,15 @@ func (r *limitsImpl) NewOrdersPerAccount() RateLimitPolicy { + return r.rlPolicy.NewOrdersPerAccount + } + ++func (r *limitsImpl) RateLimitsURL() string { ++ r.RLock() ++ defer r.RUnlock() ++ if r.rlPolicy == nil { ++ return "" ++ } ++ return r.rlPolicy.RateLimitsURL ++} ++ + // LoadPolicies loads various rate limiting policies from a byte array of + // YAML configuration (typically read from disk by a reloader) + func (r *limitsImpl) LoadPolicies(contents []byte) error { +@@ -171,6 +181,8 @@ type rateLimitConfig struct { + // lower threshold and smaller window), so that clients don't have to wait + // a long time after a small burst of accidental duplicate issuance. + CertificatesPerFQDNSetFast RateLimitPolicy `yaml:"certificatesPerFQDNSetFast"` ++ // URL to show in error messages when a rate-limit error is shown ++ RateLimitsURL string `yaml:"rateLimitsURL"` + } + + // RateLimitPolicy describes a general limiting policy