From f14a2636c5025237b0743ddbe32c49db08a45747 Mon Sep 17 00:00:00 2001 From: Arjan H Date: Mon, 10 Feb 2025 19:38:38 +0100 Subject: [PATCH] Bump boulder version to release-2025-02-04; add redis container Let's Encrypt has changed the rate limiter to require redis, so we can no longer remove it from the docker compose filei completely. But at least we can run it once instead of four instances. --- build/build.sh | 2 +- build/tmp.patch | 18 +++++-- commander | 17 ++++-- dev/versions | 2 +- gui/apply-boulder | 8 +++ gui/dashboard.go | 23 ++++++-- gui/main.go | 18 ++++++- install | 5 +- patch-cfg.sh | 12 ++--- patch.sh | 10 +--- patches/boulder-ra_main.patch | 13 +++++ patches/ca_ca.patch | 4 +- patches/ca_ca_keytype_hack.patch | 4 +- patches/cert-checker_main.patch | 8 +-- patches/cmd_config.patch | 4 +- patches/config_ra.patch | 4 +- patches/docker-compose-redis.patch | 81 ----------------------------- patches/docker-compose.patch | 50 ++++++++++++++++-- patches/errors_errors.patch | 17 ------ patches/ocsp-responder_main.patch | 8 +-- patches/ra_ra.patch | 6 +-- patches/ratelimit_rate-limits.patch | 35 ------------- patches/ratelimits_names.patch | 47 ++++++++++++----- patches/test_config_ca.patch | 40 ++++++++++---- patches/updater_updater.patch | 8 +-- patches/wfe2_main.patch | 49 +++++++++++++++-- patches/wfe2_wfe.patch | 55 +++++++------------- utils.sh | 4 ++ 28 files changed, 291 insertions(+), 261 deletions(-) create mode 100644 patches/boulder-ra_main.patch delete mode 100644 patches/docker-compose-redis.patch delete mode 100644 patches/errors_errors.patch delete mode 100644 patches/ratelimit_rate-limits.patch diff --git a/build/build.sh b/build/build.sh index f8d71b4..41beacd 100755 --- a/build/build.sh +++ b/build/build.sh @@ -8,7 +8,7 @@ TMP_DIR=$(pwd)/tmp rm -rf $TMP_DIR && mkdir -p $TMP_DIR/{admin,bin,logs,src} boulderDir=$TMP_DIR/src -boulderTag="release-2025-01-06" +boulderTag="release-2025-02-04" boulderUrl="https://github.com/letsencrypt/boulder/" cloneDir=$(pwd)/.. diff --git a/build/tmp.patch b/build/tmp.patch index a0c3641..3a4c4b3 100644 --- a/build/tmp.patch +++ b/build/tmp.patch @@ -1,5 +1,5 @@ diff --git a/docker-compose.yml b/docker-compose.yml -index 81a92bbe6..49e3c2797 100644 +index d90c629af..607ef1c7d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ services: @@ -27,7 +27,17 @@ index 81a92bbe6..49e3c2797 100644 networks: bouldernet: ipv4_address: 10.77.77.77 -@@ -91,35 +90,37 @@ services: +@@ -92,7 +91,8 @@ services: + bredis: + image: redis:6.2.7 + volumes: +- - /home/labca/boulder_labca:/opt/boulder/labca ++ - boulder_data:/opt/boulder/labca ++ - certificates:/opt/boulder/labca/certs + command: redis-server /opt/boulder/labca/redis-ratelimits.config + networks: + redisnet: +@@ -103,35 +103,37 @@ services: depends_on: - control volumes: @@ -76,7 +86,7 @@ index 81a92bbe6..49e3c2797 100644 logging: driver: "json-file" options: -@@ -136,30 +137,28 @@ services: +@@ -148,30 +150,28 @@ services: - 80:80 - 443:443 volumes: @@ -121,7 +131,7 @@ index 81a92bbe6..49e3c2797 100644 expose: - 3030 environment: -@@ -177,6 +176,15 @@ services: +@@ -189,6 +189,15 @@ services: volumes: dbdata: diff --git a/commander b/commander index 59422f0..ee6c5b8 100755 --- a/commander +++ b/commander @@ -39,7 +39,7 @@ read txt case $txt in "docker-restart") cd /opt/boulder - COMPOSE_HTTP_TIMEOUT=120 docker compose restart boulder bmysql bconsul bpkilint gui nginx &>>$LOGFILE + COMPOSE_HTTP_TIMEOUT=120 docker compose restart boulder bmysql bconsul bpkilint bredis gui nginx &>>$LOGFILE sleep 45 wait_up $PS_MYSQL &>>$LOGFILE wait_up $PS_CONSUL 2 &>>$LOGFILE @@ -170,7 +170,8 @@ case $txt in mysql=$(docker inspect $(docker ps --format "{{.Names}}" | grep -- -bmysql-) | grep -i started | grep -v depends_on | sed -e "s/[^:]*:\(.*\)/\1/" | sed -e "s/.*\"\(.*\)\".*/\1/") consul=$(docker inspect $(docker ps --format "{{.Names}}" | grep -- -bconsul-) | grep -i started | grep -v depends_on | sed -e "s/[^:]*:\(.*\)/\1/" | sed -e "s/.*\"\(.*\)\".*/\1/") pkilint=$(docker inspect $(docker ps --format "{{.Names}}" | grep -- -bpkilint-) | grep -i started | grep -v depends_on | sed -e "s/[^:]*:\(.*\)/\1/" | sed -e "s/.*\"\(.*\)\".*/\1/") - echo "$nginx|$svc|$boulder|$labca|$mysql|$consul|$pkilint" + redis=$(docker inspect $(docker ps --format "{{.Names}}" | grep -- -bredis-) | grep -i started | grep -v depends_on | sed -e "s/[^:]*:\(.*\)/\1/" | sed -e "s/.*\"\(.*\)\".*/\1/") + echo "$nginx|$svc|$boulder|$labca|$mysql|$consul|$pkilint|$redis" exit 0 ;; "log-uptime") @@ -195,7 +196,7 @@ case $txt in ;; "boulder-start") cd /opt/boulder - COMPOSE_HTTP_TIMEOUT=120 docker compose up -d bmysql bconsul bpkilint + COMPOSE_HTTP_TIMEOUT=120 docker compose up -d bmysql bconsul bpkilint bredis wait_up $PS_MYSQL &>>$LOGFILE wait_up $PS_CONSUL 2 &>>$LOGFILE wait_up $PS_PKILINT &>>$LOGFILE @@ -205,7 +206,7 @@ case $txt in "boulder-stop") cd /opt/boulder docker compose stop boulder - docker compose stop bmysql bconsul bpkilint + docker compose stop bmysql bconsul bpkilint bredis wait_down $PS_MYSQL &>>$LOGFILE wait_down $PS_CONSUL &>>$LOGFILE wait_down $PS_PKILINT &>>$LOGFILE @@ -213,7 +214,7 @@ case $txt in ;; "boulder-restart") cd /opt/boulder - COMPOSE_HTTP_TIMEOUT=120 docker compose restart boulder bmysql bconsul bpkilint &>>$LOGFILE + COMPOSE_HTTP_TIMEOUT=120 docker compose restart boulder bmysql bconsul bpkilint bredis &>>$LOGFILE sleep 30 wait_up $PS_MYSQL &>>$LOGFILE wait_up $PS_CONSUL 2 &>>$LOGFILE @@ -244,6 +245,12 @@ case $txt in COMPOSE_HTTP_TIMEOUT=120 docker compose restart bpkilint set -e ;; +"redis-restart") + cd /opt/boulder + set +e + COMPOSE_HTTP_TIMEOUT=120 docker compose restart bredis + set -e + ;; "log-backups") ls -1tr /opt/backup || /bin/true exit 0 diff --git a/dev/versions b/dev/versions index 762907a..06cab31 100755 --- a/dev/versions +++ b/dev/versions @@ -126,7 +126,7 @@ grep GO_VERSION ../boulder/docker-compose.yml | sed -e "s/\s*GO_VERSION:/ /" colorGoVersion2 build/docker-compose.yml $goversion echo -db_migrs=$(ls -1 ../boulder/sa/db/boulder_sa/ | wc -l) +db_migrs=$(ls -1 ../boulder/sa/db/boulder_sa/ | grep -v 20240304000000_CertificateProfiles.sql | grep -v 20250115000000_AuthzProfiles.sql | wc -l) db_patches=$(ls -1 ../labca/patches/db_migrations* | wc -l) echo -n "Database migrations " colorEqual $db_migrs $db_patches diff --git a/gui/apply-boulder b/gui/apply-boulder index 6d39c5b..645c658 100755 --- a/gui/apply-boulder +++ b/gui/apply-boulder @@ -79,6 +79,7 @@ if ([ "$PKI_DOMAIN_MODE" == "lockdown" ] && [ "$PKI_LOCKDOWN_DOMAINS" != "" ]) | perl -i -p0e "s/(\"badResultsOnly\":[^\n]*).*?(\s+)(\"checkPeriod\":)/\1\2\"skipForbiddenDomains\": true,\2\3/igs" config/cert-checker.json perl -i -p0e "s/(\"ignoredLints\": \[).*?(\s+)(\"w_subject_common_name_included\")/\1\2\"e_dnsname_not_valid_tld\",\2\"w_sub_cert_aia_contains_internal_names\",\2\3/igs" config/cert-checker.json perl -i -p0e "s/(\"ignoredLints\": \[).*?(\s+)(\"w_subject_common_name_included\")/\1\2\"e_dnsname_not_valid_tld\",\2\"w_sub_cert_aia_contains_internal_names\",\2\3/igs" config/ca.json + perl -i -p0e "s/(\"modern\".*)(\"ignoredLints\": \[).*?(\s+)(\"w_ext_subject_key_identifier_missing_sub_cert\")/\1\2\3\"e_dnsname_not_valid_tld\",\3\"w_sub_cert_aia_contains_internal_names\",\3\4/igs" config/ca.json perl -i -p0e "s/(\"SubscriberKeyUsageValidator:cabf.serverauth.subscriber_rsa_digitalsignature_and_keyencipherment_present\",).*(\])/\1\n \"GeneralNameDnsNameInternalDomainNameValidator:cabf.internal_domain_name\",\n \"GeneralNameUriInternalDomainNameValidator:cabf.internal_domain_name\",\n\2/igs" config/zlint.toml fi @@ -120,10 +121,12 @@ if [ "$PKI_DOMAIN_MODE" == "lockdown" ] || [ "$PKI_DOMAIN_MODE" == "whitelist" ] sed -i -e "s/\(\"w_subject_common_name_included\"\).*\]/\1,\"e_dnsname_not_valid_tld\"\]/" config/ca.json REPLACEMENT="" + REPLACEMENT2="" LABCA_DOMAINS="" if [ "$PKI_DOMAIN_MODE" == "lockdown" ] && [ "$PKI_LOCKDOWN_DOMAINS" != "" ]; then for d in $(echo $PKI_LOCKDOWN_DOMAINS | sed -e "s/\\\r/ /g" | sed -e "s/\\\n/ /g" | tr '\r' ' '); do REPLACEMENT+=" $d: 10000\r" + REPLACEMENT2+=" - id: $d\r comment: LabCA lockdown domain\r" if [ "$LABCA_DOMAINS" != "" ]; then LABCA_DOMAINS+=",\n" fi @@ -133,6 +136,7 @@ if [ "$PKI_DOMAIN_MODE" == "lockdown" ] || [ "$PKI_DOMAIN_MODE" == "whitelist" ] if [ "$PKI_DOMAIN_MODE" == "whitelist" ] && [ "$PKI_WHITELIST_DOMAINS" != "" ]; then for d in $(echo $PKI_WHITELIST_DOMAINS | sed -e "s/\\\r/ /g" | sed -e "s/\\\n/ /g" | tr '\r' ' '); do REPLACEMENT+=" $d: 10000\r" + REPLACEMENT2+=" - id: $d\r comment: LabCA whitelist domain\r" if [ "$LABCA_DOMAINS" != "" ]; then LABCA_DOMAINS+=",\n" fi @@ -142,6 +146,10 @@ if [ "$PKI_DOMAIN_MODE" == "lockdown" ] || [ "$PKI_DOMAIN_MODE" == "whitelist" ] cat rate-limit-policies.yml | tr '\n' '\r' | sed -e "s/\(must-staple.le.wtf: 10000\).*\( registrationOverrides:\)/\1\n$REPLACEMENT\2/" | tr '\r' '\n' > rate-limit-policies.yml.bak && mv rate-limit-policies.yml.bak rate-limit-policies.yml cat rate-limit-policies.yml | tr '\n' '\r' | sed -e "s|\(certificatesPerFQDNSet:.*must-staple.le.wtf: 10000\).*\(certificatesPerFQDNSetFast:.*\)|\1\n${REPLACEMENT}rateLimitsURL: http://$PKI_FQDN/rate-limits\n\2|" | tr '\r' '\n' > rate-limit-policies.yml.bak && mv rate-limit-policies.yml.bak rate-limit-policies.yml + cat config/wfe2-ratelimit-overrides.yml | tr '\n' '\r' | sed -e "s/\(- CertificatesPerDomain:.*ratelimit.me.*\)\(- CertificatesPerDomain:\)/\2/" | tr '\r' '\n' > config/wfe2-ratelimit-overrides.yml.bak && mv config/wfe2-ratelimit-overrides.yml.bak config/wfe2-ratelimit-overrides.yml + cat config/wfe2-ratelimit-overrides.yml | tr '\n' '\r' | sed -e "s/\(- CertificatesPerDomain:.*ids:\)\(.*\)\(- CertificatesPerFQDNSet:\)/\1\r$REPLACEMENT2\3/" | tr '\r' '\n' > config/wfe2-ratelimit-overrides.yml.bak && mv config/wfe2-ratelimit-overrides.yml.bak config/wfe2-ratelimit-overrides.yml + cat config/wfe2-ratelimit-overrides.yml | tr '\n' '\r' | sed -e "s/\(- CertificatesPerFQDNSet:.*ids:\)\(.*\)/\1\r$REPLACEMENT2/" | tr '\r' '\n' > config/wfe2-ratelimit-overrides.yml.bak && mv config/wfe2-ratelimit-overrides.yml.bak config/wfe2-ratelimit-overrides.yml + perl -i -p0e "s/(\"labcaDomains\": \[\n).*?(\])/\1$LABCA_DOMAINS\n\t\t\2/igs" config/remoteva-a.json perl -i -p0e "s/(\"labcaDomains\": \[\n).*?(\])/\1$LABCA_DOMAINS\n\t\t\2/igs" config/remoteva-b.json perl -i -p0e "s/(\"labcaDomains\": \[\n).*?(\])/\1$LABCA_DOMAINS\n\t\t\2/igs" config/va.json diff --git a/gui/dashboard.go b/gui/dashboard.go index 587ca00..fe12018 100644 --- a/gui/dashboard.go +++ b/gui/dashboard.go @@ -225,10 +225,11 @@ func _parseComponents(data string) []Component { parts := strings.Split(data, "|") - if len(parts) < 6 { + if len(parts) < 7 { components = append(components, Component{Name: "Boulder (ACME)"}) - components = append(components, Component{Name: "Consul (Boulder)"}) + components = append(components, Component{Name: "consul (Boulder)"}) components = append(components, Component{Name: "pkilint (Boulder)"}) + components = append(components, Component{Name: "redis (Boulder)"}) components = append(components, Component{Name: "LabCA Application"}) components = append(components, Component{Name: "LabCA Controller"}) components = append(components, Component{Name: "MySQL Database"}) @@ -306,9 +307,20 @@ func _parseComponents(data string) []Component { pkilintClass = "" } + redis, err := time.Parse(time.RFC3339Nano, parts[7]) + redisReal := "" + redisNice := "stopped" + redisClass := "error" + if err == nil { + redisReal = redis.Format("02-Jan-2006 15:04:05 MST") + redisNice = humanize.RelTime(redis, time.Now(), "", "") + redisClass = "" + } + components = append(components, Component{Name: "Boulder (ACME)", Timestamp: boulderReal, TimestampRel: boulderNice, Class: boulderClass}) - components = append(components, Component{Name: "Consul (Boulder)", Timestamp: consulReal, TimestampRel: consulNice, Class: consulClass}) + components = append(components, Component{Name: "consul (Boulder)", Timestamp: consulReal, TimestampRel: consulNice, Class: consulClass}) components = append(components, Component{Name: "pkilint (Boulder)", Timestamp: pkilintReal, TimestampRel: pkilintNice, Class: pkilintClass}) + components = append(components, Component{Name: "redis (Boulder)", Timestamp: redisReal, TimestampRel: redisNice, Class: redisClass}) components = append(components, Component{Name: "LabCA Application", Timestamp: labcaReal, TimestampRel: labcaNice, Class: labcaClass}) components = append(components, Component{Name: "LabCA Controller", Timestamp: svcReal, TimestampRel: svcNice, Class: svcClass}) components = append(components, Component{Name: "MySQL Database", Timestamp: mysqlReal, TimestampRel: mysqlNice, Class: mysqlClass}) @@ -516,11 +528,14 @@ func parseDockerStats(data string) []AjaxStat { stat.Name = "Boulder (ACME)" } if strings.Contains(docker.Name, "-bconsul-") { - stat.Name = "Consul (Boulder)" + stat.Name = "consul (Boulder)" } if strings.Contains(docker.Name, "-bpkilint-") { stat.Name = "pkilint (Boulder)" } + if strings.Contains(docker.Name, "-bredis-") { + stat.Name = "redis (Boulder)" + } if strings.Contains(docker.Name, "labca-gui-") { stat.Name = "LabCA Application" } diff --git a/gui/main.go b/gui/main.go index c5b539a..706d863 100644 --- a/gui/main.go +++ b/gui/main.go @@ -1123,8 +1123,9 @@ func (res *Result) ManageComponents(w http.ResponseWriter, r *http.Request, acti (components[i].Name == "LabCA Controller" && action == "svc-restart") || (components[i].Name == "Boulder (ACME)" && (action == "boulder-start" || action == "boulder-stop" || action == "boulder-restart")) || (components[i].Name == "LabCA Application" && action == "labca-restart") || - (components[i].Name == "Consul (Boulder)" && action == "consul-restart") || + (components[i].Name == "consul (Boulder)" && action == "consul-restart") || (components[i].Name == "pkilint (Boulder)" && action == "pkilint-restart") || + (components[i].Name == "redis (Boulder)" && action == "redis-restart") || (components[i].Name == "MySQL Database" && action == "mysql-restart") { res.Timestamp = components[i].Timestamp res.TimestampRel = components[i].TimestampRel @@ -1353,6 +1354,7 @@ func _managePost(w http.ResponseWriter, r *http.Request) { "mysql-restart", "consul-restart", "pkilint-restart", + "redis-restart", "nginx-reload", "nginx-restart", "svc-restart", @@ -1534,7 +1536,7 @@ func _manageGet(w http.ResponseWriter, r *http.Request) { components[i].Buttons = append(components[i].Buttons, btn) } - if components[i].Name == "Consul (Boulder)" { + if components[i].Name == "consul (Boulder)" { components[i].LogURL = "" components[i].LogTitle = "" @@ -1558,6 +1560,18 @@ func _manageGet(w http.ResponseWriter, r *http.Request) { components[i].Buttons = append(components[i].Buttons, btn) } + if components[i].Name == "redis (Boulder)" { + components[i].LogURL = "" + components[i].LogTitle = "" + + btn := make(map[string]interface{}) + btn["Class"] = "btn-warning" + btn["Id"] = "redis-restart" + btn["Title"] = "Restart the internal redis helper" + btn["Label"] = "Restart" + components[i].Buttons = append(components[i].Buttons, btn) + } + if components[i].Name == "MySQL Database" { components[i].LogURL = "" components[i].LogTitle = "" diff --git a/install b/install index 959ca2f..29fa819 100755 --- a/install +++ b/install @@ -30,10 +30,7 @@ dockerComposeVersion="v2.5.0" labcaUrl="https://github.com/hakwerk/labca/" boulderUrl="https://github.com/letsencrypt/boulder/" -boulderTag="release-2025-01-06" - -# Feature flags -flag_skip_redis=true +boulderTag="release-2025-02-04" # # Color configuration diff --git a/patch-cfg.sh b/patch-cfg.sh index a2e8b76..6fa0e81 100755 --- a/patch-cfg.sh +++ b/patch-cfg.sh @@ -2,7 +2,6 @@ set -e -flag_skip_redis=true cloneDir=$(dirname $0) # For legacy mode, when called from the install script... @@ -37,9 +36,7 @@ perl -i -p0e "s/(\"accountURIPrefixes\": \[\n.*?\s+\])/\1,\n\t\t\"labcaDomains\" perl -i -p0e "s/(\"accountURIPrefixes\": \[\n.*?\s+\])/\1,\n\t\t\"labcaDomains\": [\n\t\t]/igs" $boulderLabCADir/config/remoteva-b.json perl -i -p0e "s/(\"accountURIPrefixes\": \[\n.*?\s+\])/\1,\n\t\t\"labcaDomains\": [\n\t\t]/igs" $boulderLabCADir/config/va.json -if [ "$flag_skip_redis" == true ]; then - perl -i -p0e "s/\n \"redis\": \{\n.*? \},//igs" $boulderLabCADir/config/ocsp-responder.json -fi +perl -i -p0e "s/\n \"redis\": \{\n.*? \},//igs" $boulderLabCADir/config/ocsp-responder.json for f in $(grep -l boulder-proxysql $boulderLabCADir/secrets/*); do sed -i -e "s/proxysql:6033/mysql:3306/" $f; done @@ -71,11 +68,10 @@ sed -i -e "s/\"stdoutlevel\": 4,/\"stdoutlevel\": 6,/" config/remoteva-b.json sed -i -e "s/\"endpoint\": \".*\"/\"endpoint\": \"\"/" config/sfe.json sed -i -e "s/sleep 1/sleep 5/g" wait-for-it.sh -sed -i -e "s|test/certs|/opt/boulder/labca/certs|" consul/config.hcl +perl -i -p0e "s/(services {\s*id\s*=\s*\"bredis4\".*?}\n\n)//igs" consul/config.hcl -if [ "$flag_skip_redis" == true ]; then - sed -i -e "s/^\(.*wait-for-it.sh.*4218\)/#\1/" entrypoint.sh -fi +sed -i -e "s|test/certs|/opt/boulder/labca/certs|" consul/config.hcl +sed -i -e "s|/test/certs|/opt/boulder/labca/certs|" redis-ratelimits.config for file in `find . -type f | grep -v .git`; do sed -i -e "s|test/|labca/|g" $file diff --git a/patch.sh b/patch.sh index 4069de1..611b3bf 100755 --- a/patch.sh +++ b/patch.sh @@ -2,16 +2,12 @@ set -e -flag_skip_redis=true cloneDir=$(dirname $0) # For legacy mode, when called from the install script... SUDO="$1" -if [ "$flag_skip_redis" == true ]; then - $SUDO patch -p1 < $cloneDir/patches/docker-compose-redis.patch -fi $SUDO patch -p1 < $cloneDir/patches/docker-compose.patch if [ "$SUDO" == "" ]; then # TODO: should incorporate this into docker-compose.patch @@ -19,6 +15,7 @@ if [ "$SUDO" == "" ]; then fi $SUDO patch -p1 < $cloneDir/patches/bad-key-revoker_main.patch +$SUDO patch -p1 < $cloneDir/patches/boulder-ra_main.patch $SUDO patch -p1 < $cloneDir/patches/boulder-va_main.patch $SUDO patch -p1 < $cloneDir/patches/ca_ca.patch $SUDO patch -p1 < $cloneDir/patches/ca_ca_keytype_hack.patch @@ -38,7 +35,6 @@ $SUDO patch -p1 < $cloneDir/patches/db_migrations2.patch $SUDO patch -p1 < $cloneDir/patches/db_migrations3.patch $SUDO patch -p1 < $cloneDir/patches/db_migrations4.patch $SUDO patch -p1 < $cloneDir/patches/db_migrations5.patch -$SUDO patch -p1 < $cloneDir/patches/errors_errors.patch $SUDO patch -p1 < $cloneDir/patches/expiration-mailer_main.patch $SUDO patch -p1 < $cloneDir/patches/issuance_crl.patch $SUDO patch -p1 < $cloneDir/patches/linter_linter.patch @@ -51,7 +47,6 @@ $SUDO patch -p1 < $cloneDir/patches/notify-mailer_main.patch $SUDO patch -p1 < $cloneDir/patches/ocsp-responder_main.patch $SUDO patch -p1 < $cloneDir/patches/policy_pa.patch $SUDO patch -p1 < $cloneDir/patches/ra_ra.patch -$SUDO patch -p1 < $cloneDir/patches/ratelimit_rate-limits.patch $SUDO patch -p1 < $cloneDir/patches/ratelimits_names.patch $SUDO patch -p1 < $cloneDir/patches/remoteva_main.patch $SUDO patch -p1 < $cloneDir/patches/start.patch @@ -71,9 +66,6 @@ $SUDO patch -p1 < $cloneDir/patches/wfe2_wfe.patch sed -i -e "s|./test|./labca|" start.py -sed -i -e "s/berrors.RateLimitError(/berrors.RateLimitError(ra.rlPolicies.RateLimitsURL(), /g" ra/ra.go -sed -i -e "s/berrors.RateLimitError(/berrors.RateLimitError(\"\", /g" ratelimits/limiter.go - sed -i -e "s/proxysql:6033/mysql:3306/" sa/db/dbconfig.yml mkdir -p "cmd/mail-tester" diff --git a/patches/boulder-ra_main.patch b/patches/boulder-ra_main.patch new file mode 100644 index 0000000..6a0b8bc --- /dev/null +++ b/patches/boulder-ra_main.patch @@ -0,0 +1,13 @@ +diff --git a/cmd/boulder-ra/main.go b/cmd/boulder-ra/main.go +index b16c05c4e..99fe7d601 100644 +--- a/cmd/boulder-ra/main.go ++++ b/cmd/boulder-ra/main.go +@@ -317,6 +317,8 @@ func main() { + limiterRedis, err = bredis.NewRingFromConfig(*c.RA.Limiter.Redis, scope, logger) + cmd.FailOnError(err, "Failed to create Redis ring") + ++ // Set Policy Authority for ratelimits ++ ratelimits.PA = pa + source := ratelimits.NewRedisSource(limiterRedis.Ring, clk, scope) + limiter, err = ratelimits.NewLimiter(clk, source, scope) + cmd.FailOnError(err, "Failed to create rate limiter") diff --git a/patches/ca_ca.patch b/patches/ca_ca.patch index eec2c0b..4fbfe84 100644 --- a/patches/ca_ca.patch +++ b/patches/ca_ca.patch @@ -1,8 +1,8 @@ diff --git a/ca/ca.go b/ca/ca.go -index 87a6fc52c..739ce53e7 100644 +index d58f5ddd3..da8f823f4 100644 --- a/ca/ca.go +++ b/ca/ca.go -@@ -177,10 +177,10 @@ func makeIssuerMaps(issuers []*issuance.Issuer) (issuerMaps, error) { +@@ -176,10 +176,10 @@ func makeIssuerMaps(issuers []*issuance.Issuer) (issuerMaps, error) { } } if i, ok := issuersByAlg[x509.ECDSA]; !ok || len(i) == 0 { diff --git a/patches/ca_ca_keytype_hack.patch b/patches/ca_ca_keytype_hack.patch index aee1cb2..2b2d4f8 100644 --- a/patches/ca_ca_keytype_hack.patch +++ b/patches/ca_ca_keytype_hack.patch @@ -1,8 +1,8 @@ diff --git a/ca/ca.go b/ca/ca.go -index 739ce53e7..2ccb11969 100644 +index da8f823f4..25263580b 100644 --- a/ca/ca.go +++ b/ca/ca.go -@@ -177,10 +177,14 @@ func makeIssuerMaps(issuers []*issuance.Issuer) (issuerMaps, error) { +@@ -176,10 +176,14 @@ func makeIssuerMaps(issuers []*issuance.Issuer) (issuerMaps, error) { } } if i, ok := issuersByAlg[x509.ECDSA]; !ok || len(i) == 0 { diff --git a/patches/cert-checker_main.patch b/patches/cert-checker_main.patch index c73f323..2dd2ca9 100644 --- a/patches/cert-checker_main.patch +++ b/patches/cert-checker_main.patch @@ -1,5 +1,5 @@ diff --git a/cmd/cert-checker/main.go b/cmd/cert-checker/main.go -index 975922c58..3767e83bb 100644 +index 883378779..679a794ed 100644 --- a/cmd/cert-checker/main.go +++ b/cmd/cert-checker/main.go @@ -106,6 +106,7 @@ type certChecker struct { @@ -35,7 +35,7 @@ index 975922c58..3767e83bb 100644 // For defense-in-depth, even if the PA was willing to issue for a name // we double check it against a list of forbidden domains. This way even // if the hostnamePolicyFile malfunctions we will flag the forbidden -@@ -487,9 +490,10 @@ type Config struct { +@@ -489,9 +492,10 @@ type Config struct { Workers int `validate:"required,min=1"` // Deprecated: this is ignored, and cert checker always checks both expired and unexpired. @@ -49,7 +49,7 @@ index 975922c58..3767e83bb 100644 // AcceptableValidityDurations is a list of durations which are // acceptable for certificates we issue. -@@ -544,6 +548,8 @@ func main() { +@@ -546,6 +550,8 @@ func main() { acceptableValidityDurations[ninetyDays] = true } @@ -58,7 +58,7 @@ index 975922c58..3767e83bb 100644 // Validate PA config and set defaults if needed. cmd.FailOnError(config.PA.CheckChallenges(), "Invalid PA configuration") -@@ -578,6 +584,7 @@ func main() { +@@ -580,6 +586,7 @@ func main() { config.CertChecker.CheckPeriod.Duration, acceptableValidityDurations, logger, diff --git a/patches/cmd_config.patch b/patches/cmd_config.patch index 7b13e3e..580139d 100644 --- a/patches/cmd_config.patch +++ b/patches/cmd_config.patch @@ -1,8 +1,8 @@ diff --git a/cmd/config.go b/cmd/config.go -index 1a3edabff..09369bf88 100644 +index 3072f206c..f7271cb7c 100644 --- a/cmd/config.go +++ b/cmd/config.go -@@ -455,7 +455,7 @@ type GRPCServerConfig struct { +@@ -456,7 +456,7 @@ type GRPCServerConfig struct { // this controls how long it takes before a client learns about changes to its // backends. // https://pkg.go.dev/google.golang.org/grpc/keepalive#ServerParameters diff --git a/patches/config_ra.patch b/patches/config_ra.patch index 88a7e38..1cd7fe1 100644 --- a/patches/config_ra.patch +++ b/patches/config_ra.patch @@ -1,8 +1,8 @@ diff --git a/test/config/ra.json b/test/config/ra.json -index e13ca9cf8..cda9192ab 100644 +index 7a5befada..335f69b14 100644 --- a/test/config/ra.json +++ b/test/config/ra.json -@@ -12,12 +12,7 @@ +@@ -36,12 +36,7 @@ }, "orderLifetime": "168h", "issuerCerts": [ diff --git a/patches/docker-compose-redis.patch b/patches/docker-compose-redis.patch deleted file mode 100644 index cb691a8..0000000 --- a/patches/docker-compose-redis.patch +++ /dev/null @@ -1,81 +0,0 @@ -diff --git a/docker-compose.yml b/docker-compose.yml -index f25309579..79ed8c4e4 100644 ---- a/docker-compose.yml -+++ b/docker-compose.yml -@@ -26,8 +26,6 @@ services: - ipv4_address: 10.77.77.77 - integrationtestnet: - ipv4_address: 10.88.88.88 -- redisnet: -- ipv4_address: 10.33.33.33 - consulnet: - ipv4_address: 10.55.55.55 - # Use consul as a backup to Docker's embedded DNS server. If there's a name -@@ -51,10 +49,6 @@ services: - depends_on: - - bmysql - - bproxysql -- - bredis_1 -- - bredis_2 -- - bredis_3 -- - bredis_4 - - bconsul - - bjaeger - - bpkilint -@@ -107,42 +101,6 @@ services: - aliases: - - boulder-proxysql - -- bredis_1: -- image: redis:6.2.7 -- volumes: -- - ./test/:/test/:cached -- command: redis-server /test/redis-ocsp.config -- networks: -- redisnet: -- ipv4_address: 10.33.33.2 -- -- bredis_2: -- image: redis:6.2.7 -- volumes: -- - ./test/:/test/:cached -- command: redis-server /test/redis-ocsp.config -- networks: -- redisnet: -- ipv4_address: 10.33.33.3 -- -- bredis_3: -- image: redis:6.2.7 -- volumes: -- - ./test/:/test/:cached -- command: redis-server /test/redis-ratelimits.config -- networks: -- redisnet: -- ipv4_address: 10.33.33.4 -- -- bredis_4: -- image: redis:6.2.7 -- volumes: -- - ./test/:/test/:cached -- command: redis-server /test/redis-ratelimits.config -- networks: -- redisnet: -- ipv4_address: 10.33.33.5 -- - bconsul: - image: hashicorp/consul:1.15.4 - volumes: -@@ -194,13 +152,6 @@ networks: - config: - - subnet: 10.88.88.0/24 - -- redisnet: -- driver: bridge -- ipam: -- driver: default -- config: -- - subnet: 10.33.33.0/24 -- - consulnet: - driver: bridge - ipam: diff --git a/patches/docker-compose.patch b/patches/docker-compose.patch index 0182358..a55e494 100644 --- a/patches/docker-compose.patch +++ b/patches/docker-compose.patch @@ -1,5 +1,5 @@ diff --git a/docker-compose.yml b/docker-compose.yml -index d0a439f0f..81a92bbe6 100644 +index 2dfa6c278..d90c629af 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,4 @@ @@ -26,11 +26,16 @@ index d0a439f0f..81a92bbe6 100644 networks: bouldernet: ipv4_address: 10.77.77.77 -@@ -48,29 +51,22 @@ services: +@@ -50,33 +53,23 @@ services: - 4003:4003 # SFE depends_on: - bmysql - - bproxysql +- - bredis_1 +- - bredis_2 +- - bredis_3 +- - bredis_4 ++ - bredis - bconsul - - bjaeger - bpkilint @@ -67,7 +72,7 @@ index d0a439f0f..81a92bbe6 100644 networks: bouldernet: aliases: -@@ -84,46 +80,103 @@ services: +@@ -90,82 +83,112 @@ services: # small. command: mysqld --bind-address=0.0.0.0 --slow-query-log --log-output=TABLE --log-queries-not-using-indexes=ON logging: @@ -93,6 +98,45 @@ index d0a439f0f..81a92bbe6 100644 + max-file: "5" + restart: always +- bredis_1: ++ bredis: + image: redis:6.2.7 + volumes: +- - ./test/:/test/:cached +- command: redis-server /test/redis-ocsp.config +- networks: +- redisnet: +- ipv4_address: 10.33.33.2 +- +- bredis_2: +- image: redis:6.2.7 +- volumes: +- - ./test/:/test/:cached +- command: redis-server /test/redis-ocsp.config +- networks: +- redisnet: +- ipv4_address: 10.33.33.3 +- +- bredis_3: +- image: redis:6.2.7 +- volumes: +- - ./test/:/test/:cached +- command: redis-server /test/redis-ratelimits.config ++ - /home/labca/boulder_labca:/opt/boulder/labca ++ command: redis-server /opt/boulder/labca/redis-ratelimits.config + networks: + redisnet: + ipv4_address: 10.33.33.4 + +- bredis_4: +- image: redis:6.2.7 +- volumes: +- - ./test/:/test/:cached +- command: redis-server /test/redis-ratelimits.config +- networks: +- redisnet: +- ipv4_address: 10.33.33.5 +- bconsul: image: hashicorp/consul:1.15.4 + depends_on: diff --git a/patches/errors_errors.patch b/patches/errors_errors.patch deleted file mode 100644 index d0c6129..0000000 --- a/patches/errors_errors.patch +++ /dev/null @@ -1,17 +0,0 @@ -diff --git a/errors/errors.go b/errors/errors.go -index d7328b08d..00bd834d8 100644 ---- a/errors/errors.go -+++ b/errors/errors.go -@@ -171,10 +171,10 @@ func NotFoundError(msg string, args ...interface{}) error { - return New(NotFound, msg, args...) - } - --func RateLimitError(retryAfter time.Duration, msg string, args ...interface{}) error { -+func RateLimitError(errURL string, retryAfter time.Duration, msg string, args ...interface{}) error { - return &BoulderError{ - Type: RateLimit, -- Detail: fmt.Sprintf(msg+": see https://letsencrypt.org/docs/rate-limits/", args...), -+ Detail: fmt.Sprintf(msg+": see "+errURL, args...), - RetryAfter: retryAfter, - } - } diff --git a/patches/ocsp-responder_main.patch b/patches/ocsp-responder_main.patch index 2b710c3..662ad18 100644 --- a/patches/ocsp-responder_main.patch +++ b/patches/ocsp-responder_main.patch @@ -1,8 +1,8 @@ diff --git a/cmd/ocsp-responder/main.go b/cmd/ocsp-responder/main.go -index cede54ee3..1360ed1d6 100644 +index ec03eb05f..1cfe3e20e 100644 --- a/cmd/ocsp-responder/main.go +++ b/cmd/ocsp-responder/main.go -@@ -88,7 +88,7 @@ type Config struct { +@@ -91,7 +91,7 @@ type Config struct { // Configuration for using Redis as a cache. This configuration should // allow for both read and write access. @@ -11,7 +11,7 @@ index cede54ee3..1360ed1d6 100644 // TLS client certificate, private key, and trusted root bundle. TLS cmd.TLSConfig `validate:"required_without=Source,structonly"` -@@ -162,7 +162,7 @@ as generated by Boulder's ceremony command. +@@ -165,7 +165,7 @@ as generated by Boulder's ceremony command. } source, err = responder.NewMemorySourceFromFile(filename, logger) cmd.FailOnError(err, fmt.Sprintf("Couldn't read file: %s", url.Path)) @@ -20,7 +20,7 @@ index cede54ee3..1360ed1d6 100644 // Set up the redis source and the combined multiplex source. rocspRWClient, err := rocsp_config.MakeClient(c.OCSPResponder.Redis, clk, scope) cmd.FailOnError(err, "Could not make redis client") -@@ -206,6 +206,19 @@ as generated by Boulder's ceremony command. +@@ -209,6 +209,19 @@ as generated by Boulder's ceremony command. source, err = redis_responder.NewCheckedRedisSource(rocspSource, dbMap, sac, scope, logger) cmd.FailOnError(err, "Could not create checkedRedis source") diff --git a/patches/ra_ra.patch b/patches/ra_ra.patch index 8afefed..6e02295 100644 --- a/patches/ra_ra.patch +++ b/patches/ra_ra.patch @@ -1,8 +1,8 @@ diff --git a/ra/ra.go b/ra/ra.go -index 3c0f53e22..8c245358d 100644 +index e9f842777..a0d254720 100644 --- a/ra/ra.go +++ b/ra/ra.go -@@ -43,7 +43,6 @@ import ( +@@ -44,7 +44,6 @@ import ( "github.com/letsencrypt/boulder/issuance" blog "github.com/letsencrypt/boulder/log" "github.com/letsencrypt/boulder/metrics" @@ -10,7 +10,7 @@ index 3c0f53e22..8c245358d 100644 "github.com/letsencrypt/boulder/probs" pubpb "github.com/letsencrypt/boulder/publisher/proto" rapb "github.com/letsencrypt/boulder/ra/proto" -@@ -468,7 +467,7 @@ func (ra *RegistrationAuthorityImpl) validateContacts(contacts []string) error { +@@ -584,7 +583,7 @@ func (ra *RegistrationAuthorityImpl) validateContacts(contacts []string) error { if !core.IsASCII(contact) { return berrors.InvalidEmailError("contact email contains non-ASCII characters") } diff --git a/patches/ratelimit_rate-limits.patch b/patches/ratelimit_rate-limits.patch deleted file mode 100644 index 9c0a9ec..0000000 --- a/patches/ratelimit_rate-limits.patch +++ /dev/null @@ -1,35 +0,0 @@ -diff --git a/ratelimit/rate-limits.go b/ratelimit/rate-limits.go -index 3c6bd75d0..ad849a4a5 100644 ---- a/ratelimit/rate-limits.go -+++ b/ratelimit/rate-limits.go -@@ -56,6 +56,7 @@ type Limits interface { - CertificatesPerFQDNSetFast() RateLimitPolicy - NewOrdersPerAccount() RateLimitPolicy - LoadPolicies(contents []byte) error -+ RateLimitsURL() string - } - - // limitsImpl is an unexported implementation of the Limits interface. It acts -@@ -120,6 +121,13 @@ func (r *limitsImpl) NewOrdersPerAccount() RateLimitPolicy { - return r.rlPolicy.NewOrdersPerAccount - } - -+func (r *limitsImpl) RateLimitsURL() string { -+ if r.rlPolicy == nil { -+ return "" -+ } -+ return r.rlPolicy.RateLimitsURL -+} -+ - // LoadPolicies loads various rate limiting policies from a byte array of - // YAML configuration. - func (r *limitsImpl) LoadPolicies(contents []byte) error { -@@ -171,6 +179,8 @@ type rateLimitConfig struct { - // lower threshold and smaller window), so that clients don't have to wait - // a long time after a small burst of accidental duplicate issuance. - CertificatesPerFQDNSetFast RateLimitPolicy `yaml:"certificatesPerFQDNSetFast"` -+ // URL to show in error messages when a rate-limit error is shown -+ RateLimitsURL string `yaml:"rateLimitsURL"` - } - - // RateLimitPolicy describes a general limiting policy diff --git a/patches/ratelimits_names.patch b/patches/ratelimits_names.patch index 2df7b8e..69821fd 100644 --- a/patches/ratelimits_names.patch +++ b/patches/ratelimits_names.patch @@ -1,41 +1,62 @@ diff --git a/ratelimits/names.go b/ratelimits/names.go -index 99221ae0c..6106a34e7 100644 +index 99221ae0c..9abc0d512 100644 --- a/ratelimits/names.go +++ b/ratelimits/names.go -@@ -162,7 +162,11 @@ func validateRegId(id string) error { +@@ -101,6 +101,9 @@ var nameToString = map[Name]string{ + FailedAuthorizationsForPausingPerDomainPerAccount: "FailedAuthorizationsForPausingPerDomainPerAccount", + } + ++// Policy Authority singleton ++var PA *policy.AuthorityImpl ++ + // isValid returns true if the Name is a valid rate limit name. + func (n Name) isValid() bool { + return n > Unknown && n < Name(len(nameToString)) +@@ -162,7 +165,15 @@ func validateRegId(id string) error { // validateDomain validates that the provided string is formatted 'domain', // where domain is a domain name. func validateDomain(id string) error { - err := policy.ValidDomain(id) -+ pa, err := policy.New(nil, nil) -+ if err != nil { -+ return fmt.Errorf("cannot create policy authority implementation") ++ pa := PA ++ var err error ++ if pa == nil { ++ pa, err = policy.New(nil, nil) ++ if err != nil { ++ return fmt.Errorf("cannot create policy authority implementation") ++ } + } + err = pa.ValidDomain(id) if err != nil { return fmt.Errorf("invalid domain, %q must be formatted 'domain': %w", id, err) } -@@ -183,7 +187,11 @@ func validateRegIdDomain(id string) error { +@@ -183,7 +194,14 @@ func validateRegIdDomain(id string) error { return fmt.Errorf( "invalid regId, %q must be formatted 'regId:domain'", id) } - err = policy.ValidDomain(regIdDomain[1]) -+ pa, err := policy.New(nil, nil) -+ if err != nil { -+ return fmt.Errorf("cannot create policy authority implementation") ++ pa := PA ++ if pa == nil { ++ pa, err = policy.New(nil, nil) ++ if err != nil { ++ return fmt.Errorf("cannot create policy authority implementation") ++ } + } + err = pa.ValidDomain(regIdDomain[1]) if err != nil { return fmt.Errorf( "invalid domain, %q must be formatted 'regId:domain': %w", id, err) -@@ -199,7 +207,11 @@ func validateFQDNSet(id string) error { +@@ -199,7 +217,15 @@ func validateFQDNSet(id string) error { return fmt.Errorf( "invalid fqdnSet, %q must be formatted 'fqdnSet'", id) } - return policy.WellFormedDomainNames(domains) -+ pa, err := policy.New(nil, nil) -+ if err != nil { -+ return fmt.Errorf("cannot create policy authority implementation") ++ pa := PA ++ var err error ++ if pa == nil { ++ pa, err = policy.New(nil, nil) ++ if err != nil { ++ return fmt.Errorf("cannot create policy authority implementation") ++ } + } + return pa.WellFormedDomainNames(domains) } diff --git a/patches/test_config_ca.patch b/patches/test_config_ca.patch index 24ed612..1800138 100644 --- a/patches/test_config_ca.patch +++ b/patches/test_config_ca.patch @@ -1,16 +1,34 @@ diff --git a/test/config/ca.json b/test/config/ca.json -index cc4728363..2eb95ad81 100644 +index a61df7e7c..2db5a771d 100644 --- a/test/config/ca.json +++ b/test/config/ca.json -@@ -58,39 +58,6 @@ - "maxValidityBackdate": "1h5m" +@@ -50,7 +50,7 @@ + "allowMustStaple": true, + "maxValidityPeriod": "7776000s", + "maxValidityBackdate": "1h5m", +- "lintConfig": "test/config-next/zlint.toml", ++ "lintConfig": "test/config/zlint.toml", + "ignoredLints": [ + "w_subject_common_name_included", + "w_ext_subject_key_identifier_not_recommended_subscriber" +@@ -64,7 +64,7 @@ + "omitSKID": true, + "maxValidityPeriod": "583200s", + "maxValidityBackdate": "1h5m", +- "lintConfig": "test/config-next/zlint.toml", ++ "lintConfig": "test/config/zlint.toml", + "ignoredLints": [ + "w_ext_subject_key_identifier_missing_sub_cert" + ] +@@ -75,39 +75,6 @@ + "maxBackdate": "1h5m" }, "issuers": [ - { - "active": true, - "issuerURL": "http://ca.example.org:4502/int-ecdsa-a", - "ocspURL": "http://ca.example.org:4002/", -- "crlURLBase": "http://ca.example.org:4501/ecdsa-a/", +- "crlURLBase": "http://ca.example.org:4501/lets-encrypt-crls/43104258997432926/", - "location": { - "configFile": "test/certs/webpki/int-ecdsa-a.pkcs11.json", - "certFile": "test/certs/webpki/int-ecdsa-a.cert.pem", @@ -21,7 +39,7 @@ index cc4728363..2eb95ad81 100644 - "active": true, - "issuerURL": "http://ca.example.org:4502/int-ecdsa-b", - "ocspURL": "http://ca.example.org:4002/", -- "crlURLBase": "http://ca.example.org:4501/ecdsa-b/", +- "crlURLBase": "http://ca.example.org:4501/lets-encrypt-crls/17302365692836921/", - "location": { - "configFile": "test/certs/webpki/int-ecdsa-b.pkcs11.json", - "certFile": "test/certs/webpki/int-ecdsa-b.cert.pem", @@ -32,7 +50,7 @@ index cc4728363..2eb95ad81 100644 - "active": false, - "issuerURL": "http://ca.example.org:4502/int-ecdsa-c", - "ocspURL": "http://ca.example.org:4002/", -- "crlURLBase": "http://ca.example.org:4501/ecdsa-c/", +- "crlURLBase": "http://ca.example.org:4501/lets-encrypt-crls/56560759852043581/", - "location": { - "configFile": "test/certs/webpki/int-ecdsa-c.pkcs11.json", - "certFile": "test/certs/webpki/int-ecdsa-c.cert.pem", @@ -42,7 +60,7 @@ index cc4728363..2eb95ad81 100644 { "active": true, "issuerURL": "http://ca.example.org:4502/int-rsa-a", -@@ -101,28 +68,6 @@ +@@ -118,28 +85,6 @@ "certFile": "test/certs/webpki/int-rsa-a.cert.pem", "numSessions": 2 } @@ -51,7 +69,7 @@ index cc4728363..2eb95ad81 100644 - "active": true, - "issuerURL": "http://ca.example.org:4502/int-rsa-b", - "ocspURL": "http://ca.example.org:4002/", -- "crlURLBase": "http://ca.example.org:4501/rsa-b/", +- "crlURLBase": "http://ca.example.org:4501/lets-encrypt-crls/6762885421992935/", - "location": { - "configFile": "test/certs/webpki/int-rsa-b.pkcs11.json", - "certFile": "test/certs/webpki/int-rsa-b.cert.pem", @@ -62,12 +80,12 @@ index cc4728363..2eb95ad81 100644 - "active": false, - "issuerURL": "http://ca.example.org:4502/int-rsa-c", - "ocspURL": "http://ca.example.org:4002/", -- "crlURLBase": "http://ca.example.org:4501/rsa-c/", +- "crlURLBase": "http://ca.example.org:4501/lets-encrypt-crls/56183656833365902/", - "location": { - "configFile": "test/certs/webpki/int-rsa-c.pkcs11.json", - "certFile": "test/certs/webpki/int-rsa-c.cert.pem", - "numSessions": 2 - } } - ], - "lintConfig": "test/config/zlint.toml", + ] + }, diff --git a/patches/updater_updater.patch b/patches/updater_updater.patch index a5097dc..ecf6859 100644 --- a/patches/updater_updater.patch +++ b/patches/updater_updater.patch @@ -1,13 +1,13 @@ diff --git a/crl/updater/updater.go b/crl/updater/updater.go -index fec242794..ecda37738 100644 +index 2f76672fd..143b66a5e 100644 --- a/crl/updater/updater.go +++ b/crl/updater/updater.go -@@ -231,7 +231,7 @@ func (cu *crlUpdater) updateShard(ctx context.Context, atTime time.Time, issuerN - crlEntries = append(crlEntries, entry) +@@ -300,7 +300,7 @@ func (cu *crlUpdater) updateShard(ctx context.Context, atTime time.Time, issuerN + return fmt.Errorf("streaming GetRevokedCerts: %w", err) } - cu.log.Infof( + cu.log.Debugf( "Queried SA for CRL shard: id=[%s] expiresAfter=[%s] expiresBefore=[%s] numEntries=[%d]", - crlID, chunk.start, chunk.end, len(crlEntries)) + crlID, chunk.start, chunk.end, n) } diff --git a/patches/wfe2_main.patch b/patches/wfe2_main.patch index f436761..6260f8e 100644 --- a/patches/wfe2_main.patch +++ b/patches/wfe2_main.patch @@ -1,8 +1,24 @@ diff --git a/cmd/boulder-wfe2/main.go b/cmd/boulder-wfe2/main.go -index 699ed0d78..01ae1f741 100644 +index 625026cfe..9234d0b97 100644 --- a/cmd/boulder-wfe2/main.go +++ b/cmd/boulder-wfe2/main.go -@@ -105,7 +105,7 @@ type Config struct { +@@ -12,6 +12,7 @@ import ( + + "github.com/letsencrypt/boulder/cmd" + "github.com/letsencrypt/boulder/config" ++ "github.com/letsencrypt/boulder/core" + "github.com/letsencrypt/boulder/features" + "github.com/letsencrypt/boulder/goodkey" + "github.com/letsencrypt/boulder/goodkey/sagoodkey" +@@ -19,6 +20,7 @@ import ( + "github.com/letsencrypt/boulder/grpc/noncebalancer" + "github.com/letsencrypt/boulder/issuance" + "github.com/letsencrypt/boulder/nonce" ++ "github.com/letsencrypt/boulder/policy" + rapb "github.com/letsencrypt/boulder/ra/proto" + "github.com/letsencrypt/boulder/ratelimits" + bredis "github.com/letsencrypt/boulder/redis" +@@ -108,7 +110,7 @@ type Config struct { // DirectoryCAAIdentity is used for the /directory response's "meta" // element's "caaIdentities" field. It should match the VA's "issuerDomain" // configuration value (this value is the one used to enforce CAA) @@ -11,7 +27,7 @@ index 699ed0d78..01ae1f741 100644 // DirectoryWebsite is used for the /directory response's "meta" element's // "website" field. DirectoryWebsite string `validate:"required,url"` -@@ -192,6 +192,8 @@ type Config struct { +@@ -195,6 +197,8 @@ type Config struct { // to enable the pausing feature. URL string `validate:"omitempty,required_with=HMACKey JWTLifetime,url,startswith=https://,endsnotwith=/"` } @@ -20,11 +36,34 @@ index 699ed0d78..01ae1f741 100644 } Syslog cmd.SyslogConfig -@@ -403,6 +405,7 @@ func main() { +@@ -360,11 +364,22 @@ func main() { + var limiter *ratelimits.Limiter + var txnBuilder *ratelimits.TransactionBuilder + var limiterRedis *bredis.Ring ++ var pa *policy.AuthorityImpl + if c.WFE.Limiter.Defaults != "" { + // Setup rate limiting. + limiterRedis, err = bredis.NewRingFromConfig(*c.WFE.Limiter.Redis, stats, logger) + cmd.FailOnError(err, "Failed to create Redis ring") + ++ // Set Policy Authority for ratelimits ++ pa, err = policy.New(map[core.AcmeChallenge]bool{}, logger) ++ cmd.FailOnError(err, "Couldn't create PA") ++ if c.WFE.HostnamePolicyFile == "" { ++ cmd.Fail("HostnamePolicyFile must be provided.") ++ } ++ err = pa.LoadHostnamePolicyFile(c.WFE.HostnamePolicyFile) ++ cmd.FailOnError(err, "Couldn't load hostname policy file") ++ ratelimits.PA = pa ++ + source := ratelimits.NewRedisSource(limiterRedis.Ring, clk, stats) + limiter, err = ratelimits.NewLimiter(clk, source, stats) + cmd.FailOnError(err, "Failed to create rate limiter") +@@ -406,6 +421,7 @@ func main() { unpauseSigner, c.WFE.Unpause.JWTLifetime.Duration, c.WFE.Unpause.URL, -+ c.WFE.HostnamePolicyFile, ++ pa, ) cmd.FailOnError(err, "Unable to create WFE") diff --git a/patches/wfe2_wfe.patch b/patches/wfe2_wfe.patch index 4228628..5dd2e5d 100644 --- a/patches/wfe2_wfe.patch +++ b/patches/wfe2_wfe.patch @@ -1,64 +1,49 @@ diff --git a/wfe2/wfe.go b/wfe2/wfe.go -index a41472e54..42d2974c4 100644 +index d15a661e8..e45d5789a 100644 --- a/wfe2/wfe.go +++ b/wfe2/wfe.go -@@ -24,6 +24,7 @@ import ( - "google.golang.org/protobuf/types/known/durationpb" - "google.golang.org/protobuf/types/known/emptypb" - -+ "github.com/letsencrypt/boulder/cmd" - "github.com/letsencrypt/boulder/core" - corepb "github.com/letsencrypt/boulder/core/proto" - berrors "github.com/letsencrypt/boulder/errors" -@@ -171,6 +172,8 @@ type WebFrontEndImpl struct { +@@ -171,6 +171,8 @@ type WebFrontEndImpl struct { // descriptions (perhaps including URLs) of those profiles. NewOrder // Requests with a profile name not present in this map will be rejected. certProfiles map[string]string + -+ hostnamePolicyFile string ++ pa *policy.AuthorityImpl } // NewWebFrontEndImpl constructs a web service for Boulder -@@ -198,6 +201,7 @@ func NewWebFrontEndImpl( +@@ -198,6 +200,7 @@ func NewWebFrontEndImpl( unpauseSigner unpause.JWTSigner, unpauseJWTLifetime time.Duration, unpauseURL string, -+ hostnamePolicyFile string, ++ pa *policy.AuthorityImpl, ) (WebFrontEndImpl, error) { if len(issuerCertificates) == 0 { return WebFrontEndImpl{}, errors.New("must provide at least one issuer certificate") -@@ -239,6 +243,7 @@ func NewWebFrontEndImpl( +@@ -215,6 +218,10 @@ func NewWebFrontEndImpl( + return WebFrontEndImpl{}, errors.New("must provide a service for nonce redemption") + } + ++ if pa == nil { ++ return WebFrontEndImpl{}, errors.New("must provide a policy authority") ++ } ++ + wfe := WebFrontEndImpl{ + log: logger, + clk: clk, +@@ -239,6 +246,7 @@ func NewWebFrontEndImpl( unpauseSigner: unpauseSigner, unpauseJWTLifetime: unpauseJWTLifetime, unpauseURL: unpauseURL, -+ hostnamePolicyFile: hostnamePolicyFile, ++ pa: pa, } return wfe, nil -@@ -2302,8 +2307,25 @@ func (wfe *WebFrontEndImpl) NewOrder( - names[i] = ident.Value +@@ -2274,7 +2282,7 @@ func (wfe *WebFrontEndImpl) NewOrder( } -+ logger := cmd.NewLogger(cmd.SyslogConfig{StdoutLevel: 7}) -+ pa, err := policy.New(map[core.AcmeChallenge]bool{}, logger) -+ if err != nil { -+ wfe.sendError(response, logEvent, probs.Malformed("cannot create policy authority implementation"), nil) -+ return -+ } -+ -+ if wfe.hostnamePolicyFile == "" { -+ wfe.sendError(response, logEvent, probs.Malformed("HostnamePolicyFile must be provided in config"), nil) -+ return -+ } -+ err = pa.LoadHostnamePolicyFile(wfe.hostnamePolicyFile) -+ if err != nil { -+ wfe.sendError(response, logEvent, probs.Malformed("couldn't load hostname policy file"), nil) -+ return -+ } -+ names = core.UniqueLowerNames(names) - err = policy.WellFormedDomainNames(names) -+ err = pa.WellFormedDomainNames(names) ++ err = wfe.pa.WellFormedDomainNames(names) if err != nil { wfe.sendError(response, logEvent, web.ProblemDetailsForError(err, "Invalid identifiers requested"), nil) return diff --git a/utils.sh b/utils.sh index 103e693..6748a79 100644 --- a/utils.sh +++ b/utils.sh @@ -10,6 +10,7 @@ export PS_CONTROL="tcpserver" export PS_NGINX="nginx:" export PS_CONSUL="consul" export PS_PKILINT="pkilint" +export PS_REDIS="redis-server" LOOPCOUNT=120 @@ -36,6 +37,9 @@ count() { $PS_PKILINT) prefix="docker exec $(docker ps --format "{{.Names}}" | grep -- -bpkilint-) " ;; + $PS_REDIS) + prefix="" + ;; *) ;; esac