Merge branch 'release/0.8.6'

* release/0.8.6:
  Fix logging; fix patch file
  Optional extension of timeout values
  Optionally use different branch than master
  Bump boulder version to release-2018-06-28
  Make golint happy

docker-compose.patch

@@ -1,20 +1,22 @@
 diff --git a/docker-compose.yml b/docker-compose.yml
-index b0e2c1f1..30426452 100644
+index 87840f02..fc6eae34 100644
 --- a/docker-compose.yml
 +++ b/docker-compose.yml
-@@ -6,9 +6,10 @@ services:
+@@ -6,11 +6,12 @@ services:
          environment:
-             FAKE_DNS: 127.0.0.1
+             FAKE_DNS: 10.77.77.77
              PKCS11_PROXY_SOCKET: tcp://boulder-hsm:5657
 -            BOULDER_CONFIG_DIR: test/config
 +            BOULDER_CONFIG_DIR: labca/config
+             GO111MODULE: "on"
+             GOFLAGS: "-mod=vendor"
          volumes:
            - .:/go/src/github.com/letsencrypt/boulder
 +          - /home/labca/boulder_labca:/go/src/github.com/letsencrypt/boulder/labca
            - ./.gocache:/root/.cache/go-build
          networks:
            bluenet:
-@@ -47,8 +48,14 @@ services:
+@@ -51,8 +52,14 @@ services:
          depends_on:
            - bhsm
            - bmysql
@@ -29,8 +31,8 @@ index b0e2c1f1..30426452 100644
 +        restart: always
      bhsm:
          # To minimize fetching this should be the same version used above
-         image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.11.2}:2018-11-19
-@@ -61,8 +68,16 @@ services:
+         image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.12}:2019-04-08
+@@ -65,8 +72,16 @@ services:
            bluenet:
              aliases:
                - boulder-hsm
@@ -47,19 +49,22 @@ index b0e2c1f1..30426452 100644
          networks:
            bluenet:
              aliases:
-@@ -72,16 +87,37 @@ services:
+@@ -75,20 +90,36 @@ services:
+             MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
          command: mysqld --bind-address=0.0.0.0
          logging:
-             driver: none
+-            driver: none
 -    netaccess:
-+        logging:
 +          driver: "json-file"
 +          options:
 +            max-size: "500k"
 +            max-file: "5"
 +        restart: always
 +    labca:
-         image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.11.2}:2018-11-19
+         image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.12}:2019-04-08
+-        environment:
+-            GO111MODULE: "on"
+-            GOFLAGS: "-mod=vendor"
          networks:
            - bluenet
          volumes:

gui/apply-boulder

@@ -2,20 +2,38 @@
 
 set -e
 
-perl -i -p0e "s/(\"dnsResolvers\": \[\n).*?(\s+\],)/\1      \"$PKI_DNS\"\2/igs" config/ra.json
 perl -i -p0e "s/(\"dnsResolvers\": \[\n).*?(\s+\],)/\1      \"$PKI_DNS\"\2/igs" config/va.json
 perl -i -p0e "s/(\"dnsResolvers\": \[\n).*?(\s+\],)/\1      \"$PKI_DNS\"\2/igs" config/expiration-mailer.json
 sed -i -e "s/\"issuerDomain\": \".*\"/\"issuerDomain\": \"$PKI_DOMAIN\"/" config/va.json
 sed -i -e "s/\"directoryCAAIdentity\": \".*\"/\"directoryCAAIdentity\": \"$PKI_DOMAIN\"/" config/wfe.json
 sed -i -e "s/\"directoryCAAIdentity\": \".*\"/\"directoryCAAIdentity\": \"$PKI_DOMAIN\"/" config/wfe2.json
 
-[ -e ../test/hostname-policy.json ] && cp ../test/hostname-policy.json ./ || true
-[ -e ../boulder/test/hostname-policy.json ] && cp ../boulder/test/hostname-policy.json ./ || true
+[ -e ../test/hostname-policy.yaml ] && cp ../test/hostname-policy.yaml ./ || true
+[ -e ../boulder/test/hostname-policy.yaml ] && cp ../boulder/test/hostname-policy.yaml ./ || true
+[ -e hostname-policy.json ] && rm hostname-policy.json || true
 if [ "$PKI_DOMAIN_MODE" == "lockdown" ]; then
-    sed -i -e "s/  \]$/  \],\n  \"Lockdown\": \[\n    \"$PKI_LOCKDOWN_DOMAINS\"\n&/" hostname-policy.json
+    echo "Lockdown:" >> hostname-policy.yaml
+    echo "  - \"$PKI_LOCKDOWN_DOMAINS\"" >> hostname-policy.yaml
 fi
 if [ "$PKI_DOMAIN_MODE" == "whitelist" ]; then
-    sed -i -e "s/  \]$/  \],\n  \"Whitelist\": \[\n    \"$PKI_WHITELIST_DOMAINS\"\n&/" hostname-policy.json
+    echo "Whitelist:" >> hostname-policy.yaml
+    echo "  - \"$PKI_LOCKDOWN_DOMAINS\"" >> hostname-policy.yaml
+fi
+
+if [ "$PKI_EXTENDED_TIMEOUT" == "1" ]; then
+    sed -i -e "s/\"timeout\": \"15s\"/\"timeout\": \"30s\"/" config/ca-a.json
+    sed -i -e "s/\"timeout\": \"15s\"/\"timeout\": \"30s\"/" config/ca-b.json
+    sed -i -e "s/\"timeout\": \"15s\"/\"timeout\": \"30s\"/" config/admin-revoker.json
+    sed -i -e "s/\"timeout\": \"15s\"/\"timeout\": \"30s\"/" config/wfe.json
+    sed -i -e "s/\"timeout\": \"20s\"/\"timeout\": \"40s\"/" config/wfe.json
+    sed -i -e "s/\"timeout\": \"15s\"/\"timeout\": \"30s\"/" config/wfe2.json
+    sed -i -e "s/\"timeout\": \"20s\"/\"timeout\": \"40s\"/" config/wfe2.json
+    sed -i -e "s/\"timeout\": \"15s\"/\"timeout\": \"30s\"/" config/ca.json
+    sed -i -e "s/\"timeout\": \"15s\"/\"timeout\": \"30s\"/" config/expiration-mailer.json
+    sed -i -e "s/\"timeout\": \"15s\"/\"timeout\": \"30s\"/" config/ra.json
+    sed -i -e "s/\"timeout\": \"20s\"/\"timeout\": \"40s\"/" config/ra.json
+    sed -i -e "s/\"timeout\": \"15s\"/\"timeout\": \"30s\"/" config/ocsp-updater.json
+    sed -i -e "s/\"timeout\": \"15s\"/\"timeout\": \"30s\"/" config/orphan-finder.json
 fi
 
 sed -i -e "s/\"server\": \".*\"/\"server\": \"$PKI_EMAIL_SERVER\"/" config/expiration-mailer.json

gui/certificate.go

@@ -322,11 +322,8 @@ func (ci *CertificateInfo) Upload(path string, certBase string, tmpKey string, t
 		return reportError(err)
 	}
 
-	if err := ioutil.WriteFile(tmpCert, []byte(ci.Certificate), 0644); err != nil {
-		return err
-	}
-
-	return nil
+	err := ioutil.WriteFile(tmpCert, []byte(ci.Certificate), 0644)
+	return err
 }
 
 // ImportCerts imports both the root and the issuer certificates

gui/main.go

@@ -150,6 +150,7 @@ type SetupConfig struct {
 	DomainMode       string
 	LockdownDomains  string
 	WhitelistDomains string
+	ExtendedTimeout  bool
 	RequestBase      string
 	Errors           map[string]string
 }
@@ -428,6 +429,7 @@ func _configUpdateHandler(w http.ResponseWriter, r *http.Request) {
 		DomainMode:       r.Form.Get("domain_mode"),
 		LockdownDomains:  r.Form.Get("lockdown_domains"),
 		WhitelistDomains: r.Form.Get("whitelist_domains"),
+		ExtendedTimeout:  (r.Form.Get("extended_timeout") == "true"),
 	}
 
 	res := struct {
@@ -477,6 +479,12 @@ func _configUpdateHandler(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 
+		extendedTimeout := cfg.ExtendedTimeout
+		if extendedTimeout != viper.GetBool("labca.extended_timeout") {
+			delta = true
+			viper.Set("labca.extended_timeout", cfg.ExtendedTimeout)
+		}
+
 		if delta {
 			viper.WriteConfig()
 
@@ -994,6 +1002,7 @@ func _manageGet(w http.ResponseWriter, r *http.Request) {
 	if domainMode == "whitelist" {
 		manageData["WhitelistDomains"] = viper.GetString("labca.whitelist")
 	}
+	manageData["ExtendedTimeout"] = viper.GetBool("labca.extended_timeout")
 
 	manageData["DoEmail"] = viper.GetBool("labca.email.enable")
 	manageData["Server"] = viper.GetString("labca.email.server")
@@ -1437,6 +1446,11 @@ func _applyConfig() error {
 	os.Setenv("PKI_DOMAIN_MODE", viper.GetString("labca.domain_mode"))
 	os.Setenv("PKI_LOCKDOWN_DOMAINS", viper.GetString("labca.lockdown"))
 	os.Setenv("PKI_WHITELIST_DOMAINS", viper.GetString("labca.whitelist"))
+	if viper.GetBool("labca.extended_timeout") {
+		os.Setenv("PKI_EXTENDED_TIMEOUT", "1")
+	} else {
+		os.Setenv("PKI_EXTENDED_TIMEOUT", "0")
+	}
 	if viper.GetBool("labca.email.enable") {
 		os.Setenv("PKI_EMAIL_SERVER", viper.GetString("labca.email.server"))
 		os.Setenv("PKI_EMAIL_PORT", viper.GetString("labca.email.port"))

gui/templates/base.tmpl

@@ -32,7 +32,7 @@
                         <small>{{ if .Version }}Version {{ .Version }}{{ end }}</small>
                     </div>
                     <div class="col-sm-6 footer text-muted text-right" id="footer">
-                        <small>Copyright &copy; 2018 LabCA</small>
+                        <small>Copyright &copy; 2018-2019 LabCA</small>
                     </div>
                 </div>
             </div>

gui/templates/views/manage.tmpl

@@ -163,6 +163,11 @@
 
         <input type="radio" id="standard" name="domain_mode" value="standard" {{ if eq .DomainMode "standard"}}checked{{ end }}/> Standard - any official domains<br/>
       </div>
+      <div class="form-group">
+        <label>Extended timeout:</label><br/>
+        <input type="checkbox" class="extended_timeout" id="extended_timeout" value="extend" {{ if .ExtendedTimeout }}checked{{ end }}></input>
+        &nbsp;If you see timeout related errors on the Dashboard / Audit Log, try checking this box.
+      </div>
       <div class="form-group">
         <span class="hidden" id="update-config-result"></span>
         <button class="btn btn-default" type="button" id="update-config" title="Update the system configuration">Update</button>
@@ -489,6 +494,7 @@
                             domain_mode: ($("#standard").prop('checked') ? 'standard' : ($("#whitelist").prop('checked') ? 'whitelist' : 'lockdown')),
                             lockdown_domains: $("#lockdown_domains").val(),
                             whitelist_domains: $("#whitelist_domains").val(),
+                            extended_timeout: $("#extended_timeout").prop("checked"),
                         },
                     })
                     .done(function(data) {

install

@@ -24,7 +24,7 @@ dockerComposeVersion="1.22.0"
 
 labcaUrl="https://github.com/hakwerk/labca/"
 boulderUrl="https://github.com/letsencrypt/boulder/"
-boulderTag="release-2018-12-10"
+boulderTag="release-2019-06-28"
 
 #
 # Color configuration
@@ -49,6 +49,9 @@ wait_up() {
 dn=$(dirname $0)
 source "$dn/utils.sh" &>/dev/null || true
 
+cmdlineFqdn=""
+cmdlineBranch=""
+
 #
 # Helper functions for informing the user and logging to file
 #
@@ -136,14 +139,22 @@ labca_user() {
 clone_repo() {
     local dir="$1"
     local url="$2"
+    local branch="$3"
 
     local msg="Clone $url to $dir"
     msg_info "$msg"
     sudo -u labca git clone -q "$url" "$dir" &>>$installLog && msg_ok "$msg" || msg_fatal "Could not clone git repository"
+
+    if [ "$branch" != "" ]; then
+        cd "$dir"
+        sudo -u labca git checkout $branch &>>$installLog
+        cd - >/dev/null
+    fi
 }
 
 pull_repo() {
     local dir="$1"
+    local branch="$2"
 
     cd "$dir" &>>$installLog || msg_fatal "Could not switch to directory '$dir'"
 
@@ -152,11 +163,18 @@ pull_repo() {
     sudo -u labca git stash --all --quiet &>>$installLog || true
     sudo -u labca git clean --quiet --force -d &>>$installLog || true
     sudo -u labca git pull --quiet &>>$installLog && msg_ok "$msg" || msg_fatal "Could not update local repository"
+
+    if [ "$branch" != "" ]; then
+        cd "$dir"
+        sudo -u labca git checkout $branch &>>$installLog
+        cd - >/dev/null
+    fi
 }
 
 clone_or_pull() {
     local dir="$1"
     local url="$2"
+    local branch="$3"
 
     local parentdir=$(dirname "$dir")
     local dirbase=$(basename "$dir")
@@ -168,12 +186,12 @@ clone_or_pull() {
         if [ $rc -gt 0 ]; then
             cd "$parentdir"
             mv "$dirbase" "${dirbase}_${runId}" && msg_ok "Backup existing non-git directory '$dir'"
-            clone_repo "$dir" "$url"
+            clone_repo "$dir" "$url" "$branch"
         else
-            pull_repo "$dir"
+            pull_repo "$dir" "$branch"
         fi
     else
-        clone_repo "$dir" "$url"
+        clone_repo "$dir" "$url" "$branch"
     fi
 }
 
@@ -209,22 +227,20 @@ prompt_and_export() {
     fi
 }
 
-# Determine the remote address of this machine from (in order): commandline parameter,
-# existing configuration or full hostname.
-get_fqdn() {
-    local cfgFile="$adminDir/data/config.json"
-    local cfgFqdn=$(grep fqdn $cfgFile 2>/dev/null | grep -v LABCA_FQDN | cut -d ":" -f 2- | tr -d " \",")
-    LABCA_FQDN=${cfgFqdn:-$(hostname -f)}
-
-    local parsed=$(getopt --options=n: --longoptions=name:,fqdn: --name "$0" -- "$@" 2>>$installLog) || msg_fatal "Could not process commandline parameters"
+# Parse the command line options, if any
+parse_cmdline() {
+    local parsed=$(getopt --options=n:,b: --longoptions=name:,fqdn:,branch: --name "$0" -- "$@" 2>>$installLog) || msg_fatal "Could not process commandline parameters"
     eval set -- "$parsed"
-    local cmdlineFqdn
     while true; do
         case "$1" in
             -n|--name|--fqdn)
                 cmdlineFqdn="$2"
                 shift 2
                 ;;
+            -b|--branch)
+                cmdlineBranch="$2"
+                shift 2
+                ;;
             --)
                 shift
                 break
@@ -234,6 +250,14 @@ get_fqdn() {
                 ;;
         esac
     done
+}
+
+# Determine the remote address of this machine from (in order): commandline parameter,
+# existing configuration or full hostname.
+get_fqdn() {
+    local cfgFile="$adminDir/data/config.json"
+    local cfgFqdn=$(grep fqdn $cfgFile 2>/dev/null | grep -v LABCA_FQDN | cut -d ":" -f 2- | tr -d " \",")
+    LABCA_FQDN=${cfgFqdn:-$(hostname -f)}
 
     if [ "$cfgFqdn" == "" ]; then
         if [ "$cmdlineFqdn" != "" ]; then
@@ -417,6 +441,8 @@ get_boulder() {
 
     cd "$boulderDir"
     sudo -u labca git reset --hard $boulderTag &>>$installLog
+    sudo -u labca cp sa/_db-next/migrations/20190221140139_AddAuthz2.sql sa/_db/migrations/
+    sudo -u labca cp sa/_db-next/migrations/20190524120239_AddAuthz2ExpiresIndex.sql sa/_db/migrations/
     msg_ok "Boulder checkout '$boulderTag'"
 }
 
@@ -480,6 +506,7 @@ config_boulder() {
     sed -i -e "s/test-ca2.pem/test-ca.pem/" config/ocsp-responder.json
     sed -i -e "s/test-ca2.pem/test-ca.pem/" config/ocsp-updater.json
     sed -i -e "s/test-ca2.pem/test-ca.pem/" config/publisher.json
+    sed -i -e "s/test-ca2.pem/test-ca.pem/" config/ra.json
     sed -i -e "s/test-ca2.pem/test-ca.pem/" config/wfe.json
     sed -i -e "s/test-ca2.pem/test-ca.pem/" config/wfe2.json
     sed -i -e "s/5001/443/g" config/va.json
@@ -637,10 +664,12 @@ main() {
     [ -e $this ] || this="$curdir/$0"
     local checksum=$(md5sum $this 2>/dev/null | cut -d' ' -f1)
     [ ! -e "$cloneDir/cron_d" ] || chown labca:labca "$cloneDir/cron_d"
-    clone_or_pull "$cloneDir" "$labcaUrl"
+
+    parse_cmdline "$@"
+    clone_or_pull "$cloneDir" "$labcaUrl" "$cmdlineBranch"
     restart_if_updated "$checksum"
 
-    get_fqdn "$@"
+    get_fqdn
     copy_admin
 
     update_upgrade

policy_pa.patch

@@ -1,47 +1,48 @@
 diff --git a/policy/pa.go b/policy/pa.go
-index a8337bf7..83150102 100644
+index 3d097365..53cf6020 100644
 --- a/policy/pa.go
 +++ b/policy/pa.go
-@@ -29,6 +29,8 @@ type AuthorityImpl struct {
- 	blacklist              map[string]bool
- 	exactBlacklist         map[string]bool
- 	wildcardExactBlacklist map[string]bool
+@@ -30,6 +30,8 @@ type AuthorityImpl struct {
+ 	blocklist              map[string]bool
+ 	exactBlocklist         map[string]bool
+ 	wildcardExactBlocklist map[string]bool
 +	whitelist              map[string]bool
 +	lockdown               map[string]bool
- 	blacklistMu            sync.RWMutex
+ 	blocklistMu            sync.RWMutex
  
- 	enabledChallenges          map[string]bool
-@@ -53,6 +55,8 @@ func New(challengeTypes map[string]bool) (*AuthorityImpl, error) {
- type blacklistJSON struct {
- 	Blacklist      []string
- 	ExactBlacklist []string
-+	Whitelist      []string
-+	Lockdown       []string
+ 	enabledChallenges map[string]bool
+@@ -70,6 +72,9 @@ type blockedNamesPolicy struct {
+ 	// time above and beyond the high-risk domains. Managing these entries separately
+ 	// from HighRiskBlockedNames makes it easier to vet changes accurately.
+ 	AdminBlockedNames []string `yaml:"AdminBlockedNames"`
++
++	Whitelist []string `yaml:"Whitelist"`
++	Lockdown  []string `yaml:"Lockdown"`
  }
  
  // SetHostnamePolicyFile will load the given policy file, returning error if it
-@@ -103,10 +107,20 @@ func (pa *AuthorityImpl) loadHostnamePolicy(b []byte) error {
+@@ -138,10 +143,20 @@ func (pa *AuthorityImpl) processHostnamePolicy(policy blockedNamesPolicy) error
  		// wildcardNameMap to block issuance for `*.`+parts[1]
  		wildcardNameMap[parts[1]] = true
  	}
 +	whiteMap := make(map[string]bool)
-+	for _, v := range bl.Whitelist {
++	for _, v := range policy.Whitelist {
 +		whiteMap[v] = true
 +	}
 +	lockMap := make(map[string]bool)
-+	for _, v := range bl.Lockdown {
++	for _, v := range policy.Lockdown {
 +		lockMap[v] = true
 +	}
- 	pa.blacklistMu.Lock()
- 	pa.blacklist = nameMap
- 	pa.exactBlacklist = exactNameMap
- 	pa.wildcardExactBlacklist = wildcardNameMap
+ 	pa.blocklistMu.Lock()
+ 	pa.blocklist = nameMap
+ 	pa.exactBlocklist = exactNameMap
+ 	pa.wildcardExactBlocklist = wildcardNameMap
 +	pa.whitelist = whiteMap
 +	pa.lockdown = lockMap
- 	pa.blacklistMu.Unlock()
+ 	pa.blocklistMu.Unlock()
  	return nil
  }
-@@ -288,6 +302,14 @@ func (pa *AuthorityImpl) WillingToIssue(id core.AcmeIdentifier) error {
+@@ -287,6 +302,14 @@ func (pa *AuthorityImpl) WillingToIssue(id identifier.ACMEIdentifier) error {
  		}
  	}
  
@@ -54,15 +55,15 @@ index a8337bf7..83150102 100644
 +	}
 +
  	// Names must end in an ICANN TLD, but they must not be equal to an ICANN TLD.
- 	icannTLD, err := extractDomainIANASuffix(domain)
+ 	icannTLD, err := iana.ExtractSuffix(domain)
  	if err != nil {
-@@ -413,6 +435,31 @@ func (pa *AuthorityImpl) checkHostLists(domain string) error {
+@@ -304,6 +327,31 @@ func (pa *AuthorityImpl) WillingToIssue(id identifier.ACMEIdentifier) error {
  	return nil
  }
  
-+func (pa *AuthorityImpl) checkWhitelist(domain string) (bool,error) {
-+	pa.blacklistMu.RLock()
-+	defer pa.blacklistMu.RUnlock()
++func (pa *AuthorityImpl) checkWhitelist(domain string) (bool, error) {
++	pa.blocklistMu.RLock()
++	defer pa.blocklistMu.RUnlock()
 +
 +	if (pa.whitelist == nil) || (pa.lockdown == nil) {
 +		return false, fmt.Errorf("Hostname policy not yet loaded.")
@@ -78,13 +79,13 @@ index a8337bf7..83150102 100644
 +
 +	if len(pa.lockdown) > 0 {
 +		// In Lockdown mode, the domain MUST be in the list, so return an error if not found
-+		return false, errBlacklisted
++		return false, errPolicyForbidden
 +	} else {
 +		// In Whitelist mode, if the domain is not in the list, continue with the other checks
 +		return false, nil
 +	}
 +}
 +
- // ChallengesFor makes a decision of what challenges, and combinations, are
- // acceptable for the given identifier. If the TLSSNIRevalidation feature flag
- // is set, create TLS-SNI-01 challenges for revalidation requests even if
+ // WillingToIssueWildcards is an extension of WillingToIssue that accepts DNS
+ // identifiers for well formed wildcard domains in addition to regular
+ // identifiers.