Run nginx as docker container instead of on the host system (#36)

This commit is contained in:
Arjan H
2022-04-02 13:01:52 +02:00
parent 8f97390494
commit 954d9bb014
56 changed files with 117 additions and 91 deletions

View File

@@ -97,7 +97,7 @@ The end users in your organization / lab can visit the public pages of you LabCA
## Troubleshooting
After installing sometimes the application is not starting up properly and it can be quite hard to figure out why. Some log files to check in case of issues are:
* /etc/nginx/ssl/acme_tiny.log
* /home/labca/nginx_data/ssl/acme_tiny.log
* /home/labca/logs/commander.log
* cd /home/labca/boulder; docker-compose logs labca
* cd /home/labca/boulder; docker-compose logs boulder
@@ -106,7 +106,7 @@ After installing sometimes the application is not starting up properly and it ca
### Common error messages
If you get "**No valid IP addresses found for <hostname>**" in /etc/nginx/ssl/acme_tiny.log, solve it by entering the hostname in your local DNS. Same for "**Could not resolve host: <hostname>**" in /var/log/labca.err.
If you get "**No valid IP addresses found for <hostname>**" in /home/labca/nginx_data/ssl/acme_tiny.log, solve it by entering the hostname in your local DNS. Same for "**Could not resolve host: <hostname>**" in /var/log/labca.err.
When issuing a certificate, LabCA/boulder checks for CAA (Certification Authority Authorization) records in DNS, which specify what CAs are allowed to issue certificates for the domain. If you get an error like "**SERVFAIL looking up CAA for internal**" or "**CAA record for ca01.foo.internal prevents issuance**", you can try to add something like this to your DNS domain:
```

View File

@@ -202,7 +202,7 @@ if __name__ == "__main__": # pragma: no cover
## openssl genrsa 4096 > account.key
## openssl genrsa 4096 > domain.key
## openssl req -new -sha256 -key domain.key -subj "/" -reqexts SAN -config <(cat /etc/ssl/openssl.cnf <(printf "[SAN]\nsubjectAltName=DNS:tessie.hakwerk.local")) > domain.csr
## python acme_tiny.py --account-key ./account.key --csr ./domain.csr --acme-dir /var/www/html/.well-known/acme-challenge/ > domain_chain.crt
## python acme_tiny.py --account-key ./account.key --csr ./domain.csr --acme-dir /home/labca/nginx_data/static/.well-known/acme-challenge/ > domain_chain.crt
## cp domain_chain.crt ~/boulder/test/
## docker exec -ti boulder_boulder_1 /bin/bash

2
backup
View File

@@ -15,7 +15,7 @@ mkdir -p /home/labca/backup
cd /home/labca/boulder
docker-compose exec -T bmysql mysqldump boulder_sa_integration >$TMPDIR/boulder_sa_integration.sql
cp -p /etc/nginx/ssl/*key* /etc/nginx/ssl/*cert.pem /etc/nginx/ssl/*.csr $TMPDIR/
cp -p /home/labca/nginx_data/ssl/*key* /home/labca/nginx_data/ssl/*cert.pem /home/labca/nginx_data/ssl/*.csr $TMPDIR/
cp -rp /home/labca/admin/data $TMPDIR/

View File

@@ -38,7 +38,7 @@ function wait_server() {
read txt
case $txt in
"trust-store")
cp /etc/nginx/ssl/labca_cert.pem /usr/local/share/ca-certificates/labca_cert.crt
cp /home/labca/nginx_data/ssl/labca_cert.pem /usr/local/share/ca-certificates/labca_cert.crt
cp ~labca/admin/data/root-ca.pem /usr/local/share/ca-certificates/root-ca.crt
update-ca-certificates &>>$LOGFILE
echo "Waiting for initial startup of the docker containers..." &>>$LOGFILE
@@ -58,12 +58,11 @@ case $txt in
wait_up $PS_BOULDER $PS_BOULDER_COUNT &>>$LOGFILE
;;
"acme-request")
cd /etc/nginx/ssl
cd /home/labca/nginx_data/ssl
[ -e account.key ] || openssl genrsa 4096 > account.key
[ -e labca_key.pem ] || openssl genrsa 4096 > labca_key.pem
san=$(openssl x509 -noout -text -in labca_cert.pem | grep DNS:)
openssl req -new -sha256 -key labca_key.pem -subj "/" -reqexts SAN -config <(cat /etc/ssl/openssl.cnf <(printf "[SAN]\nsubjectAltName=$san")) > domain.csr
chown -R www-data:www-data *
url=$(grep 'DEFAULT_DIRECTORY_URL =' /home/labca/acme_tiny.py | sed -e 's/.*=[ ]*//' | sed -e 's/\"//g')
wait_server $url
sleep 10
@@ -72,16 +71,18 @@ case $txt in
ln -sf /home/labca/labca/logrotate_d /etc/logrotate.d/labca
;;
"nginx-remove-redirect")
perl -i -p0e 's/\n # BEGIN temporary redirect\n location = \/ \{\n return 302 \/admin\/;\n }\n # END temporary redirect\n//igs' /etc/nginx/sites-available/labca
perl -i -p0e 's/\n # BEGIN temporary redirect\n location = \/ \{\n return 302 \/admin\/;\n }\n # END temporary redirect\n//igs' /home/labca/nginx_data/conf.d/labca.conf
;;
"nginx-reload")
service nginx reload
cd /home/labca/boulder
docker-compose exec -T nginx nginx -s reload &>>$LOGFILE
;;
"nginx-restart")
service nginx restart
cd /home/labca/boulder
docker-compose restart nginx &>>$LOGFILE
;;
"log-cert")
[ -f /etc/nginx/ssl/acme_tiny.log ] && tail -200 /etc/nginx/ssl/acme_tiny.log || /bin/true
[ -f /home/labca/nginx_data/ssl/acme_tiny.log ] && tail -200 /home/labca/nginx_data/ssl/acme_tiny.log || /bin/true
exit 0
;;
"log-commander")
@@ -120,11 +121,8 @@ case $txt in
exit 0
;;
"log-web")
tail -f -n 50 /var/log/nginx/access.log
;;
"log-weberr")
[ -f /var/log/nginx/error.log ] && tail -200 /var/log/nginx/error.log || /bin/true
exit 0
cd /home/labca/boulder
docker-compose logs -f --no-color --tail=50 nginx
;;
"log-components")
timezone=$(cat /etc/timezone)

View File

@@ -16,8 +16,6 @@ cp $PKI_ROOT_CERT_BASE.der certs/
cp $PKI_INT_CERT_BASE.pem certs/
cp $PKI_INT_CERT_BASE.der certs/
chown -R www-data:www-data .
cd /boulder/labca
$PKI_PWD/apply-boulder

View File

@@ -18,5 +18,3 @@ sed -i -e "s|\[PKI_ROOT_FINGERPRINT\]|$PKI_ROOT_FINGERPRINT|g" cps/index.html
sed -i -e "s|\[PKI_ROOT_VALIDITY\]|$PKI_ROOT_VALIDITY|g" cps/index.html
sed -i -e "s|\[PKI_COMPANY_NAME\]|$PKI_DEFAULT_O|g" terms/v1.html
chown -R www-data:www-data .

View File

@@ -232,7 +232,7 @@ func errorHandler(w http.ResponseWriter, r *http.Request, err error, status int)
var FileErrors []interface{}
data := getLog(w, r, "cert")
if data != "" {
FileErrors = append(FileErrors, map[string]interface{}{"FileName": "/etc/nginx/ssl/acme_tiny.log", "Content": data})
FileErrors = append(FileErrors, map[string]interface{}{"FileName": "/home/labca/nginx_data/ssl/acme_tiny.log", "Content": data})
}
data = getLog(w, r, "commander")
if data != "" {
@@ -1006,7 +1006,7 @@ func _manageGet(w http.ResponseWriter, r *http.Request) {
components := _parseComponents(getLog(w, r, "components"))
for i := 0; i < len(components); i++ {
if components[i].Name == "NGINX Webserver" {
components[i].LogURL = r.Header.Get("X-Request-Base") + "/logs/weberr"
components[i].LogURL = r.Header.Get("X-Request-Base") + "/logs/web"
components[i].LogTitle = "Web Error Log"
btn := make(map[string]interface{})
@@ -1185,11 +1185,6 @@ func logsHandler(w http.ResponseWriter, r *http.Request) {
case "web":
name = "Web Access Log"
message = "Live view on the NGINX web server access log."
case "weberr":
name = "Web Error Log"
message = "Log file for the NGINX web server error log."
wsurl = ""
data = getLog(w, r, logType)
default:
errorHandler(w, r, fmt.Errorf("unknown log type '%s'", logType), http.StatusBadRequest)
return
@@ -2272,21 +2267,13 @@ func activeNav(active string, uri string, requestBase string) []navItem {
},
}
web := navItem{
Name: "Web Access",
Name: "Web Server",
Icon: "fa-globe",
Attrs: map[template.HTMLAttr]string{
"href": requestBase + "/logs/web",
"title": "Live view on the NGINX web server access log",
},
}
weberr := navItem{
Name: "Web Error",
Icon: "fa-times",
Attrs: map[template.HTMLAttr]string{
"href": requestBase + "/logs/weberr",
"title": "Log file for the NGINX web server error log",
},
}
logs := navItem{
Name: "Logs",
Icon: "fa-files-o",
@@ -2295,7 +2282,7 @@ func activeNav(active string, uri string, requestBase string) []navItem {
"title": "Log Files",
},
IsActive: strings.HasPrefix(uri, "/logs/"),
SubMenu: []navItem{cert, boulder, audit, labca, web, weberr},
SubMenu: []navItem{cert, boulder, audit, labca, web},
}
manage := navItem{
Name: "Manage",
@@ -2491,13 +2478,13 @@ func main() {
r.NotFoundHandler = http.HandlerFunc(notFoundHandler)
if isDev {
r.PathPrefix("/accounts/static/").Handler(http.StripPrefix("/accounts/static/", http.FileServer(http.Dir("../www"))))
r.PathPrefix("/authz/static/").Handler(http.StripPrefix("/authz/static/", http.FileServer(http.Dir("../www"))))
r.PathPrefix("/challenges/static/").Handler(http.StripPrefix("/challenges/static/", http.FileServer(http.Dir("../www"))))
r.PathPrefix("/certificates/static/").Handler(http.StripPrefix("/certificates/static/", http.FileServer(http.Dir("../www"))))
r.PathPrefix("/orders/static/").Handler(http.StripPrefix("/orders/static/", http.FileServer(http.Dir("../www"))))
r.PathPrefix("/logs/static/").Handler(http.StripPrefix("/logs/static/", http.FileServer(http.Dir("../www"))))
r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir("../www"))))
r.PathPrefix("/accounts/static/").Handler(http.StripPrefix("/accounts/static/", http.FileServer(http.Dir("../static"))))
r.PathPrefix("/authz/static/").Handler(http.StripPrefix("/authz/static/", http.FileServer(http.Dir("../static"))))
r.PathPrefix("/challenges/static/").Handler(http.StripPrefix("/challenges/static/", http.FileServer(http.Dir("../static"))))
r.PathPrefix("/certificates/static/").Handler(http.StripPrefix("/certificates/static/", http.FileServer(http.Dir("../static"))))
r.PathPrefix("/orders/static/").Handler(http.StripPrefix("/orders/static/", http.FileServer(http.Dir("../static"))))
r.PathPrefix("/logs/static/").Handler(http.StripPrefix("/logs/static/", http.FileServer(http.Dir("../static"))))
r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir("../static"))))
}
r.Use(authorized)

60
install
View File

@@ -203,7 +203,7 @@ clone_or_pull() {
# Checkout the latest release tag
checkout_release() {
local branch="$1"
if [ "$branch" == "" ] || [ "$branch" == "master" ]; then
if [ "$branch" == "" ] || [ "$branch" == "master" ] || [ "$branch" == "main" ]; then
cd "$cloneDir"
TAG=$(git describe --tags $(git rev-list --tags --max-count=1))
sudo -u labca -H git reset --hard $TAG &>>$installLog
@@ -393,7 +393,7 @@ install_pkg() {
}
install_extra() {
local packages=(apt-transport-https ca-certificates curl gnupg2 net-tools nginx software-properties-common tzdata ucspi-tcp zip python)
local packages=(apt-transport-https ca-certificates curl gnupg2 net-tools software-properties-common tzdata ucspi-tcp zip python)
for package in "${packages[@]}"; do
install_pkg "$package"
done
@@ -425,11 +425,22 @@ static_web() {
local msg="Static web pages"
msg_info "$msg"
[ -e /etc/nginx/sites-available/labca ] || cp $cloneDir/nginx.conf /etc/nginx/sites-available/labca
[ -e /etc/nginx/sites-enabled/labca ] || ln -s ../sites-available/labca /etc/nginx/sites-enabled/
rm -f /etc/nginx/sites-enabled/default
if [ -d /etc/nginx ]; then
# Migrate cert from host nginx to dockerized nginx
[ -d /home/labca/nginx_data/ssl ] || mkdir -p /home/labca/nginx_data/ssl
mv /etc/nginx/ssl/* /home/labca/nginx_data/ssl/
mv /etc/nginx /etc/nginx.backup
fi
cd /var/www/html
[ -d /home/labca/nginx_data/conf.d ] || mkdir -p /home/labca/nginx_data/conf.d
[ -d /home/labca/nginx_data/ssl ] || mkdir -p /home/labca/nginx_data/ssl
cp $cloneDir/nginx.conf /home/labca/nginx_data/conf.d/labca.conf
if [ -f "$boulderLabCADir/setup_complete" ]; then
perl -i -p0e 's/\n # BEGIN temporary redirect\n location = \/ \{\n return 302 \/admin\/;\n }\n # END temporary redirect\n//igs' /home/labca/nginx_data/conf.d/labca.conf
fi
[ -d /home/labca/nginx_data/static ] || mkdir /home/labca/nginx_data/static
cd /home/labca/nginx_data/static
git status --short &> /dev/null || rc=$?
if [ $rc -gt 0 ]; then
git init >>$installLog
@@ -438,9 +449,10 @@ static_web() {
git commit --all --quiet -m "LabCA before update $runId" &>>$installLog && { msg_ok "Commit existing modifications of $adminDir"; msg_info "$msg"; } || true
mkdir -p .well-known/acme-challenge
find .well-known/acme-challenge/ -mtime +10 -exec rm {} \; # Clean up files older than 10 days
mkdir -p crl
[ -e cert ] || ln -s certs cert
cp -rp $cloneDir/www/* .
cp -rp $cloneDir/static/* .
sed -i -e "s|\[LABCA_CPS_LOCATION\]|http://$LABCA_FQDN/cps/|g" cps/index.html
sed -i -e "s|\[LABCA_CERTS_LOCATION\]|http://$LABCA_FQDN/certs/|g" cps/index.html
@@ -451,8 +463,6 @@ static_web() {
export PKI_DEFAULT_O=$(grep organization $adminDir/data/config.json | sed -e 's/.*:[ ]*//' | sed -e 's/\",//g' | sed -e 's/\"//g')
$adminDir/apply-nginx
else
chown -R www-data:www-data .
fi
git add --all &>/dev/null || true
@@ -463,19 +473,16 @@ static_web() {
# Create a temporary self-signed certificate if there is no certificate yet
selfsigned_cert() {
if [ -e /etc/nginx/ssl/labca_cert.pem ]; then
if [ -e /home/labca/nginx_data/ssl/labca_cert.pem ]; then
msg_ok "Certificate is present"
else
local msg="Create self-signed certificate"
msg_info "$msg"
mkdir -p /etc/nginx/ssl
cd /etc/nginx/ssl
mkdir -p /home/labca/nginx_data/ssl
cd /home/labca/nginx_data/ssl
openssl req -x509 -nodes -sha256 -newkey rsa:2048 -keyout labca_key.pem -out labca_cert.pem -days 7 \
-subj "/O=LabCA/CN=$LABCA_FQDN" -reqexts SAN -extensions SAN \
-config <(cat /etc/ssl/openssl.cnf <(printf "\n[SAN]\nbasicConstraints=CA:FALSE\nnsCertType=server\nsubjectAltName=DNS:$LABCA_FQDN")) &>>$installLog
chown -R www-data:www-data labca_*
service nginx restart &>>$installLog
msg_ok "$msg"
fi
}
@@ -720,13 +727,15 @@ cleanup() {
local msg="Cleaning up obsolete files"
msg_info "$msg"
rm -f /var/www/html/css/skeleton.css
rm -f /var/www/html/css/skeleton-tabs.css
rm -f /var/www/html/css/normalize.css
rm -f /var/www/html/css/font.css
rm -f /var/www/html/img/favicon.ico
rm -f /var/www/html/js/jquery-3.3.1.min.js
rm -f /var/www/html/js/skeleton-tabs.js
if [ -d /var/www/html ]; then
rm -f /var/www/html/css/skeleton.css
rm -f /var/www/html/css/skeleton-tabs.css
rm -f /var/www/html/css/normalize.css
rm -f /var/www/html/css/font.css
rm -f /var/www/html/img/favicon.ico
rm -f /var/www/html/js/jquery-3.3.1.min.js
rm -f /var/www/html/js/skeleton-tabs.js
fi
rm -f $adminDir/templates/cert.tmpl
rm -f $adminDir/templates/error.tmpl
rm -f $adminDir/templates/final.tmpl
@@ -739,6 +748,11 @@ cleanup() {
rm -f $adminDir/templates/setup.tmpl
rm -f $adminDir/templates/wrapup.tmpl
# Remove host nginx if installed, as we are now using the docker container
systemctl stop nginx &>>$installLog || true
systemctl disable nginx &>>$installLog || true
apt remove -y nginx &>>$installLog
msg_ok "$msg"
}
@@ -784,7 +798,7 @@ startup() {
# If the nginx certificate is self-signed then show extra text
first_time() {
local certFile="/etc/nginx/ssl/labca_cert.pem"
local certFile="/home/labca/nginx_data/ssl/labca_cert.pem"
[ -e "$certFile" ] || msg_fatal "The SSL certificate $certFile does not exist"
local subject=$(openssl x509 -noout -in "$certFile" -subject_hash)

View File

@@ -1,4 +1,4 @@
/etc/nginx/ssl/*.log
/home/labca/nginx_data/ssl/*.log
/home/labca/logs/cron-*.log
{
rotate 4

View File

@@ -19,8 +19,11 @@ server {
}
location /ocsp/ {
include proxy_params;
proxy_pass http://127.0.0.1:4002/;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://boulder:4002/;
}
location /rate-limits {
@@ -52,33 +55,48 @@ server {
}
location /admin/ {
include proxy_params;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Request-Base "/admin";
proxy_pass http://127.0.0.1:3000/;
proxy_pass http://labca:3000/;
error_page 502 504 /502.html;
}
location /admin/ws {
include proxy_params;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Request-Base "/admin";
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "Upgrade";
proxy_pass http://127.0.0.1:3000/ws;
proxy_pass http://labca:3000/ws;
}
location /acme/ {
include proxy_params;
proxy_pass http://127.0.0.1:4001;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://boulder:4001;
}
location /directory {
include proxy_params;
proxy_pass http://127.0.0.1:4001;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://boulder:4001;
}
location /ocsp/ {
include proxy_params;
proxy_pass http://127.0.0.1:4002/;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://boulder:4002/;
}
location /rate-limits {

View File

@@ -40,7 +40,7 @@ index b7e5656c5..d771aa011 100644
networks:
bluenet:
aliases:
@@ -56,21 +65,38 @@ services:
@@ -56,21 +65,51 @@ services:
# small.
command: mysqld --bind-address=0.0.0.0 --slow-query-log --log-output=TABLE --log-queries-not-using-indexes=ON
logging:
@@ -64,7 +64,7 @@ index b7e5656c5..d771aa011 100644
volumes:
+ - /home/labca/admin:/go/src/labca
+ - ./.gocache:/root/.cache/go-build
+ - /var/www/html:/wwwstatic
+ - /home/labca/nginx_data/static:/wwwstatic
- .:/boulder
- working_dir: *boulder_working_dir
- entrypoint: test/entrypoint-netaccess.sh
@@ -82,6 +82,19 @@ index b7e5656c5..d771aa011 100644
+ max-file: "5"
+ restart: always
+
+ nginx:
+ image: nginx:1.21.6
+ restart: always
+ networks:
+ - bluenet
+ ports:
+ - 80:80
+ - 443:443
+ volumes:
+ - /home/labca/nginx_data/conf.d:/etc/nginx/conf.d
+ - /home/labca/nginx_data/ssl:/etc/nginx/ssl
+ - /home/labca/nginx_data/static:/var/www/html
+
+volumes:
+ dbdata:

8
renew
View File

@@ -2,10 +2,10 @@
set -e
cd /etc/nginx/ssl
cd /home/labca/nginx_data/ssl
date >> acme_tiny.log
python ~labca/acme_tiny.py --account-key ./account.key --csr ./domain.csr --acme-dir /var/www/html/.well-known/acme-challenge/ > domain_chain.crt 2>> acme_tiny.log || exit 1
python ~labca/acme_tiny.py --account-key ./account.key --csr ./domain.csr --acme-dir /home/labca/nginx_data/static/.well-known/acme-challenge/ > domain_chain.crt 2>> acme_tiny.log || exit 1
mv domain_chain.crt labca_cert.pem
chown -R www-data:www-data labca_cert.pem
service nginx reload
cd /home/labca/boulder
docker-compose restart nginx

View File

@@ -16,7 +16,7 @@ tar xzf $FILE
cd /home/labca/boulder
docker-compose exec bmysql mysql boulder_sa_integration <$TMPDIR/boulder_sa_integration.sql
mv -f $TMPDIR/*key* $TMPDIR/*cert.pem $TMPDIR/*.csr /etc/nginx/ssl/
mv -f $TMPDIR/*key* $TMPDIR/*cert.pem $TMPDIR/*.csr /home/labca/nginx_data/ssl/
rm -rf /home/labca/admin/data && mv $TMPDIR/data /home/labca/admin/

View File

@@ -5,10 +5,10 @@ set -e
RENEW=30
TODAY=`date '+%Y_%m_%d'`
echo $TODAY >> /etc/nginx/ssl/cron.log
echo $TODAY >> /home/labca/nginx_data/ssl/cron.log
if ! expires=`openssl x509 -checkend $[ 86400 * $RENEW ] -noout -in /etc/nginx/ssl/labca_cert.pem`; then
echo " renewing!" >> /etc/nginx/ssl/cron.log
cp /etc/nginx/ssl/labca_cert.pem /etc/nginx/ssl/labca_cert_$TODAY.pem
if ! expires=`openssl x509 -checkend $[ 86400 * $RENEW ] -noout -in /home/labca/nginx_data/ssl/labca_cert.pem`; then
echo " renewing!" >> /home/labca/nginx_data/ssl/cron.log
cp /home/labca/nginx_data/ssl/labca_cert.pem /home/labca/nginx_data/ssl/labca_cert_$TODAY.pem
~labca/labca/renew
fi

View File

Before

Width:  |  Height:  |  Size: 382 KiB

After

Width:  |  Height:  |  Size: 382 KiB

View File

Before

Width:  |  Height:  |  Size: 106 KiB

After

Width:  |  Height:  |  Size: 106 KiB

View File

Before

Width:  |  Height:  |  Size: 39 KiB

After

Width:  |  Height:  |  Size: 39 KiB

View File

Before

Width:  |  Height:  |  Size: 39 KiB

After

Width:  |  Height:  |  Size: 39 KiB

View File

Before

Width:  |  Height:  |  Size: 16 KiB

After

Width:  |  Height:  |  Size: 16 KiB

View File

Before

Width:  |  Height:  |  Size: 561 B

After

Width:  |  Height:  |  Size: 561 B