signer scripts: retry more than once when mounting

The retry logic has brought down the number of flakes significantly (from
multiple errors a day to ~one every other day).  But let's up the retry
count, and have it sleep longer after each failure, so hopefully we can
bring down the flake count even further.

BUG=chrome-os-partner:8156
TEST=`./signing_poller.py -s` signs local images fine

Change-Id: I98bc947836514d8b931568f87f7f9a373f771b79
Reviewed-on: https://gerrit.chromium.org/gerrit/21468
Reviewed-by: Gaurav Shah <gauravsh@chromium.org>
Tested-by: Mike Frysinger <vapier@chromium.org>
Commit-Ready: Mike Frysinger <vapier@chromium.org>
This commit is contained in:
Mike Frysinger
2012-04-30 12:31:33 -04:00
committed by Gerrit
parent aca0a135ac
commit 81f704edad

View File

@@ -168,7 +168,7 @@ _mount_image_partition_retry() {
local mount_dir=$3 local mount_dir=$3
local ro=$4 local ro=$4
local offset=$(( $(partoffset "$image" "$partnum") * 512 )) local offset=$(( $(partoffset "$image" "$partnum") * 512 ))
local out local out try
if [ "$ro" != "ro" ]; then if [ "$ro" != "ro" ]; then
# Forcibly call enable_rw_mount. It should fail on unsupported # Forcibly call enable_rw_mount. It should fail on unsupported
@@ -178,19 +178,30 @@ _mount_image_partition_retry() {
set -- sudo LC_ALL=C mount -o loop,offset=${offset},${ro} \ set -- sudo LC_ALL=C mount -o loop,offset=${offset},${ro} \
"${image}" "${mount_dir}" "${image}" "${mount_dir}"
try=1
while [ ${try} -le 5 ]; do
if ! out=$("$@" 2>&1); then if ! out=$("$@" 2>&1); then
if [ "${out}" = "mount: you must specify the filesystem type" ]; then if [ "${out}" = "mount: you must specify the filesystem type" ]; then
echo "WARNING: mounting ${image} at ${mount_dir} failed; retrying" printf 'WARNING: mounting %s at %s failed (try %i); retrying\n' \
sleep 5 "${image}" "${mount_dir}" "${try}"
"$@" # Try to "quiet" the disks and sleep a little to reduce contention.
sync
sleep $(( try * 5 ))
else else
# Failed for a different reason; abort!
break
fi
else
# It worked!
return 0
fi
: $(( try += 1 ))
done
echo "ERROR: mounting ${image} at ${mount_dir} failed:" echo "ERROR: mounting ${image} at ${mount_dir} failed:"
echo "${out}" echo "${out}"
# We don't preserve the exact exit code of `mount`, but since # We don't preserve the exact exit code of `mount`, but since
# no one in this code base seems to check it, it's a moot point. # no one in this code base seems to check it, it's a moot point.
return 1 return 1
fi
fi
} }
# Mount a partition read-only from an image into a local directory # Mount a partition read-only from an image into a local directory