Message ID | 20190319001732.GB4916@magnolia |
---|---|
State | Superseded |
Headers | show |
Series | e2scrub_all: refactor device probe loop | expand |
On Mon, Mar 18, 2019 at 05:17:32PM -0700, Darrick J. Wong wrote: > From: Darrick J. Wong <darrick.wong@oracle.com> > > Paul Menzel reported that the e2scrub_all reaper service that runs at > startup takes a long time to run, and Ted T'so pointed out that we could > do a lot less work by using lvs as the outer loop in the ext4 filesystem > probe function so that we only have to lsblk the lvm devices containing > ext4 filesystems. > > Therefore, refactor the loops to put lvs first, which should boost speed > a bit. > > Reported-by: Paul Menzel <pmenzel@molgen.mpg.de> > Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com> > --- > scrub/e2scrub_all.in | 51 ++++++++++++++++++++++++++++++++++++-------------- > 1 file changed, 37 insertions(+), 14 deletions(-) > > diff --git a/scrub/e2scrub_all.in b/scrub/e2scrub_all.in > index 23d122d25..41420d03d 100644 > --- a/scrub/e2scrub_all.in > +++ b/scrub/e2scrub_all.in > @@ -19,6 +19,7 @@ > # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. > > scrub_all=0 > +reap=0 > conffile="@root_sysconfdir@/e2scrub.conf" > > test -f "${conffile}" && . "${conffile}" > @@ -61,7 +62,7 @@ exitcode() { > while getopts "ArV" opt; do > case "${opt}" in > "A") scrub_all=1;; > - "r") scrub_args="${scrub_args} -r";; > + "r") scrub_args="${scrub_args} -r"; reap=1;; > "V") print_version; exitcode 0;; > *) print_help; exitcode 2;; > esac > @@ -69,27 +70,27 @@ done > shift "$((OPTIND - 1))" > > # Find scrub targets, make sure we only do this once. > -ls_scrub_targets() { > - lsblk -o NAME,FSTYPE,MOUNTPOINT -p -P -n | while read vars; do > +ls_scan_targets() { > + lvs --name-prefixes -o vg_name,lv_name,lv_path \ > + -S lv_active=active,lv_role=public --noheadings | \ You're not using vg_name, nor lv_name so you can drop it maybe ? Also you're missing lv_role since you're checking it later, however you can do this instead -S lv_active=active,lv_role=public -S lv_role!=snapshot so you only need to ask for lv_path > + while read vars; do > eval "${vars}" > + eval "$(lsblk -o NAME,FSTYPE,MOUNTPOINT -p -P -n "${LVM2_LV_PATH}")" Now that you only have one lv filed you can maybe pass this directly to the lsblk ? This looks funny but works fine for me :) lsblk -o MOUNTPOINT,NAME,FSTYPE -p -n `lvs -o lv_path -S lv_active=active,lv_role=public -S lv_role!=snapshot --noheadings` | grep 'ext[234]' | awk '{print $1}' if you only want mounted file systems than you just add | grep -v -E '^/dev/' > > - # Skip non-ext[234] > + # Skip unless ext* > case "${FSTYPE}" in > - ext[234]) ;; > - *) continue;; > + ext*) ;; > + *) continue;; > esac > > + # Don't run against a snapshot ever > + echo "${LVM2_LV_ROLE}" | grep -q "snapshot" && continue > + > # Skip unmounted filesystems unless -A > if [ "${scrub_all}" -eq 0 ] && [ -z "${MOUNTPOINT}" ]; then > continue; > fi > > - # Skip non-lvm devices and lvm snapshots > - lvm_vars="$(lvs --nameprefixes -o vg_name,lv_name,lv_role --noheadings "${NAME}" 2> /dev/null)" > - test $? -ne 0 && continue > - eval "${lvm_vars}" > - echo "${LVM2_LV_ROLE}" | grep -q "snapshot" && continue > - > if [ -n "${MOUNTPOINT}" ]; then > echo "${MOUNTPOINT}" > else > @@ -98,6 +99,28 @@ ls_scrub_targets() { > done | sort | uniq > } > > +# Find leftover scrub snapshots > +ls_reap_targets() { > + lvs --name-prefixes -o vg_name,lv_name,lv_path,origin \ You only seem to be using lv_name and lv_path ? also I think you can do this: lvs -o lv_path -S lv_role=snapshot -S lv_name=~\(e2scrub$\) --noheadings Also since you only ask for one field I tihnk it's already sorted so no need to sort, or add -O -Lukas > + -S lv_role=snapshot --noheadings | while read vars; do > + eval "${vars}" > + > + # Filter out anything except our snapshots > + case "${LVM2_LV_NAME}" in > + *.e2scrub) echo "${LVM2_LV_PATH}";; > + esac > + done | sort | uniq > +} > + > +# Figure out what we're targeting > +ls_targets() { > + if [ "${reap}" -eq 1 ]; then > + ls_reap_targets > + else > + ls_scan_targets > + fi > +} > + > # systemd doesn't know to do path escaping on the instance variable we pass > # to the e2scrub service, which breaks things if there is a dash in the path > # name. Therefore, do the path escaping ourselves if needed. > @@ -118,10 +141,10 @@ escape_path_for_systemd() { > > # Scrub any mounted fs on lvm by creating a snapshot and fscking that. > stdin="$(realpath /dev/stdin)" > -ls_scrub_targets | while read tgt; do > +ls_targets | while read tgt; do > # If we're not reaping and systemd is present, try invoking the > # systemd service. > - if [ -z "${scrub_args}" ] && type systemctl > /dev/null 2>&1; then > + if [ "${reap}" -ne 1 ] && type systemctl > /dev/null 2>&1; then > tgt_esc="$(escape_path_for_systemd "${tgt}")" > ${DBG} systemctl start "e2scrub@${tgt_esc}" 2> /dev/null < "${stdin}" > res=$?
On Tue, Mar 19, 2019 at 11:58:24AM +0100, Lukas Czerner wrote: > > # Find scrub targets, make sure we only do this once. > > -ls_scrub_targets() { > > - lsblk -o NAME,FSTYPE,MOUNTPOINT -p -P -n | while read vars; do > > +ls_scan_targets() { > > + lvs --name-prefixes -o vg_name,lv_name,lv_path \ > > + -S lv_active=active,lv_role=public --noheadings | \ > > You're not using vg_name, nor lv_name so you can drop it maybe ? Also > you're missing lv_role since you're checking it later, however you can > do this instead > > -S lv_active=active,lv_role=public -S lv_role!=snapshot We don't need to check lv_role at all, since the command already included: -S lv_active=active,lv_role=public And we need to do lv_role=public and not lv_role!=snapshot, since we need to also exclude thinpools. Speaking of thinpools, although we *can* take a thick snapshot of a thinp volume, in the future we might want to add support for taking a thinp snapshot. (Although that will make the check which I've been working on to make sure there's enough free space a bit more complicated.) I have some other patches queued for e2scrub, so I'll take a whack and trying to make a reivison of this patch, and then send out a stack of patches against e2scrub for folks to look at. - Ted
On Wed, Mar 20, 2019 at 12:17:08PM -0400, Theodore Ts'o wrote: > On Tue, Mar 19, 2019 at 11:58:24AM +0100, Lukas Czerner wrote: > > > # Find scrub targets, make sure we only do this once. > > > -ls_scrub_targets() { > > > - lsblk -o NAME,FSTYPE,MOUNTPOINT -p -P -n | while read vars; do > > > +ls_scan_targets() { > > > + lvs --name-prefixes -o vg_name,lv_name,lv_path \ > > > + -S lv_active=active,lv_role=public --noheadings | \ > > > > You're not using vg_name, nor lv_name so you can drop it maybe ? Also > > you're missing lv_role since you're checking it later, however you can > > do this instead > > > > -S lv_active=active,lv_role=public -S lv_role!=snapshot > > We don't need to check lv_role at all, since the command already > included: > > -S lv_active=active,lv_role=public You actually need to check the lv_role for snapshot, because snapshot can be "public" as well. > > And we need to do lv_role=public and not lv_role!=snapshot, since we > need to also exclude thinpools. No, you need to check both if you want to exclude snapshots. I think that specifying -S twice tricked you, sorry about that. But again you can do: -S lv_active=active,lv_role=public,lv_role!=snapshot Thanks! -Lukas
diff --git a/scrub/e2scrub_all.in b/scrub/e2scrub_all.in index 23d122d25..41420d03d 100644 --- a/scrub/e2scrub_all.in +++ b/scrub/e2scrub_all.in @@ -19,6 +19,7 @@ # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. scrub_all=0 +reap=0 conffile="@root_sysconfdir@/e2scrub.conf" test -f "${conffile}" && . "${conffile}" @@ -61,7 +62,7 @@ exitcode() { while getopts "ArV" opt; do case "${opt}" in "A") scrub_all=1;; - "r") scrub_args="${scrub_args} -r";; + "r") scrub_args="${scrub_args} -r"; reap=1;; "V") print_version; exitcode 0;; *) print_help; exitcode 2;; esac @@ -69,27 +70,27 @@ done shift "$((OPTIND - 1))" # Find scrub targets, make sure we only do this once. -ls_scrub_targets() { - lsblk -o NAME,FSTYPE,MOUNTPOINT -p -P -n | while read vars; do +ls_scan_targets() { + lvs --name-prefixes -o vg_name,lv_name,lv_path \ + -S lv_active=active,lv_role=public --noheadings | \ + while read vars; do eval "${vars}" + eval "$(lsblk -o NAME,FSTYPE,MOUNTPOINT -p -P -n "${LVM2_LV_PATH}")" - # Skip non-ext[234] + # Skip unless ext* case "${FSTYPE}" in - ext[234]) ;; - *) continue;; + ext*) ;; + *) continue;; esac + # Don't run against a snapshot ever + echo "${LVM2_LV_ROLE}" | grep -q "snapshot" && continue + # Skip unmounted filesystems unless -A if [ "${scrub_all}" -eq 0 ] && [ -z "${MOUNTPOINT}" ]; then continue; fi - # Skip non-lvm devices and lvm snapshots - lvm_vars="$(lvs --nameprefixes -o vg_name,lv_name,lv_role --noheadings "${NAME}" 2> /dev/null)" - test $? -ne 0 && continue - eval "${lvm_vars}" - echo "${LVM2_LV_ROLE}" | grep -q "snapshot" && continue - if [ -n "${MOUNTPOINT}" ]; then echo "${MOUNTPOINT}" else @@ -98,6 +99,28 @@ ls_scrub_targets() { done | sort | uniq } +# Find leftover scrub snapshots +ls_reap_targets() { + lvs --name-prefixes -o vg_name,lv_name,lv_path,origin \ + -S lv_role=snapshot --noheadings | while read vars; do + eval "${vars}" + + # Filter out anything except our snapshots + case "${LVM2_LV_NAME}" in + *.e2scrub) echo "${LVM2_LV_PATH}";; + esac + done | sort | uniq +} + +# Figure out what we're targeting +ls_targets() { + if [ "${reap}" -eq 1 ]; then + ls_reap_targets + else + ls_scan_targets + fi +} + # systemd doesn't know to do path escaping on the instance variable we pass # to the e2scrub service, which breaks things if there is a dash in the path # name. Therefore, do the path escaping ourselves if needed. @@ -118,10 +141,10 @@ escape_path_for_systemd() { # Scrub any mounted fs on lvm by creating a snapshot and fscking that. stdin="$(realpath /dev/stdin)" -ls_scrub_targets | while read tgt; do +ls_targets | while read tgt; do # If we're not reaping and systemd is present, try invoking the # systemd service. - if [ -z "${scrub_args}" ] && type systemctl > /dev/null 2>&1; then + if [ "${reap}" -ne 1 ] && type systemctl > /dev/null 2>&1; then tgt_esc="$(escape_path_for_systemd "${tgt}")" ${DBG} systemctl start "e2scrub@${tgt_esc}" 2> /dev/null < "${stdin}" res=$?