Re: [BUG] Call trace during snapshot start/stop sequence

["Arkadiusz Bubała" <arkadiusz.bubala@open-e.com>]

Hello,
thank you for valuable information.

On 28.11.2013 00:06, Dave Chinner wrote:
>
>> Running a custom built 3.4.63 kernel with a bunch of out of tree
>> modules installed. can you reproduce this on a vanilla 3.12 kernel?
>>
>>      
Ok, we'll try.

> The script is full of bugs, and i don't have time to debug it - it
> hard codes /dev/sda in places despite taking the device as a CLI
> parameter. It has hard coded mount points.  It sometimes fails to
> make the filesystem on the base LV after it's been created.
> start_snap() appears to fail for some reason, as it doesn't result
> in mounted snapshots. stop_snap fails as well:
>
> Starting snap19 : Thursday 28 November  10:01:26 EST 2013
>    Logical volume lv1+snap19 converted to snapshot.
> [ FAIL ] Can't umount snapshot
> [ FAIL ] Can't remove snapshot
> [ FAIL ] Can't umount snapshot
> [ FAIL ] Can't remove snapshot
> [ FAIL ] Can't umount snapshot
> [ FAIL ] Can't remove snapshot
> [ FAIL ] Can't umount snapshot
> [ FAIL ] Can't remove snapshot
> [ FAIL ] Can't umount snapshot
> [ FAIL ] Can't remove snapshot
> [ FAIL ] Can't umount snapshot
> [ FAIL ] Can't remove snapshot
> [ FAIL ] Can't umount snapshot
> [ FAIL ] Can't remove snapshot
> [ OK ] lv1+snap19 activated.
> Starting time : 37 s.
> ---------------------------
> Stopping snap0 : Thursday 28 November  10:02:06 EST 2013
> [ FAIL ] Can't umount snapshot
> [ FAIL ] Can't remove snapshot
> [ FAIL ] lv0+snap00 still active !!!
> [ OK ] lv0+snap00 umounted.
> Stopping time : 0 s.
>
> I've got no idea is this is intended behaviour, but it sure doesn't
> seem right to me...
>
>
>    
Yes, sometimes umount and remove operations fail. This script tests 
system stability and these messages are debug info only.
I've fixed it. Now it takes two parameters: device and mount point.

#!/bin/bash

DEV=$1
MOUNTPOINT=$2

function print_usage()
{
     echo "Usage: $0 device mountpoint"
     exit 1;
}

if [ -z "$DEV" ]; then
     print_usage
fi

if [ -z "$MOUNTPOINT" ]; then
     print_usage
fi

function overload()
{
COUNT=$1
temp_COUNT=$COUNT;

while [ -f ./run ]; do
     while [ $COUNT -ge 1 ]; do
         if [ -f ./run ]; then
             dd bs=1024 count=102400 if=/dev/zero of=/$2/"_"$COUNT &> 
/dev/null
         fi;
         let COUNT=$COUNT-1
     done;

     rm $2/*;
     COUNT=$temp_COUNT;

done;
}


function create_vg()
{
     #create physical volume
     pvcreate $DEV
     if [[ $? -gt 0 ]]; then
         echo "[ FAIL ] Unable to create physical volume"
         exit 1
     fi
     #create volume group
     vgcreate -v -s 32M $VG $DEV
     if [[ $? -gt 0 ]]; then
         echo "[ FAIL ] Unable to create volume group"
         exit 1
     fi
}

function create_lv()
{
     local LV="$1"
     #create logical volume
     lvcreate -l 500 -n "$VG+$LV" "$VG"
     if [[ $? -gt 0 ]]; then
         echo "[ FAIL ] Unable to create LV"
         exit 1
     fi
     mkfs -t xfs -f -l lazy-count=0 /dev/$VG/"$VG+$LV" &>/dev/null
     if [[ $? -gt 0 ]]; then
         echo "[ FAIL ] Can't create filesystem"
         exit 1
     fi
}

function create_snapshots()
{
     for ((i=0; i < 20; i++)); do
         if [[ $i -lt 10 ]]; then
             lvcreate -l "64" -n "snap0$i" "$VG"
             if [[ $? -gt 0 ]]; then
                 echo "[ FAIL ] Unable to create snapshot LV"
                 exit 1
             fi
         else
             lvcreate -l "64" -n "snap$i" "$VG"
             if [[ $? -gt 0 ]]; then
                 echo "[ FAIL ] Unable to create snapshot LV"
                 exit 1
             fi
         fi
     done
}

function assign_snapshots()
{
     for ((i=0; i < 20; i++)); do
         if [[ $i -lt 10 ]]; then
             lvrename "$VG" "snap0$i" "lv0+snap0$i"
             if [[ $? -gt 0 ]]; then
                 echo "[ FAIL ] Unable to rename snapshot LV"
                 exit 1
             fi
         else
             lvrename "$VG" "snap$i" "lv1+snap$i"
             if [[ $? -gt 0 ]]; then
                 echo "[ FAIL ] Unable to rename snapshot LV"
                 exit 1
             fi
         fi
     done
}

function mount_volume()
{
     local MVG=$1
     local MLV=$2
     mkdir -p "$MOUNTPOINT/$MVG+$MLV"
     if [[ $? -gt 0 ]]; then
         echo "[ FAIL ] Unable to create mounting point"
         exit 1
     fi
     mount -t xfs -o 
defaults,usrquota,grpquota,nouuid,noatime,nodiratime 
"/dev/$MVG/$MVG+$MLV" "$MOUNTPOINT/$MVG+$MLV"
     if [[ $? -gt 0 ]]; then
         echo "[ FAIL ] Unable to mount LV"
         exit 1
     fi
}

function start_overload()
{
     touch ./run
     mkdir -p "$MOUNTPOINT/$1+$2/test"
     overload 50 "$MOUNTPOINT/$1+$2/test" $3 &
     echo "overload $1 $MOUNTPOINT/$1+$2/test $3 &"
     sleep 4;
     echo "[ OK ] copying files to $2 started"
}

function get_snapshot_status()
{
     lvdisplay /dev/$1/$2 | awk ' $0~"LV snapshot status" { print $4 } '
}

function remove_snapshot()
{
     local LVG=$1
     local LLV=$2
     local LSNAP=$3
     umount "$MOUNTPOINT/$LSNAP"
     if [[ $? -gt 0 ]]; then
         echo "[ FAIL ] Can't umount snapshot"
     fi
     lvremove -sf  "/dev/$LVG/$LSNAP"
     if [[ $? -gt 0 ]]; then
         echo "[ FAIL ] Can't remove snapshot"
     fi
}

function create_snapshot()
{
     local LVG=$1
     local LLV=$2
     local LSNAP=$3
     for((it=0; it<7; it++)); do
         local ERROR=0
         local STATUS=`get_snapshot_status $LVG $LSNAP`
         if [[ "$STATUS" == "active" ]]; then
             remove_snapshot $LVG "$LLV+$LSNAP"
         fi
         STATUS=`get_snapshot_status $LVG $LSNAP`
         if [[ "$STATUS" == "active" ]]; then
             remove_snapshot  $LVG "$LLV+$LSNAP"
         fi
         CHUNKSIZE=512
         for ((ile=0;ile<it/2;ile++)); do
             CHUNKSIZE=$((CHUNKSIZE/2))
         done
         lvconvert -s -c $CHUNKSIZE "/dev/$LVG/$LVG+$LLV" "/dev/$LVG/$LSNAP"
         if [[ $? -gt 0 ]]; then
             ERROR=1
         fi
         #mount snapshot
         mkdir -p "$MOUNTPOINT/$LSNAP"
         mount -t xfs -o nouuid,noatime "/dev/$LVG/$LSNAP" 
"$MOUNTPOINT/$LSNAP"
         if [[ $? -gt 0 ]]; then
             ERROR=2
         fi
         create_time=`date "+%Y-%m-%d %H:%M:%S"`
         if [ $ERROR -ne 0 ]; then
             remove_snapshot $LVG $LLV $LSNAP
             sleep 5
         else
             break;
         fi
     done

}

function start_snap()
{
     local i;
     for((i=0; i<20; i++)); do
         echo "Starting snap$i : `date`"
         local START=`date +%s`
         if [[ $i -lt 10 ]]; then
         snapname="lv0+snap0"$i
         create_snapshot $VG "lv0" $snapname
         else
             snapname="lv1+snap"$i
         create_snapshot $VG "lv1" $snapname
         fi
         if [ -z "`lvs | grep $snapname | grep $VG+lv`" ]; then
             echo "[ FAIL ] $snapname not activated !!!"
         else
             echo "[ OK ] $snapname activated."
         fi
         if [ -z "`mount | grep $snapname`" ]; then
             echo "[ FAIL ] $snapname not mounted !!!" >> $LOGFILE
         else
             echo "[ OK ] $snapname mounted."
         fi
         local STOP=$[`date +%s`-$START]
         echo "Starting time : $STOP s."
         echo "---------------------------"
         sleep 2
     done;
}

function stop_snap()
{
     local i
     for((i=0; i<20; i++)); do
         echo "Stopping snap$i : `date`"
         local START=`date +%s`
         if [[ $i -lt 10 ]]; then
             snapname="lv0+snap0"$i
             remove_snapshot $VG "lv0" $snapname
         else
             snapname="lv1+snap"$i
             remove_snapshot $VG "lv1" $snapname
         fi
         if [ "`lvs | grep $snapname | grep $VG+lv`" ]; then
             echo "[ FAIL ] $snapname still active !!!"
         else
             echo "[ OK ] $snapname deactivated."
         fi;
         if [  "`mount | grep $snapname`" ]; then
             echo "[ FAIL ] $snapname still mounted !!!" >> $LOGFILE
         else
             echo "[ OK ] $snapname umounted."
         fi;
         local STOP=$[`date +%s`-$START]
         echo "Stopping time : $STOP s."
         echo "---------------------------"
         sleep 2
     done;
}

VG="vg0"

echo "-------- Creating $VG on $DEV..."
create_vg
echo "[ OK ] Volume group created successfully"
echo "-------- Creating logical volumes on $VG..."
create_lv "lv0"
create_lv "lv1"
echo "[ OK ] Logical volumes created successfully"
echo "-------- Mounting logical volumes..."
mount_volume "$VG" "lv0"
mount_volume "$VG" "lv1"
echo "[ OK ] Logical volumes mounted successfully"
echo "-------- Creating snapshots..."
create_snapshots
echo "[ OK ] Snapshots created successfully"
echo "-------- Assigning snapshots..."
assign_snapshots
echo "[ OK ] Snapshots assigned successfully"
echo "-------- Start overload..."
start_overload "$VG" "lv0"
start_overload "$VG" "lv1"

while true; do
     start_snap 2> /dev/null
     stop_snap  2> /dev/null
done

rm ./run

-- 
Best regards
Arkadiusz Bubała
Open-E Poland Sp. z o.o.
www.open-e.com

_______________________________________________
xfs mailing list
xfs@oss.sgi.com
http://oss.sgi.com/mailman/listinfo/xfs