Re: [BUG] MD/RAID1 hung forever on freeze_array

[Jinpu Wang <jinpu.wang@profitbricks.com>]

On Fri, Dec 9, 2016 at 7:01 AM, NeilBrown <neilb@suse.com> wrote:
> On Thu, Dec 08 2016, Jinpu Wang wrote:
>
>
> This number:
>
>>   nr_pending = {
>>     counter = 1
>>   },
>
>
>
> and this number:
>
>>   nr_pending = {
>>     counter = 856
>>   },
>
> might be interesting.
>
> There are 855 requested on the list.  Add the one that is currently
> being retried give 856, which is nr_pending for the device that failed.
> But nr_pending on the device that didn't fail is 1.  I would expect
> zero.
> When a read or write requests succeeds, rdev_dec_pending() is called
> immediately so this should quickly go to zero.
>
> It seems as though there must be a request to the loop device that is
> stuck somewhere between the atomic_inc(&rdev->nr_pending) (possibly
> inside read_balance) and the call to generic_make_request().
> I cannot yet see how that would happen.
>
> Can you check if the is a repeatable observation?  Is nr_pending.counter
> always '1' on the loop device?
>
> Thanks,
> NeilBrown

Hi Neil,

Yes, it's repreatable observation. I triggered again, this time
  nr_pending = 1203,
  nr_waiting = 8,
  nr_queued = 1201,

conf->retry_list has 1175 entries.
on conf->bio_end_io_list has 26 entries.
Totol is 1201, match nr_queued.

in md_rdev healthy one loop1 has 1 nr_pending.
faulty one ibnbd1 has 1076.

crash> struct md_rdev 0xffff880228880400
struct md_rdev {
  same_set = {
    next = 0xffff88023202a200,
    prev = 0xffff8800b64c6018
  },
  sectors = 2095104,
  mddev = 0xffff8800b64c6000,
  last_events = 17764573,
  meta_bdev = 0x0,
  bdev = 0xffff8800b60ce080,
  sb_page = 0xffffea0002bd3040,
  bb_page = 0xffffea0002dc76c0,
  sb_loaded = 1,
  sb_events = 166,
  data_offset = 2048,
  new_data_offset = 2048,
  sb_start = 8,
  sb_size = 512,
  preferred_minor = 65535,
  kobj = {
    name = 0xffff880037962af0 "dev-ibnbd0",
    entry = {
      next = 0xffff880228880480,
      prev = 0xffff880228880480
    },
    parent = 0xffff8800b64c6050,
    kset = 0x0,
    ktype = 0xffffffffa0501300 <rdev_ktype>,
    sd = 0xffff88022bfc12d0,
    kref = {
      refcount = {
        counter = 1
      }
    },
    state_initialized = 1,
    state_in_sysfs = 1,
    state_add_uevent_sent = 0,
    state_remove_uevent_sent = 0,
    uevent_suppress = 0
  },
  flags = 581,
  blocked_wait = {
    lock = {
      {
        rlock = {
          raw_lock = {
            val = {
              counter = 0
            }
          }
        }
      }
    },
    task_list = {
      next = 0xffff8802288804c8,
      prev = 0xffff8802288804c8
    }
  },
  desc_nr = 0,
  raid_disk = 0,
  new_raid_disk = 0,
  saved_raid_disk = -1,
  {
    recovery_offset = 18446744073709551615,
    journal_tail = 18446744073709551615
  },
  nr_pending = {
    counter = 1176
  },
  read_errors = {
    counter = 0
  },
  last_read_error = {
    tv_sec = 0,
    tv_nsec = 0
  },
  corrected_errors = {
    counter = 0
  },
  del_work = {
    data = {
      counter = 0
    },
    entry = {
      next = 0x0,
      prev = 0x0
    },
    func = 0x0
  },
  sysfs_state = 0xffff88022bfc1348,
  badblocks = {
    count = 0,
    unacked_exist = 0,
    shift = 0,
    page = 0xffff8802289aa000,
    changed = 0,
    lock = {
      seqcount = {
        sequence = 0
      },
      lock = {
        {
          rlock = {
            raw_lock = {
              val = {
                counter = 0
              }
            }
          }
        }
      }
    },
    sector = 0,
    size = 0
  }
}

crash> struct md_rdev 0xffff88023202a200
struct md_rdev {
  same_set = {
    next = 0xffff8800b64c6018,
    prev = 0xffff880228880400
  },
  sectors = 2095104,
  mddev = 0xffff8800b64c6000,
  last_events = 37178561,
  meta_bdev = 0x0,
  bdev = 0xffff8800b60d09c0,
  sb_page = 0xffffea0008af7580,
  bb_page = 0xffffea0002e69380,
  sb_loaded = 1,
  sb_events = 167,
  data_offset = 2048,
  new_data_offset = 2048,
  sb_start = 8,
  sb_size = 512,
  preferred_minor = 65535,
  kobj = {
    name = 0xffff88023521ec30 "dev-loop1",
    entry = {
      next = 0xffff88023202a280,
      prev = 0xffff88023202a280
    },
    parent = 0xffff8800b64c6050,
    kset = 0x0,
    ktype = 0xffffffffa0501300 <rdev_ktype>,
    sd = 0xffff88022bc0a708,
    kref = {
      refcount = {
        counter = 1
      }
    },
    state_initialized = 1,
    state_in_sysfs = 1,
    state_add_uevent_sent = 0,
    state_remove_uevent_sent = 0,
    uevent_suppress = 0
  },
  flags = 2,
  blocked_wait = {
    lock = {
      {
        rlock = {
          raw_lock = {
            val = {
              counter = 0
            }
          }
        }
      }
    },
    task_list = {
      next = 0xffff88023202a2c8,
      prev = 0xffff88023202a2c8
    }
  },
crash> struct md_rdev 0xffff88023202a200
struct md_rdev {
  same_set = {
    next = 0xffff8800b64c6018,
    prev = 0xffff880228880400
  },
  sectors = 2095104,
  mddev = 0xffff8800b64c6000,
  last_events = 37178561,
  meta_bdev = 0x0,
  bdev = 0xffff8800b60d09c0,
  sb_page = 0xffffea0008af7580,
  bb_page = 0xffffea0002e69380,
  sb_loaded = 1,
  sb_events = 167,
  data_offset = 2048,
  new_data_offset = 2048,
  sb_start = 8,
  sb_size = 512,
  preferred_minor = 65535,
  kobj = {
    name = 0xffff88023521ec30 "dev-loop1",
    entry = {
      next = 0xffff88023202a280,
      prev = 0xffff88023202a280
    },
    parent = 0xffff8800b64c6050,
    kset = 0x0,
    ktype = 0xffffffffa0501300 <rdev_ktype>,
    sd = 0xffff88022bc0a708,
    kref = {
      refcount = {
        counter = 1
      }
    },
    state_initialized = 1,
    state_in_sysfs = 1,
    state_add_uevent_sent = 0,
    state_remove_uevent_sent = 0,
    uevent_suppress = 0
  },
  flags = 2,
  blocked_wait = {
    lock = {
      {
        rlock = {
          raw_lock = {
            val = {
              counter = 0
            }
          }
        }
      }
    },
    task_list = {
      next = 0xffff88023202a2c8,
      prev = 0xffff88023202a2c8
    }
  },
  desc_nr = 1,
  raid_disk = 1,
  new_raid_disk = 0,
  saved_raid_disk = -1,
  {
    recovery_offset = 18446744073709551615,
    journal_tail = 18446744073709551615
  },
  nr_pending = {
    counter = 1
  },
  read_errors = {
    counter = 0
  },
  last_read_error = {
    tv_sec = 0,
    tv_nsec = 0
  },
  corrected_errors = {
    counter = 0
  },
  del_work = {
    data = {
      counter = 0
    },
    entry = {
      next = 0x0,
      prev = 0x0
    },
    func = 0x0
  },
  sysfs_state = 0xffff88022bc0a780,
  badblocks = {
    count = 0,
    unacked_exist = 0,
    shift = 0,
    page = 0xffff88022bff0000,
    changed = 0,
    lock = {
      seqcount = {
        sequence = 164
      },
      lock = {
        {
          rlock = {
            raw_lock = {
              val = {
                counter = 0
              }
            }
          }
        }
      }
    },
    sector = 0,
    size = 0
  }
}



Thanks

-- 
Jinpu Wang
Linux Kernel Developer

ProfitBricks GmbH
Greifswalder Str. 207
D - 10405 Berlin

Tel:       +49 30 577 008  042
Fax:      +49 30 577 008 299
Email:    jinpu.wang@profitbricks.com
URL:      https://www.profitbricks.de

Sitz der Gesellschaft: Berlin
Registergericht: Amtsgericht Charlottenburg, HRB 125506 B
Geschäftsführer: Achim Weiss