[PATCH v18 54/83] sg: unlikely likely

[Douglas Gilbert <dgilbert@interlog.com>]

Apply 'unlikely' qualifier (or 'likely' qualifier) across
almost all functions in the driver where error path departs
from the fast path.

Other small changes:
  - move sg_rep_rq_state_fail() definition before use
  - remove some remnants of when SG_IOSUBMIT and SG_IORECEIVE
    accepted both v3 and v4 interfaces. Hence no need for
    array u8 to hold either interface, simply use correct
    interface type
  - refactor some abort request code

Signed-off-by: Douglas Gilbert <dgilbert@interlog.com>
---
 drivers/scsi/sg.c | 576 +++++++++++++++++++++++++---------------------
 1 file changed, 314 insertions(+), 262 deletions(-)

diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c
index d30c0034d767..27d9ac801f11 100644
--- a/drivers/scsi/sg.c
+++ b/drivers/scsi/sg.c
@@ -369,7 +369,7 @@ static const char *sg_shr_str(enum sg_shr_var sh_var, bool long_str);
 
 /* There is a assert that SZ_SG_IO_V4 >= SZ_SG_IO_HDR in first function */
 
-#define SG_IS_DETACHING(sdp) test_bit(SG_FDEV_DETACHING, (sdp)->fdev_bm)
+#define SG_IS_DETACHING(sdp) unlikely(test_bit(SG_FDEV_DETACHING, (sdp)->fdev_bm))
 #define SG_HAVE_EXCLUDE(sdp) test_bit(SG_FDEV_EXCLUDE, (sdp)->fdev_bm)
 #define SG_IS_O_NONBLOCK(sfp) (!!((sfp)->filp->f_flags & O_NONBLOCK))
 #define SG_RQ_ACTIVE(srp) (atomic_read(&(srp)->rq_st) != SG_RQ_INACTIVE)
@@ -430,12 +430,12 @@ sg_check_file_access(struct file *filp, const char *caller)
 	compiletime_assert(SZ_SG_IO_V4 >= SZ_SG_IO_HDR,
 			   "struct sg_io_v4 should be larger than sg_io_hdr");
 
-	if (filp->f_cred != current_real_cred()) {
+	if (unlikely(filp->f_cred != current_real_cred())) {
 		pr_err_once("%s: process %d (%s) changed security contexts after opening file descriptor, this is not allowed.\n",
 			caller, task_tgid_vnr(current), current->comm);
 		return -EPERM;
 	}
-	if (uaccess_kernel()) {
+	if (unlikely(uaccess_kernel())) {
 		pr_err_once("%s: process %d (%s) called from kernel context, this is not allowed.\n",
 			caller, task_tgid_vnr(current), current->comm);
 		return -EACCES;
@@ -454,7 +454,7 @@ sg_wait_open_event(struct sg_device *sdp, bool o_excl)
 			mutex_unlock(&sdp->open_rel_lock);
 			res = wait_event_interruptible
 					(sdp->open_wait,
-					 (unlikely(SG_IS_DETACHING(sdp)) ||
+					 (SG_IS_DETACHING(sdp) ||
 					  atomic_read(&sdp->open_cnt) == 0));
 			mutex_lock(&sdp->open_rel_lock);
 
@@ -468,7 +468,7 @@ sg_wait_open_event(struct sg_device *sdp, bool o_excl)
 			mutex_unlock(&sdp->open_rel_lock);
 			res = wait_event_interruptible
 					(sdp->open_wait,
-					 (unlikely(SG_IS_DETACHING(sdp)) ||
+					 (SG_IS_DETACHING(sdp) ||
 					  !SG_HAVE_EXCLUDE(sdp)));
 			mutex_lock(&sdp->open_rel_lock);
 
@@ -532,36 +532,36 @@ sg_open(struct inode *inode, struct file *filp)
 
 	/* Prevent the device driver from vanishing while we sleep */
 	res = scsi_device_get(sdp->device);
-	if (res)
+	if (unlikely(res))
 		goto sg_put;
 	res = scsi_autopm_get_device(sdp->device);
-	if (res)
+	if (unlikely(res))
 		goto sdp_put;
 	res = sg_allow_if_err_recovery(sdp, non_block);
-	if (res)
+	if (unlikely(res))
 		goto error_out;
 
 	mutex_lock(&sdp->open_rel_lock);
 	if (op_flags & O_NONBLOCK) {
-		if (o_excl) {
+		if (unlikely(o_excl)) {
 			if (atomic_read(&sdp->open_cnt) > 0) {
 				res = -EBUSY;
 				goto error_mutex_locked;
 			}
 		} else {
-			if (SG_HAVE_EXCLUDE(sdp)) {
+			if (unlikely(SG_HAVE_EXCLUDE(sdp))) {
 				res = -EBUSY;
 				goto error_mutex_locked;
 			}
 		}
 	} else {
 		res = sg_wait_open_event(sdp, o_excl);
-		if (res) /* -ERESTARTSYS or -ENODEV */
+		if (unlikely(res)) /* -ERESTARTSYS or -ENODEV */
 			goto error_mutex_locked;
 	}
 
 	/* N.B. at this point we are holding the open_rel_lock */
-	if (o_excl)
+	if (unlikely(o_excl))
 		set_bit(SG_FDEV_EXCLUDE, sdp->fdev_bm);
 
 	o_count = atomic_inc_return(&sdp->open_cnt);
@@ -586,7 +586,7 @@ sg_open(struct inode *inode, struct file *filp)
 	return res;
 
 out_undo:
-	if (o_excl) {		/* undo if error */
+	if (unlikely(o_excl)) {		/* undo if error */
 		clear_bit(SG_FDEV_EXCLUDE, sdp->fdev_bm);
 		wake_up_interruptible(&sdp->open_wait);
 	}
@@ -640,7 +640,7 @@ sg_release(struct inode *inode, struct file *filp)
 	if (unlikely(!sdp))
 		return -ENXIO;
 
-	if (xa_get_mark(&sdp->sfp_arr, sfp->idx, SG_XA_FD_FREE)) {
+	if (unlikely(xa_get_mark(&sdp->sfp_arr, sfp->idx, SG_XA_FD_FREE))) {
 		SG_LOG(1, sfp, "%s: sfp erased!!!\n", __func__);
 		return 0;	/* get out but can't fail */
 	}
@@ -648,11 +648,11 @@ sg_release(struct inode *inode, struct file *filp)
 	mutex_lock(&sdp->open_rel_lock);
 	o_count = atomic_read(&sdp->open_cnt);
 	SG_LOG(3, sfp, "%s: open count before=%d\n", __func__, o_count);
-	if (test_and_set_bit(SG_FFD_RELEASE, sfp->ffd_bm))
+	if (unlikely(test_and_set_bit(SG_FFD_RELEASE, sfp->ffd_bm)))
 		SG_LOG(1, sfp, "%s: second release on this fd ? ?\n",
 		       __func__);
 	scsi_autopm_put_device(sdp->device);
-	if (!xa_get_mark(&sdp->sfp_arr, sfp->idx, SG_XA_FD_FREE) &&
+	if (likely(!xa_get_mark(&sdp->sfp_arr, sfp->idx, SG_XA_FD_FREE)) &&
 	    sg_fd_is_shared(sfp))
 		sg_remove_sfp_share(sfp, xa_get_mark(&sdp->sfp_arr, sfp->idx,
 						     SG_XA_FD_RS_SHARE));
@@ -696,16 +696,16 @@ sg_write(struct file *filp, const char __user *p, size_t count, loff_t *ppos)
 	struct sg_comm_wr_t cwr;
 
 	res = sg_check_file_access(filp, __func__);
-	if (res)
+	if (unlikely(res))
 		return res;
 
 	sfp = filp->private_data;
 	sdp = sfp->parentdp;
 	SG_LOG(3, sfp, "%s: write(3rd arg) count=%d\n", __func__, (int)count);
 	res = sg_allow_if_err_recovery(sdp, !!(filp->f_flags & O_NONBLOCK));
-	if (res)
+	if (unlikely(res))
 		return res;
-	if (count < SZ_SG_HEADER || count > SG_WRITE_COUNT_LIMIT)
+	if (unlikely(count < SZ_SG_HEADER || count > SG_WRITE_COUNT_LIMIT))
 		return -EIO;
 #ifdef CONFIG_COMPAT
 	if (in_compat_syscall())
@@ -732,10 +732,10 @@ sg_write(struct file *filp, const char __user *p, size_t count, loff_t *ppos)
 #else
 			lt = (count < sizeof(struct sg_io_hdr));
 #endif
-			if (lt)
+			if (unlikely(lt))
 				return -EIO;
 			get_v3_hdr = true;
-			if (get_sg_io_hdr(h3p, p))
+			if (unlikely(get_sg_io_hdr(h3p, p)))
 				return -EFAULT;
 		}
 	}
@@ -758,13 +758,13 @@ sg_write(struct file *filp, const char __user *p, size_t count, loff_t *ppos)
 	}
 to_v2:
 	/* v1 and v2 interfaces processed below this point */
-	if (count < (SZ_SG_HEADER + 6))
+	if (unlikely(count < SZ_SG_HEADER + 6))
 		return -EIO;    /* minimum scsi command length is 6 bytes */
 	p += SZ_SG_HEADER;
-	if (get_user(opcode, p))
+	if (unlikely(get_user(opcode, p)))
 		return -EFAULT;
 	mutex_lock(&sfp->f_mutex);
-	if (sfp->next_cmd_len > 0) {
+	if (unlikely(sfp->next_cmd_len > 0)) {
 		cmd_size = sfp->next_cmd_len;
 		sfp->next_cmd_len = 0;	/* reset, only this write() effected */
 	} else {
@@ -779,7 +779,7 @@ sg_write(struct file *filp, const char __user *p, size_t count, loff_t *ppos)
 	mxsize = max_t(int, input_size, ohp->reply_len);
 	mxsize -= SZ_SG_HEADER;
 	input_size -= SZ_SG_HEADER;
-	if (input_size < 0)
+	if (unlikely(input_size < 0))
 		return -EIO; /* Insufficient bytes passed for this command. */
 	memset(h3p, 0, sizeof(*h3p));
 	h3p->interface_id = '\0';/* indicate v1 or v2 interface (tunnelled) */
@@ -808,7 +808,7 @@ sg_write(struct file *filp, const char __user *p, size_t count, loff_t *ppos)
 	 * but it is possible that the app intended SG_DXFER_TO_DEV, because
 	 * there is a non-zero input_size, so emit a warning.
 	 */
-	if (h3p->dxfer_direction == SG_DXFER_TO_FROM_DEV) {
+	if (unlikely(h3p->dxfer_direction == SG_DXFER_TO_FROM_DEV)) {
 		printk_ratelimited
 			(KERN_WARNING
 			 "%s: data in/out %d/%d bytes for SCSI command 0x%x-- guessing data in;\n"
@@ -832,13 +832,13 @@ sg_chk_mmap(struct sg_fd *sfp, int rq_flags, int len)
 {
 	if (unlikely(sfp->mmap_sz == 0))
 		return -EBADFD;
-	if (atomic_read(&sfp->submitted) > 0)
+	if (unlikely(atomic_read(&sfp->submitted) > 0))
 		return -EBUSY;  /* already active requests on fd */
-	if (len > sfp->rsv_srp->sgat_h.buflen)
+	if (unlikely(len > sfp->rsv_srp->sgat_h.buflen))
 		return -ENOMEM; /* MMAP_IO size must fit in reserve */
 	if (unlikely(len > sfp->mmap_sz))
 		return -ENOMEM; /* MMAP_IO size can't exceed mmap() size */
-	if (rq_flags & SG_FLAG_DIRECT_IO)
+	if (unlikely(rq_flags & SG_FLAG_DIRECT_IO))
 		return -EINVAL; /* not both MMAP_IO and DIRECT_IO */
 	return 0;
 }
@@ -846,7 +846,7 @@ sg_chk_mmap(struct sg_fd *sfp, int rq_flags, int len)
 static int
 sg_fetch_cmnd(struct sg_fd *sfp, const u8 __user *u_cdbp, int len, u8 *cdbp)
 {
-	if (!u_cdbp || len < 6 || len > SG_MAX_CDB_SIZE)
+	if (unlikely(!u_cdbp || len < 6 || len > SG_MAX_CDB_SIZE))
 		return -EMSGSIZE;
 	if (copy_from_user(cdbp, u_cdbp, len))
 		return -EFAULT;
@@ -872,9 +872,9 @@ sg_submit_v3(struct sg_fd *sfp, struct sg_io_hdr *hp, bool sync,
 	struct sg_comm_wr_t cwr;
 
 	/* now doing v3 blocking (sync) or non-blocking submission */
-	if (hp->flags & SGV4_FLAG_MULTIPLE_REQS)
+	if (unlikely(hp->flags & SGV4_FLAG_MULTIPLE_REQS))
 		return -ERANGE;		/* need to use v4 interface */
-	if (hp->flags & SG_FLAG_MMAP_IO) {
+	if (unlikely(hp->flags & SG_FLAG_MMAP_IO)) {
 		int res = sg_chk_mmap(sfp, hp->flags, hp->dxfer_len);
 
 		if (unlikely(res))
@@ -928,9 +928,9 @@ sg_mrq_arr_flush(struct sg_io_v4 *cop, struct sg_io_v4 *a_hds, u32 tot_reqs,
 	u32 sz = min(tot_reqs * SZ_SG_IO_V4, cop->din_xfer_len);
 	void __user *p = uptr64(cop->din_xferp);
 
-	if (s_res)
+	if (unlikely(s_res))
 		cop->spare_out = -s_res;
-	if (!p)
+	if (unlikely(!p))
 		return 0;
 	if (sz > 0) {
 		if (copy_to_user(p, a_hds, sz))
@@ -947,14 +947,14 @@ sg_mrq_1complet(struct sg_io_v4 *cop, struct sg_io_v4 *a_hds,
 	struct sg_io_v4 *siv4p;
 
 	SG_LOG(3, w_sfp, "%s: start, tot_reqs=%d\n", __func__, tot_reqs);
-	if (!srp)
+	if (unlikely(!srp))
 		return -EPROTO;
 	indx = srp->s_hdr4.mrq_ind;
-	if (indx < 0 || indx >= tot_reqs)
+	if (unlikely(indx < 0 || indx >= tot_reqs))
 		return -EPROTO;
 	siv4p = a_hds + indx;
 	s_res = sg_receive_v4(w_sfp, srp, NULL, siv4p);
-	if (s_res == -EFAULT)
+	if (unlikely(s_res == -EFAULT))
 		return s_res;
 	siv4p->info |= SG_INFO_MRQ_FINI;
 	if (w_sfp->async_qp && (siv4p->flags & SGV4_FLAG_SIGNAL)) {
@@ -1067,32 +1067,32 @@ sg_mrq_sanity(struct sg_device *sdp, struct sg_io_v4 *cop,
 			       __func__, k, "bad guard");
 			return -ERANGE;
 		}
-		if (flags & SGV4_FLAG_MULTIPLE_REQS) {
+		if (unlikely(flags & SGV4_FLAG_MULTIPLE_REQS)) {
 			SG_LOG(1, sfp, "%s: %s %u: no nested multi-reqs\n",
 			       __func__, rip, k);
 			return -ERANGE;
 		}
 		if (immed) {	/* only accept async submits on current fd */
-			if (flags & SGV4_FLAG_DO_ON_OTHER) {
+			if (unlikely(flags & SGV4_FLAG_DO_ON_OTHER)) {
 				SG_LOG(1, sfp, "%s: %s %u, %s\n", __func__,
 				       rip, k, "no IMMED with ON_OTHER");
 				return -ERANGE;
-			} else if (flags & SGV4_FLAG_SHARE) {
+			} else if (unlikely(flags & SGV4_FLAG_SHARE)) {
 				SG_LOG(1, sfp, "%s: %s %u, %s\n", __func__,
 				       rip, k, "no IMMED with FLAG_SHARE");
 				return -ERANGE;
-			} else if (flags & SGV4_FLAG_COMPLETE_B4) {
+			} else if (unlikely(flags & SGV4_FLAG_COMPLETE_B4)) {
 				SG_LOG(1, sfp, "%s: %s %u, %s\n", __func__,
 				       rip, k, "no IMMED with COMPLETE_B4");
 				return -ERANGE;
 			}
 		}
 		if (!sg_fd_is_shared(sfp)) {
-			if (flags & SGV4_FLAG_SHARE) {
+			if (unlikely(flags & SGV4_FLAG_SHARE)) {
 				SG_LOG(1, sfp, "%s: %s %u, no share\n",
 				       __func__, rip, k);
 				return -ERANGE;
-			} else if (flags & SGV4_FLAG_DO_ON_OTHER) {
+			} else if (unlikely(flags & SGV4_FLAG_DO_ON_OTHER)) {
 				SG_LOG(1, sfp, "%s: %s %u, %s do on\n",
 				       __func__, rip, k, "no other fd to");
 				return -ERANGE;
@@ -1188,13 +1188,13 @@ sg_do_multi_req(struct sg_comm_wr_t *cwrp, bool blocking)
 		cdb_mxlen = 0;
 	}
 
-	if (unlikely(SG_IS_DETACHING(sdp)))
+	if (SG_IS_DETACHING(sdp))
 		return -ENODEV;
 	else if (unlikely(o_sfp && SG_IS_DETACHING((o_sfp->parentdp))))
 		return -ENODEV;
 
 	a_hds = kcalloc(tot_reqs, SZ_SG_IO_V4, GFP_KERNEL | __GFP_NOWARN);
-	if (!a_hds)
+	if (unlikely(!a_hds))
 		return -ENOMEM;
 	n = tot_reqs * SZ_SG_IO_V4;
 	if (copy_from_user(a_hds, cuptr64(cop->dout_xferp), n)) {
@@ -1261,10 +1261,6 @@ sg_do_multi_req(struct sg_comm_wr_t *cwrp, bool blocking)
 			}
 			break;
 		}
-		if (!srp) {
-			s_res = -EPROTO;
-			break;
-		}
 		++num_cmpl;
 		hp->info |= SG_INFO_MRQ_FINI;
 		if (stop_if && (hp->driver_status || hp->transport_status ||
@@ -1292,14 +1288,14 @@ sg_do_multi_req(struct sg_comm_wr_t *cwrp, bool blocking)
 	if (immed)
 		goto fini;
 
-	if (res == 0 && (this_fp_sent + other_fp_sent) > 0) {
+	if (likely(res == 0 && (this_fp_sent + other_fp_sent) > 0)) {
 		s_res = sg_mrq_complets(cop, a_hds, fp, o_sfp, tot_reqs,
 					this_fp_sent, other_fp_sent);
-		if (s_res == -EFAULT || s_res == -ERESTARTSYS)
+		if (unlikely(s_res == -EFAULT || s_res == -ERESTARTSYS))
 			res = s_res;	/* this may leave orphans */
 	}
 fini:
-	if (res == 0 && !immed)
+	if (likely(res == 0) && !immed)
 		res = sg_mrq_arr_flush(cop, a_hds, tot_reqs, s_res);
 	kfree(cdb_ap);
 	kfree(a_hds);
@@ -1329,7 +1325,7 @@ sg_submit_v4(struct sg_fd *sfp, void __user *p, struct sg_io_v4 *h4p,
 		res = sg_do_multi_req(&cwr, sync);
 		if (unlikely(res))
 			return res;
-		if (p) {
+		if (likely(p)) {
 			/* Write back sg_io_v4 object for error/warning info */
 			if (copy_to_user(p, h4p, SZ_SG_IO_V4))
 				return -EFAULT;
@@ -1368,8 +1364,8 @@ sg_submit_v4(struct sg_fd *sfp, void __user *p, struct sg_io_v4 *h4p,
 		u64 gen_tag = srp->tag;
 		struct sg_io_v4 __user *h4_up = (struct sg_io_v4 __user *)p;
 
-		if (unlikely(copy_to_user(&h4_up->generated_tag, &gen_tag,
-					  sizeof(gen_tag))))
+		if (copy_to_user(&h4_up->generated_tag, &gen_tag,
+				 sizeof(gen_tag)))
 			return -EFAULT;
 	}
 	return res;
@@ -1384,11 +1380,11 @@ sg_ctl_iosubmit(struct sg_fd *sfp, void __user *p)
 	struct sg_device *sdp = sfp->parentdp;
 
 	res = sg_allow_if_err_recovery(sdp, SG_IS_O_NONBLOCK(sfp));
-	if (res)
+	if (unlikely(res))
 		return res;
 	if (copy_from_user(h4p, p, SZ_SG_IO_V4))
 		return -EFAULT;
-	if (h4p->guard == 'Q')
+	if (likely(h4p->guard == 'Q'))
 		return sg_submit_v4(sfp, p, h4p, false, NULL);
 	return -EPERM;
 }
@@ -1406,7 +1402,7 @@ sg_ctl_iosubmit_v3(struct sg_fd *sfp, void __user *p)
 		return res;
 	if (copy_from_user(h3p, p, SZ_SG_IO_HDR))
 		return -EFAULT;
-	if (h3p->interface_id == 'S')
+	if (likely(h3p->interface_id == 'S'))
 		return sg_submit_v3(sfp, h3p, false, NULL);
 	return -EPERM;
 }
@@ -1428,26 +1424,26 @@ sg_share_chk_flags(struct sg_fd *sfp, u32 rq_flags, int dxfer_len, int dir,
 	enum sg_shr_var sh_var = SG_SHR_NONE;
 
 	if (rq_flags & SGV4_FLAG_SHARE) {
-		if (rq_flags & SG_FLAG_DIRECT_IO)
+		if (unlikely(rq_flags & SG_FLAG_DIRECT_IO))
 			result = -EINVAL; /* since no control of data buffer */
-		else if (dxfer_len < 1)
+		else if (unlikely(dxfer_len < 1))
 			result = -ENODATA;
 		else if (is_read_side) {
 			sh_var = SG_SHR_RS_RQ;
-			if (dir != SG_DXFER_FROM_DEV)
+			if (unlikely(dir != SG_DXFER_FROM_DEV))
 				result = -ENOMSG;
 			if (rq_flags & SGV4_FLAG_NO_DXFER) {
 				/* rule out some contradictions */
-				if (rq_flags & SG_FL_MMAP_DIRECT)
+				if (unlikely(rq_flags & SG_FL_MMAP_DIRECT))
 					result = -ENODATA;
 			}
 		} else {			/* fd is write-side */
 			sh_var = SG_SHR_WS_RQ;
-			if (dir != SG_DXFER_TO_DEV)
+			if (unlikely(dir != SG_DXFER_TO_DEV))
 				result = -ENOMSG;
-			if (!(rq_flags & SGV4_FLAG_NO_DXFER))
+			if (unlikely(!(rq_flags & SGV4_FLAG_NO_DXFER)))
 				result = -ENOMSG;
-			if (rq_flags & SG_FL_MMAP_DIRECT)
+			if (unlikely(rq_flags & SG_FL_MMAP_DIRECT))
 				result = -ENODATA;
 		}
 	} else if (is_read_side) {
@@ -1515,7 +1511,7 @@ sg_rq_chg_state_ulck(struct sg_request *srp, enum sg_rq_state old_st,
 		sg_rq_state_mul2arr[(int)new_st];
 	act_old_st = (enum sg_rq_state)atomic_cmpxchg(&srp->rq_st, old_st,
 						      new_st);
-	if (act_old_st != old_st) {
+	if (unlikely(act_old_st != old_st)) {
 #if IS_ENABLED(SG_LOG_ACTIVE)
 		SG_LOG(1, srp->parentfp, "%s: unexpected old state: %s\n",
 		       __func__, sg_rq_st_str(act_old_st, false));
@@ -1671,14 +1667,14 @@ sg_common_write(struct sg_comm_wr_t *cwrp)
 	struct sg_io_hdr *hi_p;
 	struct sg_io_v4 *h4p;
 
-	if (test_bit(SG_FRQ_IS_V4I, cwrp->frq_bm)) {
+	if (likely(test_bit(SG_FRQ_IS_V4I, cwrp->frq_bm))) {
 		h4p = cwrp->h4p;
 		hi_p = NULL;
 		dxfr_len = 0;
 		dir = SG_DXFER_NONE;
 		rq_flags = h4p->flags;
 		pack_id = h4p->request_extra;
-		if (h4p->din_xfer_len && h4p->dout_xfer_len) {
+		if (unlikely(h4p->din_xfer_len && h4p->dout_xfer_len)) {
 			return ERR_PTR(-EOPNOTSUPP);
 		} else if (h4p->din_xfer_len) {
 			dxfr_len = h4p->din_xfer_len;
@@ -1695,7 +1691,7 @@ sg_common_write(struct sg_comm_wr_t *cwrp)
 		rq_flags = hi_p->flags;
 		pack_id = hi_p->pack_id;
 	}
-	if (rq_flags & SGV4_FLAG_MULTIPLE_REQS)
+	if (unlikely(rq_flags & SGV4_FLAG_MULTIPLE_REQS))
 		return ERR_PTR(-ERANGE);
 	if (sg_fd_is_shared(fp)) {
 		res = sg_share_chk_flags(fp, rq_flags, dxfr_len, dir, &sh_var);
@@ -1706,7 +1702,7 @@ sg_common_write(struct sg_comm_wr_t *cwrp)
 		if (rq_flags & SGV4_FLAG_SHARE)
 			return ERR_PTR(-ENOMSG);
 	}
-	if (dxfr_len >= SZ_256M)
+	if (unlikely(dxfr_len >= SZ_256M))
 		return ERR_PTR(-EINVAL);
 
 	srp = sg_setup_req(cwrp, sh_var, dxfr_len);
@@ -1715,7 +1711,7 @@ sg_common_write(struct sg_comm_wr_t *cwrp)
 	srp->rq_flags = rq_flags;
 	srp->pack_id = pack_id;
 
-	if (h4p) {
+	if (likely(h4p)) {
 		srp->s_hdr4.usr_ptr = h4p->usr_ptr;
 		srp->s_hdr4.sbp = uptr64(h4p->response);
 		srp->s_hdr4.max_sb_len = h4p->max_response_len;
@@ -1726,11 +1722,11 @@ sg_common_write(struct sg_comm_wr_t *cwrp)
 		memcpy(&srp->s_hdr3, hi_p, sizeof(srp->s_hdr3));
 	}
 	res = sg_start_req(srp, cwrp, dir);
-	if (res < 0)		/* probably out of space --> -ENOMEM */
+	if (unlikely(res < 0))	/* probably out of space --> -ENOMEM */
 		goto err_out;
 	SG_LOG(4, fp, "%s: opcode=0x%02x, cdb_sz=%d, pack_id=%d\n", __func__,
 	       srp->cmd_opcode, cwrp->cmd_len, pack_id);
-	if (unlikely(SG_IS_DETACHING(sdp))) {
+	if (SG_IS_DETACHING(sdp)) {
 		res = -ENODEV;
 		goto err_out;
 	}
@@ -1761,7 +1757,7 @@ sg_get_ready_srp(struct sg_fd *sfp, struct sg_request **srpp, int id,
 {
 	struct sg_request *srp;
 
-	if (unlikely(SG_IS_DETACHING(sfp->parentdp))) {
+	if (SG_IS_DETACHING(sfp->parentdp)) {
 		*srpp = ERR_PTR(-ENODEV);
 		return true;
 	}
@@ -1782,8 +1778,8 @@ sg_copy_sense(struct sg_request *srp, bool v4_active)
 
 	/* If need be, copy the sense buffer to the user space */
 	scsi_stat = srp->rq_result & 0xff;
-	if ((scsi_stat & SAM_STAT_CHECK_CONDITION) ||
-	    (driver_byte(srp->rq_result) & DRIVER_SENSE)) {
+	if (unlikely((scsi_stat & SAM_STAT_CHECK_CONDITION) ||
+		     (driver_byte(srp->rq_result) & DRIVER_SENSE))) {
 		int sb_len = min_t(int, SCSI_SENSE_BUFFERSIZE, srp->sense_len);
 		int mx_sb_len;
 		u8 *sbp = srp->sense_bp;
@@ -1821,12 +1817,12 @@ sg_rec_state_v3v4(struct sg_fd *sfp, struct sg_request *srp, bool v4_active)
 	if (unlikely(srp->rq_result & 0xff)) {
 		int sb_len_wr = sg_copy_sense(srp, v4_active);
 
-		if (sb_len_wr < 0)
+		if (unlikely(sb_len_wr < 0))
 			return sb_len_wr;
 	}
 	if (rq_res & SG_ML_RESULT_MSK)
 		srp->rq_info |= SG_INFO_CHECK;
-	if (test_bit(SG_FRQ_ABORTING, srp->frq_bm))
+	if (unlikely(test_bit(SG_FRQ_ABORTING, srp->frq_bm)))
 		srp->rq_info |= SG_INFO_ABORTED;
 
 	sh_sfp = sg_fd_share_ptr(sfp);
@@ -1856,7 +1852,7 @@ sg_rec_state_v3v4(struct sg_fd *sfp, struct sg_request *srp, bool v4_active)
 			break;	/* nothing to do */
 		}
 	}
-	if (unlikely(SG_IS_DETACHING(sfp->parentdp)))
+	if (SG_IS_DETACHING(sfp->parentdp))
 		srp->rq_info |= SG_INFO_DEVICE_DETACHING;
 	return err;
 }
@@ -1873,7 +1869,8 @@ sg_complete_v3v4(struct sg_fd *sfp, struct sg_request *srp, bool other_err)
 			int poll_type = POLL_OUT;
 			struct sg_fd *ws_sfp = sg_fd_share_ptr(sfp);
 
-			if ((srp->rq_result & SG_ML_RESULT_MSK) || other_err) {
+			if (unlikely((srp->rq_result & SG_ML_RESULT_MSK) ||
+				     other_err)) {
 				set_bit(SG_FFD_READ_SIDE_ERR, sfp->ffd_bm);
 				if (sr_st != SG_RQ_BUSY)
 					sg_rq_chg_state_force(srp, SG_RQ_BUSY);
@@ -1892,7 +1889,7 @@ sg_complete_v3v4(struct sg_fd *sfp, struct sg_request *srp, bool other_err)
 		{
 			struct sg_fd *rs_sfp = sg_fd_share_ptr(sfp);
 
-			if (rs_sfp) {
+			if (likely(rs_sfp)) {
 				rs_sfp->ws_srp = NULL;
 				if (rs_sfp->rsv_srp)
 					rs_sfp->rsv_srp->sh_var =
@@ -1955,7 +1952,7 @@ sg_receive_v4(struct sg_fd *sfp, struct sg_request *srp, void __user *p,
 	sg_complete_v3v4(sfp, srp, err < 0);
 	sg_finish_scsi_blk_rq(srp);
 	sg_deact_request(sfp, srp);
-	return err < 0 ? err : 0;
+	return unlikely(err < 0) ? err : 0;
 }
 
 /*
@@ -2017,16 +2014,16 @@ sg_mrq_ioreceive(struct sg_fd *sfp, struct sg_io_v4 *cop, void __user *p,
 
 	SG_LOG(3, sfp, "%s: non_block=%d\n", __func__, !!non_block);
 	n = cop->din_xfer_len;
-	if (n > SG_MAX_MULTI_REQ_SZ)
+	if (unlikely(n > SG_MAX_MULTI_REQ_SZ))
 		return -E2BIG;
-	if (!cop->din_xferp || n < SZ_SG_IO_V4 || (n % SZ_SG_IO_V4))
+	if (unlikely(!cop->din_xferp || n < SZ_SG_IO_V4 || (n % SZ_SG_IO_V4)))
 		return -ERANGE;
 	n /= SZ_SG_IO_V4;
 	len = n * SZ_SG_IO_V4;
 	SG_LOG(3, sfp, "%s: %s, num_reqs=%u\n", __func__,
 	       (non_block ? "IMMED" : "blocking"), n);
 	rsp_v4_arr = kcalloc(n, SZ_SG_IO_V4, GFP_KERNEL);
-	if (!rsp_v4_arr)
+	if (unlikely(!rsp_v4_arr))
 		return -ENOMEM;
 
 	sg_sgv4_out_zero(cop);
@@ -2040,7 +2037,7 @@ sg_mrq_ioreceive(struct sg_fd *sfp, struct sg_io_v4 *cop, void __user *p,
 		return -EFAULT;
 	res = 0;
 	pp = uptr64(cop->din_xferp);
-	if (pp) {
+	if (likely(pp)) {
 		if (copy_to_user(pp, rsp_v4_arr, len))
 			res = -EFAULT;
 	} else {
@@ -2066,19 +2063,20 @@ sg_ctl_ioreceive(struct sg_fd *sfp, void __user *p)
 	int res, id;
 	int pack_id = SG_PACK_ID_WILDCARD;
 	int tag = SG_TAG_WILDCARD;
-	u8 v4_holder[SZ_SG_IO_V4];
-	struct sg_io_v4 *h4p = (struct sg_io_v4 *)v4_holder;
+	struct sg_io_v4 h4;
+	struct sg_io_v4 *h4p = &h4;
 	struct sg_device *sdp = sfp->parentdp;
 	struct sg_request *srp;
 
 	res = sg_allow_if_err_recovery(sdp, non_block);
-	if (res)
+	if (unlikely(res))
 		return res;
 	/* Get first three 32 bit integers: guard, proto+subproto */
 	if (copy_from_user(h4p, p, SZ_SG_IO_V4))
 		return -EFAULT;
 	/* for v4: protocol=0 --> SCSI;  subprotocol=0 --> SPC++ */
-	if (h4p->guard != 'Q' || h4p->protocol != 0 || h4p->subprotocol != 0)
+	if (unlikely(h4p->guard != 'Q' || h4p->protocol != 0 ||
+		     h4p->subprotocol != 0))
 		return -EPERM;
 	if (h4p->flags & SGV4_FLAG_IMMED)
 		non_block = true;	/* set by either this or O_NONBLOCK */
@@ -2097,14 +2095,14 @@ sg_ctl_ioreceive(struct sg_fd *sfp, void __user *p)
 try_again:
 	srp = sg_find_srp_by_id(sfp, id, use_tag);
 	if (!srp) {     /* nothing available so wait on packet or */
-		if (unlikely(SG_IS_DETACHING(sdp)))
+		if (SG_IS_DETACHING(sdp))
 			return -ENODEV;
 		if (non_block)
 			return -EAGAIN;
 		res = wait_event_interruptible
 				(sfp->cmpl_wait,
 				 sg_get_ready_srp(sfp, &srp, id, use_tag));
-		if (res)
+		if (unlikely(res))
 			return res;	/* signal --> -ERESTARTSYS */
 		if (IS_ERR(srp))
 			return PTR_ERR(srp);
@@ -2129,8 +2127,8 @@ sg_ctl_ioreceive_v3(struct sg_fd *sfp, void __user *p)
 	bool non_block = SG_IS_O_NONBLOCK(sfp);
 	int res;
 	int pack_id = SG_PACK_ID_WILDCARD;
-	u8 v3_holder[SZ_SG_IO_HDR];
-	struct sg_io_hdr *h3p = (struct sg_io_hdr *)v3_holder;
+	struct sg_io_hdr h3;
+	struct sg_io_hdr *h3p = &h3;
 	struct sg_device *sdp = sfp->parentdp;
 	struct sg_request *srp;
 
@@ -2141,12 +2139,12 @@ sg_ctl_ioreceive_v3(struct sg_fd *sfp, void __user *p)
 	if (copy_from_user(h3p, p, SZ_SG_IO_HDR))
 		return -EFAULT;
 	/* for v3: interface_id=='S' (in a 32 bit int) */
-	if (h3p->interface_id != 'S')
+	if (unlikely(h3p->interface_id != 'S'))
 		return -EPERM;
 	if (h3p->flags & SGV4_FLAG_IMMED)
 		non_block = true;	/* set by either this or O_NONBLOCK */
 	SG_LOG(3, sfp, "%s: non_block(+IMMED)=%d\n", __func__, non_block);
-	if (h3p->flags & SGV4_FLAG_MULTIPLE_REQS)
+	if (unlikely(h3p->flags & SGV4_FLAG_MULTIPLE_REQS))
 		return -EINVAL;
 
 	if (test_bit(SG_FFD_FORCE_PACKID, sfp->ffd_bm))
@@ -2154,7 +2152,7 @@ sg_ctl_ioreceive_v3(struct sg_fd *sfp, void __user *p)
 try_again:
 	srp = sg_find_srp_by_id(sfp, pack_id, false);
 	if (!srp) {     /* nothing available so wait on packet or */
-		if (unlikely(SG_IS_DETACHING(sdp)))
+		if (SG_IS_DETACHING(sdp))
 			return -ENODEV;
 		if (non_block)
 			return -EAGAIN;
@@ -2193,9 +2191,9 @@ sg_read_v1v2(void __user *buf, int count, struct sg_fd *sfp,
 	h2p->target_status = status_byte(rq_result);
 	h2p->host_status = host_byte(rq_result);
 	h2p->driver_status = driver_byte(rq_result);
-	if ((CHECK_CONDITION & status_byte(rq_result)) ||
-	    (DRIVER_SENSE & driver_byte(rq_result))) {
-		if (srp->sense_bp) {
+	if (unlikely((CHECK_CONDITION & status_byte(rq_result)) ||
+		     (DRIVER_SENSE & driver_byte(rq_result)))) {
+		if (likely(srp->sense_bp)) {
 			u8 *sbp = srp->sense_bp;
 
 			srp->sense_bp = NULL;
@@ -2204,7 +2202,7 @@ sg_read_v1v2(void __user *buf, int count, struct sg_fd *sfp,
 			mempool_free(sbp, sg_sense_pool);
 		}
 	}
-	switch (host_byte(rq_result)) {
+	switch (unlikely(host_byte(rq_result))) {
 	/*
 	 * This following setting of 'result' is for backward compatibility
 	 * and is best ignored by the user who should use target, host and
@@ -2246,7 +2244,7 @@ sg_read_v1v2(void __user *buf, int count, struct sg_fd *sfp,
 			count = h2p->reply_len;
 		if (count > SZ_SG_HEADER) {
 			res = sg_read_append(srp, buf, count - SZ_SG_HEADER);
-			if (res)
+			if (unlikely(res))
 				goto fini;
 		}
 	} else {
@@ -2282,14 +2280,14 @@ sg_read(struct file *filp, char __user *p, size_t count, loff_t *ppos)
 	 * file descriptor to free up any resources being held.
 	 */
 	ret = sg_check_file_access(filp, __func__);
-	if (ret)
+	if (unlikely(ret))
 		return ret;
 
 	sfp = filp->private_data;
 	sdp = sfp->parentdp;
 	SG_LOG(3, sfp, "%s: read() count=%d\n", __func__, (int)count);
 	ret = sg_allow_if_err_recovery(sdp, non_block);
-	if (ret)
+	if (unlikely(ret))
 		return ret;
 
 	could_be_v3 = (count >= SZ_SG_IO_HDR);
@@ -2307,12 +2305,12 @@ sg_read(struct file *filp, char __user *p, size_t count, loff_t *ppos)
 		if (h2p->reply_len < 0 && could_be_v3) {
 			struct sg_io_hdr *v3_hdr = (struct sg_io_hdr *)h2p;
 
-			if (v3_hdr->interface_id == 'S') {
+			if (likely(v3_hdr->interface_id == 'S')) {
 				struct sg_io_hdr __user *h3_up;
 
 				h3_up = (struct sg_io_hdr __user *)p;
 				ret = get_user(want_id, &h3_up->pack_id);
-				if (ret)
+				if (unlikely(ret))
 					return ret;
 				if (!non_block) {
 					int flgs;
@@ -2337,14 +2335,14 @@ sg_read(struct file *filp, char __user *p, size_t count, loff_t *ppos)
 try_again:
 	srp = sg_find_srp_by_id(sfp, want_id, false);
 	if (!srp) {	/* nothing available so wait on packet to arrive or */
-		if (unlikely(SG_IS_DETACHING(sdp)))
+		if (SG_IS_DETACHING(sdp))
 			return -ENODEV;
 		if (non_block) /* O_NONBLOCK or v3::flags & SGV4_FLAG_IMMED */
 			return -EAGAIN;
 		ret = wait_event_interruptible
 				(sfp->cmpl_wait,
 				 sg_get_ready_srp(sfp, &srp, want_id, false));
-		if (ret)	/* -ERESTARTSYS as signal hit process */
+		if (unlikely(ret))  /* -ERESTARTSYS as signal hit process */
 			return ret;
 		if (IS_ERR(srp))
 			return PTR_ERR(srp);
@@ -2365,9 +2363,11 @@ sg_read(struct file *filp, char __user *p, size_t count, loff_t *ppos)
 		}
 		ret = sg_receive_v3(sfp, srp, p);
 	}
-	if (ret < 0)
+#if IS_ENABLED(SG_LOG_ACTIVE)
+	if (unlikely(ret < 0))
 		SG_LOG(1, sfp, "%s: negated errno: %d\n", __func__, ret);
-	return ret < 0 ? ret : (int)count;
+#endif
+	return unlikely(ret < 0) ? ret : (int)count;
 }
 
 /*
@@ -2496,7 +2496,7 @@ sg_change_after_read_side_rq(struct sg_fd *sfp, bool fini1_again0)
 		case SG_RQ_BUSY:
 			res = -EAGAIN;
 			break;
-		default:	/* read-side in SG_RQ_SHR_SWAIT is bad */
+		default:
 			res = -EINVAL;
 			break;
 		}
@@ -2589,14 +2589,15 @@ sg_remove_sfp_share(struct sg_fd *sfp, bool is_rd_side)
 	if (is_rd_side) {
 		bool set_inactive = false;
 
-		if (!xa_get_mark(xadp, sfp->idx, SG_XA_FD_RS_SHARE)) {
+		if (unlikely(!xa_get_mark(xadp, sfp->idx,
+					  SG_XA_FD_RS_SHARE))) {
 			xa_unlock_irqrestore(xadp, iflags);
 			return;
 		}
 		rsv_srp = sfp->rsv_srp;
-		if (!rsv_srp)
+		if (unlikely(!rsv_srp))
 			goto fini;
-		if (rsv_srp->sh_var != SG_SHR_RS_RQ)
+		if (unlikely(rsv_srp->sh_var != SG_SHR_RS_RQ))
 			goto fini;
 		sr_st = atomic_read(&rsv_srp->rq_st);
 		switch (sr_st) {
@@ -2626,7 +2627,7 @@ sg_remove_sfp_share(struct sg_fd *sfp, bool is_rd_side)
 			sg_unshare_ws_fd(sh_sfp, sdp != sh_sdp);
 		sg_unshare_rs_fd(sfp, false);
 	} else {
-		if (!sg_fd_is_shared(sfp)) {
+		if (unlikely(!sg_fd_is_shared(sfp))) {
 			xa_unlock_irqrestore(xadp, iflags);
 			return;
 		} else if (!xa_get_mark(&sh_sdp->sfp_arr, sh_sfp->idx,
@@ -2788,8 +2789,7 @@ sg_fill_request_element(struct sg_fd *sfp, struct sg_request *srp,
 static inline bool
 sg_rq_landed(struct sg_device *sdp, struct sg_request *srp)
 {
-	return atomic_read_acquire(&srp->rq_st) != SG_RQ_INFLIGHT ||
-	       unlikely(SG_IS_DETACHING(sdp));
+	return atomic_read_acquire(&srp->rq_st) != SG_RQ_INFLIGHT || SG_IS_DETACHING(sdp);
 }
 
 /*
@@ -2827,7 +2827,7 @@ sg_wait_event_srp(struct sg_fd *sfp, void __user *p, struct sg_io_v4 *h4p,
 		return res;
 	}
 skip_wait:
-	if (unlikely(SG_IS_DETACHING(sdp))) {
+	if (SG_IS_DETACHING(sdp)) {
 		sg_rq_chg_state_force(srp, SG_RQ_INACTIVE);
 		atomic_inc(&sfp->inactives);
 		return -ENODEV;
@@ -2865,15 +2865,25 @@ sg_ctl_sg_io(struct sg_device *sdp, struct sg_fd *sfp, void __user *p)
 	SG_LOG(3, sfp, "%s:  SG_IO%s\n", __func__,
 	       (SG_IS_O_NONBLOCK(sfp) ? " O_NONBLOCK ignored" : ""));
 	res = sg_allow_if_err_recovery(sdp, false);
-	if (res)
+	if (unlikely(res))
 		return res;
-	if (get_sg_io_hdr(h3p, p))
+	if (unlikely(get_sg_io_hdr(h3p, p)))
 		return -EFAULT;
 	if (h3p->interface_id == 'Q') {
 		/* copy in rest of sg_io_v4 object */
-		if (copy_from_user(hu8arr + SZ_SG_IO_HDR,
-				   ((u8 __user *)p) + SZ_SG_IO_HDR,
-				   SZ_SG_IO_V4 - SZ_SG_IO_HDR))
+		int v3_len;
+
+#ifdef CONFIG_COMPAT
+		if (in_compat_syscall())
+			v3_len = sizeof(struct compat_sg_io_hdr);
+		else
+			v3_len = SZ_SG_IO_HDR;
+#else
+		v3_len = SZ_SG_IO_HDR;
+#endif
+		if (copy_from_user(hu8arr + v3_len,
+				   ((u8 __user *)p) + v3_len,
+				   SZ_SG_IO_V4 - v3_len))
 			return -EFAULT;
 		res = sg_submit_v4(sfp, p, h4p, true, &srp);
 	} else if (h3p->interface_id == 'S') {
@@ -2932,6 +2942,61 @@ sg_match_request(struct sg_fd *sfp, bool use_tag, int id)
 	return NULL;
 }
 
+static int
+sg_abort_req(struct sg_fd *sfp, struct sg_request *srp)
+		__must_hold(&sfp->srp_arr->xa_lock)
+{
+	int res = 0;
+	enum sg_rq_state rq_st;
+
+	if (test_and_set_bit(SG_FRQ_ABORTING, srp->frq_bm)) {
+		SG_LOG(1, sfp, "%s: already aborting req pack_id/tag=%d/%d\n",
+		       __func__, srp->pack_id, srp->tag);
+		goto fini;	/* skip quietly if already aborted */
+	}
+	rq_st = atomic_read(&srp->rq_st);
+	SG_LOG(3, sfp, "%s: req pack_id/tag=%d/%d, status=%s\n", __func__,
+	       srp->pack_id, srp->tag, sg_rq_st_str(rq_st, false));
+	switch (rq_st) {
+	case SG_RQ_BUSY:
+		clear_bit(SG_FRQ_ABORTING, srp->frq_bm);
+		res = -EBUSY;	/* should not occur often */
+		break;
+	case SG_RQ_INACTIVE:	/* perhaps done already */
+		clear_bit(SG_FRQ_ABORTING, srp->frq_bm);
+		break;
+	case SG_RQ_AWAIT_RCV:	/* user should still do completion */
+	case SG_RQ_SHR_SWAP:
+	case SG_RQ_SHR_IN_WS:
+		clear_bit(SG_FRQ_ABORTING, srp->frq_bm);
+		break;		/* nothing to do here, return 0 */
+	case SG_RQ_INFLIGHT:	/* only attempt abort if inflight */
+		srp->rq_result |= (DRIVER_SOFT << 24);
+		{
+			struct request *rqq = READ_ONCE(srp->rqq);
+
+			if (likely(rqq)) {
+				SG_LOG(5, sfp, "%s: -->blk_abort_request srp=0x%pK\n",
+				       __func__, srp);
+				blk_abort_request(rqq);
+			}
+		}
+		break;
+	default:
+		clear_bit(SG_FRQ_ABORTING, srp->frq_bm);
+		break;
+	}
+fini:
+	return res;
+}
+
+/*
+ * Tries to abort an inflight request/command. First it checks the current fd
+ * for a match on pack_id or tag. If there is a match, aborts that match.
+ * Otherwise, if SGV4_FLAG_DEV_SCOPE is set, the rest of the file descriptors
+ * belonging to the current device are similarly checked. If there is no match
+ * then -ENODATA is returned.
+ */
 static int
 sg_ctl_abort(struct sg_device *sdp, struct sg_fd *sfp, void __user *p)
 		__must_hold(sfp->f_mutex)
@@ -2973,37 +3038,7 @@ sg_ctl_abort(struct sg_device *sdp, struct sg_fd *sfp, void __user *p)
 		if (!srp)
 			return -ENODATA;
 	}
-
-	if (test_and_set_bit(SG_FRQ_ABORTING, srp->frq_bm))
-		goto fini;
-
-	switch (atomic_read(&srp->rq_st)) {
-	case SG_RQ_BUSY:
-		clear_bit(SG_FRQ_ABORTING, srp->frq_bm);
-		res = -EBUSY;	/* should not occur often */
-		break;
-	case SG_RQ_INACTIVE:	/* perhaps done already */
-		clear_bit(SG_FRQ_ABORTING, srp->frq_bm);
-		break;
-	case SG_RQ_AWAIT_RCV:	/* user should still do completion */
-	case SG_RQ_SHR_SWAP:
-	case SG_RQ_SHR_IN_WS:
-		clear_bit(SG_FRQ_ABORTING, srp->frq_bm);
-		break;		/* nothing to do here, return 0 */
-	case SG_RQ_INFLIGHT:	/* only attempt abort if inflight */
-		srp->rq_result |= (DRIVER_SOFT << 24);
-		{
-			struct request *rqq = READ_ONCE(srp->rqq);
-
-			if (rqq)
-				blk_abort_request(rqq);
-		}
-		break;
-	default:
-		clear_bit(SG_FRQ_ABORTING, srp->frq_bm);
-		break;
-	}
-fini:
+	res = sg_abort_req(sfp, srp);
 	xa_unlock_irqrestore(&sfp->srp_arr, iflags);
 	return res;
 }
@@ -3102,7 +3137,7 @@ sg_find_sfp_by_fd(const struct file *search_for, int search_fd,
 				continue;       /* not this one */
 			res = sg_find_sfp_helper(from_sfp, sfp,
 						 from_is_rd_side, search_fd);
-			if (res == 0) {
+			if (likely(res == 0)) {
 				found = true;
 				break;
 			}
@@ -3217,6 +3252,7 @@ sg_fd_reshare(struct sg_fd *rs_sfp, int new_ws_fd)
 	bool found = false;
 	int res = 0;
 	int retry_count = 0;
+	enum sg_rq_state rq_st;
 	struct file *filp;
 	struct sg_fd *ws_sfp = sg_fd_share_ptr(rs_sfp);
 
@@ -3228,6 +3264,17 @@ sg_fd_reshare(struct sg_fd *rs_sfp, int new_ws_fd)
 	if (unlikely(!xa_get_mark(&rs_sfp->parentdp->sfp_arr, rs_sfp->idx,
 				  SG_XA_FD_RS_SHARE)))
 		return -EINVAL;
+	if (unlikely(!ws_sfp))
+		return -EINVAL;
+	if (unlikely(!rs_sfp->rsv_srp))
+		res = -EPROTO;	/* Internal error */
+	rq_st = atomic_read(&rs_sfp->rsv_srp->rq_st);
+	if (!(rq_st == SG_RQ_INACTIVE || rq_st == SG_RQ_SHR_SWAP))
+		res = -EBUSY;		/* read-side reserve buffer busy */
+	if (rs_sfp->ws_srp)
+		res = -EBUSY;	/* previous write-side request not finished */
+	if (unlikely(res))
+		return res;
 
 	/* Alternate approach: fcheck_files(current->files, m_fd) */
 	filp = fget(new_ws_fd);
@@ -3289,7 +3336,7 @@ sg_set_reserved_sz(struct sg_fd *sfp, int want_rsv_sz)
 	if (unlikely(sg_fd_is_shared(sfp)))
 		return -EBUSY;	/* this fd can't be either side of share */
 	o_srp = sfp->rsv_srp;
-	if (!o_srp)
+	if (unlikely(!o_srp))
 		return -EPROTO;
 	new_sz = min_t(int, want_rsv_sz, sdp->max_sgat_sz);
 	new_sz = max_t(int, new_sz, sfp->sgat_elem_sz);
@@ -3304,11 +3351,11 @@ sg_set_reserved_sz(struct sg_fd *sfp, int want_rsv_sz)
 	/* new sg_request object, sized correctly is now available */
 try_again:
 	o_srp = sfp->rsv_srp;
-	if (!o_srp) {
+	if (unlikely(!o_srp)) {
 		res = -EPROTO;
 		goto fini;
 	}
-	if (SG_RQ_ACTIVE(o_srp) || sfp->mmap_sz > 0) {
+	if (unlikely(SG_RQ_ACTIVE(o_srp) || sfp->mmap_sz > 0)) {
 		res = -EBUSY;
 		goto fini;
 	}
@@ -3615,7 +3662,7 @@ sg_ctl_extended(struct sg_fd *sfp, void __user *p)
 	s_wr_mask = seip->sei_wr_mask;
 	s_rd_mask = seip->sei_rd_mask;
 	or_masks = s_wr_mask | s_rd_mask;
-	if (or_masks == 0) {
+	if (unlikely(or_masks == 0)) {
 		SG_LOG(2, sfp, "%s: both masks 0, do nothing\n", __func__);
 		return 0;
 	}
@@ -3653,7 +3700,7 @@ sg_ctl_extended(struct sg_fd *sfp, void __user *p)
 		mutex_lock(&sfp->f_mutex);
 		if (s_wr_mask & SG_SEIM_SHARE_FD) {
 			result = sg_fd_share(sfp, (int)seip->share_fd);
-			if (ret == 0 && result)
+			if (ret == 0 && unlikely(result))
 				ret = result;
 		}
 		/* if share then yield device number of (other) read-side */
@@ -3670,7 +3717,7 @@ sg_ctl_extended(struct sg_fd *sfp, void __user *p)
 		mutex_lock(&sfp->f_mutex);
 		if (s_wr_mask & SG_SEIM_CHG_SHARE_FD) {
 			result = sg_fd_reshare(sfp, (int)seip->share_fd);
-			if (ret == 0 && result)
+			if (ret == 0 && unlikely(result))
 				ret = result;
 		}
 		/* if share then yield device number of (other) write-side */
@@ -3718,7 +3765,7 @@ sg_ctl_extended(struct sg_fd *sfp, void __user *p)
 	if (s_wr_mask & SG_SEIM_RESERVED_SIZE) {
 		mutex_lock(&sfp->f_mutex);
 		result = sg_set_reserved_sz(sfp, (int)seip->reserved_sz);
-		if (ret == 0 && result)
+		if (ret == 0 && unlikely(result))
 			ret = result;
 		mutex_unlock(&sfp->f_mutex);
 	}
@@ -3743,7 +3790,8 @@ sg_ctl_extended(struct sg_fd *sfp, void __user *p)
 static int
 sg_ctl_req_tbl(struct sg_fd *sfp, void __user *p)
 {
-	int k, result, val;
+	int k, val;
+	int result = 0;
 	unsigned long idx;
 	struct sg_request *srp;
 	struct sg_req_info *rinfop;
@@ -3751,11 +3799,11 @@ sg_ctl_req_tbl(struct sg_fd *sfp, void __user *p)
 	SG_LOG(3, sfp, "%s:    SG_GET_REQUEST_TABLE\n", __func__);
 	k = SG_MAX_QUEUE;
 	rinfop = kcalloc(k, SZ_SG_REQ_INFO, GFP_KERNEL);
-	if (!rinfop)
+	if (unlikely(!rinfop))
 		return -ENOMEM;
 	val = 0;
 	xa_for_each(&sfp->srp_arr, idx, srp) {
-		if (val >= SG_MAX_QUEUE)
+		if (unlikely(val >= SG_MAX_QUEUE))
 			break;
 		if (xa_get_mark(&sfp->srp_arr, idx, SG_XA_RQ_INACTIVE))
 			continue;
@@ -3763,7 +3811,7 @@ sg_ctl_req_tbl(struct sg_fd *sfp, void __user *p)
 		val++;
 	}
 	xa_for_each(&sfp->srp_arr, idx, srp) {
-		if (val >= SG_MAX_QUEUE)
+		if (unlikely(val >= SG_MAX_QUEUE))
 			break;
 		if (!xa_get_mark(&sfp->srp_arr, idx, SG_XA_RQ_INACTIVE))
 			continue;
@@ -3825,34 +3873,34 @@ sg_ioctl_common(struct file *filp, struct sg_device *sdp, struct sg_fd *sfp,
 
 	switch (cmd_in) {
 	case SG_IO:
-		if (unlikely(SG_IS_DETACHING(sdp)))
+		if (SG_IS_DETACHING(sdp))
 			return -ENODEV;
 		return sg_ctl_sg_io(sdp, sfp, p);
 	case SG_IOSUBMIT:
 		SG_LOG(3, sfp, "%s:    SG_IOSUBMIT\n", __func__);
-		if (unlikely(SG_IS_DETACHING(sdp)))
+		if (SG_IS_DETACHING(sdp))
 			return -ENODEV;
 		return sg_ctl_iosubmit(sfp, p);
 	case SG_IOSUBMIT_V3:
 		SG_LOG(3, sfp, "%s:    SG_IOSUBMIT_V3\n", __func__);
-		if (unlikely(SG_IS_DETACHING(sdp)))
+		if (SG_IS_DETACHING(sdp))
 			return -ENODEV;
 		return sg_ctl_iosubmit_v3(sfp, p);
 	case SG_IORECEIVE:
 		SG_LOG(3, sfp, "%s:    SG_IORECEIVE\n", __func__);
-		if (unlikely(SG_IS_DETACHING(sdp)))
+		if (SG_IS_DETACHING(sdp))
 			return -ENODEV;
 		return sg_ctl_ioreceive(sfp, p);
 	case SG_IORECEIVE_V3:
 		SG_LOG(3, sfp, "%s:    SG_IORECEIVE_V3\n", __func__);
-		if (unlikely(SG_IS_DETACHING(sdp)))
+		if (SG_IS_DETACHING(sdp))
 			return -ENODEV;
 		return sg_ctl_ioreceive_v3(sfp, p);
 	case SG_IOABORT:
 		SG_LOG(3, sfp, "%s:    SG_IOABORT\n", __func__);
-		if (unlikely(SG_IS_DETACHING(sdp)))
+		if (SG_IS_DETACHING(sdp))
 			return -ENODEV;
-		if (read_only)
+		if (unlikely(read_only))
 			return -EPERM;
 		mutex_lock(&sfp->f_mutex);
 		res = sg_ctl_abort(sdp, sfp, p);
@@ -3866,7 +3914,7 @@ sg_ioctl_common(struct file *filp, struct sg_device *sdp, struct sg_fd *sfp,
 	case SG_SET_FORCE_PACK_ID:
 		SG_LOG(3, sfp, "%s:    SG_SET_FORCE_PACK_ID\n", __func__);
 		res = get_user(val, ip);
-		if (res)
+		if (unlikely(res))
 			return res;
 		assign_bit(SG_FFD_FORCE_PACKID, sfp->ffd_bm, !!val);
 		return 0;
@@ -3905,8 +3953,8 @@ sg_ioctl_common(struct file *filp, struct sg_device *sdp, struct sg_fd *sfp,
 		return put_user(sdp->max_sgat_sz, ip);
 	case SG_SET_RESERVED_SIZE:
 		res = get_user(val, ip);
-		if (!res) {
-			if (val >= 0 && val <= (1024 * 1024 * 1024)) {
+		if (likely(!res)) {
+			if (likely(val >= 0 && val <= (1024 * 1024 * 1024))) {
 				mutex_lock(&sfp->f_mutex);
 				res = sg_set_reserved_sz(sfp, val);
 				mutex_unlock(&sfp->f_mutex);
@@ -3921,14 +3969,14 @@ sg_ioctl_common(struct file *filp, struct sg_device *sdp, struct sg_fd *sfp,
 		val = min_t(int, sfp->rsv_srp->sgatp->buflen,
 			    sdp->max_sgat_sz);
 		mutex_unlock(&sfp->f_mutex);
-		SG_LOG(3, sfp, "%s:    SG_GET_RESERVED_SIZE=%d\n",
-		       __func__, val);
+		SG_LOG(3, sfp, "%s:    SG_GET_RESERVED_SIZE=%d\n", __func__,
+		       val);
 		res = put_user(val, ip);
 		return res;
 	case SG_SET_COMMAND_Q:
 		SG_LOG(3, sfp, "%s:    SG_SET_COMMAND_Q\n", __func__);
 		res = get_user(val, ip);
-		if (res)
+		if (unlikely(res))
 			return res;
 		assign_bit(SG_FFD_CMD_Q, sfp->ffd_bm, !!val);
 		return 0;
@@ -3938,7 +3986,7 @@ sg_ioctl_common(struct file *filp, struct sg_device *sdp, struct sg_fd *sfp,
 	case SG_SET_KEEP_ORPHAN:
 		SG_LOG(3, sfp, "%s:    SG_SET_KEEP_ORPHAN\n", __func__);
 		res = get_user(val, ip);
-		if (res)
+		if (unlikely(res))
 			return res;
 		assign_bit(SG_FFD_KEEP_ORPHAN, sfp->ffd_bm, !!val);
 		return 0;
@@ -3957,9 +4005,9 @@ sg_ioctl_common(struct file *filp, struct sg_device *sdp, struct sg_fd *sfp,
 	case SG_SET_TIMEOUT:
 		SG_LOG(3, sfp, "%s:    SG_SET_TIMEOUT\n", __func__);
 		res = get_user(val, ip);
-		if (res)
+		if (unlikely(res))
 			return res;
-		if (val < 0)
+		if (unlikely(val < 0))
 			return -EIO;
 		if (val >= mult_frac((s64)INT_MAX, USER_HZ, HZ))
 			val = min_t(s64, mult_frac((s64)INT_MAX, USER_HZ, HZ),
@@ -3985,9 +4033,9 @@ sg_ioctl_common(struct file *filp, struct sg_device *sdp, struct sg_fd *sfp,
 	case SG_NEXT_CMD_LEN:	/* active only in v2 interface */
 		SG_LOG(3, sfp, "%s:    SG_NEXT_CMD_LEN\n", __func__);
 		res = get_user(val, ip);
-		if (res)
+		if (unlikely(res))
 			return res;
-		if (val > SG_MAX_CDB_SIZE)
+		if (unlikely(val > SG_MAX_CDB_SIZE))
 			return -ENOMEM;
 		mutex_lock(&sfp->f_mutex);
 		sfp->next_cmd_len = max_t(int, val, 0);
@@ -4000,7 +4048,7 @@ sg_ioctl_common(struct file *filp, struct sg_device *sdp, struct sg_fd *sfp,
 		return put_user(val, ip);
 	case SG_EMULATED_HOST:
 		SG_LOG(3, sfp, "%s:    SG_EMULATED_HOST\n", __func__);
-		if (unlikely(SG_IS_DETACHING(sdp)))
+		if (SG_IS_DETACHING(sdp))
 			return -ENODEV;
 		return put_user(sdev->host->hostt->emulated, ip);
 	case SCSI_IOCTL_SEND_COMMAND:
@@ -4010,7 +4058,7 @@ sg_ioctl_common(struct file *filp, struct sg_device *sdp, struct sg_fd *sfp,
 	case SG_SET_DEBUG:
 		SG_LOG(3, sfp, "%s:    SG_SET_DEBUG\n", __func__);
 		res = get_user(val, ip);
-		if (res)
+		if (unlikely(res))
 			return res;
 		assign_bit(SG_FDEV_LOG_SENSE, sdp->fdev_bm, !!val);
 		if (val == 0)	/* user can force recalculation */
@@ -4072,7 +4120,7 @@ sg_ioctl(struct file *filp, unsigned int cmd_in, unsigned long arg)
 
 	sfp = filp->private_data;
 	sdp = sfp->parentdp;
-	if (!sdp)
+	if (unlikely(!sdp))
 		return -ENXIO;
 
 	ret = sg_ioctl_common(filp, sdp, sfp, cmd_in, p);
@@ -4189,7 +4237,7 @@ sg_poll(struct file *filp, poll_table *wait)
 	if (num > 0)
 		p_res = EPOLLIN | EPOLLRDNORM;
 
-	if (unlikely(SG_IS_DETACHING(sfp->parentdp)))
+	if (SG_IS_DETACHING(sfp->parentdp))
 		p_res |= EPOLLHUP;
 	else if (likely(test_bit(SG_FFD_CMD_Q, sfp->ffd_bm)))
 		p_res |= EPOLLOUT | EPOLLWRNORM;
@@ -4248,29 +4296,29 @@ sg_vma_fault(struct vm_fault *vmf)
 	struct sg_fd *sfp;
 	const char *nbp = "==NULL, bad";
 
-	if (!vma) {
+	if (unlikely(!vma)) {
 		pr_warn("%s: vma%s\n", __func__, nbp);
 		goto out_err;
 	}
 	sfp = vma->vm_private_data;
-	if (!sfp) {
+	if (unlikely(!sfp)) {
 		pr_warn("%s: sfp%s\n", __func__, nbp);
 		goto out_err;
 	}
 	sdp = sfp->parentdp;
-	if (sdp && unlikely(SG_IS_DETACHING(sdp))) {
+	if (sdp && SG_IS_DETACHING(sdp)) {
 		SG_LOG(1, sfp, "%s: device detaching\n", __func__);
 		goto out_err;
 	}
 	srp = sfp->rsv_srp;
-	if (!srp) {
+	if (unlikely(!srp)) {
 		SG_LOG(1, sfp, "%s: srp%s\n", __func__, nbp);
 		goto out_err;
 	}
 	mutex_lock(&sfp->f_mutex);
 	rsv_schp = srp->sgatp;
 	offset = vmf->pgoff << PAGE_SHIFT;
-	if (offset >= (unsigned int)rsv_schp->buflen) {
+	if (unlikely(offset >= (unsigned int)rsv_schp->buflen)) {
 		SG_LOG(1, sfp, "%s: offset[%lu] >= rsv.buflen\n", __func__,
 		       offset);
 		goto out_err_unlock;
@@ -4309,10 +4357,10 @@ sg_mmap(struct file *filp, struct vm_area_struct *vma)
 	struct sg_fd *sfp;
 	struct sg_request *srp;
 
-	if (!filp || !vma)
+	if (unlikely(!filp || !vma))
 		return -ENXIO;
 	sfp = filp->private_data;
-	if (!sfp) {
+	if (unlikely(!sfp)) {
 		pr_warn("sg: %s: sfp is NULL\n", __func__);
 		return -ENXIO;
 	}
@@ -4330,7 +4378,7 @@ sg_mmap(struct file *filp, struct vm_area_struct *vma)
 		res = -EBUSY;
 		goto fini;
 	}
-	if (req_sz > SG_WRITE_COUNT_LIMIT) {	/* sanity check */
+	if (unlikely(req_sz > SG_WRITE_COUNT_LIMIT)) {	/* sanity check */
 		res = -ENOMEM;
 		goto fini;
 	}
@@ -4368,17 +4416,17 @@ sg_rq_end_io_usercontext(struct work_struct *work)
 					      ew_orph.work);
 	struct sg_fd *sfp;
 
-	if (!srp) {
+	if (unlikely(!srp)) {
 		WARN_ONCE(1, "%s: srp unexpectedly NULL\n", __func__);
 		return;
 	}
 	sfp = srp->parentfp;
-	if (!sfp) {
+	if (unlikely(!sfp)) {
 		WARN_ONCE(1, "%s: sfp unexpectedly NULL\n", __func__);
 		return;
 	}
 	SG_LOG(3, sfp, "%s: srp=0x%pK\n", __func__, srp);
-	if (test_bit(SG_FRQ_DEACT_ORPHAN, srp->frq_bm)) {
+	if (unlikely(test_bit(SG_FRQ_DEACT_ORPHAN, srp->frq_bm))) {
 		sg_finish_scsi_blk_rq(srp);	/* clean up orphan case */
 		sg_deact_request(sfp, srp);
 	}
@@ -4411,7 +4459,7 @@ sg_rq_end_io(struct request *rqq, blk_status_t status)
 	slen = min_t(int, scsi_rp->sense_len, SCSI_SENSE_BUFFERSIZE);
 	a_resid = scsi_rp->resid_len;
 
-	if (a_resid) {
+	if (unlikely(a_resid)) {
 		if (test_bit(SG_FRQ_IS_V4I, srp->frq_bm)) {
 			if (rq_data_dir(rqq) == READ)
 				srp->in_resid = a_resid;
@@ -4421,7 +4469,7 @@ sg_rq_end_io(struct request *rqq, blk_status_t status)
 			srp->in_resid = a_resid;
 		}
 	}
-	if (test_bit(SG_FRQ_ABORTING, srp->frq_bm) && rq_result == 0)
+	if (unlikely(test_bit(SG_FRQ_ABORTING, srp->frq_bm)) && rq_result == 0)
 		srp->rq_result |= (DRIVER_HARD << 24);
 
 	SG_LOG(6, sfp, "%s: pack/tag_id=%d/%d, cmd=0x%x, res=0x%x\n", __func__,
@@ -4435,11 +4483,12 @@ sg_rq_end_io(struct request *rqq, blk_status_t status)
 		    (rq_result & 0xff) == SAM_STAT_COMMAND_TERMINATED)
 			__scsi_print_sense(sdp->device, __func__, scsi_rp->sense, slen);
 	}
-	if (slen > 0) {
-		if (scsi_rp->sense && !srp->sense_bp) {
-			srp->sense_bp = mempool_alloc(sg_sense_pool,
-						      GFP_ATOMIC);
-			if (srp->sense_bp) {
+	if (unlikely(slen > 0)) {
+		if (likely(scsi_rp->sense && !srp->sense_bp)) {
+			srp->sense_bp =
+				mempool_alloc(sg_sense_pool,
+					      GFP_ATOMIC   /* <-- leave */);
+			if (likely(srp->sense_bp)) {
 				memcpy(srp->sense_bp, scsi_rp->sense, slen);
 				if (slen < SCSI_SENSE_BUFFERSIZE)
 					memset(srp->sense_bp + slen, 0,
@@ -4449,7 +4498,7 @@ sg_rq_end_io(struct request *rqq, blk_status_t status)
 				pr_warn("%s: sense but can't alloc buffer\n",
 					__func__);
 			}
-		} else if (srp->sense_bp) {
+		} else if (unlikely(srp->sense_bp)) {
 			slen = 0;
 			pr_warn("%s: non-NULL srp->sense_bp ? ?\n", __func__);
 		} else {
@@ -4536,14 +4585,14 @@ sg_add_device_helper(struct gendisk *disk, struct scsi_device *scsidp)
 	unsigned long iflags;
 
 	sdp = kzalloc(sizeof(*sdp), GFP_KERNEL);
-	if (!sdp)
+	if (unlikely(!sdp))
 		return ERR_PTR(-ENOMEM);
 
 	idr_preload(GFP_KERNEL);
 	write_lock_irqsave(&sg_index_lock, iflags);
 
 	error = idr_alloc(&sg_index_idr, sdp, 0, SG_MAX_DEVS, GFP_NOWAIT);
-	if (error < 0) {
+	if (unlikely(error < 0)) {
 		if (error == -ENOSPC) {
 			sdev_printk(KERN_WARNING, scsidp,
 				    "Unable to attach sg device type=%d, minor number exceeds %d\n",
@@ -4577,7 +4626,7 @@ sg_add_device_helper(struct gendisk *disk, struct scsi_device *scsidp)
 	write_unlock_irqrestore(&sg_index_lock, iflags);
 	idr_preload_end();
 
-	if (error) {
+	if (unlikely(error)) {
 		kfree(sdp);
 		return ERR_PTR(error);
 	}
@@ -4595,7 +4644,7 @@ sg_add_device(struct device *cl_dev, struct class_interface *cl_intf)
 	unsigned long iflags;
 
 	disk = alloc_disk(1);
-	if (!disk) {
+	if (unlikely(!disk)) {
 		pr_warn("%s: alloc_disk failed\n", __func__);
 		return -ENOMEM;
 	}
@@ -4603,7 +4652,7 @@ sg_add_device(struct device *cl_dev, struct class_interface *cl_intf)
 
 	error = -ENOMEM;
 	cdev = cdev_alloc();
-	if (!cdev) {
+	if (unlikely(!cdev)) {
 		pr_warn("%s: cdev_alloc failed\n", __func__);
 		goto out;
 	}
@@ -4617,7 +4666,7 @@ sg_add_device(struct device *cl_dev, struct class_interface *cl_intf)
 	}
 
 	error = cdev_add(cdev, MKDEV(SCSI_GENERIC_MAJOR, sdp->index), 1);
-	if (error)
+	if (unlikely(error))
 		goto cdev_add_err;
 
 	sdp->cdev = cdev;
@@ -4693,7 +4742,7 @@ sg_remove_device(struct device *cl_dev, struct class_interface *cl_intf)
 	unsigned long idx;
 	struct sg_fd *sfp;
 
-	if (!sdp)
+	if (unlikely(!sdp))
 		return;
 	/* set this flag as soon as possible as it could be a surprise */
 	if (test_and_set_bit(SG_FDEV_DETACHING, sdp->fdev_bm))
@@ -4734,21 +4783,21 @@ init_sg(void)
 
 	rc = register_chrdev_region(MKDEV(SCSI_GENERIC_MAJOR, 0),
 				    SG_MAX_DEVS, "sg");
-	if (rc)
+	if (unlikely(rc))
 		return rc;
 
 	sg_sense_cache = kmem_cache_create_usercopy
 				("sg_sense", SCSI_SENSE_BUFFERSIZE, 0,
 				 SLAB_HWCACHE_ALIGN, 0,
 				 SCSI_SENSE_BUFFERSIZE, NULL);
-	if (!sg_sense_cache) {
+	if (unlikely(!sg_sense_cache)) {
 		pr_err("sg: can't init sense cache\n");
 		rc = -ENOMEM;
 		goto err_out_unreg;
 	}
 	sg_sense_pool = mempool_create_slab_pool(SG_MEMPOOL_MIN_NR,
 						 sg_sense_cache);
-	if (!sg_sense_pool) {
+	if (unlikely(!sg_sense_pool)) {
 		pr_err("sg: can't init sense pool\n");
 		rc = -ENOMEM;
 		goto err_out_cache;
@@ -4764,7 +4813,7 @@ init_sg(void)
 	}
 	sg_sysfs_valid = true;
 	rc = scsi_register_interface(&sg_interface);
-	if (rc == 0) {
+	if (likely(rc == 0)) {
 		sg_proc_init();
 		sg_dfs_init();
 		return 0;
@@ -4888,13 +4937,13 @@ sg_start_req(struct sg_request *srp, struct sg_comm_wr_t *cwrp, int dxfer_dir)
 	sdp = sfp->parentdp;
 	if (cwrp->cmd_len > BLK_MAX_CDB) {	/* for longer SCSI cdb_s */
 		long_cmdp = kzalloc(cwrp->cmd_len, GFP_KERNEL);
-		if (!long_cmdp) {
+		if (unlikely(!long_cmdp)) {
 			res = -ENOMEM;
 			goto err_out;
 		}
 		SG_LOG(5, sfp, "%s: long_cmdp=0x%pK ++\n", __func__, long_cmdp);
 	}
-	if (test_bit(SG_FRQ_IS_V4I, srp->frq_bm)) {
+	if (likely(test_bit(SG_FRQ_IS_V4I, srp->frq_bm))) {
 		struct sg_io_v4 *h4p = cwrp->h4p;
 
 		if (dxfer_dir == SG_DXFER_TO_DEV) {
@@ -4947,11 +4996,11 @@ sg_start_req(struct sg_request *srp, struct sg_comm_wr_t *cwrp, int dxfer_dir)
 	if (cwrp->u_cmdp)
 		res = sg_fetch_cmnd(sfp, cwrp->u_cmdp, cwrp->cmd_len,
 				    scsi_rp->cmd);
-	else if (cwrp->cmdp)
+	else if (likely(cwrp->cmdp))
 		memcpy(scsi_rp->cmd, cwrp->cmdp, cwrp->cmd_len);
 	else
 		res = -EPROTO;
-	if (res)
+	if (unlikely(res))
 		goto err_out;
 	scsi_rp->cmd_len = cwrp->cmd_len;
 	srp->cmd_opcode = scsi_rp->cmd[0];
@@ -4966,7 +5015,7 @@ sg_start_req(struct sg_request *srp, struct sg_comm_wr_t *cwrp, int dxfer_dir)
 		SG_LOG(4, sfp, "%s: no data xfer [0x%pK]\n", __func__, srp);
 		clear_bit(SG_FRQ_US_XFER, srp->frq_bm);
 		goto fini;	/* path of reqs with no din nor dout */
-	} else if ((rq_flags & SG_FLAG_DIRECT_IO) && iov_count == 0 &&
+	} else if (unlikely(rq_flags & SG_FLAG_DIRECT_IO) && iov_count == 0 &&
 		   !sdp->device->host->unchecked_isa_dma &&
 		   blk_rq_aligned(q, (unsigned long)up, dxfer_len)) {
 		srp->rq_info |= SG_INFO_DIRECT_IO;
@@ -4980,7 +5029,8 @@ sg_start_req(struct sg_request *srp, struct sg_comm_wr_t *cwrp, int dxfer_dir)
 	if (likely(md)) {	/* normal, "indirect" IO */
 		if (unlikely(rq_flags & SG_FLAG_MMAP_IO)) {
 			/* mmap IO must use and fit in reserve request */
-			if (!reserved || dxfer_len > req_schp->buflen)
+			if (unlikely(!reserved ||
+				     dxfer_len > req_schp->buflen))
 				res = reserved ? -ENOMEM : -EBUSY;
 		} else if (req_schp->buflen == 0) {
 			int up_sz = max_t(int, dxfer_len, sfp->sgat_elem_sz);
@@ -5003,7 +5053,7 @@ sg_start_req(struct sg_request *srp, struct sg_comm_wr_t *cwrp, int dxfer_dir)
 			goto fini;
 
 		iov_iter_truncate(&i, dxfer_len);
-		if (!iov_iter_count(&i)) {
+		if (unlikely(!iov_iter_count(&i))) {
 			kfree(iov);
 			res = -EINVAL;
 			goto fini;
@@ -5016,9 +5066,11 @@ sg_start_req(struct sg_request *srp, struct sg_comm_wr_t *cwrp, int dxfer_dir)
 			cp = "iov_count > 0";
 	} else if (us_xfer) { /* setup for transfer data to/from user space */
 		res = blk_rq_map_user(q, rqq, md, up, dxfer_len, GFP_ATOMIC);
-		if (IS_ENABLED(CONFIG_SCSI_PROC_FS) && res)
+#if IS_ENABLED(SG_LOG_ACTIVE)
+		if (unlikely(res))
 			SG_LOG(1, sfp, "%s: blk_rq_map_user() res=%d\n",
 			       __func__, res);
+#endif
 	} else {	/* transfer data to/from kernel buffers */
 		res = sg_rq_map_kern(srp, q, rqq, r0w);
 	}
@@ -5067,14 +5119,14 @@ sg_finish_scsi_blk_rq(struct sg_request *srp)
 			scsi_req_free_cmd(scsi_req(rqq));
 		blk_put_request(rqq);
 	}
-	if (srp->bio) {
+	if (likely(srp->bio)) {
 		bool us_xfer = test_bit(SG_FRQ_US_XFER, srp->frq_bm);
 		struct bio *bio = srp->bio;
 
 		srp->bio = NULL;
 		if (us_xfer && bio) {
 			ret = blk_rq_unmap_user(bio);
-			if (ret) {	/* -EINTR (-4) can be ignored */
+			if (unlikely(ret)) {	/* -EINTR (-4) can be ignored */
 				SG_LOG(6, sfp,
 				       "%s: blk_rq_unmap_user() --> %d\n",
 				       __func__, ret);
@@ -5282,7 +5334,7 @@ sg_find_srp_by_id(struct sg_fd *sfp, int id, bool is_tag)
 		     srp = xa_find_after(xafp, &idx, end_idx, SG_XA_RQ_AWAIT)) {
 			if (test_bit(SG_FRQ_SYNC_INVOC, srp->frq_bm))
 				continue;
-			if (is_tag) {
+			if (unlikely(is_tag)) {
 				if (srp->tag != id)
 					continue;
 			} else {
@@ -5359,7 +5411,7 @@ sg_find_srp_by_id(struct sg_fd *sfp, int id, bool is_tag)
 			__maybe_unused const char *cptp = is_tag ? "tag=" :
 								   "pack_id=";
 
-			if (is_bad_st)
+			if (unlikely(is_bad_st))
 				SG_LOG(1, sfp, "%s: %s%d wrong state: %s\n",
 				       __func__, cptp, id,
 				       sg_rq_st_str(bad_sr_st, true));
@@ -5391,7 +5443,7 @@ sg_mrq_get_ready_srp(struct sg_fd *sfp, struct sg_request **srpp)
 	struct sg_request *srp;
 	struct xarray *xafp = &sfp->srp_arr;
 
-	if (unlikely(SG_IS_DETACHING(sfp->parentdp))) {
+	if (SG_IS_DETACHING(sfp->parentdp)) {
 		*srpp = ERR_PTR(-ENODEV);
 		return true;
 	}
@@ -5450,7 +5502,7 @@ sg_mk_srp(struct sg_fd *sfp, bool first)
 		srp = kzalloc(sizeof(*srp), gfp | GFP_KERNEL);
 	else
 		srp = kzalloc(sizeof(*srp), gfp | GFP_ATOMIC);
-	if (srp) {
+	if (likely(srp)) {
 		atomic_set(&srp->rq_st, SG_RQ_BUSY);
 		srp->sh_var = SG_SHR_NONE;
 		srp->parentfp = sfp;
@@ -5505,7 +5557,7 @@ sg_build_reserve(struct sg_fd *sfp, int buflen)
 			go_out = true;
 		}
 		res = sg_mk_sgat(srp, sfp, buflen);
-		if (res == 0) {
+		if (likely(res == 0)) {
 			SG_LOG(4, sfp, "%s: final buflen=%d, srp=0x%pK ++\n",
 			       __func__, buflen, srp);
 			return srp;
@@ -5575,7 +5627,7 @@ sg_setup_req(struct sg_comm_wr_t *cwrp, enum sg_shr_var sh_var, int dxfr_len)
 		break;
 	case SG_SHR_WS_RQ:
 		rs_sfp = sg_fd_share_ptr(fp);
-		if (!sg_fd_is_shared(fp)) {
+		if (unlikely(!rs_sfp)) {
 			r_srp = ERR_PTR(-EPROTO);
 			break;
 		}
@@ -5589,7 +5641,7 @@ sg_setup_req(struct sg_comm_wr_t *cwrp, enum sg_shr_var sh_var, int dxfr_len)
 		rs_sr_st = atomic_read(&rs_rsv_srp->rq_st);
 		switch (rs_sr_st) {
 		case SG_RQ_AWAIT_RCV:
-			if (rs_rsv_srp->rq_result & SG_ML_RESULT_MSK) {
+			if (unlikely(rs_rsv_srp->rq_result & SG_ML_RESULT_MSK)) {
 				/* read-side done but error occurred */
 				r_srp = ERR_PTR(-ENOSTR);
 				break;
@@ -5597,7 +5649,7 @@ sg_setup_req(struct sg_comm_wr_t *cwrp, enum sg_shr_var sh_var, int dxfr_len)
 			fallthrough;
 		case SG_RQ_SHR_SWAP:
 			ws_rq = true;
-			if (rs_sr_st == SG_RQ_AWAIT_RCV)
+			if (unlikely(rs_sr_st == SG_RQ_AWAIT_RCV))
 				break;
 			res = sg_rq_chg_state(rs_rsv_srp, rs_sr_st, SG_RQ_SHR_IN_WS);
 			if (unlikely(res))
@@ -5633,8 +5685,8 @@ sg_setup_req(struct sg_comm_wr_t *cwrp, enum sg_shr_var sh_var, int dxfr_len)
 	}
 	cp = "";
 
-	if (ws_rq) {	/* write-side dlen may be smaller than read-side's dlen */
-		if (dxfr_len > rs_rsv_srp->sgatp->dlen) {
+	if (ws_rq) {	/* write-side dlen may be <= read-side's dlen */
+		if (unlikely(dxfr_len > rs_rsv_srp->sgatp->dlen)) {
 			SG_LOG(4, fp, "%s: write-side dlen [%d] > read-side dlen\n",
 			       __func__, dxfr_len);
 			r_srp = ERR_PTR(-E2BIG);
@@ -5650,7 +5702,7 @@ sg_setup_req(struct sg_comm_wr_t *cwrp, enum sg_shr_var sh_var, int dxfr_len)
 		mk_new_srp = true;
 	} else if (atomic_read(&fp->inactives) <= 0) {
 		mk_new_srp = true;
-	} else if (!try_harder && dxfr_len < SG_DEF_SECTOR_SZ) {
+	} else if (likely(!try_harder) && dxfr_len < SG_DEF_SECTOR_SZ) {
 		l_used_idx = READ_ONCE(fp->low_used_idx);
 		s_idx = (l_used_idx < 0) ? 0 : l_used_idx;
 		if (l_used_idx >= 0 && xa_get_mark(xafp, s_idx, SG_XA_RQ_INACTIVE)) {
@@ -5675,7 +5727,7 @@ sg_setup_req(struct sg_comm_wr_t *cwrp, enum sg_shr_var sh_var, int dxfr_len)
 		/* If dxfr_len is small, use lowest inactive request */
 		if (low_srp) {
 			r_srp = low_srp;
-			if (sg_rq_chg_state(r_srp, SG_RQ_INACTIVE, SG_RQ_BUSY))
+			if (unlikely(sg_rq_chg_state(r_srp, SG_RQ_INACTIVE, SG_RQ_BUSY)))
 				goto start_again; /* gone to another thread */
 			atomic_dec(&fp->inactives);
 			cp = "lowest inactive in srp_arr";
@@ -5732,7 +5784,7 @@ sg_setup_req(struct sg_comm_wr_t *cwrp, enum sg_shr_var sh_var, int dxfr_len)
 			goto fini;
 		} else if (fp->tot_fd_thresh > 0) {
 			sum_dlen = atomic_read(&fp->sum_fd_dlens) + dxfr_len;
-			if (sum_dlen > (u32)fp->tot_fd_thresh) {
+			if (unlikely(sum_dlen > (u32)fp->tot_fd_thresh)) {
 				r_srp = ERR_PTR(-E2BIG);
 				SG_LOG(2, fp, "%s: sum_of_dlen(%u) > %s\n",
 				       __func__, sum_dlen, "tot_fd_thresh");
@@ -5750,7 +5802,7 @@ sg_setup_req(struct sg_comm_wr_t *cwrp, enum sg_shr_var sh_var, int dxfr_len)
 		xa_lock_irqsave(xafp, iflags);
 		res = __xa_alloc(xafp, &n_idx, r_srp, xa_limit_32b, GFP_KERNEL);
 		xa_unlock_irqrestore(xafp, iflags);
-		if (res < 0) {
+		if (unlikely(res < 0)) {
 			sg_remove_sgat(r_srp);
 			kfree(r_srp);
 			r_srp = ERR_PTR(-EPROTOTYPE);
@@ -5838,7 +5890,7 @@ sg_add_sfp(struct sg_device *sdp, struct file *filp)
 	struct xarray *xafp;
 
 	sfp = kzalloc(sizeof(*sfp), GFP_ATOMIC | __GFP_NOWARN);
-	if (!sfp)
+	if (unlikely(!sfp))
 		return ERR_PTR(-ENOMEM);
 	init_waitqueue_head(&sfp->cmpl_wait);
 	xa_init_flags(&sfp->srp_arr, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
@@ -5869,7 +5921,7 @@ sg_add_sfp(struct sg_device *sdp, struct file *filp)
 	atomic_set(&sfp->waiting, 0);
 	atomic_set(&sfp->inactives, 0);
 
-	if (unlikely(SG_IS_DETACHING(sdp))) {
+	if (SG_IS_DETACHING(sdp)) {
 		SG_LOG(1, sfp, "%s: detaching\n", __func__);
 		kfree(sfp);
 		return ERR_PTR(-ENODEV);
@@ -5918,7 +5970,7 @@ sg_add_sfp(struct sg_device *sdp, struct file *filp)
 	xa_lock_irqsave(xadp, iflags);
 	res = __xa_alloc(xadp, &idx, sfp, xa_limit_32b, GFP_KERNEL);
 	xa_unlock_irqrestore(xadp, iflags);
-	if (res < 0) {
+	if (unlikely(res < 0)) {
 		pr_warn("%s: xa_alloc(sdp) bad, o_count=%d, errno=%d\n",
 			__func__, atomic_read(&sdp->open_cnt), -res);
 		if (srp) {
@@ -5958,7 +6010,7 @@ sg_remove_sfp_usercontext(struct work_struct *work)
 	struct xarray *xafp = &sfp->srp_arr;
 	struct xarray *xadp;
 
-	if (!sfp) {
+	if (unlikely(!sfp)) {
 		pr_warn("sg: %s: sfp is NULL\n", __func__);
 		return;
 	}
@@ -5971,14 +6023,14 @@ sg_remove_sfp_usercontext(struct work_struct *work)
 			sg_finish_scsi_blk_rq(srp);
 		if (srp->sgatp->buflen > 0)
 			sg_remove_sgat(srp);
-		if (srp->sense_bp) {
+		if (unlikely(srp->sense_bp)) {
 			mempool_free(srp->sense_bp, sg_sense_pool);
 			srp->sense_bp = NULL;
 		}
 		xa_lock_irqsave(xafp, iflags);
 		e_srp = __xa_erase(xafp, srp->rq_idx);
 		xa_unlock_irqrestore(xafp, iflags);
-		if (srp != e_srp)
+		if (unlikely(srp != e_srp))
 			SG_LOG(1, sfp, "%s: xa_erase() return unexpected\n",
 			       __func__);
 		SG_LOG(6, sfp, "%s: kfree: srp=%pK --\n", __func__, srp);
@@ -6033,7 +6085,7 @@ sg_get_dev(int min_dev)
 
 	read_lock_irqsave(&sg_index_lock, iflags);
 	sdp = sg_lookup_dev(min_dev);
-	if (!sdp)
+	if (unlikely(!sdp))
 		sdp = ERR_PTR(-ENXIO);
 	else if (SG_IS_DETACHING(sdp)) {
 		/* If detaching, then the refcount may already be 0, in
@@ -6139,7 +6191,7 @@ dev_seq_start(struct seq_file *s, loff_t *pos)
 	struct sg_proc_deviter *it = kzalloc(sizeof(*it), GFP_KERNEL);
 
 	s->private = it;
-	if (!it)
+	if (unlikely(!it))
 		return NULL;
 
 	it->index = *pos;
@@ -6189,10 +6241,10 @@ sg_proc_write_adio(struct file *filp, const char __user *buffer,
 	int err;
 	unsigned long num;
 
-	if (!capable(CAP_SYS_ADMIN) || !capable(CAP_SYS_RAWIO))
+	if (unlikely(!capable(CAP_SYS_ADMIN) || !capable(CAP_SYS_RAWIO)))
 		return -EACCES;
 	err = kstrtoul_from_user(buffer, count, 0, &num);
-	if (err)
+	if (unlikely(err))
 		return err;
 	sg_allow_dio = num ? 1 : 0;
 	return count;
@@ -6211,13 +6263,13 @@ sg_proc_write_dressz(struct file *filp, const char __user *buffer,
 	int err;
 	unsigned long k = ULONG_MAX;
 
-	if (!capable(CAP_SYS_ADMIN) || !capable(CAP_SYS_RAWIO))
+	if (unlikely(!capable(CAP_SYS_ADMIN) || !capable(CAP_SYS_RAWIO)))
 		return -EACCES;
 
 	err = kstrtoul_from_user(buffer, count, 0, &k);
-	if (err)
+	if (unlikely(err))
 		return err;
-	if (k <= 1048576) {	/* limit "big buff" to 1 MB */
+	if (likely(k <= 1048576)) {	/* limit "big buff" to 1 MB */
 		sg_big_buff = k;
 		return count;
 	}
@@ -6249,7 +6301,7 @@ sg_proc_seq_show_dev(struct seq_file *s, void *v)
 
 	read_lock_irqsave(&sg_index_lock, iflags);
 	sdp = it ? sg_lookup_dev(it->index) : NULL;
-	if (!sdp || !sdp->device || SG_IS_DETACHING(sdp))
+	if (unlikely(!sdp || !sdp->device || SG_IS_DETACHING(sdp)))
 		seq_puts(s, "-1\t-1\t-1\t-1\t-1\t-1\t-1\t-1\t-1\n");
 	else {
 		scsidp = sdp->device;
@@ -6301,7 +6353,7 @@ sg_proc_debug_sreq(struct sg_request *srp, int to, bool t_in_ns, char *obp,
 	const char *cp;
 	const char *tp = t_in_ns ? "ns" : "ms";
 
-	if (len < 1)
+	if (unlikely(len < 1))
 		return 0;
 	v4 = test_bit(SG_FRQ_IS_V4I, srp->frq_bm);
 	is_v3v4 = v4 ? true : (srp->s_hdr3.interface_id != '\0');
@@ -6467,14 +6519,14 @@ sg_proc_seq_show_debug(struct seq_file *s, void *v, bool reduced)
 			   (int)it->max, def_reserved_size);
 	fdi_p = it ? &it->fd_index : &k;
 	bp = kzalloc(bp_len, __GFP_NOWARN | GFP_KERNEL);
-	if (!bp) {
+	if (unlikely(!bp)) {
 		seq_printf(s, "%s: Unable to allocate %d on heap, finish\n",
 			   __func__, bp_len);
 		return -ENOMEM;
 	}
 	read_lock_irqsave(&sg_index_lock, iflags);
 	sdp = it ? sg_lookup_dev(it->index) : NULL;
-	if (!sdp)
+	if (unlikely(!sdp))
 		goto skip;
 	sd_n = dev_arr[0];
 	if (sd_n != -1 && sd_n != sdp->index && sd_n != dev_arr[1]) {
@@ -6530,7 +6582,7 @@ sg_proc_seq_show_debug(struct seq_file *s, void *v, bool reduced)
 					   "due to buffer size");
 		} else if (b1[0]) {
 			seq_puts(s, b1);
-			if (seq_has_overflowed(s))
+			if (unlikely(seq_has_overflowed(s)))
 				goto s_ovfl;
 		}
 	}
@@ -6605,7 +6657,7 @@ sg_proc_init(void)
 	struct proc_dir_entry *p;
 
 	p = proc_mkdir("scsi/sg", NULL);
-	if (!p)
+	if (unlikely(!p))
 		return 1;
 
 	proc_create("allow_dio", 0644, p, &adio_proc_ops);
@@ -6731,7 +6783,7 @@ sg_dfs_write(struct file *file, const char __user *buf, size_t count,
 	 * Attributes that only implement .seq_ops are read-only and 'attr' is
 	 * the same with 'data' in this case.
 	 */
-	if (attr == data || !attr->write)
+	if (unlikely(attr == data || !attr->write))
 		return -EPERM;
 	return attr->write(data, buf, count, ppos);
 }
-- 
2.25.1