From: Jens Axboe <axboe@kernel.dk>
To: Al Viro <viro@zeniv.linux.org.uk>
Cc: Pavel Begunkov <asml.silence@gmail.com>,
yangerkun <yangerkun@huawei.com>,
linux-fsdevel@vger.kernel.org, linux-block@vger.kernel.org,
io-uring@vger.kernel.org
Subject: Re: [PATCH] block: reexpand iov_iter after read/write
Date: Thu, 6 May 2021 13:15:01 -0600 [thread overview]
Message-ID: <b4fe4a3d-06ab-31e3-e1a2-46c23307b32a@kernel.dk> (raw)
In-Reply-To: <YJQ7jf7Twxexx31T@zeniv-ca.linux.org.uk>
[-- Attachment #1: Type: text/plain, Size: 1273 bytes --]
On 5/6/21 12:55 PM, Al Viro wrote:
> On Thu, May 06, 2021 at 11:19:03AM -0600, Jens Axboe wrote:
>
>> Doing a quick profile, on the latter run with ->write_iter() we're
>> spending 8% of the time in _copy_from_iter(), and 4% in
>> new_sync_write(). That's obviously not there at all for the first case.
>> Both have about 4% in eventfd_write(). Non-iter case spends 1% in
>> copy_from_user().
>>
>> Finally with your branch pulled in as well, iow using ->write_iter() for
>> eventfd and your iov changes:
>>
>> Executed in 485.26 millis fish external
>> usr time 103.09 millis 70.00 micros 103.03 millis
>> sys time 382.18 millis 83.00 micros 382.09 millis
>>
>> Executed in 485.16 millis fish external
>> usr time 104.07 millis 69.00 micros 104.00 millis
>> sys time 381.09 millis 94.00 micros 381.00 millis
>>
>> and there's no real difference there. We're spending less time in
>> _copy_from_iter() (8% -> 6%) and less time in new_sync_write(), but
>> doesn't seem to manifest itself in reduced runtime.
>
> Interesting... do you have instruction-level profiles for _copy_from_iter()
> and new_sync_write() on the last of those trees?
Attached output of perf annotate <func> for that last run.
--
Jens Axboe
[-- Attachment #2: nsw --]
[-- Type: text/plain, Size: 10648 bytes --]
Percent | Source code & Disassembly of vmlinux for cycles (72 samples, percent: local period)
---------------------------------------------------------------------------------------------------
:
:
:
: Disassembly of section .text:
:
: ffffffff812cef20 <new_sync_write>:
: new_sync_write():
: inc_syscr(current);
: return ret;
: }
:
: static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
: {
0.00 : ffffffff812cef20: callq ffffffff8103a8a0 <__fentry__>
0.00 : ffffffff812cef25: push %rbp
0.00 : ffffffff812cef26: mov %rdx,%r8
5.55 : ffffffff812cef29: mov %rsp,%rbp
0.00 : ffffffff812cef2c: push %r12
0.00 : ffffffff812cef2e: push %rbx
0.00 : ffffffff812cef2f: mov %rcx,%r12
0.00 : ffffffff812cef32: sub $0x68,%rsp
: struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
0.00 : ffffffff812cef36: mov %rdx,-0x70(%rbp)
: iocb_flags():
: }
:
: static inline int iocb_flags(struct file *file)
: {
: int res = 0;
: if (file->f_flags & O_APPEND)
0.00 : ffffffff812cef3a: mov 0x40(%rdi),%edx
: new_sync_write():
: {
8.33 : ffffffff812cef3d: mov %rdi,%rbx
: struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
0.00 : ffffffff812cef40: mov %rsi,-0x78(%rbp)
: iocb_flags():
0.00 : ffffffff812cef44: mov %edx,%eax
0.00 : ffffffff812cef46: shr $0x6,%eax
0.00 : ffffffff812cef49: and $0x10,%eax
: res |= IOCB_APPEND;
: if (file->f_flags & O_DIRECT)
: res |= IOCB_DIRECT;
0.00 : ffffffff812cef4c: mov %eax,%ecx
0.00 : ffffffff812cef4e: or $0x20000,%ecx
0.00 : ffffffff812cef54: test $0x40,%dh
6.94 : ffffffff812cef57: cmovne %ecx,%eax
: if ((file->f_flags & O_DSYNC) || IS_SYNC(file->f_mapping->host))
0.00 : ffffffff812cef5a: test $0x10,%dh
0.00 : ffffffff812cef5d: jne ffffffff812cef77 <new_sync_write+0x57>
0.00 : ffffffff812cef5f: mov 0xd0(%rdi),%rcx
0.00 : ffffffff812cef66: mov (%rcx),%rcx
0.00 : ffffffff812cef69: mov 0x28(%rcx),%rsi
0.00 : ffffffff812cef6d: testb $0x10,0x50(%rsi)
13.89 : ffffffff812cef71: je ffffffff812cf04c <new_sync_write+0x12c>
: res |= IOCB_DSYNC;
0.00 : ffffffff812cef77: or $0x2,%eax
: if (file->f_flags & __O_SYNC)
: res |= IOCB_SYNC;
0.00 : ffffffff812cef7a: mov %eax,%ecx
0.00 : ffffffff812cef7c: or $0x4,%ecx
0.00 : ffffffff812cef7f: and $0x100000,%edx
: file_write_hint():
: if (file->f_write_hint != WRITE_LIFE_NOT_SET)
0.00 : ffffffff812cef85: mov 0x34(%rbx),%edx
: iocb_flags():
: res |= IOCB_SYNC;
0.00 : ffffffff812cef88: cmovne %ecx,%eax
: file_write_hint():
: if (file->f_write_hint != WRITE_LIFE_NOT_SET)
0.00 : ffffffff812cef8b: test %edx,%edx
6.97 : ffffffff812cef8d: jne ffffffff812cf03c <new_sync_write+0x11c>
: return file_inode(file)->i_write_hint;
0.00 : ffffffff812cef93: mov 0x20(%rbx),%rdx
0.00 : ffffffff812cef97: movzbl 0x87(%rdx),%edx
: get_current():
:
: DECLARE_PER_CPU(struct task_struct *, current_task);
:
: static __always_inline struct task_struct *get_current(void)
: {
: return this_cpu_read_stable(current_task);
0.00 : ffffffff812cef9e: mov %gs:0x126c0,%rcx
: get_current_ioprio():
: * If the calling process has set an I/O priority, use that. Otherwise, return
: * the default I/O priority.
: */
: static inline int get_current_ioprio(void)
: {
: struct io_context *ioc = current->io_context;
0.00 : ffffffff812cefa7: mov 0x860(%rcx),%rsi
:
: if (ioc)
0.00 : ffffffff812cefae: xor %ecx,%ecx
0.00 : ffffffff812cefb0: test %rsi,%rsi
0.00 : ffffffff812cefb3: je ffffffff812cefb9 <new_sync_write+0x99>
: return ioc->ioprio;
0.00 : ffffffff812cefb5: movzwl 0x14(%rsi),%ecx
: init_sync_kiocb():
: *kiocb = (struct kiocb) {
0.00 : ffffffff812cefb9: shl $0x10,%ecx
12.50 : ffffffff812cefbc: movzwl %dx,%edx
0.00 : ffffffff812cefbf: movq $0x0,-0x38(%rbp)
0.00 : ffffffff812cefc7: movq $0x0,-0x30(%rbp)
0.00 : ffffffff812cefcf: or %ecx,%edx
0.00 : ffffffff812cefd1: movq $0x0,-0x28(%rbp)
0.00 : ffffffff812cefd9: movq $0x0,-0x18(%rbp)
0.00 : ffffffff812cefe1: mov %rbx,-0x40(%rbp)
0.00 : ffffffff812cefe5: mov %eax,-0x20(%rbp)
6.93 : ffffffff812cefe8: mov %edx,-0x1c(%rbp)
: new_sync_write():
: struct kiocb kiocb;
: struct iov_iter iter;
: ssize_t ret;
:
: init_sync_kiocb(&kiocb, filp);
: kiocb.ki_pos = (ppos ? *ppos : 0);
0.00 : ffffffff812cefeb: test %r12,%r12
0.00 : ffffffff812cefee: je ffffffff812cf05b <new_sync_write+0x13b>
: iov_iter_init(&iter, WRITE, &iov, 1, len);
0.00 : ffffffff812ceff0: mov $0x1,%esi
0.00 : ffffffff812ceff5: lea -0x68(%rbp),%rdi
0.00 : ffffffff812ceff9: mov $0x1,%ecx
0.00 : ffffffff812ceffe: lea -0x78(%rbp),%rdx
: kiocb.ki_pos = (ppos ? *ppos : 0);
0.00 : ffffffff812cf002: mov (%r12),%rax
0.00 : ffffffff812cf006: mov %rax,-0x38(%rbp)
: iov_iter_init(&iter, WRITE, &iov, 1, len);
8.33 : ffffffff812cf00a: callq ffffffff814c45e0 <iov_iter_init>
: call_write_iter():
: return file->f_op->write_iter(kio, iter);
12.51 : ffffffff812cf00f: mov 0x28(%rbx),%rax
0.00 : ffffffff812cf013: lea -0x68(%rbp),%rsi
0.00 : ffffffff812cf017: lea -0x40(%rbp),%rdi
0.00 : ffffffff812cf01b: callq *0x28(%rax)
: new_sync_write():
:
: ret = call_write_iter(filp, &kiocb, &iter);
: BUG_ON(ret == -EIOCBQUEUED);
0.00 : ffffffff812cf01e: cmp $0xfffffffffffffdef,%rax
0.00 : ffffffff812cf024: je ffffffff812cf089 <new_sync_write+0x169>
: if (ret > 0 && ppos)
0.00 : ffffffff812cf026: test %rax,%rax
0.00 : ffffffff812cf029: jle ffffffff812cf033 <new_sync_write+0x113>
: *ppos = kiocb.ki_pos;
0.00 : ffffffff812cf02b: mov -0x38(%rbp),%rdx
12.49 : ffffffff812cf02f: mov %rdx,(%r12)
: return ret;
: }
0.00 : ffffffff812cf033: add $0x68,%rsp
0.00 : ffffffff812cf037: pop %rbx
0.00 : ffffffff812cf038: pop %r12
0.00 : ffffffff812cf03a: pop %rbp
0.00 : ffffffff812cf03b: retq
: ki_hint_validate():
: if (hint <= max_hint)
0.00 : ffffffff812cf03c: xor %ecx,%ecx
0.00 : ffffffff812cf03e: cmp $0xffff,%edx
0.00 : ffffffff812cf044: cmova %ecx,%edx
0.00 : ffffffff812cf047: jmpq ffffffff812cef9e <new_sync_write+0x7e>
: iocb_flags():
: if ((file->f_flags & O_DSYNC) || IS_SYNC(file->f_mapping->host))
5.55 : ffffffff812cf04c: testb $0x1,0xc(%rcx)
0.00 : ffffffff812cf050: je ffffffff812cef7a <new_sync_write+0x5a>
0.00 : ffffffff812cf056: jmpq ffffffff812cef77 <new_sync_write+0x57>
: new_sync_write():
: iov_iter_init(&iter, WRITE, &iov, 1, len);
0.00 : ffffffff812cf05b: mov $0x1,%esi
0.00 : ffffffff812cf060: lea -0x68(%rbp),%rdi
0.00 : ffffffff812cf064: mov $0x1,%ecx
0.00 : ffffffff812cf069: lea -0x78(%rbp),%rdx
0.00 : ffffffff812cf06d: callq ffffffff814c45e0 <iov_iter_init>
: call_write_iter():
: return file->f_op->write_iter(kio, iter);
0.00 : ffffffff812cf072: mov 0x28(%rbx),%rax
0.00 : ffffffff812cf076: lea -0x68(%rbp),%rsi
0.00 : ffffffff812cf07a: lea -0x40(%rbp),%rdi
0.00 : ffffffff812cf07e: callq *0x28(%rax)
: new_sync_write():
: BUG_ON(ret == -EIOCBQUEUED);
0.00 : ffffffff812cf081: cmp $0xfffffffffffffdef,%rax
0.00 : ffffffff812cf087: jne ffffffff812cf033 <new_sync_write+0x113>
0.00 : ffffffff812cf089: ud2
[-- Attachment #3: cfi --]
[-- Type: text/plain, Size: 30346 bytes --]
Percent | Source code & Disassembly of vmlinux for cycles (113 samples, percent: local period)
----------------------------------------------------------------------------------------------------
:
:
:
: Disassembly of section .text:
:
: ffffffff814c6aa0 <_copy_from_iter>:
: _copy_from_iter():
: }
: EXPORT_SYMBOL_GPL(_copy_mc_to_iter);
: #endif /* CONFIG_ARCH_HAS_COPY_MC */
:
: size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
: {
0.00 : ffffffff814c6aa0: push %rbp
7.07 : ffffffff814c6aa1: mov %rdx,%rax
0.00 : ffffffff814c6aa4: mov %rsp,%rbp
0.00 : ffffffff814c6aa7: push %r15
0.00 : ffffffff814c6aa9: push %r14
0.00 : ffffffff814c6aab: push %r13
3.54 : ffffffff814c6aad: push %r12
0.00 : ffffffff814c6aaf: push %rbx
0.00 : ffffffff814c6ab0: sub $0x50,%rsp
0.00 : ffffffff814c6ab4: mov %rdx,-0x78(%rbp)
: iov_iter_type():
: };
: };
:
: static inline enum iter_type iov_iter_type(const struct iov_iter *i)
: {
: return i->iter_type;
0.89 : ffffffff814c6ab8: movzbl (%rdx),%edx
: _copy_from_iter():
0.00 : ffffffff814c6abb: mov %rdi,-0x68(%rbp)
: if (unlikely(iov_iter_is_pipe(i))) {
0.00 : ffffffff814c6abf: cmp $0x3,%dl
0.00 : ffffffff814c6ac2: je ffffffff814c6bd6 <_copy_from_iter+0x136>
0.00 : ffffffff814c6ac8: mov %rax,%rdi
: WARN_ON(1);
: return 0;
: }
: if (iter_is_iovec(i))
: might_fault();
: iterate_and_advance(i, bytes, base, len, off,
0.00 : ffffffff814c6acb: mov 0x10(%rax),%rax
0.00 : ffffffff814c6acf: cmp %rsi,%rax
0.00 : ffffffff814c6ad2: cmovbe %rax,%rsi
2.65 : ffffffff814c6ad6: mov %rsi,%r13
0.00 : ffffffff814c6ad9: test %rsi,%rsi
3.52 : ffffffff814c6adc: je ffffffff814c6bdd <_copy_from_iter+0x13d>
1.76 : ffffffff814c6ae2: test %dl,%dl
0.00 : ffffffff814c6ae4: jne ffffffff814c6be2 <_copy_from_iter+0x142>
0.00 : ffffffff814c6aea: mov 0x18(%rdi),%rax
0.00 : ffffffff814c6aee: mov 0x8(%rdi),%r14
0.00 : ffffffff814c6af2: xor %r15d,%r15d
0.00 : ffffffff814c6af5: mov -0x68(%rbp),%rdi
25.58 : ffffffff814c6af9: lea 0x10(%rax),%r12
0.00 : ffffffff814c6afd: jmp ffffffff814c6b0e <_copy_from_iter+0x6e>
0.00 : ffffffff814c6aff: mov -0x68(%rbp),%rax
0.00 : ffffffff814c6b03: lea (%rax,%r15,1),%rdi
0.00 : ffffffff814c6b07: add $0x10,%r12
: {
0.00 : ffffffff814c6b0b: xor %r14d,%r14d
: iterate_and_advance(i, bytes, base, len, off,
0.00 : ffffffff814c6b0e: mov -0x8(%r12),%rcx
1.09 : ffffffff814c6b13: lea -0x10(%r12),%rax
0.00 : ffffffff814c6b18: mov %r12,-0x60(%rbp)
0.00 : ffffffff814c6b1c: mov %rax,-0x70(%rbp)
0.00 : ffffffff814c6b20: mov %rcx,%rbx
0.00 : ffffffff814c6b23: sub %r14,%rbx
1.76 : ffffffff814c6b26: cmp %r13,%rbx
0.00 : ffffffff814c6b29: cmova %r13,%rbx
0.00 : ffffffff814c6b2d: test %rbx,%rbx
0.00 : ffffffff814c6b30: je ffffffff814c6b07 <_copy_from_iter+0x67>
0.00 : ffffffff814c6b32: mov -0x10(%r12),%rsi
0.00 : ffffffff814c6b37: mov %rbx,%rax
0.00 : ffffffff814c6b3a: add %r14,%rsi
: __chk_range_not_ok():
: */
: if (__builtin_constant_p(size))
: return unlikely(addr > limit - size);
:
: /* Arbitrary sizes? Be careful about overflow */
: addr += size;
0.00 : ffffffff814c6b3d: add %rsi,%rax
4.42 : ffffffff814c6b40: jb ffffffff814c6bd1 <_copy_from_iter+0x131>
: copyin():
: if (access_ok(from, n)) {
0.00 : ffffffff814c6b46: movabs $0x7ffffffff000,%rdx
0.00 : ffffffff814c6b50: cmp %rdx,%rax
3.52 : ffffffff814c6b53: ja ffffffff814c6bd1 <_copy_from_iter+0x131>
: copy_user_generic():
: /*
: * If CPU has ERMS feature, use copy_user_enhanced_fast_string.
: * Otherwise, if CPU has rep_good feature, use copy_user_generic_string.
: * Otherwise, use copy_user_generic_unrolled.
: */
: alternative_call_2(copy_user_generic_unrolled,
0.00 : ffffffff814c6b55: mov %ebx,%edx
0.00 : ffffffff814c6b57: callq ffffffff81523880 <copy_user_generic_unrolled>
: _copy_from_iter():
: iterate_and_advance(i, bytes, base, len, off,
6.18 : ffffffff814c6b5c: mov -0x8(%r12),%rcx
: copy_user_generic():
: X86_FEATURE_ERMS,
: ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from),
: "=d" (len)),
: "1" (to), "2" (from), "3" (len)
: : "memory", "rcx", "r8", "r9", "r10", "r11");
: return ret;
0.00 : ffffffff814c6b61: mov %eax,%eax
: _copy_from_iter():
0.00 : ffffffff814c6b63: cltq
0.00 : ffffffff814c6b65: mov %rbx,%rdx
0.00 : ffffffff814c6b68: sub %rbx,%r13
0.00 : ffffffff814c6b6b: sub %rax,%rdx
0.00 : ffffffff814c6b6e: add %rax,%r13
0.00 : ffffffff814c6b71: add %rdx,%r15
3.53 : ffffffff814c6b74: add %r14,%rdx
0.00 : ffffffff814c6b77: cmp %rcx,%rdx
0.00 : ffffffff814c6b7a: jb ffffffff814c6bc4 <_copy_from_iter+0x124>
0.00 : ffffffff814c6b7c: test %r13,%r13
0.00 : ffffffff814c6b7f: jne ffffffff814c6aff <_copy_from_iter+0x5f>
0.00 : ffffffff814c6b85: mov -0x78(%rbp),%rcx
2.66 : ffffffff814c6b89: mov -0x60(%rbp),%rdi
2.65 : ffffffff814c6b8d: mov %rdi,%rax
0.00 : ffffffff814c6b90: sub 0x18(%rcx),%rax
11.54 : ffffffff814c6b94: mov %r13,0x8(%rcx)
0.00 : ffffffff814c6b98: mov %rdi,0x18(%rcx)
12.36 : ffffffff814c6b9c: mov %rcx,%rdi
0.00 : ffffffff814c6b9f: sar $0x4,%rax
0.00 : ffffffff814c6ba3: sub %rax,0x20(%rcx)
0.00 : ffffffff814c6ba7: mov 0x10(%rcx),%rax
0.00 : ffffffff814c6bab: sub %r15,%rax
0.00 : ffffffff814c6bae: mov %rax,0x10(%rdi)
: copyin(addr + off, base, len),
: memcpy(addr + off, base, len)
: )
:
: return bytes;
: }
3.53 : ffffffff814c6bb2: add $0x50,%rsp
0.00 : ffffffff814c6bb6: mov %r15,%rax
0.00 : ffffffff814c6bb9: pop %rbx
0.00 : ffffffff814c6bba: pop %r12
0.00 : ffffffff814c6bbc: pop %r13
0.00 : ffffffff814c6bbe: pop %r14
0.00 : ffffffff814c6bc0: pop %r15
1.76 : ffffffff814c6bc2: pop %rbp
0.00 : ffffffff814c6bc3: retq
0.00 : ffffffff814c6bc4: mov -0x70(%rbp),%rax
0.00 : ffffffff814c6bc8: mov %rdx,%r13
0.00 : ffffffff814c6bcb: mov %rax,-0x60(%rbp)
0.00 : ffffffff814c6bcf: jmp ffffffff814c6b85 <_copy_from_iter+0xe5>
: copyin():
0.00 : ffffffff814c6bd1: mov %rbx,%rax
0.00 : ffffffff814c6bd4: jmp ffffffff814c6b63 <_copy_from_iter+0xc3>
: _copy_from_iter():
: WARN_ON(1);
0.00 : ffffffff814c6bd6: ud2
: return 0;
0.00 : ffffffff814c6bd8: xor %r15d,%r15d
0.00 : ffffffff814c6bdb: jmp ffffffff814c6bb2 <_copy_from_iter+0x112>
0.00 : ffffffff814c6bdd: xor %r15d,%r15d
0.00 : ffffffff814c6be0: jmp ffffffff814c6bb2 <_copy_from_iter+0x112>
: iterate_and_advance(i, bytes, base, len, off,
0.00 : ffffffff814c6be2: cmp $0x2,%dl
0.00 : ffffffff814c6be5: je ffffffff814c6e09 <_copy_from_iter+0x369>
0.00 : ffffffff814c6beb: cmp $0x1,%dl
0.00 : ffffffff814c6bee: je ffffffff814c6d6b <_copy_from_iter+0x2cb>
0.00 : ffffffff814c6bf4: mov %rsi,%r15
0.00 : ffffffff814c6bf7: cmp $0x4,%dl
0.00 : ffffffff814c6bfa: jne ffffffff814c6bab <_copy_from_iter+0x10b>
0.00 : ffffffff814c6bfc: mov 0x8(%rdi),%rax
0.00 : ffffffff814c6c00: add 0x20(%rdi),%rax
0.00 : ffffffff814c6c04: movl $0x0,-0x48(%rbp)
0.00 : ffffffff814c6c0b: movq $0x3,-0x40(%rbp)
0.00 : ffffffff814c6c13: movq $0x0,-0x38(%rbp)
0.00 : ffffffff814c6c1b: movq $0x0,-0x30(%rbp)
0.00 : ffffffff814c6c23: mov %eax,%ebx
0.00 : ffffffff814c6c25: shr $0xc,%rax
0.00 : ffffffff814c6c29: mov %rax,%rcx
0.00 : ffffffff814c6c2c: mov %rax,-0x60(%rbp)
0.00 : ffffffff814c6c30: mov 0x18(%rdi),%rax
0.00 : ffffffff814c6c34: and $0xfff,%ebx
0.00 : ffffffff814c6c3a: mov %rcx,-0x50(%rbp)
0.00 : ffffffff814c6c3e: mov %rax,-0x58(%rbp)
0.00 : ffffffff814c6c42: mov $0xffffffffffffffff,%rsi
0.00 : ffffffff814c6c49: lea -0x58(%rbp),%rdi
0.00 : ffffffff814c6c4d: xor %r15d,%r15d
0.00 : ffffffff814c6c50: callq ffffffff8151fe40 <xas_find>
0.00 : ffffffff814c6c55: mov %rax,%r14
0.00 : ffffffff814c6c58: test %rax,%rax
0.00 : ffffffff814c6c5b: je ffffffff814c6d51 <_copy_from_iter+0x2b1>
0.00 : ffffffff814c6c61: mov %ebx,%r12d
: xas_retry():
: * Context: Any context.
: * Return: true if the operation needs to be retried.
: */
: static inline bool xas_retry(struct xa_state *xas, const void *entry)
: {
: if (xa_is_zero(entry))
0.00 : ffffffff814c6c64: cmp $0x406,%r14
0.00 : ffffffff814c6c6b: je ffffffff814c6d1c <_copy_from_iter+0x27c>
: return true;
: if (!xa_is_retry(entry))
0.00 : ffffffff814c6c71: cmp $0x402,%r14
0.00 : ffffffff814c6c78: je ffffffff814c6f7f <_copy_from_iter+0x4df>
: _copy_from_iter():
0.00 : ffffffff814c6c7e: test $0x1,%r14b
0.00 : ffffffff814c6c82: jne ffffffff814c6f78 <_copy_from_iter+0x4d8>
0.00 : ffffffff814c6c88: mov %r14,%rdi
0.00 : ffffffff814c6c8b: callq ffffffff81296c00 <PageHuge>
0.00 : ffffffff814c6c90: mov %eax,%ebx
0.00 : ffffffff814c6c92: test %eax,%eax
0.00 : ffffffff814c6c94: jne ffffffff814c6f30 <_copy_from_iter+0x490>
0.00 : ffffffff814c6c9a: mov -0x60(%rbp),%rdi
0.00 : ffffffff814c6c9e: mov 0x20(%r14),%rax
0.00 : ffffffff814c6ca2: mov %edi,%ecx
0.00 : ffffffff814c6ca4: sub %eax,%ecx
0.00 : ffffffff814c6ca6: cmp %rdi,%rax
0.00 : ffffffff814c6ca9: cmovb %ecx,%ebx
0.00 : ffffffff814c6cac: jmp ffffffff814c6d00 <_copy_from_iter+0x260>
0.00 : ffffffff814c6cae: mov %r12d,%eax
0.00 : ffffffff814c6cb1: mov $0x1000,%edx
0.00 : ffffffff814c6cb6: movslq %ebx,%rsi
0.00 : ffffffff814c6cb9: mov -0x68(%rbp),%rcx
0.00 : ffffffff814c6cbd: sub %rax,%rdx
0.00 : ffffffff814c6cc0: cmp %r13,%rdx
0.00 : ffffffff814c6cc3: cmova %r13,%rdx
0.00 : ffffffff814c6cc7: shl $0x6,%rsi
0.00 : ffffffff814c6ccb: add %r14,%rsi
: lowmem_page_address():
: */
: #include <linux/vmstat.h>
:
: static __always_inline void *lowmem_page_address(const struct page *page)
: {
: return page_to_virt(page);
0.00 : ffffffff814c6cce: sub 0xebda73(%rip),%rsi # ffffffff82384748 <vmemmap_base>
: _copy_from_iter():
0.00 : ffffffff814c6cd5: mov %rdx,%r12
0.00 : ffffffff814c6cd8: lea (%rcx,%r15,1),%rdi
0.00 : ffffffff814c6cdc: add %r12,%r15
: lowmem_page_address():
0.00 : ffffffff814c6cdf: sar $0x6,%rsi
0.00 : ffffffff814c6ce3: shl $0xc,%rsi
0.00 : ffffffff814c6ce7: add 0xebda6a(%rip),%rsi # ffffffff82384758 <page_offset_base>
: _copy_from_iter():
0.00 : ffffffff814c6cee: add %rax,%rsi
: memcpy():
: if (q_size < size)
: __read_overflow2();
: }
: if (p_size < size || q_size < size)
: fortify_panic(__func__);
: return __underlying_memcpy(p, q, size);
0.00 : ffffffff814c6cf1: callq ffffffff81a22620 <__memcpy>
: _copy_from_iter():
0.00 : ffffffff814c6cf6: sub %r12,%r13
0.00 : ffffffff814c6cf9: je ffffffff814c6d51 <_copy_from_iter+0x2b1>
0.00 : ffffffff814c6cfb: inc %ebx
0.00 : ffffffff814c6cfd: xor %r12d,%r12d
: constant_test_bit():
: }
:
: static __always_inline bool constant_test_bit(long nr, const volatile unsigned long *addr)
: {
: return ((1UL << (nr & (BITS_PER_LONG-1))) &
: (addr[nr >> _BITOPS_LONG_SHIFT])) != 0;
0.00 : ffffffff814c6d00: mov (%r14),%rax
0.00 : ffffffff814c6d03: shr $0x10,%rax
0.00 : ffffffff814c6d07: and $0x1,%eax
: thp_nr_pages():
: */
: static inline int thp_nr_pages(struct page *page)
: {
: VM_BUG_ON_PGFLAGS(PageTail(page), page);
: if (PageHead(page))
: return HPAGE_PMD_NR;
0.00 : ffffffff814c6d0a: cmp $0x1,%al
0.00 : ffffffff814c6d0c: sbb %eax,%eax
0.00 : ffffffff814c6d0e: and $0xfffffe01,%eax
0.00 : ffffffff814c6d13: add $0x200,%eax
: _copy_from_iter():
0.00 : ffffffff814c6d18: cmp %eax,%ebx
0.00 : ffffffff814c6d1a: jl ffffffff814c6cae <_copy_from_iter+0x20e>
: xas_next_entry():
: *
: * Return: The next present entry after the one currently referred to by @xas.
: */
: static inline void *xas_next_entry(struct xa_state *xas, unsigned long max)
: {
: struct xa_node *node = xas->xa_node;
0.00 : ffffffff814c6d1c: mov -0x40(%rbp),%rdi
: xas_not_node():
: return ((unsigned long)node & 3) || !node;
0.00 : ffffffff814c6d20: test $0x3,%dil
0.00 : ffffffff814c6d24: setne %cl
0.00 : ffffffff814c6d27: test %rdi,%rdi
0.00 : ffffffff814c6d2a: sete %al
0.00 : ffffffff814c6d2d: or %al,%cl
0.00 : ffffffff814c6d2f: je ffffffff814c6ecc <_copy_from_iter+0x42c>
: xas_next_entry():
: return xas_find(xas, max);
: if (unlikely(xas->xa_offset == XA_CHUNK_MASK))
: return xas_find(xas, max);
: entry = xa_entry(xas->xa, node, xas->xa_offset + 1);
: if (unlikely(xa_is_internal(entry)))
: return xas_find(xas, max);
0.00 : ffffffff814c6d35: mov $0xffffffffffffffff,%rsi
0.00 : ffffffff814c6d3c: lea -0x58(%rbp),%rdi
0.00 : ffffffff814c6d40: callq ffffffff8151fe40 <xas_find>
0.00 : ffffffff814c6d45: mov %rax,%r14
: _copy_from_iter():
0.00 : ffffffff814c6d48: test %rax,%rax
0.00 : ffffffff814c6d4b: jne ffffffff814c6c64 <_copy_from_iter+0x1c4>
: __rcu_read_unlock():
: }
:
: static inline void __rcu_read_unlock(void)
: {
: preempt_enable();
: rcu_read_unlock_strict();
0.00 : ffffffff814c6d51: callq ffffffff810e12c0 <rcu_read_unlock_strict>
: _copy_from_iter():
0.00 : ffffffff814c6d56: mov -0x78(%rbp),%rax
0.00 : ffffffff814c6d5a: mov -0x78(%rbp),%rdi
0.00 : ffffffff814c6d5e: add %r15,0x8(%rax)
0.00 : ffffffff814c6d62: mov 0x10(%rax),%rax
0.00 : ffffffff814c6d66: jmpq ffffffff814c6bab <_copy_from_iter+0x10b>
0.00 : ffffffff814c6d6b: mov 0x18(%rdi),%rax
0.00 : ffffffff814c6d6f: xor %r15d,%r15d
0.00 : ffffffff814c6d72: mov 0x8(%rdi),%rbx
0.00 : ffffffff814c6d76: mov -0x68(%rbp),%rdi
0.00 : ffffffff814c6d7a: lea 0x10(%rax),%r12
0.00 : ffffffff814c6d7e: mov %r15,%rax
0.00 : ffffffff814c6d81: mov %r12,%r15
0.00 : ffffffff814c6d84: mov %rax,%r12
0.00 : ffffffff814c6d87: jmp ffffffff814c6d97 <_copy_from_iter+0x2f7>
0.00 : ffffffff814c6d89: mov -0x68(%rbp),%rax
0.00 : ffffffff814c6d8d: lea (%rax,%r12,1),%rdi
0.00 : ffffffff814c6d91: add $0x10,%r15
0.00 : ffffffff814c6d95: xor %ebx,%ebx
0.00 : ffffffff814c6d97: mov -0x8(%r15),%r14
0.00 : ffffffff814c6d9b: lea -0x10(%r15),%rax
0.00 : ffffffff814c6d9f: mov %r15,-0x60(%rbp)
0.00 : ffffffff814c6da3: mov %rax,-0x70(%rbp)
0.00 : ffffffff814c6da7: sub %rbx,%r14
0.00 : ffffffff814c6daa: cmp %r13,%r14
0.00 : ffffffff814c6dad: cmova %r13,%r14
0.00 : ffffffff814c6db1: test %r14,%r14
0.00 : ffffffff814c6db4: je ffffffff814c6d91 <_copy_from_iter+0x2f1>
0.00 : ffffffff814c6db6: mov -0x10(%r15),%rsi
: memcpy():
0.00 : ffffffff814c6dba: mov %r14,%rdx
: _copy_from_iter():
0.00 : ffffffff814c6dbd: add %r14,%r12
0.00 : ffffffff814c6dc0: sub %r14,%r13
0.00 : ffffffff814c6dc3: add %rbx,%rsi
: memcpy():
0.00 : ffffffff814c6dc6: callq ffffffff81a22620 <__memcpy>
: _copy_from_iter():
0.00 : ffffffff814c6dcb: lea (%rbx,%r14,1),%rcx
0.00 : ffffffff814c6dcf: cmp %rcx,-0x8(%r15)
0.00 : ffffffff814c6dd3: ja ffffffff814c6eb9 <_copy_from_iter+0x419>
0.00 : ffffffff814c6dd9: test %r13,%r13
0.00 : ffffffff814c6ddc: jne ffffffff814c6d89 <_copy_from_iter+0x2e9>
0.00 : ffffffff814c6dde: mov %r12,%r15
0.00 : ffffffff814c6de1: mov -0x78(%rbp),%rdi
0.00 : ffffffff814c6de5: mov -0x60(%rbp),%rcx
0.00 : ffffffff814c6de9: mov %rcx,%rax
0.00 : ffffffff814c6dec: sub 0x18(%rdi),%rax
0.00 : ffffffff814c6df0: mov %r13,0x8(%rdi)
0.00 : ffffffff814c6df4: mov %rcx,0x18(%rdi)
0.00 : ffffffff814c6df8: sar $0x4,%rax
0.00 : ffffffff814c6dfc: sub %rax,0x20(%rdi)
0.00 : ffffffff814c6e00: mov 0x10(%rdi),%rax
0.00 : ffffffff814c6e04: jmpq ffffffff814c6bab <_copy_from_iter+0x10b>
0.00 : ffffffff814c6e09: mov 0x18(%rdi),%r14
0.00 : ffffffff814c6e0d: mov 0x8(%rdi),%r12d
0.00 : ffffffff814c6e11: xor %r15d,%r15d
0.00 : ffffffff814c6e14: mov 0xc(%r14),%eax
0.00 : ffffffff814c6e18: mov 0x8(%r14),%edx
0.00 : ffffffff814c6e1c: mov $0x1000,%esi
0.00 : ffffffff814c6e21: mov -0x68(%rbp),%rdi
0.00 : ffffffff814c6e25: add %r12d,%eax
0.00 : ffffffff814c6e28: sub %r12d,%edx
0.00 : ffffffff814c6e2b: mov %eax,%ecx
0.00 : ffffffff814c6e2d: and $0xfff,%ecx
0.00 : ffffffff814c6e33: cmp %r13,%rdx
0.00 : ffffffff814c6e36: cmova %r13,%rdx
0.00 : ffffffff814c6e3a: sub %rcx,%rsi
0.00 : ffffffff814c6e3d: cmp %rsi,%rdx
0.00 : ffffffff814c6e40: cmovbe %rdx,%rsi
0.00 : ffffffff814c6e44: shr $0xc,%eax
0.00 : ffffffff814c6e47: add %r15,%rdi
0.00 : ffffffff814c6e4a: mov %rsi,%rbx
0.00 : ffffffff814c6e4d: mov %eax,%esi
0.00 : ffffffff814c6e4f: shl $0x6,%rsi
0.00 : ffffffff814c6e53: add (%r14),%rsi
: memcpy():
0.00 : ffffffff814c6e56: mov %rbx,%rdx
: _copy_from_iter():
0.00 : ffffffff814c6e59: add %rbx,%r15
: lowmem_page_address():
0.00 : ffffffff814c6e5c: sub 0xebd8e5(%rip),%rsi # ffffffff82384748 <vmemmap_base>
: _copy_from_iter():
0.00 : ffffffff814c6e63: add %ebx,%r12d
: lowmem_page_address():
0.00 : ffffffff814c6e66: sar $0x6,%rsi
0.00 : ffffffff814c6e6a: shl $0xc,%rsi
0.00 : ffffffff814c6e6e: add 0xebd8e3(%rip),%rsi # ffffffff82384758 <page_offset_base>
: _copy_from_iter():
0.00 : ffffffff814c6e75: add %rcx,%rsi
: memcpy():
0.00 : ffffffff814c6e78: callq ffffffff81a22620 <__memcpy>
: _copy_from_iter():
0.00 : ffffffff814c6e7d: cmp %r12d,0x8(%r14)
0.00 : ffffffff814c6e81: jne ffffffff814c6e8a <_copy_from_iter+0x3ea>
0.00 : ffffffff814c6e83: add $0x10,%r14
0.00 : ffffffff814c6e87: xor %r12d,%r12d
0.00 : ffffffff814c6e8a: sub %rbx,%r13
0.00 : ffffffff814c6e8d: jne ffffffff814c6e14 <_copy_from_iter+0x374>
0.00 : ffffffff814c6e8f: mov -0x78(%rbp),%rcx
0.00 : ffffffff814c6e93: mov %r12d,%eax
0.00 : ffffffff814c6e96: mov %rax,0x8(%rcx)
0.00 : ffffffff814c6e9a: mov %r14,%rax
0.00 : ffffffff814c6e9d: sub 0x18(%rcx),%rax
0.00 : ffffffff814c6ea1: mov %rcx,%rdi
0.00 : ffffffff814c6ea4: mov %r14,0x18(%rcx)
0.00 : ffffffff814c6ea8: sar $0x4,%rax
0.00 : ffffffff814c6eac: sub %rax,0x20(%rcx)
0.00 : ffffffff814c6eb0: mov 0x10(%rcx),%rax
0.00 : ffffffff814c6eb4: jmpq ffffffff814c6bab <_copy_from_iter+0x10b>
0.00 : ffffffff814c6eb9: mov -0x70(%rbp),%rax
0.00 : ffffffff814c6ebd: mov %r12,%r15
0.00 : ffffffff814c6ec0: mov %rcx,%r13
0.00 : ffffffff814c6ec3: mov %rax,-0x60(%rbp)
0.00 : ffffffff814c6ec7: jmpq ffffffff814c6de1 <_copy_from_iter+0x341>
: xas_next_entry():
: if (unlikely(xas_not_node(node) || node->shift ||
0.00 : ffffffff814c6ecc: cmpb $0x0,(%rdi)
0.00 : ffffffff814c6ecf: jne ffffffff814c6d35 <_copy_from_iter+0x295>
0.00 : ffffffff814c6ed5: mov -0x50(%rbp),%rsi
0.00 : ffffffff814c6ed9: movzbl -0x46(%rbp),%r9d
0.00 : ffffffff814c6ede: mov %rsi,%r8
0.00 : ffffffff814c6ee1: mov %r9,%rax
0.00 : ffffffff814c6ee4: and $0x3f,%r8d
0.00 : ffffffff814c6ee8: cmp %r8,%r9
0.00 : ffffffff814c6eeb: jne ffffffff814c6d35 <_copy_from_iter+0x295>
: if (unlikely(xas->xa_index >= max))
0.00 : ffffffff814c6ef1: cmp $0xffffffffffffffff,%rsi
0.00 : ffffffff814c6ef5: je ffffffff814c6f60 <_copy_from_iter+0x4c0>
: if (unlikely(xas->xa_offset == XA_CHUNK_MASK))
0.00 : ffffffff814c6ef7: cmp $0x3f,%al
0.00 : ffffffff814c6ef9: je ffffffff814c6f4b <_copy_from_iter+0x4ab>
: entry = xa_entry(xas->xa, node, xas->xa_offset + 1);
0.00 : ffffffff814c6efb: movzbl %al,%r8d
: xa_entry():
: return rcu_dereference_check(node->slots[offset],
0.00 : ffffffff814c6eff: add $0x5,%r8
0.00 : ffffffff814c6f03: mov 0x8(%rdi,%r8,8),%r14
: xa_is_internal():
: return ((unsigned long)entry & 3) == 2;
0.00 : ffffffff814c6f08: mov %r14,%r8
0.00 : ffffffff814c6f0b: and $0x3,%r8d
: xas_next_entry():
: if (unlikely(xa_is_internal(entry)))
0.00 : ffffffff814c6f0f: cmp $0x2,%r8
0.00 : ffffffff814c6f13: je ffffffff814c6f37 <_copy_from_iter+0x497>
: xas->xa_offset++;
0.00 : ffffffff814c6f15: inc %eax
: xas->xa_index++;
0.00 : ffffffff814c6f17: inc %rsi
: } while (!entry);
0.00 : ffffffff814c6f1a: mov $0x1,%ecx
0.00 : ffffffff814c6f1f: test %r14,%r14
0.00 : ffffffff814c6f22: je ffffffff814c6ef1 <_copy_from_iter+0x451>
0.00 : ffffffff814c6f24: mov %al,-0x46(%rbp)
0.00 : ffffffff814c6f27: mov %rsi,-0x50(%rbp)
0.00 : ffffffff814c6f2b: jmpq ffffffff814c6c64 <_copy_from_iter+0x1c4>
: _copy_from_iter():
0.00 : ffffffff814c6f30: ud2
0.00 : ffffffff814c6f32: jmpq ffffffff814c6d51 <_copy_from_iter+0x2b1>
0.00 : ffffffff814c6f37: test %cl,%cl
0.00 : ffffffff814c6f39: je ffffffff814c6d35 <_copy_from_iter+0x295>
0.00 : ffffffff814c6f3f: mov %al,-0x46(%rbp)
0.00 : ffffffff814c6f42: mov %rsi,-0x50(%rbp)
0.00 : ffffffff814c6f46: jmpq ffffffff814c6d35 <_copy_from_iter+0x295>
0.00 : ffffffff814c6f4b: test %cl,%cl
0.00 : ffffffff814c6f4d: je ffffffff814c6d35 <_copy_from_iter+0x295>
0.00 : ffffffff814c6f53: movb $0x3f,-0x46(%rbp)
0.00 : ffffffff814c6f57: mov %rsi,-0x50(%rbp)
: xas_next_entry():
: return xas_find(xas, max);
0.00 : ffffffff814c6f5b: jmpq ffffffff814c6d35 <_copy_from_iter+0x295>
0.00 : ffffffff814c6f60: test %cl,%cl
0.00 : ffffffff814c6f62: je ffffffff814c6d35 <_copy_from_iter+0x295>
0.00 : ffffffff814c6f68: mov %al,-0x46(%rbp)
0.00 : ffffffff814c6f6b: movq $0xffffffffffffffff,-0x50(%rbp)
: return xas_find(xas, max);
0.00 : ffffffff814c6f73: jmpq ffffffff814c6d35 <_copy_from_iter+0x295>
: _copy_from_iter():
0.00 : ffffffff814c6f78: ud2
0.00 : ffffffff814c6f7a: jmpq ffffffff814c6d51 <_copy_from_iter+0x2b1>
: xas_reset():
: xas->xa_node = XAS_RESTART;
0.00 : ffffffff814c6f7f: movq $0x3,-0x40(%rbp)
: xas_not_node():
: return ((unsigned long)node & 3) || !node;
0.00 : ffffffff814c6f87: jmpq ffffffff814c6d35 <_copy_from_iter+0x295>
next prev parent reply other threads:[~2021-05-06 19:15 UTC|newest]
Thread overview: 18+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-04-01 7:18 [PATCH] block: reexpand iov_iter after read/write yangerkun
2021-04-06 1:28 ` yangerkun
2021-04-06 11:04 ` Pavel Begunkov
2021-04-07 14:16 ` yangerkun
2021-04-09 14:49 ` Pavel Begunkov
2021-04-15 17:37 ` Pavel Begunkov
2021-04-15 17:39 ` Pavel Begunkov
2021-04-28 6:16 ` yangerkun
2021-04-30 12:57 ` Pavel Begunkov
2021-04-30 14:35 ` Al Viro
2021-05-06 16:57 ` Pavel Begunkov
2021-05-06 17:17 ` Al Viro
2021-05-06 17:19 ` Jens Axboe
2021-05-06 18:55 ` Al Viro
2021-05-06 19:15 ` Jens Axboe [this message]
2021-05-06 21:08 ` Al Viro
2021-05-06 21:17 ` Matthew Wilcox
2021-05-07 14:59 ` Jens Axboe
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=b4fe4a3d-06ab-31e3-e1a2-46c23307b32a@kernel.dk \
--to=axboe@kernel.dk \
--cc=asml.silence@gmail.com \
--cc=io-uring@vger.kernel.org \
--cc=linux-block@vger.kernel.org \
--cc=linux-fsdevel@vger.kernel.org \
--cc=viro@zeniv.linux.org.uk \
--cc=yangerkun@huawei.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.