[PATCH 3/5] memcg: Annotate slow paths in consume_stock() and refill_stock().

[Kuniyuki Iwashima <kuniyu@google.com>]

In consume_stock() and refill_stock(), each per-CPU stock slot
holds up to MEMCG_CHARGE_BATCH (64) pages, so depleting a slot
in consume_stock() (!stock_pages) or overflowing a slot in
refill_stock() (stock_pages > MEMCG_CHARGE_BATCH) happens at most
once per batch.

Similarly, requests larger than MEMCG_CHARGE_BATCH, local_trylock()
failure, and insufficient cached pages (stock_pages < nr_pages) are
all slow paths.

Without unlikely() annotations (when building without FDO, or when
an FDO profile does not cover a caller), the compiler treats these
branches as likely and inlines css_put() in consume_stock() and
drain_stock() in refill_stock() right into the middle of the hot path.

Consequently, the fast path has to jump over the inlined slow path
and jump again to the function epilogue after local_unlock().

Let's annotate the slow paths in consume_stock() and refill_stock()
with unlikely() so that the compiler keeps css_put() and
drain_stock() out-of-line at the cold tail and lets local_unlock()
and the function epilogue fall through on the fast path:

Before (refill_stock):

  stock_pages = stock->nr_pages[i] + nr_pages;
   b242: 43 02 74 3c 01          addb    0x1(%r12,%r15), %sil
  WRITE_ONCE(stock->nr_pages[i], stock_pages);
   b247: 43 88 74 3c 01          movb    %sil, 0x1(%r12,%r15)
  if (stock_pages > MEMCG_CHARGE_BATCH)
   b24c: 40 80 fe 41             cmpb    $0x41, %sil
   b250: 72 4f                   jb      0xb2a1 <refill_stock+0x2a1>
  struct mem_cgroup *old = stock->cached[i];   # drain_stock() inlined
   b252: 4b 8b 5c fc 08          movq    0x8(%r12,%r15,8), %rbx
   ...
   b276: e8 00 00 00 00          callq   <page_counter_uncharge>
   ...
  local_unlock(&memcg_stock.lock);
   b2a1: 65 48 8b 05 00 00 00 00 movq    %gs:(%rip), %rax
   b2a9: c6 80 00 00 00 00 00    movb    $0x0, (%rax)
   b2b0: e9 93 fd ff ff          jmp     0xb048 <refill_stock+0x48>

After (refill_stock):

  stock_pages = stock->nr_pages[i] + nr_pages;
   b278: 40 02 74 03 01          addb    0x1(%rbx,%rax), %sil
  WRITE_ONCE(stock->nr_pages[i], stock_pages);
   b27d: 40 88 74 03 01          movb    %sil, 0x1(%rbx,%rax)
  if (unlikely(stock_pages > MEMCG_CHARGE_BATCH))
   b282: 40 80 fe 41             cmpb    $0x41, %sil
   b286: 73 23                   jae     0xb2ab <refill_stock+0x27b>
  local_unlock(&memcg_stock.lock);
   b288: 65 48 8b 05 00 00 00 00 movq    %gs:(%rip), %rax
   b290: c6 80 00 00 00 00 00    movb    $0x0, (%rax)
  }
   b297: 48 83 c4 08             addq    $0x8, %rsp
   b29b: 5b                      popq    %rbx
   b29c: 41 5c                   popq    %r12
   b29e: 41 5d                   popq    %r13
   b2a0: 41 5e                   popq    %r14
   b2a2: 41 5f                   popq    %r15
   b2a4: 5d                      popq    %rbp
   b2a5: 2e e9 00 00 00 00       jmp     <__x86_return_thunk>
  drain_stock(stock, i);
   b2ab: 48 89 df                movq    %rbx, %rdi
   b2ae: 89 c6                   movl    %eax, %esi
   b2b0: e8 2b 16 00 00          callq   0xc8e0 <drain_stock>
   b2b5: eb d1                   jmp     0xb288 <refill_stock+0x258>

Signed-off-by: Kuniyuki Iwashima <kuniyu@google.com>
---
 mm/memcontrol.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 7c722a593cb5..cc3d78938b34 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2128,8 +2128,8 @@ static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
 	bool ret = false;
 	int i;
 
-	if (nr_pages > MEMCG_CHARGE_BATCH ||
-	    !local_trylock(&memcg_stock.lock))
+	if (unlikely(nr_pages > MEMCG_CHARGE_BATCH ||
+		     !local_trylock(&memcg_stock.lock)))
 		return ret;
 
 	stock = this_cpu_ptr(&memcg_stock);
@@ -2139,12 +2139,12 @@ static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
 			continue;
 
 		stock_pages = stock->nr_pages[i];
-		if (stock_pages < nr_pages)
+		if (unlikely(stock_pages < nr_pages))
 			break;
 
 		stock_pages -= nr_pages;
 		WRITE_ONCE(stock->nr_pages[i], stock_pages);
-		if (!stock_pages) {
+		if (unlikely(!stock_pages)) {
 			css_put(&memcg->css);
 			WRITE_ONCE(stock->cached[i], NULL);
 		}
@@ -2243,8 +2243,8 @@ static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
 
 	VM_WARN_ON_ONCE(mem_cgroup_is_root(memcg));
 
-	if (nr_pages > MEMCG_CHARGE_BATCH ||
-	    !local_trylock(&memcg_stock.lock)) {
+	if (unlikely(nr_pages > MEMCG_CHARGE_BATCH ||
+		     !local_trylock(&memcg_stock.lock))) {
 		/*
 		 * In case of larger than batch refill or unlikely failure to
 		 * lock the percpu memcg_stock.lock, uncharge memcg directly.
@@ -2264,7 +2264,7 @@ static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
 
 		stock_pages = stock->nr_pages[i] + nr_pages;
 		WRITE_ONCE(stock->nr_pages[i], stock_pages);
-		if (stock_pages > MEMCG_CHARGE_BATCH)
+		if (unlikely(stock_pages > MEMCG_CHARGE_BATCH))
 			drain_stock(stock, i);
 
 		local_unlock(&memcg_stock.lock);
-- 
2.56.0.212.g129ede20d9-goog