diff for duplicates of <67545647-1371-4540-84ae-e33d1c5c8465@linux.intel.com> diff --git a/a/2.txt b/N1/2.txt index 8b13789..3db3f34 100644 --- a/a/2.txt +++ b/N1/2.txt @@ -1 +1,195 @@ +>From 59e4341e041d7aa1f9339a03f876eee566768c84 Mon Sep 17 00:00:00 2001 +From: Tim Chen <tim.c.chen@linux.intel.com> +Date: Fri, 25 Aug 2017 09:13:54 -0700 +Subject: [PATCH 1/2] sched/wait: Break up long wake list walk +We encountered workloads that have very long wake up list on large +systems. A waker takes a long time to traverse the entire wake list and +execute all the wake functions. + +We saw page wait list that are up to 3700+ entries long in tests of +large 4 and 8 socket systems. It took 0.8 sec to traverse such list +during wake up. Any other CPU that contends for the list spin lock will +spin for a long time. It is a result of the numa balancing migration of +hot pages that are shared by many threads. + +Multiple CPUs waking are queued up behind the lock, and the last one +queued has to wait until all CPUs did all the wakeups. + +The page wait list is traversed with interrupt disabled, which caused +various problems. This was the original cause that triggered the NMI +watch dog timer in: https://patchwork.kernel.org/patch/9800303/ . Only +extending the NMI watch dog timer there helped. + +This patch bookmarks the waker's scan position in wake list and break +the wake up walk, to allow access to the list before the waker resume +its walk down the rest of the wait list. It lowers the interrupt and +rescheduling latency. + +This patch also provides a performance boost when combined with the next +patch to break up page wakeup list walk. We saw 22% improvement in the +will-it-scale file pread2 test on a Xeon Phi system running 256 threads. + +[ v2: Merged in Linus' changes to remove the bookmark_wake_function, and + simply access to flags. ] + +Reported-by: Kan Liang <kan.liang@intel.com> +Tested-by: Kan Liang <kan.liang@intel.com> +Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com> +Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org> +--- + include/linux/wait.h | 1 + + kernel/sched/wait.c | 78 ++++++++++++++++++++++++++++++++++++++++++---------- + 2 files changed, 64 insertions(+), 15 deletions(-) + +diff --git a/include/linux/wait.h b/include/linux/wait.h +index dc19880c02f5..78401ef02d29 100644 +--- a/include/linux/wait.h ++++ b/include/linux/wait.h +@@ -18,6 +18,7 @@ int default_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int + /* wait_queue_entry::flags */ + #define WQ_FLAG_EXCLUSIVE 0x01 + #define WQ_FLAG_WOKEN 0x02 ++#define WQ_FLAG_BOOKMARK 0x04 + + /* + * A single wait-queue entry structure: +diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c +index d6afed6d0752..70701ef50465 100644 +--- a/kernel/sched/wait.c ++++ b/kernel/sched/wait.c +@@ -53,6 +53,12 @@ void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry + } + EXPORT_SYMBOL(remove_wait_queue); + ++/* ++ * Scan threshold to break wait queue walk. ++ * This allows a waker to take a break from holding the ++ * wait queue lock during the wait queue walk. ++ */ ++#define WAITQUEUE_WALK_BREAK_CNT 64 + + /* + * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just +@@ -63,18 +69,67 @@ EXPORT_SYMBOL(remove_wait_queue); + * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns + * zero in this (rare) case, and we handle it by continuing to scan the queue. + */ +-static void __wake_up_common(struct wait_queue_head *wq_head, unsigned int mode, +- int nr_exclusive, int wake_flags, void *key) ++static int __wake_up_common(struct wait_queue_head *wq_head, unsigned int mode, ++ int nr_exclusive, int wake_flags, void *key, ++ wait_queue_entry_t *bookmark) + { + wait_queue_entry_t *curr, *next; ++ int cnt = 0; ++ ++ if (bookmark && (bookmark->flags & WQ_FLAG_BOOKMARK)) { ++ curr = list_next_entry(bookmark, entry); + +- list_for_each_entry_safe(curr, next, &wq_head->head, entry) { ++ list_del(&bookmark->entry); ++ bookmark->flags = 0; ++ } else ++ curr = list_first_entry(&wq_head->head, wait_queue_entry_t, entry); ++ ++ if (&curr->entry == &wq_head->head) ++ return nr_exclusive; ++ ++ list_for_each_entry_safe_from(curr, next, &wq_head->head, entry) { + unsigned flags = curr->flags; +- int ret = curr->func(curr, mode, wake_flags, key); ++ int ret; ++ ++ if (flags & WQ_FLAG_BOOKMARK) ++ continue; ++ ++ ret = curr->func(curr, mode, wake_flags, key); + if (ret < 0) + break; + if (ret && (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive) + break; ++ ++ if (bookmark && (++cnt > WAITQUEUE_WALK_BREAK_CNT) && ++ (&next->entry != &wq_head->head)) { ++ bookmark->flags = WQ_FLAG_BOOKMARK; ++ list_add_tail(&bookmark->entry, &next->entry); ++ break; ++ } ++ } ++ return nr_exclusive; ++} ++ ++static void __wake_up_common_lock(struct wait_queue_head *wq_head, unsigned int mode, ++ int nr_exclusive, int wake_flags, void *key) ++{ ++ unsigned long flags; ++ wait_queue_entry_t bookmark; ++ ++ bookmark.flags = 0; ++ bookmark.private = NULL; ++ bookmark.func = NULL; ++ INIT_LIST_HEAD(&bookmark.entry); ++ ++ spin_lock_irqsave(&wq_head->lock, flags); ++ nr_exclusive = __wake_up_common(wq_head, mode, nr_exclusive, wake_flags, key, &bookmark); ++ spin_unlock_irqrestore(&wq_head->lock, flags); ++ ++ while (bookmark.flags & WQ_FLAG_BOOKMARK) { ++ spin_lock_irqsave(&wq_head->lock, flags); ++ nr_exclusive = __wake_up_common(wq_head, mode, nr_exclusive, ++ wake_flags, key, &bookmark); ++ spin_unlock_irqrestore(&wq_head->lock, flags); + } + } + +@@ -91,11 +146,7 @@ static void __wake_up_common(struct wait_queue_head *wq_head, unsigned int mode, + void __wake_up(struct wait_queue_head *wq_head, unsigned int mode, + int nr_exclusive, void *key) + { +- unsigned long flags; +- +- spin_lock_irqsave(&wq_head->lock, flags); +- __wake_up_common(wq_head, mode, nr_exclusive, 0, key); +- spin_unlock_irqrestore(&wq_head->lock, flags); ++ __wake_up_common_lock(wq_head, mode, nr_exclusive, 0, key); + } + EXPORT_SYMBOL(__wake_up); + +@@ -104,13 +155,13 @@ EXPORT_SYMBOL(__wake_up); + */ + void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr) + { +- __wake_up_common(wq_head, mode, nr, 0, NULL); ++ __wake_up_common(wq_head, mode, nr, 0, NULL, NULL); + } + EXPORT_SYMBOL_GPL(__wake_up_locked); + + void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key) + { +- __wake_up_common(wq_head, mode, 1, 0, key); ++ __wake_up_common(wq_head, mode, 1, 0, key, NULL); + } + EXPORT_SYMBOL_GPL(__wake_up_locked_key); + +@@ -134,7 +185,6 @@ EXPORT_SYMBOL_GPL(__wake_up_locked_key); + void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, + int nr_exclusive, void *key) + { +- unsigned long flags; + int wake_flags = 1; /* XXX WF_SYNC */ + + if (unlikely(!wq_head)) +@@ -143,9 +193,7 @@ void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, + if (unlikely(nr_exclusive != 1)) + wake_flags = 0; + +- spin_lock_irqsave(&wq_head->lock, flags); +- __wake_up_common(wq_head, mode, nr_exclusive, wake_flags, key); +- spin_unlock_irqrestore(&wq_head->lock, flags); ++ __wake_up_common_lock(wq_head, mode, nr_exclusive, wake_flags, key); + } + EXPORT_SYMBOL_GPL(__wake_up_sync_key); + +-- +2.14.0.rc1.2.g4c8247ec3 diff --git a/N1/3.hdr b/N1/3.hdr new file mode 100644 index 0000000..af3e513 --- /dev/null +++ b/N1/3.hdr @@ -0,0 +1,6 @@ +Content-Type: text/x-patch; + name="0002-sched-wait-Introduce-wakeup-boomark-in-wake_up_page_.patch" +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; + filename*0="0002-sched-wait-Introduce-wakeup-boomark-in-wake_up_page_.pa"; + filename*1="tch" diff --git a/N1/3.txt b/N1/3.txt new file mode 100644 index 0000000..9b64bc1 --- /dev/null +++ b/N1/3.txt @@ -0,0 +1,96 @@ +>From 6a519e86f2042edcf878463ed19e37dfd774f28b Mon Sep 17 00:00:00 2001 +From: Tim Chen <tim.c.chen@linux.intel.com> +Date: Fri, 25 Aug 2017 09:13:55 -0700 +Subject: [PATCH 2/2] sched/wait: Introduce wakeup boomark in wake_up_page_bit + +Now that we have added breaks in the wait queue scan and allow bookmark +on scan position, we put this logic in the wake_up_page_bit function. + +We can have very long page wait list in large system where multiple +pages share the same wait list. We break the wake up walk here to allow +other cpus a chance to access the list, and not to disable the interrupts +when traversing the list for too long. This reduces the interrupt and +rescheduling latency, and excessive page wait queue lock hold time. + +[ v2: Remove bookmark_wake_function ] + +Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com> +Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org> +--- + include/linux/wait.h | 2 ++ + kernel/sched/wait.c | 7 +++++++ + mm/filemap.c | 22 +++++++++++++++++++++- + 3 files changed, 30 insertions(+), 1 deletion(-) + +diff --git a/include/linux/wait.h b/include/linux/wait.h +index 78401ef02d29..87c4641023fb 100644 +--- a/include/linux/wait.h ++++ b/include/linux/wait.h +@@ -185,6 +185,8 @@ __remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq + + void __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key); + void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key); ++void __wake_up_locked_key_bookmark(struct wait_queue_head *wq_head, ++ unsigned int mode, void *key, wait_queue_entry_t *bookmark); + void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key); + void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr); + void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode, int nr); +diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c +index 70701ef50465..98feab7933c7 100644 +--- a/kernel/sched/wait.c ++++ b/kernel/sched/wait.c +@@ -165,6 +165,13 @@ void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, vo + } + EXPORT_SYMBOL_GPL(__wake_up_locked_key); + ++void __wake_up_locked_key_bookmark(struct wait_queue_head *wq_head, ++ unsigned int mode, void *key, wait_queue_entry_t *bookmark) ++{ ++ __wake_up_common(wq_head, mode, 1, 0, key, bookmark); ++} ++EXPORT_SYMBOL_GPL(__wake_up_locked_key_bookmark); ++ + /** + * __wake_up_sync_key - wake up threads blocked on a waitqueue. + * @wq_head: the waitqueue +diff --git a/mm/filemap.c b/mm/filemap.c +index 0b41c8cbeabc..74123a298f53 100644 +--- a/mm/filemap.c ++++ b/mm/filemap.c +@@ -923,13 +923,33 @@ static void wake_up_page_bit(struct page *page, int bit_nr) + wait_queue_head_t *q = page_waitqueue(page); + struct wait_page_key key; + unsigned long flags; ++ wait_queue_entry_t bookmark; + + key.page = page; + key.bit_nr = bit_nr; + key.page_match = 0; + ++ bookmark.flags = 0; ++ bookmark.private = NULL; ++ bookmark.func = NULL; ++ INIT_LIST_HEAD(&bookmark.entry); ++ + spin_lock_irqsave(&q->lock, flags); +- __wake_up_locked_key(q, TASK_NORMAL, &key); ++ __wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark); ++ ++ while (bookmark.flags & WQ_FLAG_BOOKMARK) { ++ /* ++ * Take a breather from holding the lock, ++ * allow pages that finish wake up asynchronously ++ * to acquire the lock and remove themselves ++ * from wait queue ++ */ ++ spin_unlock_irqrestore(&q->lock, flags); ++ cpu_relax(); ++ spin_lock_irqsave(&q->lock, flags); ++ __wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark); ++ } ++ + /* + * It is possible for other pages to have collided on the waitqueue + * hash, so in that case check for a page match. That prevents a long- +-- +2.14.0.rc1.2.g4c8247ec3 diff --git a/a/content_digest b/N1/content_digest index 9e367cb..e05157e 100644 --- a/a/content_digest +++ b/N1/content_digest @@ -59,5 +59,299 @@ "\01:2\0" "fn\00001-sched-wait-Break-up-long-wake-list-walk.patch\0" "b\0" + ">From 59e4341e041d7aa1f9339a03f876eee566768c84 Mon Sep 17 00:00:00 2001\n" + "From: Tim Chen <tim.c.chen@linux.intel.com>\n" + "Date: Fri, 25 Aug 2017 09:13:54 -0700\n" + "Subject: [PATCH 1/2] sched/wait: Break up long wake list walk\n" + "\n" + "We encountered workloads that have very long wake up list on large\n" + "systems. A waker takes a long time to traverse the entire wake list and\n" + "execute all the wake functions.\n" + "\n" + "We saw page wait list that are up to 3700+ entries long in tests of\n" + "large 4 and 8 socket systems. It took 0.8 sec to traverse such list\n" + "during wake up. Any other CPU that contends for the list spin lock will\n" + "spin for a long time. It is a result of the numa balancing migration of\n" + "hot pages that are shared by many threads.\n" + "\n" + "Multiple CPUs waking are queued up behind the lock, and the last one\n" + "queued has to wait until all CPUs did all the wakeups.\n" + "\n" + "The page wait list is traversed with interrupt disabled, which caused\n" + "various problems. This was the original cause that triggered the NMI\n" + "watch dog timer in: https://patchwork.kernel.org/patch/9800303/ . Only\n" + "extending the NMI watch dog timer there helped.\n" + "\n" + "This patch bookmarks the waker's scan position in wake list and break\n" + "the wake up walk, to allow access to the list before the waker resume\n" + "its walk down the rest of the wait list. It lowers the interrupt and\n" + "rescheduling latency.\n" + "\n" + "This patch also provides a performance boost when combined with the next\n" + "patch to break up page wakeup list walk. We saw 22% improvement in the\n" + "will-it-scale file pread2 test on a Xeon Phi system running 256 threads.\n" + "\n" + "[ v2: Merged in Linus' changes to remove the bookmark_wake_function, and\n" + " simply access to flags. ]\n" + "\n" + "Reported-by: Kan Liang <kan.liang@intel.com>\n" + "Tested-by: Kan Liang <kan.liang@intel.com>\n" + "Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>\n" + "Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>\n" + "---\n" + " include/linux/wait.h | 1 +\n" + " kernel/sched/wait.c | 78 ++++++++++++++++++++++++++++++++++++++++++----------\n" + " 2 files changed, 64 insertions(+), 15 deletions(-)\n" + "\n" + "diff --git a/include/linux/wait.h b/include/linux/wait.h\n" + "index dc19880c02f5..78401ef02d29 100644\n" + "--- a/include/linux/wait.h\n" + "+++ b/include/linux/wait.h\n" + "@@ -18,6 +18,7 @@ int default_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int\n" + " /* wait_queue_entry::flags */\n" + " #define WQ_FLAG_EXCLUSIVE\t0x01\n" + " #define WQ_FLAG_WOKEN\t\t0x02\n" + "+#define WQ_FLAG_BOOKMARK\t0x04\n" + " \n" + " /*\n" + " * A single wait-queue entry structure:\n" + "diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c\n" + "index d6afed6d0752..70701ef50465 100644\n" + "--- a/kernel/sched/wait.c\n" + "+++ b/kernel/sched/wait.c\n" + "@@ -53,6 +53,12 @@ void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry\n" + " }\n" + " EXPORT_SYMBOL(remove_wait_queue);\n" + " \n" + "+/*\n" + "+ * Scan threshold to break wait queue walk.\n" + "+ * This allows a waker to take a break from holding the\n" + "+ * wait queue lock during the wait queue walk.\n" + "+ */\n" + "+#define WAITQUEUE_WALK_BREAK_CNT 64\n" + " \n" + " /*\n" + " * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just\n" + "@@ -63,18 +69,67 @@ EXPORT_SYMBOL(remove_wait_queue);\n" + " * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns\n" + " * zero in this (rare) case, and we handle it by continuing to scan the queue.\n" + " */\n" + "-static void __wake_up_common(struct wait_queue_head *wq_head, unsigned int mode,\n" + "-\t\t\tint nr_exclusive, int wake_flags, void *key)\n" + "+static int __wake_up_common(struct wait_queue_head *wq_head, unsigned int mode,\n" + "+\t\t\tint nr_exclusive, int wake_flags, void *key,\n" + "+\t\t\twait_queue_entry_t *bookmark)\n" + " {\n" + " \twait_queue_entry_t *curr, *next;\n" + "+\tint cnt = 0;\n" + "+\n" + "+\tif (bookmark && (bookmark->flags & WQ_FLAG_BOOKMARK)) {\n" + "+\t\tcurr = list_next_entry(bookmark, entry);\n" + " \n" + "-\tlist_for_each_entry_safe(curr, next, &wq_head->head, entry) {\n" + "+\t\tlist_del(&bookmark->entry);\n" + "+\t\tbookmark->flags = 0;\n" + "+\t} else\n" + "+\t\tcurr = list_first_entry(&wq_head->head, wait_queue_entry_t, entry);\n" + "+\n" + "+\tif (&curr->entry == &wq_head->head)\n" + "+\t\treturn nr_exclusive;\n" + "+\n" + "+\tlist_for_each_entry_safe_from(curr, next, &wq_head->head, entry) {\n" + " \t\tunsigned flags = curr->flags;\n" + "-\t\tint ret = curr->func(curr, mode, wake_flags, key);\n" + "+\t\tint ret;\n" + "+\n" + "+\t\tif (flags & WQ_FLAG_BOOKMARK)\n" + "+\t\t\tcontinue;\n" + "+\n" + "+\t\tret = curr->func(curr, mode, wake_flags, key);\n" + " \t\tif (ret < 0)\n" + " \t\t\tbreak;\n" + " \t\tif (ret && (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)\n" + " \t\t\tbreak;\n" + "+\n" + "+\t\tif (bookmark && (++cnt > WAITQUEUE_WALK_BREAK_CNT) &&\n" + "+\t\t\t\t(&next->entry != &wq_head->head)) {\n" + "+\t\t\tbookmark->flags = WQ_FLAG_BOOKMARK;\n" + "+\t\t\tlist_add_tail(&bookmark->entry, &next->entry);\n" + "+\t\t\tbreak;\n" + "+\t\t}\n" + "+\t}\n" + "+\treturn nr_exclusive;\n" + "+}\n" + "+\n" + "+static void __wake_up_common_lock(struct wait_queue_head *wq_head, unsigned int mode,\n" + "+\t\t\tint nr_exclusive, int wake_flags, void *key)\n" + "+{\n" + "+\tunsigned long flags;\n" + "+\twait_queue_entry_t bookmark;\n" + "+\n" + "+\tbookmark.flags = 0;\n" + "+\tbookmark.private = NULL;\n" + "+\tbookmark.func = NULL;\n" + "+\tINIT_LIST_HEAD(&bookmark.entry);\n" + "+\n" + "+\tspin_lock_irqsave(&wq_head->lock, flags);\n" + "+\tnr_exclusive = __wake_up_common(wq_head, mode, nr_exclusive, wake_flags, key, &bookmark);\n" + "+\tspin_unlock_irqrestore(&wq_head->lock, flags);\n" + "+\n" + "+\twhile (bookmark.flags & WQ_FLAG_BOOKMARK) {\n" + "+\t\tspin_lock_irqsave(&wq_head->lock, flags);\n" + "+\t\tnr_exclusive = __wake_up_common(wq_head, mode, nr_exclusive,\n" + "+\t\t\t\t\t\twake_flags, key, &bookmark);\n" + "+\t\tspin_unlock_irqrestore(&wq_head->lock, flags);\n" + " \t}\n" + " }\n" + " \n" + "@@ -91,11 +146,7 @@ static void __wake_up_common(struct wait_queue_head *wq_head, unsigned int mode,\n" + " void __wake_up(struct wait_queue_head *wq_head, unsigned int mode,\n" + " \t\t\tint nr_exclusive, void *key)\n" + " {\n" + "-\tunsigned long flags;\n" + "-\n" + "-\tspin_lock_irqsave(&wq_head->lock, flags);\n" + "-\t__wake_up_common(wq_head, mode, nr_exclusive, 0, key);\n" + "-\tspin_unlock_irqrestore(&wq_head->lock, flags);\n" + "+\t__wake_up_common_lock(wq_head, mode, nr_exclusive, 0, key);\n" + " }\n" + " EXPORT_SYMBOL(__wake_up);\n" + " \n" + "@@ -104,13 +155,13 @@ EXPORT_SYMBOL(__wake_up);\n" + " */\n" + " void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr)\n" + " {\n" + "-\t__wake_up_common(wq_head, mode, nr, 0, NULL);\n" + "+\t__wake_up_common(wq_head, mode, nr, 0, NULL, NULL);\n" + " }\n" + " EXPORT_SYMBOL_GPL(__wake_up_locked);\n" + " \n" + " void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key)\n" + " {\n" + "-\t__wake_up_common(wq_head, mode, 1, 0, key);\n" + "+\t__wake_up_common(wq_head, mode, 1, 0, key, NULL);\n" + " }\n" + " EXPORT_SYMBOL_GPL(__wake_up_locked_key);\n" + " \n" + "@@ -134,7 +185,6 @@ EXPORT_SYMBOL_GPL(__wake_up_locked_key);\n" + " void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode,\n" + " \t\t\tint nr_exclusive, void *key)\n" + " {\n" + "-\tunsigned long flags;\n" + " \tint wake_flags = 1; /* XXX WF_SYNC */\n" + " \n" + " \tif (unlikely(!wq_head))\n" + "@@ -143,9 +193,7 @@ void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode,\n" + " \tif (unlikely(nr_exclusive != 1))\n" + " \t\twake_flags = 0;\n" + " \n" + "-\tspin_lock_irqsave(&wq_head->lock, flags);\n" + "-\t__wake_up_common(wq_head, mode, nr_exclusive, wake_flags, key);\n" + "-\tspin_unlock_irqrestore(&wq_head->lock, flags);\n" + "+\t__wake_up_common_lock(wq_head, mode, nr_exclusive, wake_flags, key);\n" + " }\n" + " EXPORT_SYMBOL_GPL(__wake_up_sync_key);\n" + " \n" + "-- \n" + 2.14.0.rc1.2.g4c8247ec3 + "\01:3\0" + "fn\00002-sched-wait-Introduce-wakeup-boomark-in-wake_up_page_.patch\0" + "b\0" + ">From 6a519e86f2042edcf878463ed19e37dfd774f28b Mon Sep 17 00:00:00 2001\n" + "From: Tim Chen <tim.c.chen@linux.intel.com>\n" + "Date: Fri, 25 Aug 2017 09:13:55 -0700\n" + "Subject: [PATCH 2/2] sched/wait: Introduce wakeup boomark in wake_up_page_bit\n" + "\n" + "Now that we have added breaks in the wait queue scan and allow bookmark\n" + "on scan position, we put this logic in the wake_up_page_bit function.\n" + "\n" + "We can have very long page wait list in large system where multiple\n" + "pages share the same wait list. We break the wake up walk here to allow\n" + "other cpus a chance to access the list, and not to disable the interrupts\n" + "when traversing the list for too long. This reduces the interrupt and\n" + "rescheduling latency, and excessive page wait queue lock hold time.\n" + "\n" + "[ v2: Remove bookmark_wake_function ]\n" + "\n" + "Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>\n" + "Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>\n" + "---\n" + " include/linux/wait.h | 2 ++\n" + " kernel/sched/wait.c | 7 +++++++\n" + " mm/filemap.c | 22 +++++++++++++++++++++-\n" + " 3 files changed, 30 insertions(+), 1 deletion(-)\n" + "\n" + "diff --git a/include/linux/wait.h b/include/linux/wait.h\n" + "index 78401ef02d29..87c4641023fb 100644\n" + "--- a/include/linux/wait.h\n" + "+++ b/include/linux/wait.h\n" + "@@ -185,6 +185,8 @@ __remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq\n" + " \n" + " void __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);\n" + " void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);\n" + "+void __wake_up_locked_key_bookmark(struct wait_queue_head *wq_head,\n" + "+\t\tunsigned int mode, void *key, wait_queue_entry_t *bookmark);\n" + " void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);\n" + " void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr);\n" + " void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode, int nr);\n" + "diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c\n" + "index 70701ef50465..98feab7933c7 100644\n" + "--- a/kernel/sched/wait.c\n" + "+++ b/kernel/sched/wait.c\n" + "@@ -165,6 +165,13 @@ void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, vo\n" + " }\n" + " EXPORT_SYMBOL_GPL(__wake_up_locked_key);\n" + " \n" + "+void __wake_up_locked_key_bookmark(struct wait_queue_head *wq_head,\n" + "+\t\tunsigned int mode, void *key, wait_queue_entry_t *bookmark)\n" + "+{\n" + "+\t__wake_up_common(wq_head, mode, 1, 0, key, bookmark);\n" + "+}\n" + "+EXPORT_SYMBOL_GPL(__wake_up_locked_key_bookmark);\n" + "+\n" + " /**\n" + " * __wake_up_sync_key - wake up threads blocked on a waitqueue.\n" + " * @wq_head: the waitqueue\n" + "diff --git a/mm/filemap.c b/mm/filemap.c\n" + "index 0b41c8cbeabc..74123a298f53 100644\n" + "--- a/mm/filemap.c\n" + "+++ b/mm/filemap.c\n" + "@@ -923,13 +923,33 @@ static void wake_up_page_bit(struct page *page, int bit_nr)\n" + " \twait_queue_head_t *q = page_waitqueue(page);\n" + " \tstruct wait_page_key key;\n" + " \tunsigned long flags;\n" + "+\twait_queue_entry_t bookmark;\n" + " \n" + " \tkey.page = page;\n" + " \tkey.bit_nr = bit_nr;\n" + " \tkey.page_match = 0;\n" + " \n" + "+\tbookmark.flags = 0;\n" + "+\tbookmark.private = NULL;\n" + "+\tbookmark.func = NULL;\n" + "+\tINIT_LIST_HEAD(&bookmark.entry);\n" + "+\n" + " \tspin_lock_irqsave(&q->lock, flags);\n" + "-\t__wake_up_locked_key(q, TASK_NORMAL, &key);\n" + "+\t__wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark);\n" + "+\n" + "+\twhile (bookmark.flags & WQ_FLAG_BOOKMARK) {\n" + "+\t\t/*\n" + "+\t\t * Take a breather from holding the lock,\n" + "+\t\t * allow pages that finish wake up asynchronously\n" + "+\t\t * to acquire the lock and remove themselves\n" + "+\t\t * from wait queue\n" + "+\t\t */\n" + "+\t\tspin_unlock_irqrestore(&q->lock, flags);\n" + "+\t\tcpu_relax();\n" + "+\t\tspin_lock_irqsave(&q->lock, flags);\n" + "+\t\t__wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark);\n" + "+\t}\n" + "+\n" + " \t/*\n" + " \t * It is possible for other pages to have collided on the waitqueue\n" + " \t * hash, so in that case check for a page match. That prevents a long-\n" + "-- \n" + 2.14.0.rc1.2.g4c8247ec3 -97108cf2739b613b27a9da58820ea402fbc7d158d02babd257d9963f12728704 +226d27bba0f57a197e933aaa2e8d6ad9b84f8ad581d3b9dce41fa8d3ad36b98d
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.