diff --git a/include/linux/shrinker.h b/include/linux/shrinker.h index 98be3ab..3edcd7f 100644 --- a/include/linux/shrinker.h +++ b/include/linux/shrinker.h @@ -53,7 +53,7 @@ struct shrinker { /* These are for internal use */ struct list_head list; - atomic_long_t nr_in_batch; /* objs pending delete */ + atomic_long_t *nr_in_batch; /* objs pending delete, per node */ }; #define DEFAULT_SEEKS 2 /* A good number if you don't know better. */ extern void register_shrinker(struct shrinker *); diff --git a/mm/vmscan.c b/mm/vmscan.c index 35a6a9b..6dddc8d 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -159,7 +159,14 @@ static unsigned long get_lru_size(struct lruvec *lruvec, enum lru_list lru) */ void register_shrinker(struct shrinker *shrinker) { - atomic_long_set(&shrinker->nr_in_batch, 0); + int i = 0; + + shrinker->nr_in_batch = kmalloc(sizeof(atomic_long_t) * nr_node_ids, GFP_KERNEL); + BUG_ON(!shrinker->nr_in_batch); /* obviously bogus */ + + for (i = 0; i < nr_node_ids; i++) + atomic_long_set(&shrinker->nr_in_batch[i], 0); + down_write(&shrinker_rwsem); list_add_tail(&shrinker->list, &shrinker_list); up_write(&shrinker_rwsem); @@ -211,6 +218,7 @@ unsigned long shrink_slab(struct shrink_control *shrinkctl, { struct shrinker *shrinker; unsigned long freed = 0; + unsigned long nr_active_nodes = 0; if (nr_pages_scanned == 0) nr_pages_scanned = SWAP_CLUSTER_MAX; @@ -229,6 +237,7 @@ unsigned long shrink_slab(struct shrink_control *shrinkctl, long new_nr; long batch_size = shrinker->batch ? shrinker->batch : SHRINK_BATCH; + int nid; if (shrinker->scan_objects) { max_pass = shrinker->count_objects(shrinker, shrinkctl); @@ -238,12 +247,17 @@ unsigned long shrink_slab(struct shrink_control *shrinkctl, if (max_pass <= 0) continue; - /* - * copy the current shrinker scan count into a local variable - * and zero it so that other concurrent shrinker invocations - * don't also do this scanning work. - */ - nr = atomic_long_xchg(&shrinker->nr_in_batch, 0); + nr = 0; + for_each_node_mask(nid, shrinkctl->nodes_to_scan) { + /* + * copy the current shrinker scan count into a local + * variable and zero it so that other concurrent + * shrinker invocations don't also do this scanning + * work. + */ + nr += atomic_long_xchg(&shrinker->nr_in_batch[nid], 0); + nr_active_nodes++; + } total_scan = nr; delta = (4 * nr_pages_scanned) / shrinker->seeks; @@ -311,17 +325,16 @@ unsigned long shrink_slab(struct shrink_control *shrinkctl, cond_resched(); } - /* - * move the unused scan count back into the shrinker in a - * manner that handles concurrent updates. If we exhausted the - * scan, there is no need to do an update. - */ - if (total_scan > 0) - new_nr = atomic_long_add_return(total_scan, - &shrinker->nr_in_batch); - else - new_nr = atomic_long_read(&shrinker->nr_in_batch); + new_nr = 0; + total_scan /= nr_active_nodes; + for_each_node_mask(nid, shrinkctl->nodes_to_scan) { + if (total_scan > 0) + new_nr += atomic_long_add_return(total_scan / nr_active_nodes, + &shrinker->nr_in_batch[nid]); + else + new_nr += atomic_long_read(&shrinker->nr_in_batch[nid]); + } trace_mm_shrink_slab_end(shrinker, freed, nr, new_nr); } up_read(&shrinker_rwsem);