diff options
Diffstat (limited to 'mm/memcontrol.c')
-rw-r--r-- | mm/memcontrol.c | 26 |
1 files changed, 16 insertions, 10 deletions
diff --git a/mm/memcontrol.c b/mm/memcontrol.c index d72bdc3ca09b..0c4538811188 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -60,6 +60,8 @@ #include <net/sock.h> #include <net/ip.h> #include <net/tcp_memcontrol.h> +#include <linux/locallock.h> + #include "slab.h" #include <asm/uaccess.h> @@ -87,6 +89,7 @@ static int really_do_swap_account __initdata; #define do_swap_account 0 #endif +static DEFINE_LOCAL_IRQ_LOCK(event_lock); static const char * const mem_cgroup_stat_names[] = { "cache", @@ -2376,14 +2379,17 @@ static void __init memcg_stock_init(void) */ static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) { - struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock); + struct memcg_stock_pcp *stock; + int cpu = get_cpu_light(); + + stock = &per_cpu(memcg_stock, cpu); if (stock->cached != memcg) { /* reset if necessary */ drain_stock(stock); stock->cached = memcg; } stock->nr_pages += nr_pages; - put_cpu_var(memcg_stock); + put_cpu_light(); } /* @@ -2397,7 +2403,7 @@ static void drain_all_stock(struct mem_cgroup *root_memcg, bool sync) /* Notify other cpus that system-wide "drain" is running */ get_online_cpus(); - curcpu = get_cpu(); + curcpu = get_cpu_light(); for_each_online_cpu(cpu) { struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); struct mem_cgroup *memcg; @@ -2414,7 +2420,7 @@ static void drain_all_stock(struct mem_cgroup *root_memcg, bool sync) schedule_work_on(cpu, &stock->work); } } - put_cpu(); + put_cpu_light(); if (!sync) goto out; @@ -3419,12 +3425,12 @@ static int mem_cgroup_move_account(struct page *page, move_unlock_mem_cgroup(from, &flags); ret = 0; - local_irq_disable(); + local_lock_irq(event_lock); mem_cgroup_charge_statistics(to, page, nr_pages); memcg_check_events(to, page); mem_cgroup_charge_statistics(from, page, -nr_pages); memcg_check_events(from, page); - local_irq_enable(); + local_unlock_irq(event_lock); out_unlock: unlock_page(page); out: @@ -6406,10 +6412,10 @@ void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg, VM_BUG_ON_PAGE(!PageTransHuge(page), page); } - local_irq_disable(); + local_lock_irq(event_lock); mem_cgroup_charge_statistics(memcg, page, nr_pages); memcg_check_events(memcg, page); - local_irq_enable(); + local_unlock_irq(event_lock); if (do_swap_account && PageSwapCache(page)) { swp_entry_t entry = { .val = page_private(page) }; @@ -6468,14 +6474,14 @@ static void uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout, memcg_oom_recover(memcg); } - local_irq_save(flags); + local_lock_irqsave(event_lock, flags); __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS], nr_anon); __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_CACHE], nr_file); __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], nr_huge); __this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT], pgpgout); __this_cpu_add(memcg->stat->nr_page_events, nr_anon + nr_file); memcg_check_events(memcg, dummy_page); - local_irq_restore(flags); + local_unlock_irqrestore(event_lock, flags); } static void uncharge_list(struct list_head *page_list) |