Diff markup
1 /* 1 /*
2 * linux/mm/vmscan.c 2 * linux/mm/vmscan.c
3 * 3 *
4 * Copyright (C) 1991, 1992, 1993, 1994 Linu 4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
5 * 5 *
6 * Swap reorganised 29.12.95, Stephen Tweedie 6 * Swap reorganised 29.12.95, Stephen Tweedie.
7 * kswapd added: 7.1.96 sct 7 * kswapd added: 7.1.96 sct
8 * Removed kswapd_ctl limits, and swap out as 8 * Removed kswapd_ctl limits, and swap out as many pages as needed
9 * to bring the system back to freepages.high 9 * to bring the system back to freepages.high: 2.4.97, Rik van Riel.
10 * Zone aware kswapd started 02/00, Kanoj Sar 10 * Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
11 * Multiqueue VM started 5.8.00, Rik van Riel 11 * Multiqueue VM started 5.8.00, Rik van Riel.
12 */ 12 */
13 13
14 #include <linux/mm.h> 14 #include <linux/mm.h>
15 #include <linux/module.h> 15 #include <linux/module.h>
16 #include <linux/slab.h> 16 #include <linux/slab.h>
17 #include <linux/kernel_stat.h> 17 #include <linux/kernel_stat.h>
18 #include <linux/swap.h> 18 #include <linux/swap.h>
19 #include <linux/pagemap.h> 19 #include <linux/pagemap.h>
20 #include <linux/init.h> 20 #include <linux/init.h>
21 #include <linux/highmem.h> 21 #include <linux/highmem.h>
22 #include <linux/vmstat.h> 22 #include <linux/vmstat.h>
23 #include <linux/file.h> 23 #include <linux/file.h>
24 #include <linux/writeback.h> 24 #include <linux/writeback.h>
25 #include <linux/blkdev.h> 25 #include <linux/blkdev.h>
>> 26 #include <linux/interrupt.h>
26 #include <linux/buffer_head.h> /* for try_to_ 27 #include <linux/buffer_head.h> /* for try_to_release_page(),
27 buffer 28 buffer_heads_over_limit */
28 #include <linux/mm_inline.h> 29 #include <linux/mm_inline.h>
29 #include <linux/pagevec.h> 30 #include <linux/pagevec.h>
30 #include <linux/backing-dev.h> 31 #include <linux/backing-dev.h>
31 #include <linux/rmap.h> 32 #include <linux/rmap.h>
32 #include <linux/topology.h> 33 #include <linux/topology.h>
33 #include <linux/cpu.h> 34 #include <linux/cpu.h>
34 #include <linux/cpuset.h> 35 #include <linux/cpuset.h>
35 #include <linux/notifier.h> 36 #include <linux/notifier.h>
36 #include <linux/rwsem.h> 37 #include <linux/rwsem.h>
37 #include <linux/delay.h> 38 #include <linux/delay.h>
38 #include <linux/kthread.h> 39 #include <linux/kthread.h>
39 #include <linux/freezer.h> 40 #include <linux/freezer.h>
40 #include <linux/memcontrol.h> 41 #include <linux/memcontrol.h>
41 42
42 #include <asm/tlbflush.h> 43 #include <asm/tlbflush.h>
43 #include <asm/div64.h> 44 #include <asm/div64.h>
44 45
45 #include <linux/swapops.h> 46 #include <linux/swapops.h>
46 47
47 #include "internal.h" 48 #include "internal.h"
48 49
49 struct scan_control { 50 struct scan_control {
50 /* Incremented by the number of inacti 51 /* Incremented by the number of inactive pages that were scanned */
51 unsigned long nr_scanned; 52 unsigned long nr_scanned;
52 53
53 /* This context's GFP mask */ 54 /* This context's GFP mask */
54 gfp_t gfp_mask; 55 gfp_t gfp_mask;
55 56
56 int may_writepage; 57 int may_writepage;
57 58
58 /* Can pages be swapped as part of rec 59 /* Can pages be swapped as part of reclaim? */
59 int may_swap; 60 int may_swap;
60 61
61 /* This context's SWAP_CLUSTER_MAX. If 62 /* This context's SWAP_CLUSTER_MAX. If freeing memory for
62 * suspend, we effectively ignore SWAP 63 * suspend, we effectively ignore SWAP_CLUSTER_MAX.
63 * In this context, it doesn't matter 64 * In this context, it doesn't matter that we scan the
64 * whole list at once. */ 65 * whole list at once. */
65 int swap_cluster_max; 66 int swap_cluster_max;
66 67
67 int swappiness; 68 int swappiness;
68 69
69 int all_unreclaimable; 70 int all_unreclaimable;
70 71
71 int order; 72 int order;
72 73
73 /* Which cgroup do we reclaim from */ 74 /* Which cgroup do we reclaim from */
74 struct mem_cgroup *mem_cgroup; 75 struct mem_cgroup *mem_cgroup;
75 76
76 /* Pluggable isolate pages callback */ 77 /* Pluggable isolate pages callback */
77 unsigned long (*isolate_pages)(unsigne 78 unsigned long (*isolate_pages)(unsigned long nr, struct list_head *dst,
78 unsigned long *scanned 79 unsigned long *scanned, int order, int mode,
79 struct zone *z, struct 80 struct zone *z, struct mem_cgroup *mem_cont,
80 int active); 81 int active);
81 }; 82 };
82 83
83 #define lru_to_page(_head) (list_entry((_head) 84 #define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
84 85
85 #ifdef ARCH_HAS_PREFETCH 86 #ifdef ARCH_HAS_PREFETCH
86 #define prefetch_prev_lru_page(_page, _base, _ 87 #define prefetch_prev_lru_page(_page, _base, _field) \
87 do { 88 do { \
88 if ((_page)->lru.prev != _base 89 if ((_page)->lru.prev != _base) { \
89 struct page *prev; 90 struct page *prev; \
90 91 \
91 prev = lru_to_page(&(_ 92 prev = lru_to_page(&(_page->lru)); \
92 prefetch(&prev->_field 93 prefetch(&prev->_field); \
93 } 94 } \
94 } while (0) 95 } while (0)
95 #else 96 #else
96 #define prefetch_prev_lru_page(_page, _base, _ 97 #define prefetch_prev_lru_page(_page, _base, _field) do { } while (0)
97 #endif 98 #endif
98 99
99 #ifdef ARCH_HAS_PREFETCHW 100 #ifdef ARCH_HAS_PREFETCHW
100 #define prefetchw_prev_lru_page(_page, _base, 101 #define prefetchw_prev_lru_page(_page, _base, _field) \
101 do { 102 do { \
102 if ((_page)->lru.prev != _base 103 if ((_page)->lru.prev != _base) { \
103 struct page *prev; 104 struct page *prev; \
104 105 \
105 prev = lru_to_page(&(_ 106 prev = lru_to_page(&(_page->lru)); \
106 prefetchw(&prev->_fiel 107 prefetchw(&prev->_field); \
107 } 108 } \
108 } while (0) 109 } while (0)
109 #else 110 #else
110 #define prefetchw_prev_lru_page(_page, _base, 111 #define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
111 #endif 112 #endif
112 113
113 /* 114 /*
114 * From 0 .. 100. Higher means more swappy. 115 * From 0 .. 100. Higher means more swappy.
115 */ 116 */
116 int vm_swappiness = 60; 117 int vm_swappiness = 60;
117 long vm_total_pages; /* The total number of 118 long vm_total_pages; /* The total number of pages which the VM controls */
118 119
119 static LIST_HEAD(shrinker_list); 120 static LIST_HEAD(shrinker_list);
120 static DECLARE_RWSEM(shrinker_rwsem); 121 static DECLARE_RWSEM(shrinker_rwsem);
121 122
122 #ifdef CONFIG_CGROUP_MEM_RES_CTLR 123 #ifdef CONFIG_CGROUP_MEM_RES_CTLR
123 #define scan_global_lru(sc) (!(sc)->mem_cg 124 #define scan_global_lru(sc) (!(sc)->mem_cgroup)
124 #else 125 #else
125 #define scan_global_lru(sc) (1) 126 #define scan_global_lru(sc) (1)
126 #endif 127 #endif
127 128
128 /* 129 /*
129 * Add a shrinker callback to be called from t 130 * Add a shrinker callback to be called from the vm
130 */ 131 */
131 void register_shrinker(struct shrinker *shrink 132 void register_shrinker(struct shrinker *shrinker)
132 { 133 {
133 shrinker->nr = 0; 134 shrinker->nr = 0;
134 down_write(&shrinker_rwsem); 135 down_write(&shrinker_rwsem);
135 list_add_tail(&shrinker->list, &shrink 136 list_add_tail(&shrinker->list, &shrinker_list);
136 up_write(&shrinker_rwsem); 137 up_write(&shrinker_rwsem);
137 } 138 }
138 EXPORT_SYMBOL(register_shrinker); 139 EXPORT_SYMBOL(register_shrinker);
139 140
140 /* 141 /*
141 * Remove one 142 * Remove one
142 */ 143 */
143 void unregister_shrinker(struct shrinker *shri 144 void unregister_shrinker(struct shrinker *shrinker)
144 { 145 {
145 down_write(&shrinker_rwsem); 146 down_write(&shrinker_rwsem);
146 list_del(&shrinker->list); 147 list_del(&shrinker->list);
147 up_write(&shrinker_rwsem); 148 up_write(&shrinker_rwsem);
148 } 149 }
149 EXPORT_SYMBOL(unregister_shrinker); 150 EXPORT_SYMBOL(unregister_shrinker);
150 151
151 #define SHRINK_BATCH 128 152 #define SHRINK_BATCH 128
152 /* 153 /*
153 * Call the shrink functions to age shrinkable 154 * Call the shrink functions to age shrinkable caches
154 * 155 *
155 * Here we assume it costs one seek to replace 156 * Here we assume it costs one seek to replace a lru page and that it also
156 * takes a seek to recreate a cache object. W 157 * takes a seek to recreate a cache object. With this in mind we age equal
157 * percentages of the lru and ageable caches. 158 * percentages of the lru and ageable caches. This should balance the seeks
158 * generated by these structures. 159 * generated by these structures.
159 * 160 *
160 * If the vm encountered mapped pages on the L 161 * If the vm encountered mapped pages on the LRU it increase the pressure on
161 * slab to avoid swapping. 162 * slab to avoid swapping.
162 * 163 *
163 * We do weird things to avoid (scanned*seeks* 164 * We do weird things to avoid (scanned*seeks*entries) overflowing 32 bits.
164 * 165 *
165 * `lru_pages' represents the number of on-LRU 166 * `lru_pages' represents the number of on-LRU pages in all the zones which
166 * are eligible for the caller's allocation at 167 * are eligible for the caller's allocation attempt. It is used for balancing
167 * slab reclaim versus page reclaim. 168 * slab reclaim versus page reclaim.
168 * 169 *
169 * Returns the number of slab objects which we 170 * Returns the number of slab objects which we shrunk.
170 */ 171 */
171 unsigned long shrink_slab(unsigned long scanne 172 unsigned long shrink_slab(unsigned long scanned, gfp_t gfp_mask,
172 unsigned long lru_page 173 unsigned long lru_pages)
173 { 174 {
174 struct shrinker *shrinker; 175 struct shrinker *shrinker;
175 unsigned long ret = 0; 176 unsigned long ret = 0;
176 177
177 if (scanned == 0) 178 if (scanned == 0)
178 scanned = SWAP_CLUSTER_MAX; 179 scanned = SWAP_CLUSTER_MAX;
179 180
180 if (!down_read_trylock(&shrinker_rwsem 181 if (!down_read_trylock(&shrinker_rwsem))
181 return 1; /* Assume we'l 182 return 1; /* Assume we'll be able to shrink next time */
182 183
183 list_for_each_entry(shrinker, &shrinke 184 list_for_each_entry(shrinker, &shrinker_list, list) {
184 unsigned long long delta; 185 unsigned long long delta;
185 unsigned long total_scan; 186 unsigned long total_scan;
186 unsigned long max_pass = (*shr 187 unsigned long max_pass = (*shrinker->shrink)(0, gfp_mask);
187 188
188 delta = (4 * scanned) / shrink 189 delta = (4 * scanned) / shrinker->seeks;
189 delta *= max_pass; 190 delta *= max_pass;
190 do_div(delta, lru_pages + 1); 191 do_div(delta, lru_pages + 1);
191 shrinker->nr += delta; 192 shrinker->nr += delta;
192 if (shrinker->nr < 0) { 193 if (shrinker->nr < 0) {
193 printk(KERN_ERR "%s: n 194 printk(KERN_ERR "%s: nr=%ld\n",
194 __FUNC 195 __FUNCTION__, shrinker->nr);
195 shrinker->nr = max_pas 196 shrinker->nr = max_pass;
196 } 197 }
197 198
198 /* 199 /*
199 * Avoid risking looping forev 200 * Avoid risking looping forever due to too large nr value:
200 * never try to free more than 201 * never try to free more than twice the estimate number of
201 * freeable entries. 202 * freeable entries.
202 */ 203 */
203 if (shrinker->nr > max_pass * 204 if (shrinker->nr > max_pass * 2)
204 shrinker->nr = max_pas 205 shrinker->nr = max_pass * 2;
205 206
206 total_scan = shrinker->nr; 207 total_scan = shrinker->nr;
207 shrinker->nr = 0; 208 shrinker->nr = 0;
208 209
209 while (total_scan >= SHRINK_BA 210 while (total_scan >= SHRINK_BATCH) {
210 long this_scan = SHRIN 211 long this_scan = SHRINK_BATCH;
211 int shrink_ret; 212 int shrink_ret;
212 int nr_before; 213 int nr_before;
213 214
214 nr_before = (*shrinker 215 nr_before = (*shrinker->shrink)(0, gfp_mask);
215 shrink_ret = (*shrinke 216 shrink_ret = (*shrinker->shrink)(this_scan, gfp_mask);
216 if (shrink_ret == -1) 217 if (shrink_ret == -1)
217 break; 218 break;
218 if (shrink_ret < nr_be 219 if (shrink_ret < nr_before)
219 ret += nr_befo 220 ret += nr_before - shrink_ret;
220 count_vm_events(SLABS_ 221 count_vm_events(SLABS_SCANNED, this_scan);
221 total_scan -= this_sca 222 total_scan -= this_scan;
222 223
223 cond_resched(); 224 cond_resched();
224 } 225 }
225 226
226 shrinker->nr += total_scan; 227 shrinker->nr += total_scan;
227 } 228 }
228 up_read(&shrinker_rwsem); 229 up_read(&shrinker_rwsem);
229 return ret; 230 return ret;
230 } 231 }
231 232
232 /* Called without lock on whether page is mapp 233 /* Called without lock on whether page is mapped, so answer is unstable */
233 static inline int page_mapping_inuse(struct pa 234 static inline int page_mapping_inuse(struct page *page)
234 { 235 {
235 struct address_space *mapping; 236 struct address_space *mapping;
236 237
237 /* Page is in somebody's page tables. 238 /* Page is in somebody's page tables. */
238 if (page_mapped(page)) 239 if (page_mapped(page))
239 return 1; 240 return 1;
240 241
241 /* Be more reluctant to reclaim swapca 242 /* Be more reluctant to reclaim swapcache than pagecache */
242 if (PageSwapCache(page)) 243 if (PageSwapCache(page))
243 return 1; 244 return 1;
244 245
245 mapping = page_mapping(page); 246 mapping = page_mapping(page);
246 if (!mapping) 247 if (!mapping)
247 return 0; 248 return 0;
248 249
249 /* File is mmap'd by somebody? */ 250 /* File is mmap'd by somebody? */
250 return mapping_mapped(mapping); 251 return mapping_mapped(mapping);
251 } 252 }
252 253
253 static inline int is_page_cache_freeable(struc 254 static inline int is_page_cache_freeable(struct page *page)
254 { 255 {
255 return page_count(page) - !!PagePrivat 256 return page_count(page) - !!PagePrivate(page) == 2;
256 } 257 }
257 258
258 static int may_write_to_queue(struct backing_d 259 static int may_write_to_queue(struct backing_dev_info *bdi)
259 { 260 {
260 if (current->flags & PF_SWAPWRITE) 261 if (current->flags & PF_SWAPWRITE)
261 return 1; 262 return 1;
262 if (!bdi_write_congested(bdi)) 263 if (!bdi_write_congested(bdi))
263 return 1; 264 return 1;
264 if (bdi == current->backing_dev_info) 265 if (bdi == current->backing_dev_info)
265 return 1; 266 return 1;
266 return 0; 267 return 0;
267 } 268 }
268 269
269 /* 270 /*
270 * We detected a synchronous write error writi 271 * We detected a synchronous write error writing a page out. Probably
271 * -ENOSPC. We need to propagate that into th 272 * -ENOSPC. We need to propagate that into the address_space for a subsequent
272 * fsync(), msync() or close(). 273 * fsync(), msync() or close().
273 * 274 *
274 * The tricky part is that after writepage we 275 * The tricky part is that after writepage we cannot touch the mapping: nothing
275 * prevents it from being freed up. But we ha 276 * prevents it from being freed up. But we have a ref on the page and once
276 * that page is locked, the mapping is pinned. 277 * that page is locked, the mapping is pinned.
277 * 278 *
278 * We're allowed to run sleeping lock_page() h 279 * We're allowed to run sleeping lock_page() here because we know the caller has
279 * __GFP_FS. 280 * __GFP_FS.
280 */ 281 */
281 static void handle_write_error(struct address_ 282 static void handle_write_error(struct address_space *mapping,
282 struct page *p 283 struct page *page, int error)
283 { 284 {
284 lock_page(page); 285 lock_page(page);
285 if (page_mapping(page) == mapping) 286 if (page_mapping(page) == mapping)
286 mapping_set_error(mapping, err 287 mapping_set_error(mapping, error);
287 unlock_page(page); 288 unlock_page(page);
288 } 289 }
289 290
290 /* Request for sync pageout. */ 291 /* Request for sync pageout. */
291 enum pageout_io { 292 enum pageout_io {
292 PAGEOUT_IO_ASYNC, 293 PAGEOUT_IO_ASYNC,
293 PAGEOUT_IO_SYNC, 294 PAGEOUT_IO_SYNC,
294 }; 295 };
295 296
296 /* possible outcome of pageout() */ 297 /* possible outcome of pageout() */
297 typedef enum { 298 typedef enum {
298 /* failed to write page out, page is l 299 /* failed to write page out, page is locked */
299 PAGE_KEEP, 300 PAGE_KEEP,
300 /* move page to the active list, page 301 /* move page to the active list, page is locked */
301 PAGE_ACTIVATE, 302 PAGE_ACTIVATE,
302 /* page has been sent to the disk succ 303 /* page has been sent to the disk successfully, page is unlocked */
303 PAGE_SUCCESS, 304 PAGE_SUCCESS,
304 /* page is clean and locked */ 305 /* page is clean and locked */
305 PAGE_CLEAN, 306 PAGE_CLEAN,
306 } pageout_t; 307 } pageout_t;
307 308
308 /* 309 /*
309 * pageout is called by shrink_page_list() for 310 * pageout is called by shrink_page_list() for each dirty page.
310 * Calls ->writepage(). 311 * Calls ->writepage().
311 */ 312 */
312 static pageout_t pageout(struct page *page, st 313 static pageout_t pageout(struct page *page, struct address_space *mapping,
313 314 enum pageout_io sync_writeback)
314 { 315 {
315 /* 316 /*
316 * If the page is dirty, only perform 317 * If the page is dirty, only perform writeback if that write
317 * will be non-blocking. To prevent t 318 * will be non-blocking. To prevent this allocation from being
318 * stalled by pagecache activity. But 319 * stalled by pagecache activity. But note that there may be
319 * stalls if we need to run get_block( 320 * stalls if we need to run get_block(). We could test
320 * PagePrivate for that. 321 * PagePrivate for that.
321 * 322 *
322 * If this process is currently in gen 323 * If this process is currently in generic_file_write() against
323 * this page's queue, we can perform w 324 * this page's queue, we can perform writeback even if that
324 * will block. 325 * will block.
325 * 326 *
326 * If the page is swapcache, write it 327 * If the page is swapcache, write it back even if that would
327 * block, for some throttling. This ha 328 * block, for some throttling. This happens by accident, because
328 * swap_backing_dev_info is bust: it d 329 * swap_backing_dev_info is bust: it doesn't reflect the
329 * congestion state of the swapdevs. 330 * congestion state of the swapdevs. Easy to fix, if needed.
330 * See swapfile.c:page_queue_congested 331 * See swapfile.c:page_queue_congested().
331 */ 332 */
332 if (!is_page_cache_freeable(page)) 333 if (!is_page_cache_freeable(page))
333 return PAGE_KEEP; 334 return PAGE_KEEP;
334 if (!mapping) { 335 if (!mapping) {
335 /* 336 /*
336 * Some data journaling orphan 337 * Some data journaling orphaned pages can have
337 * page->mapping == NULL while 338 * page->mapping == NULL while being dirty with clean buffers.
338 */ 339 */
339 if (PagePrivate(page)) { 340 if (PagePrivate(page)) {
340 if (try_to_free_buffer 341 if (try_to_free_buffers(page)) {
341 ClearPageDirty 342 ClearPageDirty(page);
342 printk("%s: or 343 printk("%s: orphaned page\n", __FUNCTION__);
343 return PAGE_CL 344 return PAGE_CLEAN;
344 } 345 }
345 } 346 }
346 return PAGE_KEEP; 347 return PAGE_KEEP;
347 } 348 }
348 if (mapping->a_ops->writepage == NULL) 349 if (mapping->a_ops->writepage == NULL)
349 return PAGE_ACTIVATE; 350 return PAGE_ACTIVATE;
350 if (!may_write_to_queue(mapping->backi 351 if (!may_write_to_queue(mapping->backing_dev_info))
351 return PAGE_KEEP; 352 return PAGE_KEEP;
352 353
353 if (clear_page_dirty_for_io(page)) { 354 if (clear_page_dirty_for_io(page)) {
354 int res; 355 int res;
355 struct writeback_control wbc = 356 struct writeback_control wbc = {
356 .sync_mode = WB_SYNC_N 357 .sync_mode = WB_SYNC_NONE,
357 .nr_to_write = SWAP_CL 358 .nr_to_write = SWAP_CLUSTER_MAX,
358 .range_start = 0, 359 .range_start = 0,
359 .range_end = LLONG_MAX 360 .range_end = LLONG_MAX,
360 .nonblocking = 1, 361 .nonblocking = 1,
361 .for_reclaim = 1, 362 .for_reclaim = 1,
362 }; 363 };
363 364
364 SetPageReclaim(page); 365 SetPageReclaim(page);
365 res = mapping->a_ops->writepag 366 res = mapping->a_ops->writepage(page, &wbc);
366 if (res < 0) 367 if (res < 0)
367 handle_write_error(map 368 handle_write_error(mapping, page, res);
368 if (res == AOP_WRITEPAGE_ACTIV 369 if (res == AOP_WRITEPAGE_ACTIVATE) {
369 ClearPageReclaim(page) 370 ClearPageReclaim(page);
370 return PAGE_ACTIVATE; 371 return PAGE_ACTIVATE;
371 } 372 }
372 373
373 /* 374 /*
374 * Wait on writeback if reques 375 * Wait on writeback if requested to. This happens when
375 * direct reclaiming a large c 376 * direct reclaiming a large contiguous area and the
376 * first attempt to free a ran 377 * first attempt to free a range of pages fails.
377 */ 378 */
378 if (PageWriteback(page) && syn 379 if (PageWriteback(page) && sync_writeback == PAGEOUT_IO_SYNC)
379 wait_on_page_writeback 380 wait_on_page_writeback(page);
380 381
381 if (!PageWriteback(page)) { 382 if (!PageWriteback(page)) {
382 /* synchronous write o 383 /* synchronous write or broken a_ops? */
383 ClearPageReclaim(page) 384 ClearPageReclaim(page);
384 } 385 }
385 inc_zone_page_state(page, NR_V 386 inc_zone_page_state(page, NR_VMSCAN_WRITE);
386 return PAGE_SUCCESS; 387 return PAGE_SUCCESS;
387 } 388 }
388 389
389 return PAGE_CLEAN; 390 return PAGE_CLEAN;
390 } 391 }
391 392
392 /* 393 /*
393 * Attempt to detach a locked page from its -> 394 * Attempt to detach a locked page from its ->mapping. If it is dirty or if
394 * someone else has a ref on the page, abort a 395 * someone else has a ref on the page, abort and return 0. If it was
395 * successfully detached, return 1. Assumes t 396 * successfully detached, return 1. Assumes the caller has a single ref on
396 * this page. 397 * this page.
397 */ 398 */
398 int remove_mapping(struct address_space *mappi 399 int remove_mapping(struct address_space *mapping, struct page *page)
399 { 400 {
400 BUG_ON(!PageLocked(page)); 401 BUG_ON(!PageLocked(page));
401 BUG_ON(mapping != page_mapping(page)); 402 BUG_ON(mapping != page_mapping(page));
402 403
403 write_lock_irq(&mapping->tree_lock); !! 404 lock_page_ref_irq(page);
404 /* 405 /*
405 * The non racy check for a busy page. 406 * The non racy check for a busy page.
406 * 407 *
407 * Must be careful with the order of t 408 * Must be careful with the order of the tests. When someone has
408 * a ref to the page, it may be possib 409 * a ref to the page, it may be possible that they dirty it then
409 * drop the reference. So if PageDirty 410 * drop the reference. So if PageDirty is tested before page_count
410 * here, then the following race may o 411 * here, then the following race may occur:
411 * 412 *
412 * get_user_pages(&page); 413 * get_user_pages(&page);
413 * [user mapping goes away] 414 * [user mapping goes away]
414 * write_to(page); 415 * write_to(page);
415 * !PageD 416 * !PageDirty(page) [good]
416 * SetPageDirty(page); 417 * SetPageDirty(page);
417 * put_page(page); 418 * put_page(page);
418 * !page_ 419 * !page_count(page) [good, discard it]
419 * 420 *
420 * [oops, our write_to data is lost] 421 * [oops, our write_to data is lost]
421 * 422 *
422 * Reversing the order of the tests en 423 * Reversing the order of the tests ensures such a situation cannot
423 * escape unnoticed. The smp_rmb is ne 424 * escape unnoticed. The smp_rmb is needed to ensure the page->flags
424 * load is not satisfied before that o 425 * load is not satisfied before that of page->_count.
425 * 426 *
426 * Note that if SetPageDirty is always 427 * Note that if SetPageDirty is always performed via set_page_dirty,
427 * and thus under tree_lock, then this 428 * and thus under tree_lock, then this ordering is not required.
428 */ 429 */
429 if (unlikely(page_count(page) != 2)) 430 if (unlikely(page_count(page) != 2))
430 goto cannot_free; 431 goto cannot_free;
431 smp_rmb(); 432 smp_rmb();
432 if (unlikely(PageDirty(page))) 433 if (unlikely(PageDirty(page)))
433 goto cannot_free; 434 goto cannot_free;
434 435
435 if (PageSwapCache(page)) { 436 if (PageSwapCache(page)) {
436 swp_entry_t swap = { .val = pa 437 swp_entry_t swap = { .val = page_private(page) };
437 __delete_from_swap_cache(page) 438 __delete_from_swap_cache(page);
438 write_unlock_irq(&mapping->tre <<
439 swap_free(swap); 439 swap_free(swap);
440 __put_page(page); /* The !! 440 goto free_it;
441 return 1; <<
442 } 441 }
443 442
444 __remove_from_page_cache(page); 443 __remove_from_page_cache(page);
445 write_unlock_irq(&mapping->tree_lock); !! 444
446 __put_page(page); !! 445 free_it:
>> 446 unlock_page_ref_irq(page);
>> 447 __put_page(page); /* The pagecache ref */
447 return 1; 448 return 1;
448 449
449 cannot_free: 450 cannot_free:
450 write_unlock_irq(&mapping->tree_lock); !! 451 unlock_page_ref_irq(page);
451 return 0; 452 return 0;
452 } 453 }
453 454
454 /* 455 /*
455 * shrink_page_list() returns the number of re 456 * shrink_page_list() returns the number of reclaimed pages
456 */ 457 */
457 static unsigned long shrink_page_list(struct l 458 static unsigned long shrink_page_list(struct list_head *page_list,
458 struct 459 struct scan_control *sc,
459 enum p 460 enum pageout_io sync_writeback)
460 { 461 {
461 LIST_HEAD(ret_pages); 462 LIST_HEAD(ret_pages);
462 struct pagevec freed_pvec; 463 struct pagevec freed_pvec;
463 int pgactivate = 0; 464 int pgactivate = 0;
464 unsigned long nr_reclaimed = 0; 465 unsigned long nr_reclaimed = 0;
465 466
466 cond_resched(); 467 cond_resched();
467 468
468 pagevec_init(&freed_pvec, 1); 469 pagevec_init(&freed_pvec, 1);
469 while (!list_empty(page_list)) { 470 while (!list_empty(page_list)) {
470 struct address_space *mapping; 471 struct address_space *mapping;
471 struct page *page; 472 struct page *page;
472 int may_enter_fs; 473 int may_enter_fs;
473 int referenced; 474 int referenced;
474 475
475 cond_resched(); 476 cond_resched();
476 477
477 page = lru_to_page(page_list); 478 page = lru_to_page(page_list);
478 list_del(&page->lru); 479 list_del(&page->lru);
479 480
480 if (TestSetPageLocked(page)) 481 if (TestSetPageLocked(page))
481 goto keep; 482 goto keep;
482 483
483 VM_BUG_ON(PageActive(page)); 484 VM_BUG_ON(PageActive(page));
484 485
485 sc->nr_scanned++; 486 sc->nr_scanned++;
486 487
487 if (!sc->may_swap && page_mapp 488 if (!sc->may_swap && page_mapped(page))
488 goto keep_locked; 489 goto keep_locked;
489 490
490 /* Double the slab pressure fo 491 /* Double the slab pressure for mapped and swapcache pages */
491 if (page_mapped(page) || PageS 492 if (page_mapped(page) || PageSwapCache(page))
492 sc->nr_scanned++; 493 sc->nr_scanned++;
493 494
494 may_enter_fs = (sc->gfp_mask & 495 may_enter_fs = (sc->gfp_mask & __GFP_FS) ||
495 (PageSwapCache(page) & 496 (PageSwapCache(page) && (sc->gfp_mask & __GFP_IO));
496 497
497 if (PageWriteback(page)) { 498 if (PageWriteback(page)) {
498 /* 499 /*
499 * Synchronous reclaim 500 * Synchronous reclaim is performed in two passes,
500 * first an asynchrono 501 * first an asynchronous pass over the list to
501 * start parallel writ 502 * start parallel writeback, and a second synchronous
502 * pass to wait for th 503 * pass to wait for the IO to complete. Wait here
503 * for any page for wh 504 * for any page for which writeback has already
504 * started. 505 * started.
505 */ 506 */
506 if (sync_writeback == 507 if (sync_writeback == PAGEOUT_IO_SYNC && may_enter_fs)
507 wait_on_page_w 508 wait_on_page_writeback(page);
508 else 509 else
509 goto keep_lock 510 goto keep_locked;
510 } 511 }
511 512
512 referenced = page_referenced(p 513 referenced = page_referenced(page, 1, sc->mem_cgroup);
513 /* In active use or really unf 514 /* In active use or really unfreeable? Activate it. */
514 if (sc->order <= PAGE_ALLOC_CO 515 if (sc->order <= PAGE_ALLOC_COSTLY_ORDER &&
515 refere 516 referenced && page_mapping_inuse(page))
516 goto activate_locked; 517 goto activate_locked;
517 518
518 #ifdef CONFIG_SWAP 519 #ifdef CONFIG_SWAP
519 /* 520 /*
520 * Anonymous process memory ha 521 * Anonymous process memory has backing store?
521 * Try to allocate it some swa 522 * Try to allocate it some swap space here.
522 */ 523 */
523 if (PageAnon(page) && !PageSwa 524 if (PageAnon(page) && !PageSwapCache(page))
524 if (!add_to_swap(page, 525 if (!add_to_swap(page, GFP_ATOMIC))
525 goto activate_ 526 goto activate_locked;
526 #endif /* CONFIG_SWAP */ 527 #endif /* CONFIG_SWAP */
527 528
528 mapping = page_mapping(page); 529 mapping = page_mapping(page);
529 530
530 /* 531 /*
531 * The page is mapped into the 532 * The page is mapped into the page tables of one or more
532 * processes. Try to unmap it 533 * processes. Try to unmap it here.
533 */ 534 */
534 if (page_mapped(page) && mappi 535 if (page_mapped(page) && mapping) {
535 switch (try_to_unmap(p 536 switch (try_to_unmap(page, 0)) {
536 case SWAP_FAIL: 537 case SWAP_FAIL:
537 goto activate_ 538 goto activate_locked;
538 case SWAP_AGAIN: 539 case SWAP_AGAIN:
539 goto keep_lock 540 goto keep_locked;
540 case SWAP_SUCCESS: 541 case SWAP_SUCCESS:
541 ; /* try to fr 542 ; /* try to free the page below */
542 } 543 }
543 } 544 }
544 545
545 if (PageDirty(page)) { 546 if (PageDirty(page)) {
546 if (sc->order <= PAGE_ 547 if (sc->order <= PAGE_ALLOC_COSTLY_ORDER && referenced)
547 goto keep_lock 548 goto keep_locked;
548 if (!may_enter_fs) 549 if (!may_enter_fs)
549 goto keep_lock 550 goto keep_locked;
550 if (!sc->may_writepage 551 if (!sc->may_writepage)
551 goto keep_lock 552 goto keep_locked;
552 553
553 /* Page is dirty, try 554 /* Page is dirty, try to write it out here */
554 switch (pageout(page, 555 switch (pageout(page, mapping, sync_writeback)) {
555 case PAGE_KEEP: 556 case PAGE_KEEP:
556 goto keep_lock 557 goto keep_locked;
557 case PAGE_ACTIVATE: 558 case PAGE_ACTIVATE:
558 goto activate_ 559 goto activate_locked;
559 case PAGE_SUCCESS: 560 case PAGE_SUCCESS:
560 if (PageWriteb 561 if (PageWriteback(page) || PageDirty(page))
561 goto k 562 goto keep;
562 /* 563 /*
563 * A synchrono 564 * A synchronous write - probably a ramdisk. Go
564 * ahead and t 565 * ahead and try to reclaim the page.
565 */ 566 */
566 if (TestSetPag 567 if (TestSetPageLocked(page))
567 goto k 568 goto keep;
568 if (PageDirty( 569 if (PageDirty(page) || PageWriteback(page))
569 goto k 570 goto keep_locked;
570 mapping = page 571 mapping = page_mapping(page);
571 case PAGE_CLEAN: 572 case PAGE_CLEAN:
572 ; /* try to fr 573 ; /* try to free the page below */
573 } 574 }
574 } 575 }
575 576
576 /* 577 /*
577 * If the page has buffers, tr 578 * If the page has buffers, try to free the buffer mappings
578 * associated with this page. 579 * associated with this page. If we succeed we try to free
579 * the page as well. 580 * the page as well.
580 * 581 *
581 * We do this even if the page 582 * We do this even if the page is PageDirty().
582 * try_to_release_page() does 583 * try_to_release_page() does not perform I/O, but it is
583 * possible for a page to have 584 * possible for a page to have PageDirty set, but it is actually
584 * clean (all its buffers are 585 * clean (all its buffers are clean). This happens if the
585 * buffers were written out di 586 * buffers were written out directly, with submit_bh(). ext3
586 * will do this, as well as th 587 * will do this, as well as the blockdev mapping.
587 * try_to_release_page() will 588 * try_to_release_page() will discover that cleanness and will
588 * drop the buffers and mark t 589 * drop the buffers and mark the page clean - it can be freed.
589 * 590 *
590 * Rarely, pages can have buff 591 * Rarely, pages can have buffers and no ->mapping. These are
591 * the pages which were not su 592 * the pages which were not successfully invalidated in
592 * truncate_complete_page(). 593 * truncate_complete_page(). We try to drop those buffers here
593 * and if that worked, and the 594 * and if that worked, and the page is no longer mapped into
594 * process address space (page 595 * process address space (page_count == 1) it can be freed.
595 * Otherwise, leave the page o 596 * Otherwise, leave the page on the LRU so it is swappable.
596 */ 597 */
597 if (PagePrivate(page)) { 598 if (PagePrivate(page)) {
598 if (!try_to_release_pa 599 if (!try_to_release_page(page, sc->gfp_mask))
599 goto activate_ 600 goto activate_locked;
600 if (!mapping && page_c 601 if (!mapping && page_count(page) == 1)
601 goto free_it; 602 goto free_it;
602 } 603 }
603 604
604 if (!mapping || !remove_mappin 605 if (!mapping || !remove_mapping(mapping, page))
605 goto keep_locked; 606 goto keep_locked;
606 607
607 free_it: 608 free_it:
608 unlock_page(page); 609 unlock_page(page);
609 nr_reclaimed++; 610 nr_reclaimed++;
610 if (!pagevec_add(&freed_pvec, 611 if (!pagevec_add(&freed_pvec, page))
611 __pagevec_release_nonl 612 __pagevec_release_nonlru(&freed_pvec);
612 continue; 613 continue;
613 614
614 activate_locked: 615 activate_locked:
615 SetPageActive(page); 616 SetPageActive(page);
616 pgactivate++; 617 pgactivate++;
617 keep_locked: 618 keep_locked:
618 unlock_page(page); 619 unlock_page(page);
619 keep: 620 keep:
620 list_add(&page->lru, &ret_page 621 list_add(&page->lru, &ret_pages);
621 VM_BUG_ON(PageLRU(page)); 622 VM_BUG_ON(PageLRU(page));
622 } 623 }
623 list_splice(&ret_pages, page_list); 624 list_splice(&ret_pages, page_list);
624 if (pagevec_count(&freed_pvec)) 625 if (pagevec_count(&freed_pvec))
625 __pagevec_release_nonlru(&free 626 __pagevec_release_nonlru(&freed_pvec);
626 count_vm_events(PGACTIVATE, pgactivate 627 count_vm_events(PGACTIVATE, pgactivate);
627 return nr_reclaimed; 628 return nr_reclaimed;
628 } 629 }
629 630
630 /* LRU Isolation modes. */ 631 /* LRU Isolation modes. */
631 #define ISOLATE_INACTIVE 0 /* Isolate ina 632 #define ISOLATE_INACTIVE 0 /* Isolate inactive pages. */
632 #define ISOLATE_ACTIVE 1 /* Isolate act 633 #define ISOLATE_ACTIVE 1 /* Isolate active pages. */
633 #define ISOLATE_BOTH 2 /* Isolate bot 634 #define ISOLATE_BOTH 2 /* Isolate both active and inactive pages. */
634 635
635 /* 636 /*
636 * Attempt to remove the specified page from i 637 * Attempt to remove the specified page from its LRU. Only take this page
637 * if it is of the appropriate PageActive stat 638 * if it is of the appropriate PageActive status. Pages which are being
638 * freed elsewhere are also ignored. 639 * freed elsewhere are also ignored.
639 * 640 *
640 * page: page to consider 641 * page: page to consider
641 * mode: one of the LRU isolation modes 642 * mode: one of the LRU isolation modes defined above
642 * 643 *
643 * returns 0 on success, -ve errno on failure. 644 * returns 0 on success, -ve errno on failure.
644 */ 645 */
645 int __isolate_lru_page(struct page *page, int 646 int __isolate_lru_page(struct page *page, int mode)
646 { 647 {
647 int ret = -EINVAL; 648 int ret = -EINVAL;
648 649
649 /* Only take pages on the LRU. */ 650 /* Only take pages on the LRU. */
650 if (!PageLRU(page)) 651 if (!PageLRU(page))
651 return ret; 652 return ret;
652 653
653 /* 654 /*
654 * When checking the active state, we 655 * When checking the active state, we need to be sure we are
655 * dealing with comparible boolean val 656 * dealing with comparible boolean values. Take the logical not
656 * of each. 657 * of each.
657 */ 658 */
658 if (mode != ISOLATE_BOTH && (!PageActi 659 if (mode != ISOLATE_BOTH && (!PageActive(page) != !mode))
659 return ret; 660 return ret;
660 661
661 ret = -EBUSY; 662 ret = -EBUSY;
662 if (likely(get_page_unless_zero(page)) 663 if (likely(get_page_unless_zero(page))) {
663 /* 664 /*
664 * Be careful not to clear Pag 665 * Be careful not to clear PageLRU until after we're
665 * sure the page is not being 666 * sure the page is not being freed elsewhere -- the
666 * page release code relies on 667 * page release code relies on it.
667 */ 668 */
668 ClearPageLRU(page); 669 ClearPageLRU(page);
669 ret = 0; 670 ret = 0;
670 } 671 }
671 672
672 return ret; 673 return ret;
673 } 674 }
674 675
675 /* 676 /*
676 * zone->lru_lock is heavily contended. Some 677 * zone->lru_lock is heavily contended. Some of the functions that
677 * shrink the lists perform better by taking o 678 * shrink the lists perform better by taking out a batch of pages
678 * and working on them outside the LRU lock. 679 * and working on them outside the LRU lock.
679 * 680 *
680 * For pagecache intensive workloads, this fun 681 * For pagecache intensive workloads, this function is the hottest
681 * spot in the kernel (apart from copy_*_user 682 * spot in the kernel (apart from copy_*_user functions).
682 * 683 *
683 * Appropriate locks must be held before calli 684 * Appropriate locks must be held before calling this function.
684 * 685 *
685 * @nr_to_scan: The number of pages to look th 686 * @nr_to_scan: The number of pages to look through on the list.
686 * @src: The LRU list to pull pages off 687 * @src: The LRU list to pull pages off.
687 * @dst: The temp list to put pages on 688 * @dst: The temp list to put pages on to.
688 * @scanned: The number of pages that were 689 * @scanned: The number of pages that were scanned.
689 * @order: The caller's attempted allocat 690 * @order: The caller's attempted allocation order
690 * @mode: One of the LRU isolation modes 691 * @mode: One of the LRU isolation modes
691 * 692 *
692 * returns how many pages were moved onto *@ds 693 * returns how many pages were moved onto *@dst.
693 */ 694 */
694 static unsigned long isolate_lru_pages(unsigne 695 static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
695 struct list_head *src, struct 696 struct list_head *src, struct list_head *dst,
696 unsigned long *scanned, int or 697 unsigned long *scanned, int order, int mode)
697 { 698 {
698 unsigned long nr_taken = 0; 699 unsigned long nr_taken = 0;
699 unsigned long scan; 700 unsigned long scan;
700 701
701 for (scan = 0; scan < nr_to_scan && !l 702 for (scan = 0; scan < nr_to_scan && !list_empty(src); scan++) {
702 struct page *page; 703 struct page *page;
703 unsigned long pfn; 704 unsigned long pfn;
704 unsigned long end_pfn; 705 unsigned long end_pfn;
705 unsigned long page_pfn; 706 unsigned long page_pfn;
706 int zone_id; 707 int zone_id;
707 708
708 page = lru_to_page(src); 709 page = lru_to_page(src);
709 prefetchw_prev_lru_page(page, 710 prefetchw_prev_lru_page(page, src, flags);
710 711
711 VM_BUG_ON(!PageLRU(page)); 712 VM_BUG_ON(!PageLRU(page));
712 713
713 switch (__isolate_lru_page(pag 714 switch (__isolate_lru_page(page, mode)) {
714 case 0: 715 case 0:
715 list_move(&page->lru, 716 list_move(&page->lru, dst);
716 nr_taken++; 717 nr_taken++;
717 break; 718 break;
718 719
719 case -EBUSY: 720 case -EBUSY:
720 /* else it is being fr 721 /* else it is being freed elsewhere */
721 list_move(&page->lru, 722 list_move(&page->lru, src);
722 continue; 723 continue;
723 724
724 default: 725 default:
725 BUG(); 726 BUG();
726 } 727 }
727 728
728 if (!order) 729 if (!order)
729 continue; 730 continue;
730 731
731 /* 732 /*
732 * Attempt to take all pages i 733 * Attempt to take all pages in the order aligned region
733 * surrounding the tag page. 734 * surrounding the tag page. Only take those pages of
734 * the same active state as th 735 * the same active state as that tag page. We may safely
735 * round the target page pfn d 736 * round the target page pfn down to the requested order
736 * as the mem_map is guarentee 737 * as the mem_map is guarenteed valid out to MAX_ORDER,
737 * where that page is in a dif 738 * where that page is in a different zone we will detect
738 * it from its zone id and abo 739 * it from its zone id and abort this block scan.
739 */ 740 */
740 zone_id = page_zone_id(page); 741 zone_id = page_zone_id(page);
741 page_pfn = page_to_pfn(page); 742 page_pfn = page_to_pfn(page);
742 pfn = page_pfn & ~((1 << order 743 pfn = page_pfn & ~((1 << order) - 1);
743 end_pfn = pfn + (1 << order); 744 end_pfn = pfn + (1 << order);
744 for (; pfn < end_pfn; pfn++) { 745 for (; pfn < end_pfn; pfn++) {
745 struct page *cursor_pa 746 struct page *cursor_page;
746 747
747 /* The target page is 748 /* The target page is in the block, ignore it. */
748 if (unlikely(pfn == pa 749 if (unlikely(pfn == page_pfn))
749 continue; 750 continue;
750 751
751 /* Avoid holes within 752 /* Avoid holes within the zone. */
752 if (unlikely(!pfn_vali 753 if (unlikely(!pfn_valid_within(pfn)))
753 break; 754 break;
754 755
755 cursor_page = pfn_to_p 756 cursor_page = pfn_to_page(pfn);
756 /* Check that we have 757 /* Check that we have not crossed a zone boundary. */
757 if (unlikely(page_zone 758 if (unlikely(page_zone_id(cursor_page) != zone_id))
758 continue; 759 continue;
759 switch (__isolate_lru_ 760 switch (__isolate_lru_page(cursor_page, mode)) {
760 case 0: 761 case 0:
761 list_move(&cur 762 list_move(&cursor_page->lru, dst);
762 nr_taken++; 763 nr_taken++;
763 scan++; 764 scan++;
764 break; 765 break;
765 766
766 case -EBUSY: 767 case -EBUSY:
767 /* else it is 768 /* else it is being freed elsewhere */
768 list_move(&cur 769 list_move(&cursor_page->lru, src);
769 default: 770 default:
770 break; 771 break;
771 } 772 }
772 } 773 }
773 } 774 }
774 775
775 *scanned = scan; 776 *scanned = scan;
776 return nr_taken; 777 return nr_taken;
777 } 778 }
778 779
779 static unsigned long isolate_pages_global(unsi 780 static unsigned long isolate_pages_global(unsigned long nr,
780 struct 781 struct list_head *dst,
781 unsign 782 unsigned long *scanned, int order,
782 int mo 783 int mode, struct zone *z,
783 struct 784 struct mem_cgroup *mem_cont,
784 int ac 785 int active)
785 { 786 {
786 if (active) 787 if (active)
787 return isolate_lru_pages(nr, & 788 return isolate_lru_pages(nr, &z->active_list, dst,
788 789 scanned, order, mode);
789 else 790 else
790 return isolate_lru_pages(nr, & 791 return isolate_lru_pages(nr, &z->inactive_list, dst,
791 792 scanned, order, mode);
792 } 793 }
793 794
794 /* 795 /*
795 * clear_active_flags() is a helper for shrink 796 * clear_active_flags() is a helper for shrink_active_list(), clearing
796 * any active bits from the pages in the list. 797 * any active bits from the pages in the list.
797 */ 798 */
798 static unsigned long clear_active_flags(struct 799 static unsigned long clear_active_flags(struct list_head *page_list)
799 { 800 {
800 int nr_active = 0; 801 int nr_active = 0;
801 struct page *page; 802 struct page *page;
802 803
803 list_for_each_entry(page, page_list, l 804 list_for_each_entry(page, page_list, lru)
804 if (PageActive(page)) { 805 if (PageActive(page)) {
805 ClearPageActive(page); 806 ClearPageActive(page);
806 nr_active++; 807 nr_active++;
807 } 808 }
808 809
809 return nr_active; 810 return nr_active;
810 } 811 }
811 812
812 /* 813 /*
813 * shrink_inactive_list() is a helper for shri 814 * shrink_inactive_list() is a helper for shrink_zone(). It returns the number
814 * of reclaimed pages 815 * of reclaimed pages
815 */ 816 */
816 static unsigned long shrink_inactive_list(unsi 817 static unsigned long shrink_inactive_list(unsigned long max_scan,
817 struct zone *z 818 struct zone *zone, struct scan_control *sc)
818 { 819 {
819 LIST_HEAD(page_list); 820 LIST_HEAD(page_list);
820 struct pagevec pvec; 821 struct pagevec pvec;
821 unsigned long nr_scanned = 0; 822 unsigned long nr_scanned = 0;
822 unsigned long nr_reclaimed = 0; 823 unsigned long nr_reclaimed = 0;
823 824
824 pagevec_init(&pvec, 1); 825 pagevec_init(&pvec, 1);
825 826
826 lru_add_drain(); 827 lru_add_drain();
827 spin_lock_irq(&zone->lru_lock); 828 spin_lock_irq(&zone->lru_lock);
828 do { 829 do {
829 struct page *page; 830 struct page *page;
830 unsigned long nr_taken; 831 unsigned long nr_taken;
831 unsigned long nr_scan; 832 unsigned long nr_scan;
832 unsigned long nr_freed; 833 unsigned long nr_freed;
833 unsigned long nr_active; 834 unsigned long nr_active;
834 835
835 nr_taken = sc->isolate_pages(s 836 nr_taken = sc->isolate_pages(sc->swap_cluster_max,
836 &page_list, &nr_s 837 &page_list, &nr_scan, sc->order,
837 (sc->order > PAGE 838 (sc->order > PAGE_ALLOC_COSTLY_ORDER)?
838 I 839 ISOLATE_BOTH : ISOLATE_INACTIVE,
839 zone, sc->mem_ 840 zone, sc->mem_cgroup, 0);
840 nr_active = clear_active_flags 841 nr_active = clear_active_flags(&page_list);
841 __count_vm_events(PGDEACTIVATE 842 __count_vm_events(PGDEACTIVATE, nr_active);
842 843
843 __mod_zone_page_state(zone, NR 844 __mod_zone_page_state(zone, NR_ACTIVE, -nr_active);
844 __mod_zone_page_state(zone, NR 845 __mod_zone_page_state(zone, NR_INACTIVE,
845 846 -(nr_taken - nr_active));
846 if (scan_global_lru(sc)) 847 if (scan_global_lru(sc))
847 zone->pages_scanned += 848 zone->pages_scanned += nr_scan;
848 spin_unlock_irq(&zone->lru_loc 849 spin_unlock_irq(&zone->lru_lock);
849 850
850 nr_scanned += nr_scan; 851 nr_scanned += nr_scan;
851 nr_freed = shrink_page_list(&p 852 nr_freed = shrink_page_list(&page_list, sc, PAGEOUT_IO_ASYNC);
852 853
853 /* 854 /*
854 * If we are direct reclaiming 855 * If we are direct reclaiming for contiguous pages and we do
855 * not reclaim everything in t 856 * not reclaim everything in the list, try again and wait
856 * for IO to complete. This wi 857 * for IO to complete. This will stall high-order allocations
857 * but that should be acceptab 858 * but that should be acceptable to the caller
858 */ 859 */
859 if (nr_freed < nr_taken && !cu 860 if (nr_freed < nr_taken && !current_is_kswapd() &&
860 sc->or 861 sc->order > PAGE_ALLOC_COSTLY_ORDER) {
861 congestion_wait(WRITE, 862 congestion_wait(WRITE, HZ/10);
862 863
863 /* 864 /*
864 * The attempt at page 865 * The attempt at page out may have made some
865 * of the pages active 866 * of the pages active, mark them inactive again.
866 */ 867 */
867 nr_active = clear_acti 868 nr_active = clear_active_flags(&page_list);
868 count_vm_events(PGDEAC 869 count_vm_events(PGDEACTIVATE, nr_active);
869 870
870 nr_freed += shrink_pag 871 nr_freed += shrink_page_list(&page_list, sc,
871 872 PAGEOUT_IO_SYNC);
872 } 873 }
873 874
874 nr_reclaimed += nr_freed; 875 nr_reclaimed += nr_freed;
875 local_irq_disable(); !! 876 local_irq_disable_nort();
876 if (current_is_kswapd()) { 877 if (current_is_kswapd()) {
877 __count_zone_vm_events 878 __count_zone_vm_events(PGSCAN_KSWAPD, zone, nr_scan);
878 __count_vm_events(KSWA 879 __count_vm_events(KSWAPD_STEAL, nr_freed);
879 } else if (scan_global_lru(sc) 880 } else if (scan_global_lru(sc))
880 __count_zone_vm_events 881 __count_zone_vm_events(PGSCAN_DIRECT, zone, nr_scan);
881 882
882 __count_zone_vm_events(PGSTEAL 883 __count_zone_vm_events(PGSTEAL, zone, nr_freed);
883 884
884 if (nr_taken == 0) 885 if (nr_taken == 0)
885 goto done; 886 goto done;
886 887
887 spin_lock(&zone->lru_lock); 888 spin_lock(&zone->lru_lock);
888 /* 889 /*
889 * Put back any unfreeable pag 890 * Put back any unfreeable pages.
890 */ 891 */
891 while (!list_empty(&page_list) 892 while (!list_empty(&page_list)) {
892 page = lru_to_page(&pa 893 page = lru_to_page(&page_list);
893 VM_BUG_ON(PageLRU(page 894 VM_BUG_ON(PageLRU(page));
894 SetPageLRU(page); 895 SetPageLRU(page);
895 list_del(&page->lru); 896 list_del(&page->lru);
896 if (PageActive(page)) 897 if (PageActive(page))
897 add_page_to_ac 898 add_page_to_active_list(zone, page);
898 else 899 else
899 add_page_to_in 900 add_page_to_inactive_list(zone, page);
900 if (!pagevec_add(&pvec 901 if (!pagevec_add(&pvec, page)) {
901 spin_unlock_ir 902 spin_unlock_irq(&zone->lru_lock);
902 __pagevec_rele 903 __pagevec_release(&pvec);
903 spin_lock_irq( 904 spin_lock_irq(&zone->lru_lock);
904 } 905 }
905 } 906 }
906 } while (nr_scanned < max_scan); 907 } while (nr_scanned < max_scan);
>> 908 /*
>> 909 * Non-PREEMPT_RT relies on IRQs-off protecting the page_states
>> 910 * per-CPU data. PREEMPT_RT has that data protected even in
>> 911 * __mod_page_state(), so no need to keep IRQs disabled.
>> 912 */
907 spin_unlock(&zone->lru_lock); 913 spin_unlock(&zone->lru_lock);
908 done: 914 done:
909 local_irq_enable(); !! 915 local_irq_enable_nort();
910 pagevec_release(&pvec); 916 pagevec_release(&pvec);
911 return nr_reclaimed; 917 return nr_reclaimed;
912 } 918 }
913 919
914 /* 920 /*
915 * We are about to scan this zone at a certain 921 * We are about to scan this zone at a certain priority level. If that priority
916 * level is smaller (ie: more urgent) than the 922 * level is smaller (ie: more urgent) than the previous priority, then note
917 * that priority level within the zone. This 923 * that priority level within the zone. This is done so that when the next
918 * process comes in to scan this zone, it will 924 * process comes in to scan this zone, it will immediately start out at this
919 * priority level rather than having to build 925 * priority level rather than having to build up its own scanning priority.
920 * Here, this priority affects only the reclai 926 * Here, this priority affects only the reclaim-mapped threshold.
921 */ 927 */
922 static inline void note_zone_scanning_priority 928 static inline void note_zone_scanning_priority(struct zone *zone, int priority)
923 { 929 {
924 if (priority < zone->prev_priority) 930 if (priority < zone->prev_priority)
925 zone->prev_priority = priority 931 zone->prev_priority = priority;
926 } 932 }
927 933
928 static inline int zone_is_near_oom(struct zone 934 static inline int zone_is_near_oom(struct zone *zone)
929 { 935 {
930 return zone->pages_scanned >= (zone_pa 936 return zone->pages_scanned >= (zone_page_state(zone, NR_ACTIVE)
931 + zone_page_st 937 + zone_page_state(zone, NR_INACTIVE))*3;
932 } 938 }
933 939
934 /* 940 /*
935 * Determine we should try to reclaim mapped p 941 * Determine we should try to reclaim mapped pages.
936 * This is called only when sc->mem_cgroup is 942 * This is called only when sc->mem_cgroup is NULL.
937 */ 943 */
938 static int calc_reclaim_mapped(struct scan_con 944 static int calc_reclaim_mapped(struct scan_control *sc, struct zone *zone,
939 int priority) 945 int priority)
940 { 946 {
941 long mapped_ratio; 947 long mapped_ratio;
942 long distress; 948 long distress;
943 long swap_tendency; 949 long swap_tendency;
944 long imbalance; 950 long imbalance;
945 int reclaim_mapped = 0; 951 int reclaim_mapped = 0;
946 int prev_priority; 952 int prev_priority;
947 953
948 if (scan_global_lru(sc) && zone_is_nea 954 if (scan_global_lru(sc) && zone_is_near_oom(zone))
949 return 1; 955 return 1;
950 /* 956 /*
951 * `distress' is a measure of how much 957 * `distress' is a measure of how much trouble we're having
952 * reclaiming pages. 0 -> no problems 958 * reclaiming pages. 0 -> no problems. 100 -> great trouble.
953 */ 959 */
954 if (scan_global_lru(sc)) 960 if (scan_global_lru(sc))
955 prev_priority = zone->prev_pri 961 prev_priority = zone->prev_priority;
956 else 962 else
957 prev_priority = mem_cgroup_get 963 prev_priority = mem_cgroup_get_reclaim_priority(sc->mem_cgroup);
958 964
959 distress = 100 >> min(prev_priority, p 965 distress = 100 >> min(prev_priority, priority);
960 966
961 /* 967 /*
962 * The point of this algorithm is to d 968 * The point of this algorithm is to decide when to start
963 * reclaiming mapped memory instead of 969 * reclaiming mapped memory instead of just pagecache. Work out
964 * how much memory 970 * how much memory
965 * is mapped. 971 * is mapped.
966 */ 972 */
967 if (scan_global_lru(sc)) 973 if (scan_global_lru(sc))
968 mapped_ratio = ((global_page_s 974 mapped_ratio = ((global_page_state(NR_FILE_MAPPED) +
969 global_page_st 975 global_page_state(NR_ANON_PAGES)) * 100) /
970 vm_tot 976 vm_total_pages;
971 else 977 else
972 mapped_ratio = mem_cgroup_calc 978 mapped_ratio = mem_cgroup_calc_mapped_ratio(sc->mem_cgroup);
973 979
974 /* 980 /*
975 * Now decide how much we really want 981 * Now decide how much we really want to unmap some pages. The
976 * mapped ratio is downgraded - just b 982 * mapped ratio is downgraded - just because there's a lot of
977 * mapped memory doesn't necessarily m 983 * mapped memory doesn't necessarily mean that page reclaim
978 * isn't succeeding. 984 * isn't succeeding.
979 * 985 *
980 * The distress ratio is important - w 986 * The distress ratio is important - we don't want to start
981 * going oom. 987 * going oom.
982 * 988 *
983 * A 100% value of vm_swappiness overr 989 * A 100% value of vm_swappiness overrides this algorithm
984 * altogether. 990 * altogether.
985 */ 991 */
986 swap_tendency = mapped_ratio / 2 + dis 992 swap_tendency = mapped_ratio / 2 + distress + sc->swappiness;
987 993
988 /* 994 /*
989 * If there's huge imbalance between a 995 * If there's huge imbalance between active and inactive
990 * (think active 100 times larger than 996 * (think active 100 times larger than inactive) we should
991 * become more permissive, or the syst 997 * become more permissive, or the system will take too much
992 * cpu before it start swapping during 998 * cpu before it start swapping during memory pressure.
993 * Distress is about avoiding early-oo 999 * Distress is about avoiding early-oom, this is about
994 * making swappiness graceful despite 1000 * making swappiness graceful despite setting it to low
995 * values. 1001 * values.
996 * 1002 *
997 * Avoid div by zero with nr_inactive+ 1003 * Avoid div by zero with nr_inactive+1, and max resulting
998 * value is vm_total_pages. 1004 * value is vm_total_pages.
999 */ 1005 */
1000 if (scan_global_lru(sc)) { 1006 if (scan_global_lru(sc)) {
1001 imbalance = zone_page_state( 1007 imbalance = zone_page_state(zone, NR_ACTIVE);
1002 imbalance /= zone_page_state( 1008 imbalance /= zone_page_state(zone, NR_INACTIVE) + 1;
1003 } else 1009 } else
1004 imbalance = mem_cgroup_reclai 1010 imbalance = mem_cgroup_reclaim_imbalance(sc->mem_cgroup);
1005 1011
1006 /* 1012 /*
1007 * Reduce the effect of imbalance if 1013 * Reduce the effect of imbalance if swappiness is low,
1008 * this means for a swappiness very l 1014 * this means for a swappiness very low, the imbalance
1009 * must be much higher than 100 for t 1015 * must be much higher than 100 for this logic to make
1010 * the difference. 1016 * the difference.
1011 * 1017 *
1012 * Max temporary value is vm_total_pa 1018 * Max temporary value is vm_total_pages*100.
1013 */ 1019 */
1014 imbalance *= (vm_swappiness + 1); 1020 imbalance *= (vm_swappiness + 1);
1015 imbalance /= 100; 1021 imbalance /= 100;
1016 1022
1017 /* 1023 /*
1018 * If not much of the ram is mapped, 1024 * If not much of the ram is mapped, makes the imbalance
1019 * less relevant, it's high priority 1025 * less relevant, it's high priority we refill the inactive
1020 * list with mapped pages only in pre 1026 * list with mapped pages only in presence of high ratio of
1021 * mapped pages. 1027 * mapped pages.
1022 * 1028 *
1023 * Max temporary value is vm_total_pa 1029 * Max temporary value is vm_total_pages*100.
1024 */ 1030 */
1025 imbalance *= mapped_ratio; 1031 imbalance *= mapped_ratio;
1026 imbalance /= 100; 1032 imbalance /= 100;
1027 1033
1028 /* apply imbalance feedback to swap_t 1034 /* apply imbalance feedback to swap_tendency */
1029 swap_tendency += imbalance; 1035 swap_tendency += imbalance;
1030 1036
1031 /* 1037 /*
1032 * Now use this metric to decide whet 1038 * Now use this metric to decide whether to start moving mapped
1033 * memory onto the inactive list. 1039 * memory onto the inactive list.
1034 */ 1040 */
1035 if (swap_tendency >= 100) 1041 if (swap_tendency >= 100)
1036 reclaim_mapped = 1; 1042 reclaim_mapped = 1;
1037 1043
1038 return reclaim_mapped; 1044 return reclaim_mapped;
1039 } 1045 }
1040 1046
1041 /* 1047 /*
1042 * This moves pages from the active list to t 1048 * This moves pages from the active list to the inactive list.
1043 * 1049 *
1044 * We move them the other way if the page is 1050 * We move them the other way if the page is referenced by one or more
1045 * processes, from rmap. 1051 * processes, from rmap.
1046 * 1052 *
1047 * If the pages are mostly unmapped, the proc 1053 * If the pages are mostly unmapped, the processing is fast and it is
1048 * appropriate to hold zone->lru_lock across 1054 * appropriate to hold zone->lru_lock across the whole operation. But if
1049 * the pages are mapped, the processing is sl 1055 * the pages are mapped, the processing is slow (page_referenced()) so we
1050 * should drop zone->lru_lock around each pag 1056 * should drop zone->lru_lock around each page. It's impossible to balance
1051 * this, so instead we remove the pages from 1057 * this, so instead we remove the pages from the LRU while processing them.
1052 * It is safe to rely on PG_active against th 1058 * It is safe to rely on PG_active against the non-LRU pages in here because
1053 * nobody will play with that bit on a non-LR 1059 * nobody will play with that bit on a non-LRU page.
1054 * 1060 *
1055 * The downside is that we have to touch page 1061 * The downside is that we have to touch page->_count against each page.
1056 * But we had to alter page->flags anyway. 1062 * But we had to alter page->flags anyway.
1057 */ 1063 */
1058 1064
1059 1065
1060 static void shrink_active_list(unsigned long 1066 static void shrink_active_list(unsigned long nr_pages, struct zone *zone,
1061 struct scan_c 1067 struct scan_control *sc, int priority)
1062 { 1068 {
1063 unsigned long pgmoved; 1069 unsigned long pgmoved;
1064 int pgdeactivate = 0; 1070 int pgdeactivate = 0;
1065 unsigned long pgscanned; 1071 unsigned long pgscanned;
1066 LIST_HEAD(l_hold); /* The pages 1072 LIST_HEAD(l_hold); /* The pages which were snipped off */
1067 LIST_HEAD(l_inactive); /* Pages to g 1073 LIST_HEAD(l_inactive); /* Pages to go onto the inactive_list */
1068 LIST_HEAD(l_active); /* Pages to g 1074 LIST_HEAD(l_active); /* Pages to go onto the active_list */
1069 struct page *page; 1075 struct page *page;
1070 struct pagevec pvec; 1076 struct pagevec pvec;
1071 int reclaim_mapped = 0; 1077 int reclaim_mapped = 0;
1072 1078
1073 if (sc->may_swap) 1079 if (sc->may_swap)
1074 reclaim_mapped = calc_reclaim 1080 reclaim_mapped = calc_reclaim_mapped(sc, zone, priority);
1075 1081
1076 lru_add_drain(); 1082 lru_add_drain();
1077 spin_lock_irq(&zone->lru_lock); 1083 spin_lock_irq(&zone->lru_lock);
1078 pgmoved = sc->isolate_pages(nr_pages, 1084 pgmoved = sc->isolate_pages(nr_pages, &l_hold, &pgscanned, sc->order,
1079 ISOLA 1085 ISOLATE_ACTIVE, zone,
1080 sc->m 1086 sc->mem_cgroup, 1);
1081 /* 1087 /*
1082 * zone->pages_scanned is used for de 1088 * zone->pages_scanned is used for detect zone's oom
1083 * mem_cgroup remembers nr_scan by it 1089 * mem_cgroup remembers nr_scan by itself.
1084 */ 1090 */
1085 if (scan_global_lru(sc)) 1091 if (scan_global_lru(sc))
1086 zone->pages_scanned += pgscan 1092 zone->pages_scanned += pgscanned;
1087 1093
1088 __mod_zone_page_state(zone, NR_ACTIVE 1094 __mod_zone_page_state(zone, NR_ACTIVE, -pgmoved);
1089 spin_unlock_irq(&zone->lru_lock); 1095 spin_unlock_irq(&zone->lru_lock);
1090 1096
1091 while (!list_empty(&l_hold)) { 1097 while (!list_empty(&l_hold)) {
1092 cond_resched(); 1098 cond_resched();
1093 page = lru_to_page(&l_hold); 1099 page = lru_to_page(&l_hold);
1094 list_del(&page->lru); 1100 list_del(&page->lru);
1095 if (page_mapped(page)) { 1101 if (page_mapped(page)) {
1096 if (!reclaim_mapped | 1102 if (!reclaim_mapped ||
1097 (total_swap_pages 1103 (total_swap_pages == 0 && PageAnon(page)) ||
1098 page_referenced(p 1104 page_referenced(page, 0, sc->mem_cgroup)) {
1099 list_add(&pag 1105 list_add(&page->lru, &l_active);
1100 continue; 1106 continue;
1101 } 1107 }
1102 } 1108 }
1103 list_add(&page->lru, &l_inact 1109 list_add(&page->lru, &l_inactive);
1104 } 1110 }
1105 1111
1106 pagevec_init(&pvec, 1); 1112 pagevec_init(&pvec, 1);
1107 pgmoved = 0; 1113 pgmoved = 0;
1108 spin_lock_irq(&zone->lru_lock); 1114 spin_lock_irq(&zone->lru_lock);
1109 while (!list_empty(&l_inactive)) { 1115 while (!list_empty(&l_inactive)) {
1110 page = lru_to_page(&l_inactiv 1116 page = lru_to_page(&l_inactive);
1111 prefetchw_prev_lru_page(page, 1117 prefetchw_prev_lru_page(page, &l_inactive, flags);
1112 VM_BUG_ON(PageLRU(page)); 1118 VM_BUG_ON(PageLRU(page));
1113 SetPageLRU(page); 1119 SetPageLRU(page);
1114 VM_BUG_ON(!PageActive(page)); 1120 VM_BUG_ON(!PageActive(page));
1115 ClearPageActive(page); 1121 ClearPageActive(page);
1116 1122
1117 list_move(&page->lru, &zone-> 1123 list_move(&page->lru, &zone->inactive_list);
1118 mem_cgroup_move_lists(page, f 1124 mem_cgroup_move_lists(page, false);
1119 pgmoved++; 1125 pgmoved++;
1120 if (!pagevec_add(&pvec, page) 1126 if (!pagevec_add(&pvec, page)) {
1121 __mod_zone_page_state 1127 __mod_zone_page_state(zone, NR_INACTIVE, pgmoved);
1122 spin_unlock_irq(&zone 1128 spin_unlock_irq(&zone->lru_lock);
1123 pgdeactivate += pgmov 1129 pgdeactivate += pgmoved;
1124 pgmoved = 0; 1130 pgmoved = 0;
1125 if (buffer_heads_over 1131 if (buffer_heads_over_limit)
1126 pagevec_strip 1132 pagevec_strip(&pvec);
1127 __pagevec_release(&pv 1133 __pagevec_release(&pvec);
1128 spin_lock_irq(&zone-> 1134 spin_lock_irq(&zone->lru_lock);
1129 } 1135 }
1130 } 1136 }
1131 __mod_zone_page_state(zone, NR_INACTI 1137 __mod_zone_page_state(zone, NR_INACTIVE, pgmoved);
1132 pgdeactivate += pgmoved; 1138 pgdeactivate += pgmoved;
1133 if (buffer_heads_over_limit) { 1139 if (buffer_heads_over_limit) {
1134 spin_unlock_irq(&zone->lru_lo 1140 spin_unlock_irq(&zone->lru_lock);
1135 pagevec_strip(&pvec); 1141 pagevec_strip(&pvec);
1136 spin_lock_irq(&zone->lru_lock 1142 spin_lock_irq(&zone->lru_lock);
1137 } 1143 }
1138 1144
1139 pgmoved = 0; 1145 pgmoved = 0;
1140 while (!list_empty(&l_active)) { 1146 while (!list_empty(&l_active)) {
1141 page = lru_to_page(&l_active) 1147 page = lru_to_page(&l_active);
1142 prefetchw_prev_lru_page(page, 1148 prefetchw_prev_lru_page(page, &l_active, flags);
1143 VM_BUG_ON(PageLRU(page)); 1149 VM_BUG_ON(PageLRU(page));
1144 SetPageLRU(page); 1150 SetPageLRU(page);
1145 VM_BUG_ON(!PageActive(page)); 1151 VM_BUG_ON(!PageActive(page));
1146 1152
1147 list_move(&page->lru, &zone-> 1153 list_move(&page->lru, &zone->active_list);
1148 mem_cgroup_move_lists(page, t 1154 mem_cgroup_move_lists(page, true);
1149 pgmoved++; 1155 pgmoved++;
1150 if (!pagevec_add(&pvec, page) 1156 if (!pagevec_add(&pvec, page)) {
1151 __mod_zone_page_state 1157 __mod_zone_page_state(zone, NR_ACTIVE, pgmoved);
1152 pgmoved = 0; 1158 pgmoved = 0;
1153 spin_unlock_irq(&zone 1159 spin_unlock_irq(&zone->lru_lock);
1154 __pagevec_release(&pv 1160 __pagevec_release(&pvec);
1155 spin_lock_irq(&zone-> 1161 spin_lock_irq(&zone->lru_lock);
1156 } 1162 }
1157 } 1163 }
1158 __mod_zone_page_state(zone, NR_ACTIVE 1164 __mod_zone_page_state(zone, NR_ACTIVE, pgmoved);
1159 1165
1160 __count_zone_vm_events(PGREFILL, zone 1166 __count_zone_vm_events(PGREFILL, zone, pgscanned);
1161 __count_vm_events(PGDEACTIVATE, pgdea 1167 __count_vm_events(PGDEACTIVATE, pgdeactivate);
1162 spin_unlock_irq(&zone->lru_lock); 1168 spin_unlock_irq(&zone->lru_lock);
1163 1169
1164 pagevec_release(&pvec); 1170 pagevec_release(&pvec);
1165 } 1171 }
1166 1172
1167 /* 1173 /*
1168 * This is a basic per-zone page freer. Used 1174 * This is a basic per-zone page freer. Used by both kswapd and direct reclaim.
1169 */ 1175 */
1170 static unsigned long shrink_zone(int priority 1176 static unsigned long shrink_zone(int priority, struct zone *zone,
1171 struct scan_c 1177 struct scan_control *sc)
1172 { 1178 {
1173 unsigned long nr_active; 1179 unsigned long nr_active;
1174 unsigned long nr_inactive; 1180 unsigned long nr_inactive;
1175 unsigned long nr_to_scan; 1181 unsigned long nr_to_scan;
1176 unsigned long nr_reclaimed = 0; 1182 unsigned long nr_reclaimed = 0;
1177 1183
1178 if (scan_global_lru(sc)) { 1184 if (scan_global_lru(sc)) {
1179 /* 1185 /*
1180 * Add one to nr_to_scan just 1186 * Add one to nr_to_scan just to make sure that the kernel
1181 * will slowly sift through t 1187 * will slowly sift through the active list.
1182 */ 1188 */
1183 zone->nr_scan_active += 1189 zone->nr_scan_active +=
1184 (zone_page_state(zone 1190 (zone_page_state(zone, NR_ACTIVE) >> priority) + 1;
1185 nr_active = zone->nr_scan_act 1191 nr_active = zone->nr_scan_active;
1186 zone->nr_scan_inactive += 1192 zone->nr_scan_inactive +=
1187 (zone_page_state(zone 1193 (zone_page_state(zone, NR_INACTIVE) >> priority) + 1;
1188 nr_inactive = zone->nr_scan_i 1194 nr_inactive = zone->nr_scan_inactive;
1189 if (nr_inactive >= sc->swap_c 1195 if (nr_inactive >= sc->swap_cluster_max)
1190 zone->nr_scan_inactiv 1196 zone->nr_scan_inactive = 0;
1191 else 1197 else
1192 nr_inactive = 0; 1198 nr_inactive = 0;
1193 1199
1194 if (nr_active >= sc->swap_clu 1200 if (nr_active >= sc->swap_cluster_max)
1195 zone->nr_scan_active 1201 zone->nr_scan_active = 0;
1196 else 1202 else
1197 nr_active = 0; 1203 nr_active = 0;
1198 } else { 1204 } else {
1199 /* 1205 /*
1200 * This reclaim occurs not be 1206 * This reclaim occurs not because zone memory shortage but
1201 * because memory controller 1207 * because memory controller hits its limit.
1202 * Then, don't modify zone re 1208 * Then, don't modify zone reclaim related data.
1203 */ 1209 */
1204 nr_active = mem_cgroup_calc_r 1210 nr_active = mem_cgroup_calc_reclaim_active(sc->mem_cgroup,
1205 zone, 1211 zone, priority);
1206 1212
1207 nr_inactive = mem_cgroup_calc 1213 nr_inactive = mem_cgroup_calc_reclaim_inactive(sc->mem_cgroup,
1208 zone, 1214 zone, priority);
1209 } 1215 }
1210 1216
1211 1217
1212 while (nr_active || nr_inactive) { 1218 while (nr_active || nr_inactive) {
1213 if (nr_active) { 1219 if (nr_active) {
1214 nr_to_scan = min(nr_a 1220 nr_to_scan = min(nr_active,
1215 (unsi 1221 (unsigned long)sc->swap_cluster_max);
1216 nr_active -= nr_to_sc 1222 nr_active -= nr_to_scan;
1217 shrink_active_list(nr 1223 shrink_active_list(nr_to_scan, zone, sc, priority);
1218 } 1224 }
1219 1225
1220 if (nr_inactive) { 1226 if (nr_inactive) {
1221 nr_to_scan = min(nr_i 1227 nr_to_scan = min(nr_inactive,
1222 (unsi 1228 (unsigned long)sc->swap_cluster_max);
1223 nr_inactive -= nr_to_ 1229 nr_inactive -= nr_to_scan;
1224 nr_reclaimed += shrin 1230 nr_reclaimed += shrink_inactive_list(nr_to_scan, zone,
1225 1231 sc);
1226 } 1232 }
1227 } 1233 }
1228 1234
1229 throttle_vm_writeout(sc->gfp_mask); 1235 throttle_vm_writeout(sc->gfp_mask);
1230 return nr_reclaimed; 1236 return nr_reclaimed;
1231 } 1237 }
1232 1238
1233 /* 1239 /*
1234 * This is the direct reclaim path, for page- 1240 * This is the direct reclaim path, for page-allocating processes. We only
1235 * try to reclaim pages from zones which will 1241 * try to reclaim pages from zones which will satisfy the caller's allocation
1236 * request. 1242 * request.
1237 * 1243 *
1238 * We reclaim from a zone even if that zone i 1244 * We reclaim from a zone even if that zone is over pages_high. Because:
1239 * a) The caller may be trying to free *extra 1245 * a) The caller may be trying to free *extra* pages to satisfy a higher-order
1240 * allocation or 1246 * allocation or
1241 * b) The zones may be over pages_high but th 1247 * b) The zones may be over pages_high but they must go *over* pages_high to
1242 * satisfy the `incremental min' zone defe 1248 * satisfy the `incremental min' zone defense algorithm.
1243 * 1249 *
1244 * Returns the number of reclaimed pages. 1250 * Returns the number of reclaimed pages.
1245 * 1251 *
1246 * If a zone is deemed to be full of pinned p 1252 * If a zone is deemed to be full of pinned pages then just give it a light
1247 * scan then give up on it. 1253 * scan then give up on it.
1248 */ 1254 */
1249 static unsigned long shrink_zones(int priorit 1255 static unsigned long shrink_zones(int priority, struct zone **zones,
1250 struc 1256 struct scan_control *sc)
1251 { 1257 {
1252 unsigned long nr_reclaimed = 0; 1258 unsigned long nr_reclaimed = 0;
1253 int i; 1259 int i;
1254 1260
1255 1261
1256 sc->all_unreclaimable = 1; 1262 sc->all_unreclaimable = 1;
1257 for (i = 0; zones[i] != NULL; i++) { 1263 for (i = 0; zones[i] != NULL; i++) {
1258 struct zone *zone = zones[i]; 1264 struct zone *zone = zones[i];
1259 1265
1260 if (!populated_zone(zone)) 1266 if (!populated_zone(zone))
1261 continue; 1267 continue;
1262 /* 1268 /*
1263 * Take care memory controlle 1269 * Take care memory controller reclaiming has small influence
1264 * to global LRU. 1270 * to global LRU.
1265 */ 1271 */
1266 if (scan_global_lru(sc)) { 1272 if (scan_global_lru(sc)) {
1267 if (!cpuset_zone_allo 1273 if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
1268 continue; 1274 continue;
1269 note_zone_scanning_pr 1275 note_zone_scanning_priority(zone, priority);
1270 1276
1271 if (zone_is_all_unrec 1277 if (zone_is_all_unreclaimable(zone) &&
1272 1278 priority != DEF_PRIORITY)
1273 continue; 1279 continue; /* Let kswapd poll it */
1274 sc->all_unreclaimable 1280 sc->all_unreclaimable = 0;
1275 } else { 1281 } else {
1276 /* 1282 /*
1277 * Ignore cpuset limi 1283 * Ignore cpuset limitation here. We just want to reduce
1278 * # of used pages by 1284 * # of used pages by us regardless of memory shortage.
1279 */ 1285 */
1280 sc->all_unreclaimable 1286 sc->all_unreclaimable = 0;
1281 mem_cgroup_note_recla 1287 mem_cgroup_note_reclaim_priority(sc->mem_cgroup,
1282 1288 priority);
1283 } 1289 }
1284 1290
1285 nr_reclaimed += shrink_zone(p 1291 nr_reclaimed += shrink_zone(priority, zone, sc);
1286 } 1292 }
1287 1293
1288 return nr_reclaimed; 1294 return nr_reclaimed;
1289 } 1295 }
1290 1296
1291 /* 1297 /*
1292 * This is the main entry point to direct pag 1298 * This is the main entry point to direct page reclaim.
1293 * 1299 *
1294 * If a full scan of the inactive list fails 1300 * If a full scan of the inactive list fails to free enough memory then we
1295 * are "out of memory" and something needs to 1301 * are "out of memory" and something needs to be killed.
1296 * 1302 *
1297 * If the caller is !__GFP_FS then the probab 1303 * If the caller is !__GFP_FS then the probability of a failure is reasonably
1298 * high - the zone may be full of dirty or un 1304 * high - the zone may be full of dirty or under-writeback pages, which this
1299 * caller can't do much about. We kick pdflu 1305 * caller can't do much about. We kick pdflush and take explicit naps in the
1300 * hope that some of these pages can be writt 1306 * hope that some of these pages can be written. But if the allocating task
1301 * holds filesystem locks which prevent write 1307 * holds filesystem locks which prevent writeout this might not work, and the
1302 * allocation attempt will fail. 1308 * allocation attempt will fail.
1303 */ 1309 */
1304 static unsigned long do_try_to_free_pages(str 1310 static unsigned long do_try_to_free_pages(struct zone **zones, gfp_t gfp_mask,
1305 str 1311 struct scan_control *sc)
1306 { 1312 {
1307 int priority; 1313 int priority;
1308 int ret = 0; 1314 int ret = 0;
1309 unsigned long total_scanned = 0; 1315 unsigned long total_scanned = 0;
1310 unsigned long nr_reclaimed = 0; 1316 unsigned long nr_reclaimed = 0;
1311 struct reclaim_state *reclaim_state = 1317 struct reclaim_state *reclaim_state = current->reclaim_state;
1312 unsigned long lru_pages = 0; 1318 unsigned long lru_pages = 0;
1313 int i; 1319 int i;
1314 1320
1315 if (scan_global_lru(sc)) 1321 if (scan_global_lru(sc))
1316 count_vm_event(ALLOCSTALL); 1322 count_vm_event(ALLOCSTALL);
1317 /* 1323 /*
1318 * mem_cgroup will not do shrink_slab 1324 * mem_cgroup will not do shrink_slab.
1319 */ 1325 */
1320 if (scan_global_lru(sc)) { 1326 if (scan_global_lru(sc)) {
1321 for (i = 0; zones[i] != NULL; 1327 for (i = 0; zones[i] != NULL; i++) {
1322 struct zone *zone = z 1328 struct zone *zone = zones[i];
1323 1329
1324 if (!cpuset_zone_allo 1330 if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
1325 continue; 1331 continue;
1326 1332
1327 lru_pages += zone_pag 1333 lru_pages += zone_page_state(zone, NR_ACTIVE)
1328 + zon 1334 + zone_page_state(zone, NR_INACTIVE);
1329 } 1335 }
1330 } 1336 }
1331 1337
1332 for (priority = DEF_PRIORITY; priorit 1338 for (priority = DEF_PRIORITY; priority >= 0; priority--) {
1333 sc->nr_scanned = 0; 1339 sc->nr_scanned = 0;
1334 if (!priority) 1340 if (!priority)
1335 disable_swap_token(); 1341 disable_swap_token();
1336 nr_reclaimed += shrink_zones( 1342 nr_reclaimed += shrink_zones(priority, zones, sc);
1337 /* 1343 /*
1338 * Don't shrink slabs when re 1344 * Don't shrink slabs when reclaiming memory from
1339 * over limit cgroups 1345 * over limit cgroups
1340 */ 1346 */
1341 if (scan_global_lru(sc)) { 1347 if (scan_global_lru(sc)) {
1342 shrink_slab(sc->nr_sc 1348 shrink_slab(sc->nr_scanned, gfp_mask, lru_pages);
1343 if (reclaim_state) { 1349 if (reclaim_state) {
1344 nr_reclaimed 1350 nr_reclaimed += reclaim_state->reclaimed_slab;
1345 reclaim_state 1351 reclaim_state->reclaimed_slab = 0;
1346 } 1352 }
1347 } 1353 }
1348 total_scanned += sc->nr_scann 1354 total_scanned += sc->nr_scanned;
1349 if (nr_reclaimed >= sc->swap_ 1355 if (nr_reclaimed >= sc->swap_cluster_max) {
1350 ret = 1; 1356 ret = 1;
1351 goto out; 1357 goto out;
1352 } 1358 }
1353 1359
1354 /* 1360 /*
1355 * Try to write back as many 1361 * Try to write back as many pages as we just scanned. This
1356 * tends to cause slow stream 1362 * tends to cause slow streaming writers to write data to the
1357 * disk smoothly, at the dirt 1363 * disk smoothly, at the dirtying rate, which is nice. But
1358 * that's undesirable in lapt 1364 * that's undesirable in laptop mode, where we *want* lumpy
1359 * writeout. So in laptop mo 1365 * writeout. So in laptop mode, write out the whole world.
1360 */ 1366 */
1361 if (total_scanned > sc->swap_ 1367 if (total_scanned > sc->swap_cluster_max +
1362 sc->s 1368 sc->swap_cluster_max / 2) {
1363 wakeup_pdflush(laptop 1369 wakeup_pdflush(laptop_mode ? 0 : total_scanned);
1364 sc->may_writepage = 1 1370 sc->may_writepage = 1;
1365 } 1371 }
1366 1372
1367 /* Take a nap, wait for some 1373 /* Take a nap, wait for some writeback to complete */
1368 if (sc->nr_scanned && priorit 1374 if (sc->nr_scanned && priority < DEF_PRIORITY - 2)
1369 congestion_wait(WRITE 1375 congestion_wait(WRITE, HZ/10);
1370 } 1376 }
1371 /* top priority shrink_caches still h 1377 /* top priority shrink_caches still had more to do? don't OOM, then */
1372 if (!sc->all_unreclaimable && scan_gl 1378 if (!sc->all_unreclaimable && scan_global_lru(sc))
1373 ret = 1; 1379 ret = 1;
1374 out: 1380 out:
1375 /* 1381 /*
1376 * Now that we've scanned all the zon 1382 * Now that we've scanned all the zones at this priority level, note
1377 * that level within the zone so that 1383 * that level within the zone so that the next thread which performs
1378 * scanning of this zone will immedia 1384 * scanning of this zone will immediately start out at this priority
1379 * level. This affects only the deci 1385 * level. This affects only the decision whether or not to bring
1380 * mapped pages onto the inactive lis 1386 * mapped pages onto the inactive list.
1381 */ 1387 */
1382 if (priority < 0) 1388 if (priority < 0)
1383 priority = 0; 1389 priority = 0;
1384 1390
1385 if (scan_global_lru(sc)) { 1391 if (scan_global_lru(sc)) {
1386 for (i = 0; zones[i] != NULL; 1392 for (i = 0; zones[i] != NULL; i++) {
1387 struct zone *zone = z 1393 struct zone *zone = zones[i];
1388 1394
1389 if (!cpuset_zone_allo 1395 if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
1390 continue; 1396 continue;
1391 1397
1392 zone->prev_priority = 1398 zone->prev_priority = priority;
1393 } 1399 }
1394 } else 1400 } else
1395 mem_cgroup_record_reclaim_pri 1401 mem_cgroup_record_reclaim_priority(sc->mem_cgroup, priority);
1396 1402
1397 return ret; 1403 return ret;
1398 } 1404 }
1399 1405
1400 unsigned long try_to_free_pages(struct zone * 1406 unsigned long try_to_free_pages(struct zone **zones, int order, gfp_t gfp_mask)
1401 { 1407 {
1402 struct scan_control sc = { 1408 struct scan_control sc = {
1403 .gfp_mask = gfp_mask, 1409 .gfp_mask = gfp_mask,
1404 .may_writepage = !laptop_mode 1410 .may_writepage = !laptop_mode,
1405 .swap_cluster_max = SWAP_CLUS 1411 .swap_cluster_max = SWAP_CLUSTER_MAX,
1406 .may_swap = 1, 1412 .may_swap = 1,
1407 .swappiness = vm_swappiness, 1413 .swappiness = vm_swappiness,
1408 .order = order, 1414 .order = order,
1409 .mem_cgroup = NULL, 1415 .mem_cgroup = NULL,
1410 .isolate_pages = isolate_page 1416 .isolate_pages = isolate_pages_global,
1411 }; 1417 };
1412 1418
1413 return do_try_to_free_pages(zones, gf 1419 return do_try_to_free_pages(zones, gfp_mask, &sc);
1414 } 1420 }
1415 1421
1416 #ifdef CONFIG_CGROUP_MEM_RES_CTLR 1422 #ifdef CONFIG_CGROUP_MEM_RES_CTLR
1417 1423
1418 unsigned long try_to_free_mem_cgroup_pages(st 1424 unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *mem_cont,
1419 1425 gfp_t gfp_mask)
1420 { 1426 {
1421 struct scan_control sc = { 1427 struct scan_control sc = {
1422 .gfp_mask = gfp_mask, 1428 .gfp_mask = gfp_mask,
1423 .may_writepage = !laptop_mode 1429 .may_writepage = !laptop_mode,
1424 .may_swap = 1, 1430 .may_swap = 1,
1425 .swap_cluster_max = SWAP_CLUS 1431 .swap_cluster_max = SWAP_CLUSTER_MAX,
1426 .swappiness = vm_swappiness, 1432 .swappiness = vm_swappiness,
1427 .order = 0, 1433 .order = 0,
1428 .mem_cgroup = mem_cont, 1434 .mem_cgroup = mem_cont,
1429 .isolate_pages = mem_cgroup_i 1435 .isolate_pages = mem_cgroup_isolate_pages,
1430 }; 1436 };
1431 struct zone **zones; 1437 struct zone **zones;
1432 int target_zone = gfp_zone(GFP_HIGHUS 1438 int target_zone = gfp_zone(GFP_HIGHUSER_MOVABLE);
1433 1439
1434 zones = NODE_DATA(numa_node_id())->no 1440 zones = NODE_DATA(numa_node_id())->node_zonelists[target_zone].zones;
1435 if (do_try_to_free_pages(zones, sc.gf 1441 if (do_try_to_free_pages(zones, sc.gfp_mask, &sc))
1436 return 1; 1442 return 1;
1437 return 0; 1443 return 0;
1438 } 1444 }
1439 #endif 1445 #endif
1440 1446
1441 /* 1447 /*
1442 * For kswapd, balance_pgdat() will work acro 1448 * For kswapd, balance_pgdat() will work across all this node's zones until
1443 * they are all at pages_high. 1449 * they are all at pages_high.
1444 * 1450 *
1445 * Returns the number of pages which were act 1451 * Returns the number of pages which were actually freed.
1446 * 1452 *
1447 * There is special handling here for zones w 1453 * There is special handling here for zones which are full of pinned pages.
1448 * This can happen if the pages are all mlock 1454 * This can happen if the pages are all mlocked, or if they are all used by
1449 * device drivers (say, ZONE_DMA). Or if the 1455 * device drivers (say, ZONE_DMA). Or if they are all in use by hugetlb.
1450 * What we do is to detect the case where all 1456 * What we do is to detect the case where all pages in the zone have been
1451 * scanned twice and there has been zero succ 1457 * scanned twice and there has been zero successful reclaim. Mark the zone as
1452 * dead and from now on, only perform a short 1458 * dead and from now on, only perform a short scan. Basically we're polling
1453 * the zone for when the problem goes away. 1459 * the zone for when the problem goes away.
1454 * 1460 *
1455 * kswapd scans the zones in the highmem->nor 1461 * kswapd scans the zones in the highmem->normal->dma direction. It skips
1456 * zones which have free_pages > pages_high, 1462 * zones which have free_pages > pages_high, but once a zone is found to have
1457 * free_pages <= pages_high, we scan that zon 1463 * free_pages <= pages_high, we scan that zone and the lower zones regardless
1458 * of the number of free pages in the lower z 1464 * of the number of free pages in the lower zones. This interoperates with
1459 * the page allocator fallback scheme to ensu 1465 * the page allocator fallback scheme to ensure that aging of pages is balanced
1460 * across the zones. 1466 * across the zones.
1461 */ 1467 */
1462 static unsigned long balance_pgdat(pg_data_t 1468 static unsigned long balance_pgdat(pg_data_t *pgdat, int order)
1463 { 1469 {
1464 int all_zones_ok; 1470 int all_zones_ok;
1465 int priority; 1471 int priority;
1466 int i; 1472 int i;
1467 unsigned long total_scanned; 1473 unsigned long total_scanned;
1468 unsigned long nr_reclaimed; 1474 unsigned long nr_reclaimed;
1469 struct reclaim_state *reclaim_state = 1475 struct reclaim_state *reclaim_state = current->reclaim_state;
1470 struct scan_control sc = { 1476 struct scan_control sc = {
1471 .gfp_mask = GFP_KERNEL, 1477 .gfp_mask = GFP_KERNEL,
1472 .may_swap = 1, 1478 .may_swap = 1,
1473 .swap_cluster_max = SWAP_CLUS 1479 .swap_cluster_max = SWAP_CLUSTER_MAX,
1474 .swappiness = vm_swappiness, 1480 .swappiness = vm_swappiness,
1475 .order = order, 1481 .order = order,
1476 .mem_cgroup = NULL, 1482 .mem_cgroup = NULL,
1477 .isolate_pages = isolate_page 1483 .isolate_pages = isolate_pages_global,
1478 }; 1484 };
1479 /* 1485 /*
1480 * temp_priority is used to remember 1486 * temp_priority is used to remember the scanning priority at which
1481 * this zone was successfully refille 1487 * this zone was successfully refilled to free_pages == pages_high.
1482 */ 1488 */
1483 int temp_priority[MAX_NR_ZONES]; 1489 int temp_priority[MAX_NR_ZONES];
1484 1490
1485 loop_again: 1491 loop_again:
1486 total_scanned = 0; 1492 total_scanned = 0;
1487 nr_reclaimed = 0; 1493 nr_reclaimed = 0;
1488 sc.may_writepage = !laptop_mode; 1494 sc.may_writepage = !laptop_mode;
1489 count_vm_event(PAGEOUTRUN); 1495 count_vm_event(PAGEOUTRUN);
1490 1496
1491 for (i = 0; i < pgdat->nr_zones; i++) 1497 for (i = 0; i < pgdat->nr_zones; i++)
1492 temp_priority[i] = DEF_PRIORI 1498 temp_priority[i] = DEF_PRIORITY;
1493 1499
1494 for (priority = DEF_PRIORITY; priorit 1500 for (priority = DEF_PRIORITY; priority >= 0; priority--) {
1495 int end_zone = 0; /* In 1501 int end_zone = 0; /* Inclusive. 0 = ZONE_DMA */
1496 unsigned long lru_pages = 0; 1502 unsigned long lru_pages = 0;
1497 1503
1498 /* The swap token gets in the 1504 /* The swap token gets in the way of swapout... */
1499 if (!priority) 1505 if (!priority)
1500 disable_swap_token(); 1506 disable_swap_token();
1501 1507
1502 all_zones_ok = 1; 1508 all_zones_ok = 1;
1503 1509
1504 /* 1510 /*
1505 * Scan in the highmem->dma d 1511 * Scan in the highmem->dma direction for the highest
1506 * zone which needs scanning 1512 * zone which needs scanning
1507 */ 1513 */
1508 for (i = pgdat->nr_zones - 1; 1514 for (i = pgdat->nr_zones - 1; i >= 0; i--) {
1509 struct zone *zone = p 1515 struct zone *zone = pgdat->node_zones + i;
1510 1516
1511 if (!populated_zone(z 1517 if (!populated_zone(zone))
1512 continue; 1518 continue;
1513 1519
1514 if (zone_is_all_unrec 1520 if (zone_is_all_unreclaimable(zone) &&
1515 priority != DEF_P 1521 priority != DEF_PRIORITY)
1516 continue; 1522 continue;
1517 1523
1518 if (!zone_watermark_o 1524 if (!zone_watermark_ok(zone, order, zone->pages_high,
1519 1525 0, 0)) {
1520 end_zone = i; 1526 end_zone = i;
1521 break; 1527 break;
1522 } 1528 }
1523 } 1529 }
1524 if (i < 0) 1530 if (i < 0)
1525 goto out; 1531 goto out;
1526 1532
1527 for (i = 0; i <= end_zone; i+ 1533 for (i = 0; i <= end_zone; i++) {
1528 struct zone *zone = p 1534 struct zone *zone = pgdat->node_zones + i;
1529 1535
1530 lru_pages += zone_pag 1536 lru_pages += zone_page_state(zone, NR_ACTIVE)
1531 + zon 1537 + zone_page_state(zone, NR_INACTIVE);
1532 } 1538 }
1533 1539
1534 /* 1540 /*
1535 * Now scan the zone in the d 1541 * Now scan the zone in the dma->highmem direction, stopping
1536 * at the last zone which nee 1542 * at the last zone which needs scanning.
1537 * 1543 *
1538 * We do this because the pag 1544 * We do this because the page allocator works in the opposite
1539 * direction. This prevents 1545 * direction. This prevents the page allocator from allocating
1540 * pages behind kswapd's dire 1546 * pages behind kswapd's direction of progress, which would
1541 * cause too much scanning of 1547 * cause too much scanning of the lower zones.
1542 */ 1548 */
1543 for (i = 0; i <= end_zone; i+ 1549 for (i = 0; i <= end_zone; i++) {
1544 struct zone *zone = p 1550 struct zone *zone = pgdat->node_zones + i;
1545 int nr_slab; 1551 int nr_slab;
1546 1552
1547 if (!populated_zone(z 1553 if (!populated_zone(zone))
1548 continue; 1554 continue;
1549 1555
1550 if (zone_is_all_unrec 1556 if (zone_is_all_unreclaimable(zone) &&
1551 prior 1557 priority != DEF_PRIORITY)
1552 continue; 1558 continue;
1553 1559
1554 if (!zone_watermark_o 1560 if (!zone_watermark_ok(zone, order, zone->pages_high,
1555 1561 end_zone, 0))
1556 all_zones_ok 1562 all_zones_ok = 0;
1557 temp_priority[i] = pr 1563 temp_priority[i] = priority;
1558 sc.nr_scanned = 0; 1564 sc.nr_scanned = 0;
1559 note_zone_scanning_pr 1565 note_zone_scanning_priority(zone, priority);
1560 /* 1566 /*
1561 * We put equal press 1567 * We put equal pressure on every zone, unless one
1562 * zone has way too m 1568 * zone has way too many pages free already.
1563 */ 1569 */
1564 if (!zone_watermark_o 1570 if (!zone_watermark_ok(zone, order, 8*zone->pages_high,
1565 1571 end_zone, 0))
1566 nr_reclaimed 1572 nr_reclaimed += shrink_zone(priority, zone, &sc);
1567 reclaim_state->reclai 1573 reclaim_state->reclaimed_slab = 0;
1568 nr_slab = shrink_slab 1574 nr_slab = shrink_slab(sc.nr_scanned, GFP_KERNEL,
1569 1575 lru_pages);
1570 nr_reclaimed += recla 1576 nr_reclaimed += reclaim_state->reclaimed_slab;
1571 total_scanned += sc.n 1577 total_scanned += sc.nr_scanned;
1572 if (zone_is_all_unrec 1578 if (zone_is_all_unreclaimable(zone))
1573 continue; 1579 continue;
1574 if (nr_slab == 0 && z 1580 if (nr_slab == 0 && zone->pages_scanned >=
1575 (zone_page_st 1581 (zone_page_state(zone, NR_ACTIVE)
1576 + zone_page_s 1582 + zone_page_state(zone, NR_INACTIVE)) * 6)
1577 zone_ 1583 zone_set_flag(zone,
1578 1584 ZONE_ALL_UNRECLAIMABLE);
1579 /* 1585 /*
1580 * If we've done a de 1586 * If we've done a decent amount of scanning and
1581 * the reclaim ratio 1587 * the reclaim ratio is low, start doing writepage
1582 * even in laptop mod 1588 * even in laptop mode
1583 */ 1589 */
1584 if (total_scanned > S 1590 if (total_scanned > SWAP_CLUSTER_MAX * 2 &&
1585 total_scanned > n 1591 total_scanned > nr_reclaimed + nr_reclaimed / 2)
1586 sc.may_writep 1592 sc.may_writepage = 1;
1587 } 1593 }
1588 if (all_zones_ok) 1594 if (all_zones_ok)
1589 break; /* ks 1595 break; /* kswapd: all done */
1590 /* 1596 /*
1591 * OK, kswapd is getting into 1597 * OK, kswapd is getting into trouble. Take a nap, then take
1592 * another pass across the zo 1598 * another pass across the zones.
1593 */ 1599 */
1594 if (total_scanned && priority 1600 if (total_scanned && priority < DEF_PRIORITY - 2)
1595 congestion_wait(WRITE 1601 congestion_wait(WRITE, HZ/10);
1596 1602
1597 /* 1603 /*
1598 * We do this so kswapd doesn 1604 * We do this so kswapd doesn't build up large priorities for
1599 * example when it is freeing 1605 * example when it is freeing in parallel with allocators. It
1600 * matches the direct reclaim 1606 * matches the direct reclaim path behaviour in terms of impact
1601 * on zone->*_priority. 1607 * on zone->*_priority.
1602 */ 1608 */
1603 if (nr_reclaimed >= SWAP_CLUS 1609 if (nr_reclaimed >= SWAP_CLUSTER_MAX)
1604 break; 1610 break;
1605 } 1611 }
1606 out: 1612 out:
1607 /* 1613 /*
1608 * Note within each zone the priority 1614 * Note within each zone the priority level at which this zone was
1609 * brought into a happy state. So th 1615 * brought into a happy state. So that the next thread which scans this
1610 * zone will start out at that priori 1616 * zone will start out at that priority level.
1611 */ 1617 */
1612 for (i = 0; i < pgdat->nr_zones; i++) 1618 for (i = 0; i < pgdat->nr_zones; i++) {
1613 struct zone *zone = pgdat->no 1619 struct zone *zone = pgdat->node_zones + i;
1614 1620
1615 zone->prev_priority = temp_pr 1621 zone->prev_priority = temp_priority[i];
1616 } 1622 }
1617 if (!all_zones_ok) { 1623 if (!all_zones_ok) {
1618 cond_resched(); 1624 cond_resched();
1619 1625
1620 try_to_freeze(); 1626 try_to_freeze();
1621 1627
1622 goto loop_again; 1628 goto loop_again;
1623 } 1629 }
1624 1630
1625 return nr_reclaimed; 1631 return nr_reclaimed;
1626 } 1632 }
1627 1633
1628 /* 1634 /*
1629 * The background pageout daemon, started as 1635 * The background pageout daemon, started as a kernel thread
1630 * from the init process. 1636 * from the init process.
1631 * 1637 *
1632 * This basically trickles out pages so that 1638 * This basically trickles out pages so that we have _some_
1633 * free memory available even if there is no 1639 * free memory available even if there is no other activity
1634 * that frees anything up. This is needed for 1640 * that frees anything up. This is needed for things like routing
1635 * etc, where we otherwise might have all act 1641 * etc, where we otherwise might have all activity going on in
1636 * asynchronous contexts that cannot page thi 1642 * asynchronous contexts that cannot page things out.
1637 * 1643 *
1638 * If there are applications that are active 1644 * If there are applications that are active memory-allocators
1639 * (most normal use), this basically shouldn' 1645 * (most normal use), this basically shouldn't matter.
1640 */ 1646 */
1641 static int kswapd(void *p) 1647 static int kswapd(void *p)
1642 { 1648 {
1643 unsigned long order; 1649 unsigned long order;
1644 pg_data_t *pgdat = (pg_data_t*)p; 1650 pg_data_t *pgdat = (pg_data_t*)p;
1645 struct task_struct *tsk = current; 1651 struct task_struct *tsk = current;
1646 DEFINE_WAIT(wait); 1652 DEFINE_WAIT(wait);
1647 struct reclaim_state reclaim_state = 1653 struct reclaim_state reclaim_state = {
1648 .reclaimed_slab = 0, 1654 .reclaimed_slab = 0,
1649 }; 1655 };
1650 cpumask_t cpumask; 1656 cpumask_t cpumask;
1651 1657
1652 cpumask = node_to_cpumask(pgdat->node 1658 cpumask = node_to_cpumask(pgdat->node_id);
1653 if (!cpus_empty(cpumask)) 1659 if (!cpus_empty(cpumask))
1654 set_cpus_allowed(tsk, cpumask 1660 set_cpus_allowed(tsk, cpumask);
1655 current->reclaim_state = &reclaim_sta 1661 current->reclaim_state = &reclaim_state;
1656 1662
1657 /* 1663 /*
1658 * Tell the memory management that we 1664 * Tell the memory management that we're a "memory allocator",
1659 * and that if we need more memory we 1665 * and that if we need more memory we should get access to it
1660 * regardless (see "__alloc_pages()") 1666 * regardless (see "__alloc_pages()"). "kswapd" should
1661 * never get caught in the normal pag 1667 * never get caught in the normal page freeing logic.
1662 * 1668 *
1663 * (Kswapd normally doesn't need memo 1669 * (Kswapd normally doesn't need memory anyway, but sometimes
1664 * you need a small amount of memory 1670 * you need a small amount of memory in order to be able to
1665 * page out something else, and this 1671 * page out something else, and this flag essentially protects
1666 * us from recursively trying to free 1672 * us from recursively trying to free more memory as we're
1667 * trying to free the first piece of 1673 * trying to free the first piece of memory in the first place).
1668 */ 1674 */
1669 tsk->flags |= PF_MEMALLOC | PF_SWAPWR 1675 tsk->flags |= PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD;
1670 set_freezable(); 1676 set_freezable();
1671 1677
1672 order = 0; 1678 order = 0;
1673 for ( ; ; ) { 1679 for ( ; ; ) {
1674 unsigned long new_order; 1680 unsigned long new_order;
1675 1681
1676 prepare_to_wait(&pgdat->kswap 1682 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
1677 new_order = pgdat->kswapd_max 1683 new_order = pgdat->kswapd_max_order;
1678 pgdat->kswapd_max_order = 0; 1684 pgdat->kswapd_max_order = 0;
1679 if (order < new_order) { 1685 if (order < new_order) {
1680 /* 1686 /*
1681 * Don't sleep if som 1687 * Don't sleep if someone wants a larger 'order'
1682 * allocation 1688 * allocation
1683 */ 1689 */
1684 order = new_order; 1690 order = new_order;
1685 } else { 1691 } else {
1686 if (!freezing(current 1692 if (!freezing(current))
1687 schedule(); 1693 schedule();
1688 1694
1689 order = pgdat->kswapd 1695 order = pgdat->kswapd_max_order;
1690 } 1696 }
1691 finish_wait(&pgdat->kswapd_wa 1697 finish_wait(&pgdat->kswapd_wait, &wait);
1692 1698
1693 if (!try_to_freeze()) { 1699 if (!try_to_freeze()) {
1694 /* We can speed up th 1700 /* We can speed up thawing tasks if we don't call
1695 * balance_pgdat afte 1701 * balance_pgdat after returning from the refrigerator
1696 */ 1702 */
1697 balance_pgdat(pgdat, 1703 balance_pgdat(pgdat, order);
1698 } 1704 }
1699 } 1705 }
1700 return 0; 1706 return 0;
1701 } 1707 }
1702 1708
1703 /* 1709 /*
1704 * A zone is low on free memory, so wake its 1710 * A zone is low on free memory, so wake its kswapd task to service it.
1705 */ 1711 */
1706 void wakeup_kswapd(struct zone *zone, int ord 1712 void wakeup_kswapd(struct zone *zone, int order)
1707 { 1713 {
1708 pg_data_t *pgdat; 1714 pg_data_t *pgdat;
1709 1715
1710 if (!populated_zone(zone)) 1716 if (!populated_zone(zone))
1711 return; 1717 return;
1712 1718
1713 pgdat = zone->zone_pgdat; 1719 pgdat = zone->zone_pgdat;
1714 if (zone_watermark_ok(zone, order, zo 1720 if (zone_watermark_ok(zone, order, zone->pages_low, 0, 0))
1715 return; 1721 return;
1716 if (pgdat->kswapd_max_order < order) 1722 if (pgdat->kswapd_max_order < order)
1717 pgdat->kswapd_max_order = ord 1723 pgdat->kswapd_max_order = order;
1718 if (!cpuset_zone_allowed_hardwall(zon 1724 if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
1719 return; 1725 return;
1720 if (!waitqueue_active(&pgdat->kswapd_ 1726 if (!waitqueue_active(&pgdat->kswapd_wait))
1721 return; 1727 return;
1722 wake_up_interruptible(&pgdat->kswapd_ 1728 wake_up_interruptible(&pgdat->kswapd_wait);
1723 } 1729 }
1724 1730
1725 #ifdef CONFIG_PM 1731 #ifdef CONFIG_PM
1726 /* 1732 /*
1727 * Helper function for shrink_all_memory(). 1733 * Helper function for shrink_all_memory(). Tries to reclaim 'nr_pages' pages
1728 * from LRU lists system-wide, for given pass 1734 * from LRU lists system-wide, for given pass and priority, and returns the
1729 * number of reclaimed pages 1735 * number of reclaimed pages
1730 * 1736 *
1731 * For pass > 3 we also try to shrink the LRU 1737 * For pass > 3 we also try to shrink the LRU lists that contain a few pages
1732 */ 1738 */
1733 static unsigned long shrink_all_zones(unsigne 1739 static unsigned long shrink_all_zones(unsigned long nr_pages, int prio,
1734 int pas 1740 int pass, struct scan_control *sc)
1735 { 1741 {
1736 struct zone *zone; 1742 struct zone *zone;
1737 unsigned long nr_to_scan, ret = 0; 1743 unsigned long nr_to_scan, ret = 0;
1738 1744
1739 for_each_zone(zone) { 1745 for_each_zone(zone) {
1740 1746
1741 if (!populated_zone(zone)) 1747 if (!populated_zone(zone))
1742 continue; 1748 continue;
1743 1749
1744 if (zone_is_all_unreclaimable 1750 if (zone_is_all_unreclaimable(zone) && prio != DEF_PRIORITY)
1745 continue; 1751 continue;
1746 1752
1747 /* For pass = 0 we don't shri 1753 /* For pass = 0 we don't shrink the active list */
1748 if (pass > 0) { 1754 if (pass > 0) {
1749 zone->nr_scan_active 1755 zone->nr_scan_active +=
1750 (zone_page_st 1756 (zone_page_state(zone, NR_ACTIVE) >> prio) + 1;
1751 if (zone->nr_scan_act 1757 if (zone->nr_scan_active >= nr_pages || pass > 3) {
1752 zone->nr_scan 1758 zone->nr_scan_active = 0;
1753 nr_to_scan = 1759 nr_to_scan = min(nr_pages,
1754 zone_ 1760 zone_page_state(zone, NR_ACTIVE));
1755 shrink_active 1761 shrink_active_list(nr_to_scan, zone, sc, prio);
1756 } 1762 }
1757 } 1763 }
1758 1764
1759 zone->nr_scan_inactive += 1765 zone->nr_scan_inactive +=
1760 (zone_page_state(zone 1766 (zone_page_state(zone, NR_INACTIVE) >> prio) + 1;
1761 if (zone->nr_scan_inactive >= 1767 if (zone->nr_scan_inactive >= nr_pages || pass > 3) {
1762 zone->nr_scan_inactiv 1768 zone->nr_scan_inactive = 0;
1763 nr_to_scan = min(nr_p 1769 nr_to_scan = min(nr_pages,
1764 zone_page_sta 1770 zone_page_state(zone, NR_INACTIVE));
1765 ret += shrink_inactiv 1771 ret += shrink_inactive_list(nr_to_scan, zone, sc);
1766 if (ret >= nr_pages) 1772 if (ret >= nr_pages)
1767 return ret; 1773 return ret;
1768 } 1774 }
1769 } 1775 }
1770 1776
1771 return ret; 1777 return ret;
1772 } 1778 }
1773 1779
1774 static unsigned long count_lru_pages(void) 1780 static unsigned long count_lru_pages(void)
1775 { 1781 {
1776 return global_page_state(NR_ACTIVE) + 1782 return global_page_state(NR_ACTIVE) + global_page_state(NR_INACTIVE);
1777 } 1783 }
1778 1784
1779 /* 1785 /*
1780 * Try to free `nr_pages' of memory, system-w 1786 * Try to free `nr_pages' of memory, system-wide, and return the number of
1781 * freed pages. 1787 * freed pages.
1782 * 1788 *
1783 * Rather than trying to age LRUs the aim is 1789 * Rather than trying to age LRUs the aim is to preserve the overall
1784 * LRU order by reclaiming preferentially 1790 * LRU order by reclaiming preferentially
1785 * inactive > active > active referenced > ac 1791 * inactive > active > active referenced > active mapped
1786 */ 1792 */
1787 unsigned long shrink_all_memory(unsigned long 1793 unsigned long shrink_all_memory(unsigned long nr_pages)
1788 { 1794 {
1789 unsigned long lru_pages, nr_slab; 1795 unsigned long lru_pages, nr_slab;
1790 unsigned long ret = 0; 1796 unsigned long ret = 0;
1791 int pass; 1797 int pass;
1792 struct reclaim_state reclaim_state; 1798 struct reclaim_state reclaim_state;
1793 struct scan_control sc = { 1799 struct scan_control sc = {
1794 .gfp_mask = GFP_KERNEL, 1800 .gfp_mask = GFP_KERNEL,
1795 .may_swap = 0, 1801 .may_swap = 0,
1796 .swap_cluster_max = nr_pages, 1802 .swap_cluster_max = nr_pages,
1797 .may_writepage = 1, 1803 .may_writepage = 1,
1798 .swappiness = vm_swappiness, 1804 .swappiness = vm_swappiness,
1799 .isolate_pages = isolate_page 1805 .isolate_pages = isolate_pages_global,
1800 }; 1806 };
1801 1807
1802 current->reclaim_state = &reclaim_sta 1808 current->reclaim_state = &reclaim_state;
1803 1809
1804 lru_pages = count_lru_pages(); 1810 lru_pages = count_lru_pages();
1805 nr_slab = global_page_state(NR_SLAB_R 1811 nr_slab = global_page_state(NR_SLAB_RECLAIMABLE);
1806 /* If slab caches are huge, it's bett 1812 /* If slab caches are huge, it's better to hit them first */
1807 while (nr_slab >= lru_pages) { 1813 while (nr_slab >= lru_pages) {
1808 reclaim_state.reclaimed_slab 1814 reclaim_state.reclaimed_slab = 0;
1809 shrink_slab(nr_pages, sc.gfp_ 1815 shrink_slab(nr_pages, sc.gfp_mask, lru_pages);
1810 if (!reclaim_state.reclaimed_ 1816 if (!reclaim_state.reclaimed_slab)
1811 break; 1817 break;
1812 1818
1813 ret += reclaim_state.reclaime 1819 ret += reclaim_state.reclaimed_slab;
1814 if (ret >= nr_pages) 1820 if (ret >= nr_pages)
1815 goto out; 1821 goto out;
1816 1822
1817 nr_slab -= reclaim_state.recl 1823 nr_slab -= reclaim_state.reclaimed_slab;
1818 } 1824 }
1819 1825
1820 /* 1826 /*
1821 * We try to shrink LRUs in 5 passes: 1827 * We try to shrink LRUs in 5 passes:
1822 * 0 = Reclaim from inactive_list onl 1828 * 0 = Reclaim from inactive_list only
1823 * 1 = Reclaim from active list but d 1829 * 1 = Reclaim from active list but don't reclaim mapped
1824 * 2 = 2nd pass of type 1 1830 * 2 = 2nd pass of type 1
1825 * 3 = Reclaim mapped (normal reclaim 1831 * 3 = Reclaim mapped (normal reclaim)
1826 * 4 = 2nd pass of type 3 1832 * 4 = 2nd pass of type 3
1827 */ 1833 */
1828 for (pass = 0; pass < 5; pass++) { 1834 for (pass = 0; pass < 5; pass++) {
1829 int prio; 1835 int prio;
1830 1836
1831 /* Force reclaiming mapped pa 1837 /* Force reclaiming mapped pages in the passes #3 and #4 */
1832 if (pass > 2) { 1838 if (pass > 2) {
1833 sc.may_swap = 1; 1839 sc.may_swap = 1;
1834 sc.swappiness = 100; 1840 sc.swappiness = 100;
1835 } 1841 }
1836 1842
1837 for (prio = DEF_PRIORITY; pri 1843 for (prio = DEF_PRIORITY; prio >= 0; prio--) {
1838 unsigned long nr_to_s 1844 unsigned long nr_to_scan = nr_pages - ret;
1839 1845
1840 sc.nr_scanned = 0; 1846 sc.nr_scanned = 0;
1841 ret += shrink_all_zon 1847 ret += shrink_all_zones(nr_to_scan, prio, pass, &sc);
1842 if (ret >= nr_pages) 1848 if (ret >= nr_pages)
1843 goto out; 1849 goto out;
1844 1850
1845 reclaim_state.reclaim 1851 reclaim_state.reclaimed_slab = 0;
1846 shrink_slab(sc.nr_sca 1852 shrink_slab(sc.nr_scanned, sc.gfp_mask,
1847 count 1853 count_lru_pages());
1848 ret += reclaim_state. 1854 ret += reclaim_state.reclaimed_slab;
1849 if (ret >= nr_pages) 1855 if (ret >= nr_pages)
1850 goto out; 1856 goto out;
1851 1857
1852 if (sc.nr_scanned && 1858 if (sc.nr_scanned && prio < DEF_PRIORITY - 2)
1853 congestion_wa 1859 congestion_wait(WRITE, HZ / 10);
1854 } 1860 }
1855 } 1861 }
1856 1862
1857 /* 1863 /*
1858 * If ret = 0, we could not shrink LR 1864 * If ret = 0, we could not shrink LRUs, but there may be something
1859 * in slab caches 1865 * in slab caches
1860 */ 1866 */
1861 if (!ret) { 1867 if (!ret) {
1862 do { 1868 do {
1863 reclaim_state.reclaim 1869 reclaim_state.reclaimed_slab = 0;
1864 shrink_slab(nr_pages, 1870 shrink_slab(nr_pages, sc.gfp_mask, count_lru_pages());
1865 ret += reclaim_state. 1871 ret += reclaim_state.reclaimed_slab;
1866 } while (ret < nr_pages && re 1872 } while (ret < nr_pages && reclaim_state.reclaimed_slab > 0);
1867 } 1873 }
1868 1874
1869 out: 1875 out:
1870 current->reclaim_state = NULL; 1876 current->reclaim_state = NULL;
1871 1877
1872 return ret; 1878 return ret;
1873 } 1879 }
1874 #endif 1880 #endif
1875 1881
1876 /* It's optimal to keep kswapds on the same C 1882 /* It's optimal to keep kswapds on the same CPUs as their memory, but
1877 not required for correctness. So if the l 1883 not required for correctness. So if the last cpu in a node goes
1878 away, we get changed to run anywhere: as t 1884 away, we get changed to run anywhere: as the first one comes back,
1879 restore their cpu bindings. */ 1885 restore their cpu bindings. */
1880 static int __devinit cpu_callback(struct noti 1886 static int __devinit cpu_callback(struct notifier_block *nfb,
1881 unsigned lo 1887 unsigned long action, void *hcpu)
1882 { 1888 {
1883 pg_data_t *pgdat; 1889 pg_data_t *pgdat;
1884 cpumask_t mask; 1890 cpumask_t mask;
1885 int nid; 1891 int nid;
1886 1892
1887 if (action == CPU_ONLINE || action == 1893 if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) {
1888 for_each_node_state(nid, N_HI 1894 for_each_node_state(nid, N_HIGH_MEMORY) {
1889 pgdat = NODE_DATA(nid 1895 pgdat = NODE_DATA(nid);
1890 mask = node_to_cpumas 1896 mask = node_to_cpumask(pgdat->node_id);
1891 if (any_online_cpu(ma 1897 if (any_online_cpu(mask) != NR_CPUS)
1892 /* One of our 1898 /* One of our CPUs online: restore mask */
1893 set_cpus_allo 1899 set_cpus_allowed(pgdat->kswapd, mask);
1894 } 1900 }
1895 } 1901 }
1896 return NOTIFY_OK; 1902 return NOTIFY_OK;
1897 } 1903 }
1898 1904
1899 /* 1905 /*
1900 * This kswapd start function will be called 1906 * This kswapd start function will be called by init and node-hot-add.
1901 * On node-hot-add, kswapd will moved to prop 1907 * On node-hot-add, kswapd will moved to proper cpus if cpus are hot-added.
1902 */ 1908 */
1903 int kswapd_run(int nid) 1909 int kswapd_run(int nid)
1904 { 1910 {
1905 pg_data_t *pgdat = NODE_DATA(nid); 1911 pg_data_t *pgdat = NODE_DATA(nid);
1906 int ret = 0; 1912 int ret = 0;
1907 1913
1908 if (pgdat->kswapd) 1914 if (pgdat->kswapd)
1909 return 0; 1915 return 0;
1910 1916
1911 pgdat->kswapd = kthread_run(kswapd, p 1917 pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);
1912 if (IS_ERR(pgdat->kswapd)) { 1918 if (IS_ERR(pgdat->kswapd)) {
1913 /* failure at boot is fatal * 1919 /* failure at boot is fatal */
1914 BUG_ON(system_state == SYSTEM 1920 BUG_ON(system_state == SYSTEM_BOOTING);
1915 printk("Failed to start kswap 1921 printk("Failed to start kswapd on node %d\n",nid);
1916 ret = -1; 1922 ret = -1;
1917 } 1923 }
1918 return ret; 1924 return ret;
1919 } 1925 }
1920 1926
1921 static int __init kswapd_init(void) 1927 static int __init kswapd_init(void)
1922 { 1928 {
1923 int nid; 1929 int nid;
1924 1930
1925 swap_setup(); 1931 swap_setup();
1926 for_each_node_state(nid, N_HIGH_MEMOR 1932 for_each_node_state(nid, N_HIGH_MEMORY)
1927 kswapd_run(nid); 1933 kswapd_run(nid);
1928 hotcpu_notifier(cpu_callback, 0); 1934 hotcpu_notifier(cpu_callback, 0);
1929 return 0; 1935 return 0;
1930 } 1936 }
1931 1937
1932 module_init(kswapd_init) 1938 module_init(kswapd_init)
1933 1939
1934 #ifdef CONFIG_NUMA 1940 #ifdef CONFIG_NUMA
1935 /* 1941 /*
1936 * Zone reclaim mode 1942 * Zone reclaim mode
1937 * 1943 *
1938 * If non-zero call zone_reclaim when the num 1944 * If non-zero call zone_reclaim when the number of free pages falls below
1939 * the watermarks. 1945 * the watermarks.
1940 */ 1946 */
1941 int zone_reclaim_mode __read_mostly; 1947 int zone_reclaim_mode __read_mostly;
1942 1948
1943 #define RECLAIM_OFF 0 1949 #define RECLAIM_OFF 0
1944 #define RECLAIM_ZONE (1<<0) /* Run shrink 1950 #define RECLAIM_ZONE (1<<0) /* Run shrink_cache on the zone */
1945 #define RECLAIM_WRITE (1<<1) /* Writeout p 1951 #define RECLAIM_WRITE (1<<1) /* Writeout pages during reclaim */
1946 #define RECLAIM_SWAP (1<<2) /* Swap pages 1952 #define RECLAIM_SWAP (1<<2) /* Swap pages out during reclaim */
1947 1953
1948 /* 1954 /*
1949 * Priority for ZONE_RECLAIM. This determines 1955 * Priority for ZONE_RECLAIM. This determines the fraction of pages
1950 * of a node considered for each zone_reclaim 1956 * of a node considered for each zone_reclaim. 4 scans 1/16th of
1951 * a zone. 1957 * a zone.
1952 */ 1958 */
1953 #define ZONE_RECLAIM_PRIORITY 4 1959 #define ZONE_RECLAIM_PRIORITY 4
1954 1960
1955 /* 1961 /*
1956 * Percentage of pages in a zone that must be 1962 * Percentage of pages in a zone that must be unmapped for zone_reclaim to
1957 * occur. 1963 * occur.
1958 */ 1964 */
1959 int sysctl_min_unmapped_ratio = 1; 1965 int sysctl_min_unmapped_ratio = 1;
1960 1966
1961 /* 1967 /*
1962 * If the number of slab pages in a zone grow 1968 * If the number of slab pages in a zone grows beyond this percentage then
1963 * slab reclaim needs to occur. 1969 * slab reclaim needs to occur.
1964 */ 1970 */
1965 int sysctl_min_slab_ratio = 5; 1971 int sysctl_min_slab_ratio = 5;
1966 1972
1967 /* 1973 /*
1968 * Try to free up some pages from this zone t 1974 * Try to free up some pages from this zone through reclaim.
1969 */ 1975 */
1970 static int __zone_reclaim(struct zone *zone, 1976 static int __zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
1971 { 1977 {
1972 /* Minimum pages needed in order to s 1978 /* Minimum pages needed in order to stay on node */
1973 const unsigned long nr_pages = 1 << o 1979 const unsigned long nr_pages = 1 << order;
1974 struct task_struct *p = current; 1980 struct task_struct *p = current;
1975 struct reclaim_state reclaim_state; 1981 struct reclaim_state reclaim_state;
1976 int priority; 1982 int priority;
1977 unsigned long nr_reclaimed = 0; 1983 unsigned long nr_reclaimed = 0;
1978 struct scan_control sc = { 1984 struct scan_control sc = {
1979 .may_writepage = !!(zone_recl 1985 .may_writepage = !!(zone_reclaim_mode & RECLAIM_WRITE),
1980 .may_swap = !!(zone_reclaim_m 1986 .may_swap = !!(zone_reclaim_mode & RECLAIM_SWAP),
1981 .swap_cluster_max = max_t(uns 1987 .swap_cluster_max = max_t(unsigned long, nr_pages,
1982 SWAP_ 1988 SWAP_CLUSTER_MAX),
1983 .gfp_mask = gfp_mask, 1989 .gfp_mask = gfp_mask,
1984 .swappiness = vm_swappiness, 1990 .swappiness = vm_swappiness,
1985 .isolate_pages = isolate_page 1991 .isolate_pages = isolate_pages_global,
1986 }; 1992 };
1987 unsigned long slab_reclaimable; 1993 unsigned long slab_reclaimable;
1988 1994
1989 disable_swap_token(); 1995 disable_swap_token();
1990 cond_resched(); 1996 cond_resched();
1991 /* 1997 /*
1992 * We need to be able to allocate fro 1998 * We need to be able to allocate from the reserves for RECLAIM_SWAP
1993 * and we also need to be able to wri 1999 * and we also need to be able to write out pages for RECLAIM_WRITE
1994 * and RECLAIM_SWAP. 2000 * and RECLAIM_SWAP.
1995 */ 2001 */
1996 p->flags |= PF_MEMALLOC | PF_SWAPWRIT 2002 p->flags |= PF_MEMALLOC | PF_SWAPWRITE;
1997 reclaim_state.reclaimed_slab = 0; 2003 reclaim_state.reclaimed_slab = 0;
1998 p->reclaim_state = &reclaim_state; 2004 p->reclaim_state = &reclaim_state;
1999 2005
2000 if (zone_page_state(zone, NR_FILE_PAG 2006 if (zone_page_state(zone, NR_FILE_PAGES) -
2001 zone_page_state(zone, NR_FILE 2007 zone_page_state(zone, NR_FILE_MAPPED) >
2002 zone->min_unmapped_pages) { 2008 zone->min_unmapped_pages) {
2003 /* 2009 /*
2004 * Free memory by calling shr 2010 * Free memory by calling shrink zone with increasing
2005 * priorities until we have e 2011 * priorities until we have enough memory freed.
2006 */ 2012 */
2007 priority = ZONE_RECLAIM_PRIOR 2013 priority = ZONE_RECLAIM_PRIORITY;
2008 do { 2014 do {
2009 note_zone_scanning_pr 2015 note_zone_scanning_priority(zone, priority);
2010 nr_reclaimed += shrin 2016 nr_reclaimed += shrink_zone(priority, zone, &sc);
2011 priority--; 2017 priority--;
2012 } while (priority >= 0 && nr_ 2018 } while (priority >= 0 && nr_reclaimed < nr_pages);
2013 } 2019 }
2014 2020
2015 slab_reclaimable = zone_page_state(zo 2021 slab_reclaimable = zone_page_state(zone, NR_SLAB_RECLAIMABLE);
2016 if (slab_reclaimable > zone->min_slab 2022 if (slab_reclaimable > zone->min_slab_pages) {
2017 /* 2023 /*
2018 * shrink_slab() does not cur 2024 * shrink_slab() does not currently allow us to determine how
2019 * many pages were freed in t 2025 * many pages were freed in this zone. So we take the current
2020 * number of slab pages and s 2026 * number of slab pages and shake the slab until it is reduced
2021 * by the same nr_pages that 2027 * by the same nr_pages that we used for reclaiming unmapped
2022 * pages. 2028 * pages.
2023 * 2029 *
2024 * Note that shrink_slab will 2030 * Note that shrink_slab will free memory on all zones and may
2025 * take a long time. 2031 * take a long time.
2026 */ 2032 */
2027 while (shrink_slab(sc.nr_scan 2033 while (shrink_slab(sc.nr_scanned, gfp_mask, order) &&
2028 zone_page_state(zone, 2034 zone_page_state(zone, NR_SLAB_RECLAIMABLE) >
2029 slab_reclaima 2035 slab_reclaimable - nr_pages)
2030 ; 2036 ;
2031 2037
2032 /* 2038 /*
2033 * Update nr_reclaimed by the 2039 * Update nr_reclaimed by the number of slab pages we
2034 * reclaimed from this zone. 2040 * reclaimed from this zone.
2035 */ 2041 */
2036 nr_reclaimed += slab_reclaima 2042 nr_reclaimed += slab_reclaimable -
2037 zone_page_state(zone, 2043 zone_page_state(zone, NR_SLAB_RECLAIMABLE);
2038 } 2044 }
2039 2045
2040 p->reclaim_state = NULL; 2046 p->reclaim_state = NULL;
2041 current->flags &= ~(PF_MEMALLOC | PF_ 2047 current->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE);
2042 return nr_reclaimed >= nr_pages; 2048 return nr_reclaimed >= nr_pages;
2043 } 2049 }
2044 2050
2045 int zone_reclaim(struct zone *zone, gfp_t gfp 2051 int zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
2046 { 2052 {
2047 int node_id; 2053 int node_id;
2048 int ret; 2054 int ret;
2049 2055
2050 /* 2056 /*
2051 * Zone reclaim reclaims unmapped fil 2057 * Zone reclaim reclaims unmapped file backed pages and
2052 * slab pages if we are over the defi 2058 * slab pages if we are over the defined limits.
2053 * 2059 *
2054 * A small portion of unmapped file b 2060 * A small portion of unmapped file backed pages is needed for
2055 * file I/O otherwise pages read by f 2061 * file I/O otherwise pages read by file I/O will be immediately
2056 * thrown out if the zone is overallo 2062 * thrown out if the zone is overallocated. So we do not reclaim
2057 * if less than a specified percentag 2063 * if less than a specified percentage of the zone is used by
2058 * unmapped file backed pages. 2064 * unmapped file backed pages.
2059 */ 2065 */
2060 if (zone_page_state(zone, NR_FILE_PAG 2066 if (zone_page_state(zone, NR_FILE_PAGES) -
2061 zone_page_state(zone, NR_FILE_MAP 2067 zone_page_state(zone, NR_FILE_MAPPED) <= zone->min_unmapped_pages
2062 && zone_page_state(zone, NR_SLAB_ 2068 && zone_page_state(zone, NR_SLAB_RECLAIMABLE)
2063 <= zone->min_slab_pag 2069 <= zone->min_slab_pages)
2064 return 0; 2070 return 0;
2065 2071
2066 if (zone_is_all_unreclaimable(zone)) 2072 if (zone_is_all_unreclaimable(zone))
2067 return 0; 2073 return 0;
2068 2074
2069 /* 2075 /*
2070 * Do not scan if the allocation shou 2076 * Do not scan if the allocation should not be delayed.
2071 */ 2077 */
2072 if (!(gfp_mask & __GFP_WAIT) || (curr 2078 if (!(gfp_mask & __GFP_WAIT) || (current->flags & PF_MEMALLOC))
2073 return 0; 2079 return 0;
2074 2080
2075 /* 2081 /*
2076 * Only run zone reclaim on the local 2082 * Only run zone reclaim on the local zone or on zones that do not
2077 * have associated processors. This w 2083 * have associated processors. This will favor the local processor
2078 * over remote processors and spread 2084 * over remote processors and spread off node memory allocations
2079 * as wide as possible. 2085 * as wide as possible.
2080 */ 2086 */
2081 node_id = zone_to_nid(zone); 2087 node_id = zone_to_nid(zone);
2082 if (node_state(node_id, N_CPU) && nod 2088 if (node_state(node_id, N_CPU) && node_id != numa_node_id())
2083 return 0; 2089 return 0;
2084 2090
2085 if (zone_test_and_set_flag(zone, ZONE 2091 if (zone_test_and_set_flag(zone, ZONE_RECLAIM_LOCKED))
2086 return 0; 2092 return 0;
2087 ret = __zone_reclaim(zone, gfp_mask, 2093 ret = __zone_reclaim(zone, gfp_mask, order);
2088 zone_clear_flag(zone, ZONE_RECLAIM_LO 2094 zone_clear_flag(zone, ZONE_RECLAIM_LOCKED);
2089 2095
2090 return ret; 2096 return ret;
2091 } 2097 }
2092 #endif 2098 #endif
2093 2099
|
This page was automatically generated by the
LXR engine.
|