1 #include <linux/mm.h>
2 #include <linux/highmem.h>
3 #include <linux/sched.h>
4 #include <linux/hugetlb.h>
5
6 static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
7 struct mm_walk *walk)
8 {
9 pte_t *pte;
10 int err = 0;
11
12 pte = pte_offset_map(pmd, addr);
13 for (;;) {
14 err = walk->pte_entry(pte, addr, addr + PAGE_SIZE, walk);
15 if (err)
16 break;
17 addr += PAGE_SIZE;
18 if (addr == end)
19 break;
20 pte++;
21 }
22
23 pte_unmap(pte);
24 return err;
25 }
26
27 static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
28 struct mm_walk *walk)
29 {
30 pmd_t *pmd;
31 unsigned long next;
32 int err = 0;
33
34 pmd = pmd_offset(pud, addr);
35 do {
36 next = pmd_addr_end(addr, end);
37 if (pmd_none_or_clear_bad(pmd)) {
38 if (walk->pte_hole)
39 err = walk->pte_hole(addr, next, walk);
40 if (err)
41 break;
42 continue;
43 }
44 if (walk->pmd_entry)
45 err = walk->pmd_entry(pmd, addr, next, walk);
46 if (!err && walk->pte_entry)
47 err = walk_pte_range(pmd, addr, next, walk);
48 if (err)
49 break;
50 } while (pmd++, addr = next, addr != end);
51
52 return err;
53 }
54
55 static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end,
56 struct mm_walk *walk)
57 {
58 pud_t *pud;
59 unsigned long next;
60 int err = 0;
61
62 pud = pud_offset(pgd, addr);
63 do {
64 next = pud_addr_end(addr, end);
65 if (pud_none_or_clear_bad(pud)) {
66 if (walk->pte_hole)
67 err = walk->pte_hole(addr, next, walk);
68 if (err)
69 break;
70 continue;
71 }
72 if (walk->pud_entry)
73 err = walk->pud_entry(pud, addr, next, walk);
74 if (!err && (walk->pmd_entry || walk->pte_entry))
75 err = walk_pmd_range(pud, addr, next, walk);
76 if (err)
77 break;
78 } while (pud++, addr = next, addr != end);
79
80 return err;
81 }
82
83 /**
84 * walk_page_range - walk a memory map's page tables with a callback
85 * @mm: memory map to walk
86 * @addr: starting address
87 * @end: ending address
88 * @walk: set of callbacks to invoke for each level of the tree
89 *
90 * Recursively walk the page table for the memory area in a VMA,
91 * calling supplied callbacks. Callbacks are called in-order (first
92 * PGD, first PUD, first PMD, first PTE, second PTE... second PMD,
93 * etc.). If lower-level callbacks are omitted, walking depth is reduced.
94 *
95 * Each callback receives an entry pointer and the start and end of the
96 * associated range, and a copy of the original mm_walk for access to
97 * the ->private or ->mm fields.
98 *
99 * No locks are taken, but the bottom level iterator will map PTE
100 * directories from highmem if necessary.
101 *
102 * If any callback returns a non-zero value, the walk is aborted and
103 * the return value is propagated back to the caller. Otherwise 0 is returned.
104 */
105 int walk_page_range(unsigned long addr, unsigned long end,
106 struct mm_walk *walk)
107 {
108 pgd_t *pgd;
109 unsigned long next;
110 int err = 0;
111 struct vm_area_struct *vma;
112
113 if (addr >= end)
114 return err;
115
116 if (!walk->mm)
117 return -EINVAL;
118
119 pgd = pgd_offset(walk->mm, addr);
120 do {
121 next = pgd_addr_end(addr, end);
122
123 /* skip hugetlb vma to avoid hugepage PMD being cleared
124 * in pmd_none_or_clear_bad(). */
125 vma = find_vma(walk->mm, addr);
126 if (vma && is_vm_hugetlb_page(vma)) {
127 if (vma->vm_end < next)
128 next = vma->vm_end;
129 continue;
130 }
131
132 if (pgd_none_or_clear_bad(pgd)) {
133 if (walk->pte_hole)
134 err = walk->pte_hole(addr, next, walk);
135 if (err)
136 break;
137 pgd++;
138 continue;
139 }
140 if (walk->pgd_entry)
141 err = walk->pgd_entry(pgd, addr, next, walk);
142 if (!err &&
143 (walk->pud_entry || walk->pmd_entry || walk->pte_entry))
144 err = walk_pud_range(pgd, addr, next, walk);
145 if (err)
146 break;
147 pgd++;
148 } while (addr = next, addr != end);
149
150 return err;
151 }
152
|
This page was automatically generated by the
LXR engine.
|