1 /*
2 * cpuidle.c - core cpuidle infrastructure
3 *
4 * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
5 * Shaohua Li <shaohua.li@intel.com>
6 * Adam Belay <abelay@novell.com>
7 *
8 * This code is licenced under the GPL.
9 */
10
11 #include <linux/kernel.h>
12 #include <linux/mutex.h>
13 #include <linux/sched.h>
14 #include <linux/notifier.h>
15 #include <linux/pm_qos_params.h>
16 #include <linux/cpu.h>
17 #include <linux/cpuidle.h>
18 #include <linux/ktime.h>
19 #include <linux/hrtimer.h>
20
21 #include "cpuidle.h"
22
23 DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices);
24
25 DEFINE_MUTEX(cpuidle_lock);
26 LIST_HEAD(cpuidle_detected_devices);
27 static void (*pm_idle_old)(void);
28
29 static int enabled_devices;
30
31 #if defined(CONFIG_ARCH_HAS_CPU_IDLE_WAIT)
32 static void cpuidle_kick_cpus(void)
33 {
34 cpu_idle_wait();
35 }
36 #elif defined(CONFIG_SMP)
37 # error "Arch needs cpu_idle_wait() equivalent here"
38 #else /* !CONFIG_ARCH_HAS_CPU_IDLE_WAIT && !CONFIG_SMP */
39 static void cpuidle_kick_cpus(void) {}
40 #endif
41
42 static int __cpuidle_register_device(struct cpuidle_device *dev);
43
44 /**
45 * cpuidle_idle_call - the main idle loop
46 *
47 * NOTE: no locks or semaphores should be used here
48 */
49 static void cpuidle_idle_call(void)
50 {
51 struct cpuidle_device *dev = __get_cpu_var(cpuidle_devices);
52 struct cpuidle_state *target_state;
53 int next_state;
54
55 /* check if the device is ready */
56 if (!dev || !dev->enabled) {
57 if (pm_idle_old)
58 pm_idle_old();
59 else
60 #if defined(CONFIG_ARCH_HAS_DEFAULT_IDLE)
61 default_idle();
62 #else
63 local_irq_enable();
64 #endif
65 return;
66 }
67
68 #if 0
69 /* shows regressions, re-enable for 2.6.29 */
70 /*
71 * run any timers that can be run now, at this point
72 * before calculating the idle duration etc.
73 */
74 hrtimer_peek_ahead_timers();
75 #endif
76 /* ask the governor for the next state */
77 next_state = cpuidle_curr_governor->select(dev);
78 if (need_resched()) {
79 local_irq_enable();
80 return;
81 }
82
83 target_state = &dev->states[next_state];
84
85 /* enter the state and update stats */
86 dev->last_state = target_state;
87 dev->last_residency = target_state->enter(dev, target_state);
88 if (dev->last_state)
89 target_state = dev->last_state;
90
91 target_state->time += (unsigned long long)dev->last_residency;
92 target_state->usage++;
93
94 /* give the governor an opportunity to reflect on the outcome */
95 if (cpuidle_curr_governor->reflect)
96 cpuidle_curr_governor->reflect(dev);
97 }
98
99 /**
100 * cpuidle_install_idle_handler - installs the cpuidle idle loop handler
101 */
102 void cpuidle_install_idle_handler(void)
103 {
104 if (enabled_devices && (pm_idle != cpuidle_idle_call)) {
105 /* Make sure all changes finished before we switch to new idle */
106 smp_wmb();
107 pm_idle = cpuidle_idle_call;
108 }
109 }
110
111 /**
112 * cpuidle_uninstall_idle_handler - uninstalls the cpuidle idle loop handler
113 */
114 void cpuidle_uninstall_idle_handler(void)
115 {
116 if (enabled_devices && pm_idle_old && (pm_idle != pm_idle_old)) {
117 pm_idle = pm_idle_old;
118 cpuidle_kick_cpus();
119 }
120 }
121
122 /**
123 * cpuidle_pause_and_lock - temporarily disables CPUIDLE
124 */
125 void cpuidle_pause_and_lock(void)
126 {
127 mutex_lock(&cpuidle_lock);
128 cpuidle_uninstall_idle_handler();
129 }
130
131 EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock);
132
133 /**
134 * cpuidle_resume_and_unlock - resumes CPUIDLE operation
135 */
136 void cpuidle_resume_and_unlock(void)
137 {
138 cpuidle_install_idle_handler();
139 mutex_unlock(&cpuidle_lock);
140 }
141
142 EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock);
143
144 /**
145 * cpuidle_enable_device - enables idle PM for a CPU
146 * @dev: the CPU
147 *
148 * This function must be called between cpuidle_pause_and_lock and
149 * cpuidle_resume_and_unlock when used externally.
150 */
151 int cpuidle_enable_device(struct cpuidle_device *dev)
152 {
153 int ret, i;
154
155 if (dev->enabled)
156 return 0;
157 if (!cpuidle_curr_driver || !cpuidle_curr_governor)
158 return -EIO;
159 if (!dev->state_count)
160 return -EINVAL;
161
162 if (dev->registered == 0) {
163 ret = __cpuidle_register_device(dev);
164 if (ret)
165 return ret;
166 }
167
168 if ((ret = cpuidle_add_state_sysfs(dev)))
169 return ret;
170
171 if (cpuidle_curr_governor->enable &&
172 (ret = cpuidle_curr_governor->enable(dev)))
173 goto fail_sysfs;
174
175 for (i = 0; i < dev->state_count; i++) {
176 dev->states[i].usage = 0;
177 dev->states[i].time = 0;
178 }
179 dev->last_residency = 0;
180 dev->last_state = NULL;
181
182 smp_wmb();
183
184 dev->enabled = 1;
185
186 enabled_devices++;
187 return 0;
188
189 fail_sysfs:
190 cpuidle_remove_state_sysfs(dev);
191
192 return ret;
193 }
194
195 EXPORT_SYMBOL_GPL(cpuidle_enable_device);
196
197 /**
198 * cpuidle_disable_device - disables idle PM for a CPU
199 * @dev: the CPU
200 *
201 * This function must be called between cpuidle_pause_and_lock and
202 * cpuidle_resume_and_unlock when used externally.
203 */
204 void cpuidle_disable_device(struct cpuidle_device *dev)
205 {
206 if (!dev->enabled)
207 return;
208 if (!cpuidle_curr_driver || !cpuidle_curr_governor)
209 return;
210
211 dev->enabled = 0;
212
213 if (cpuidle_curr_governor->disable)
214 cpuidle_curr_governor->disable(dev);
215
216 cpuidle_remove_state_sysfs(dev);
217 enabled_devices--;
218 }
219
220 EXPORT_SYMBOL_GPL(cpuidle_disable_device);
221
222 #ifdef CONFIG_ARCH_HAS_CPU_RELAX
223 static int poll_idle(struct cpuidle_device *dev, struct cpuidle_state *st)
224 {
225 ktime_t t1, t2;
226 s64 diff;
227 int ret;
228
229 t1 = ktime_get();
230 local_irq_enable();
231 while (!need_resched())
232 cpu_relax();
233
234 t2 = ktime_get();
235 diff = ktime_to_us(ktime_sub(t2, t1));
236 if (diff > INT_MAX)
237 diff = INT_MAX;
238
239 ret = (int) diff;
240 return ret;
241 }
242
243 static void poll_idle_init(struct cpuidle_device *dev)
244 {
245 struct cpuidle_state *state = &dev->states[0];
246
247 cpuidle_set_statedata(state, NULL);
248
249 snprintf(state->name, CPUIDLE_NAME_LEN, "C0");
250 snprintf(state->desc, CPUIDLE_DESC_LEN, "CPUIDLE CORE POLL IDLE");
251 state->exit_latency = 0;
252 state->target_residency = 0;
253 state->power_usage = -1;
254 state->flags = CPUIDLE_FLAG_POLL;
255 state->enter = poll_idle;
256 }
257 #else
258 static void poll_idle_init(struct cpuidle_device *dev) {}
259 #endif /* CONFIG_ARCH_HAS_CPU_RELAX */
260
261 /**
262 * __cpuidle_register_device - internal register function called before register
263 * and enable routines
264 * @dev: the cpu
265 *
266 * cpuidle_lock mutex must be held before this is called
267 */
268 static int __cpuidle_register_device(struct cpuidle_device *dev)
269 {
270 int ret;
271 struct sys_device *sys_dev = get_cpu_sysdev((unsigned long)dev->cpu);
272
273 if (!sys_dev)
274 return -EINVAL;
275 if (!try_module_get(cpuidle_curr_driver->owner))
276 return -EINVAL;
277
278 init_completion(&dev->kobj_unregister);
279
280 poll_idle_init(dev);
281
282 per_cpu(cpuidle_devices, dev->cpu) = dev;
283 list_add(&dev->device_list, &cpuidle_detected_devices);
284 if ((ret = cpuidle_add_sysfs(sys_dev))) {
285 module_put(cpuidle_curr_driver->owner);
286 return ret;
287 }
288
289 dev->registered = 1;
290 return 0;
291 }
292
293 /**
294 * cpuidle_register_device - registers a CPU's idle PM feature
295 * @dev: the cpu
296 */
297 int cpuidle_register_device(struct cpuidle_device *dev)
298 {
299 int ret;
300
301 mutex_lock(&cpuidle_lock);
302
303 if ((ret = __cpuidle_register_device(dev))) {
304 mutex_unlock(&cpuidle_lock);
305 return ret;
306 }
307
308 cpuidle_enable_device(dev);
309 cpuidle_install_idle_handler();
310
311 mutex_unlock(&cpuidle_lock);
312
313 return 0;
314
315 }
316
317 EXPORT_SYMBOL_GPL(cpuidle_register_device);
318
319 /**
320 * cpuidle_unregister_device - unregisters a CPU's idle PM feature
321 * @dev: the cpu
322 */
323 void cpuidle_unregister_device(struct cpuidle_device *dev)
324 {
325 struct sys_device *sys_dev = get_cpu_sysdev((unsigned long)dev->cpu);
326
327 if (dev->registered == 0)
328 return;
329
330 cpuidle_pause_and_lock();
331
332 cpuidle_disable_device(dev);
333
334 cpuidle_remove_sysfs(sys_dev);
335 list_del(&dev->device_list);
336 wait_for_completion(&dev->kobj_unregister);
337 per_cpu(cpuidle_devices, dev->cpu) = NULL;
338
339 cpuidle_resume_and_unlock();
340
341 module_put(cpuidle_curr_driver->owner);
342 }
343
344 EXPORT_SYMBOL_GPL(cpuidle_unregister_device);
345
346 #ifdef CONFIG_SMP
347
348 static void smp_callback(void *v)
349 {
350 /* we already woke the CPU up, nothing more to do */
351 }
352
353 /*
354 * This function gets called when a part of the kernel has a new latency
355 * requirement. This means we need to get all processors out of their C-state,
356 * and then recalculate a new suitable C-state. Just do a cross-cpu IPI; that
357 * wakes them all right up.
358 */
359 static int cpuidle_latency_notify(struct notifier_block *b,
360 unsigned long l, void *v)
361 {
362 smp_call_function(smp_callback, NULL, 1);
363 return NOTIFY_OK;
364 }
365
366 static struct notifier_block cpuidle_latency_notifier = {
367 .notifier_call = cpuidle_latency_notify,
368 };
369
370 static inline void latency_notifier_init(struct notifier_block *n)
371 {
372 pm_qos_add_notifier(PM_QOS_CPU_DMA_LATENCY, n);
373 }
374
375 #else /* CONFIG_SMP */
376
377 #define latency_notifier_init(x) do { } while (0)
378
379 #endif /* CONFIG_SMP */
380
381 /**
382 * cpuidle_init - core initializer
383 */
384 static int __init cpuidle_init(void)
385 {
386 int ret;
387
388 pm_idle_old = pm_idle;
389
390 ret = cpuidle_add_class_sysfs(&cpu_sysdev_class);
391 if (ret)
392 return ret;
393
394 latency_notifier_init(&cpuidle_latency_notifier);
395
396 return 0;
397 }
398
399 core_initcall(cpuidle_init);
400
|
This page was automatically generated by the
LXR engine.
|