1 /*
2 * linux/kernel/time/tick-sched.c
3 *
4 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
5 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
6 * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
7 *
8 * No idle tick implementation for low and high resolution timers
9 *
10 * Started by: Thomas Gleixner and Ingo Molnar
11 *
12 * Distribute under GPLv2.
13 */
14 #include <linux/cpu.h>
15 #include <linux/err.h>
16 #include <linux/hrtimer.h>
17 #include <linux/interrupt.h>
18 #include <linux/kernel_stat.h>
19 #include <linux/percpu.h>
20 #include <linux/profile.h>
21 #include <linux/sched.h>
22 #include <linux/tick.h>
23
24 #include <asm/irq_regs.h>
25
26 #include "tick-internal.h"
27
28 /*
29 * Per cpu nohz control structure
30 */
31 static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
32
33 /*
34 * The time, when the last jiffy update happened. Protected by xtime_lock.
35 */
36 static ktime_t last_jiffies_update;
37
38 struct tick_sched *tick_get_tick_sched(int cpu)
39 {
40 return &per_cpu(tick_cpu_sched, cpu);
41 }
42
43 /*
44 * Must be called with interrupts disabled !
45 */
46 static void tick_do_update_jiffies64(ktime_t now)
47 {
48 unsigned long ticks = 0;
49 ktime_t delta;
50
51 /* Reevalute with xtime_lock held */
52 write_seqlock(&xtime_lock);
53
54 delta = ktime_sub(now, last_jiffies_update);
55 if (delta.tv64 >= tick_period.tv64) {
56
57 delta = ktime_sub(delta, tick_period);
58 last_jiffies_update = ktime_add(last_jiffies_update,
59 tick_period);
60
61 /* Slow path for long timeouts */
62 if (unlikely(delta.tv64 >= tick_period.tv64)) {
63 s64 incr = ktime_to_ns(tick_period);
64
65 ticks = ktime_divns(delta, incr);
66
67 last_jiffies_update = ktime_add_ns(last_jiffies_update,
68 incr * ticks);
69 }
70 do_timer(++ticks);
71 }
72 write_sequnlock(&xtime_lock);
73 }
74
75 /*
76 * Initialize and return retrieve the jiffies update.
77 */
78 static ktime_t tick_init_jiffy_update(void)
79 {
80 ktime_t period;
81
82 write_seqlock(&xtime_lock);
83 /* Did we start the jiffies update yet ? */
84 if (last_jiffies_update.tv64 == 0)
85 last_jiffies_update = tick_next_period;
86 period = last_jiffies_update;
87 write_sequnlock(&xtime_lock);
88 return period;
89 }
90
91 /*
92 * NOHZ - aka dynamic tick functionality
93 */
94 #ifdef CONFIG_NO_HZ
95 /*
96 * NO HZ enabled ?
97 */
98 static int tick_nohz_enabled __read_mostly = 1;
99
100 /*
101 * Enable / Disable tickless mode
102 */
103 static int __init setup_tick_nohz(char *str)
104 {
105 if (!strcmp(str, "off"))
106 tick_nohz_enabled = 0;
107 else if (!strcmp(str, "on"))
108 tick_nohz_enabled = 1;
109 else
110 return 0;
111 return 1;
112 }
113
114 __setup("nohz=", setup_tick_nohz);
115
116 /**
117 * tick_nohz_update_jiffies - update jiffies when idle was interrupted
118 *
119 * Called from interrupt entry when the CPU was idle
120 *
121 * In case the sched_tick was stopped on this CPU, we have to check if jiffies
122 * must be updated. Otherwise an interrupt handler could use a stale jiffy
123 * value. We do this unconditionally on any cpu, as we don't know whether the
124 * cpu, which has the update task assigned is in a long sleep.
125 */
126 void tick_nohz_update_jiffies(void)
127 {
128 int cpu = smp_processor_id();
129 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
130 unsigned long flags;
131 ktime_t now;
132
133 if (!ts->tick_stopped)
134 return;
135
136 touch_softlockup_watchdog();
137
138 cpu_clear(cpu, nohz_cpu_mask);
139 now = ktime_get();
140 ts->idle_waketime = now;
141
142 local_irq_save(flags);
143 tick_do_update_jiffies64(now);
144 local_irq_restore(flags);
145 }
146
147 void tick_nohz_stop_idle(int cpu)
148 {
149 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
150
151 if (ts->idle_active) {
152 ktime_t now, delta;
153 now = ktime_get();
154 delta = ktime_sub(now, ts->idle_entrytime);
155 ts->idle_lastupdate = now;
156 ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
157 ts->idle_active = 0;
158 }
159 }
160
161 static ktime_t tick_nohz_start_idle(int cpu)
162 {
163 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
164 ktime_t now, delta;
165
166 now = ktime_get();
167 if (ts->idle_active) {
168 delta = ktime_sub(now, ts->idle_entrytime);
169 ts->idle_lastupdate = now;
170 ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
171 }
172 ts->idle_entrytime = now;
173 ts->idle_active = 1;
174 return now;
175 }
176
177 u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
178 {
179 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
180
181 *last_update_time = ktime_to_us(ts->idle_lastupdate);
182 return ktime_to_us(ts->idle_sleeptime);
183 }
184
185 /**
186 * tick_nohz_stop_sched_tick - stop the idle tick from the idle task
187 *
188 * When the next event is more than a tick into the future, stop the idle tick
189 * Called either from the idle loop or from irq_exit() when an idle period was
190 * just interrupted by an interrupt which did not cause a reschedule.
191 */
192 void tick_nohz_stop_sched_tick(void)
193 {
194 unsigned long seq, last_jiffies, next_jiffies, delta_jiffies, flags;
195 unsigned long rt_jiffies;
196 struct tick_sched *ts;
197 ktime_t last_update, expires, now;
198 struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
199 int cpu;
200
201 local_irq_save(flags);
202
203 cpu = smp_processor_id();
204 now = tick_nohz_start_idle(cpu);
205 ts = &per_cpu(tick_cpu_sched, cpu);
206
207 /*
208 * If this cpu is offline and it is the one which updates
209 * jiffies, then give up the assignment and let it be taken by
210 * the cpu which runs the tick timer next. If we don't drop
211 * this here the jiffies might be stale and do_timer() never
212 * invoked.
213 */
214 if (unlikely(!cpu_online(cpu))) {
215 if (cpu == tick_do_timer_cpu)
216 tick_do_timer_cpu = -1;
217 }
218
219 if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
220 goto end;
221
222 if (need_resched() || need_resched_delayed())
223 goto end;
224
225 cpu = smp_processor_id();
226
227 #ifndef CONFIG_PREEMPT_RT
228 if (unlikely(local_softirq_pending())) {
229 static int ratelimit;
230
231 if (ratelimit < 10) {
232 printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
233 local_softirq_pending());
234 ratelimit++;
235 }
236 }
237 #endif
238
239 ts->idle_calls++;
240 /* Read jiffies and the time when jiffies were updated last */
241 do {
242 seq = read_seqbegin(&xtime_lock);
243 last_update = last_jiffies_update;
244 last_jiffies = jiffies;
245 } while (read_seqretry(&xtime_lock, seq));
246
247 /* Get the next timer wheel timer */
248 next_jiffies = get_next_timer_interrupt(last_jiffies);
249 delta_jiffies = next_jiffies - last_jiffies;
250
251 rt_jiffies = rt_needs_cpu(cpu);
252 if (rt_jiffies && rt_jiffies < delta_jiffies)
253 delta_jiffies = rt_jiffies;
254
255 if (rcu_needs_cpu(cpu))
256 delta_jiffies = 1;
257 /*
258 * Do not stop the tick, if we are only one off
259 * or if the cpu is required for rcu
260 */
261 if (!ts->tick_stopped && delta_jiffies == 1)
262 goto out;
263
264 /* Schedule the tick, if we are at least one jiffie off */
265 if ((long)delta_jiffies >= 1) {
266
267 if (delta_jiffies > 1)
268 cpu_set(cpu, nohz_cpu_mask);
269 /*
270 * nohz_stop_sched_tick can be called several times before
271 * the nohz_restart_sched_tick is called. This happens when
272 * interrupts arrive which do not cause a reschedule. In the
273 * first call we save the current tick time, so we can restart
274 * the scheduler tick in nohz_restart_sched_tick.
275 */
276 if (!ts->tick_stopped) {
277 if (select_nohz_load_balancer(1)) {
278 /*
279 * sched tick not stopped!
280 */
281 cpu_clear(cpu, nohz_cpu_mask);
282 goto out;
283 }
284
285 ts->idle_tick = ts->sched_timer.expires;
286 ts->tick_stopped = 1;
287 ts->idle_jiffies = last_jiffies;
288 rcu_enter_nohz();
289 }
290
291 /*
292 * If this cpu is the one which updates jiffies, then
293 * give up the assignment and let it be taken by the
294 * cpu which runs the tick timer next, which might be
295 * this cpu as well. If we don't drop this here the
296 * jiffies might be stale and do_timer() never
297 * invoked.
298 */
299 if (cpu == tick_do_timer_cpu)
300 tick_do_timer_cpu = -1;
301
302 ts->idle_sleeps++;
303
304 /*
305 * delta_jiffies >= NEXT_TIMER_MAX_DELTA signals that
306 * there is no timer pending or at least extremly far
307 * into the future (12 days for HZ=1000). In this case
308 * we simply stop the tick timer:
309 */
310 if (unlikely(delta_jiffies >= NEXT_TIMER_MAX_DELTA)) {
311 ts->idle_expires.tv64 = KTIME_MAX;
312 if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
313 hrtimer_cancel(&ts->sched_timer);
314 goto out;
315 }
316
317 /*
318 * calculate the expiry time for the next timer wheel
319 * timer
320 */
321 expires = ktime_add_ns(last_update, tick_period.tv64 *
322 delta_jiffies);
323 ts->idle_expires = expires;
324
325 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
326 hrtimer_start(&ts->sched_timer, expires,
327 HRTIMER_MODE_ABS);
328 /* Check, if the timer was already in the past */
329 if (hrtimer_active(&ts->sched_timer))
330 goto out;
331 } else if (!tick_program_event(expires, 0))
332 goto out;
333 /*
334 * We are past the event already. So we crossed a
335 * jiffie boundary. Update jiffies and raise the
336 * softirq.
337 */
338 tick_do_update_jiffies64(ktime_get());
339 cpu_clear(cpu, nohz_cpu_mask);
340 }
341 raise_softirq_irqoff(TIMER_SOFTIRQ);
342 out:
343 ts->next_jiffies = next_jiffies;
344 ts->last_jiffies = last_jiffies;
345 ts->sleep_length = ktime_sub(dev->next_event, now);
346 end:
347 local_irq_restore(flags);
348 }
349
350 /**
351 * tick_nohz_get_sleep_length - return the length of the current sleep
352 *
353 * Called from power state control code with interrupts disabled
354 */
355 ktime_t tick_nohz_get_sleep_length(void)
356 {
357 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
358
359 return ts->sleep_length;
360 }
361
362 /**
363 * tick_nohz_restart_sched_tick - restart the idle tick from the idle task
364 *
365 * Restart the idle tick when the CPU is woken up from idle
366 */
367 void tick_nohz_restart_sched_tick(void)
368 {
369 int cpu = smp_processor_id();
370 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
371 unsigned long ticks;
372 ktime_t now;
373
374 local_irq_disable();
375 tick_nohz_stop_idle(cpu);
376
377 if (!ts->tick_stopped) {
378 local_irq_enable();
379 return;
380 }
381
382 rcu_exit_nohz();
383
384 /* Update jiffies first */
385 select_nohz_load_balancer(0);
386 now = ktime_get();
387 tick_do_update_jiffies64(now);
388 cpu_clear(cpu, nohz_cpu_mask);
389
390 /*
391 * We stopped the tick in idle. Update process times would miss the
392 * time we slept as update_process_times does only a 1 tick
393 * accounting. Enforce that this is accounted to idle !
394 */
395 ticks = jiffies - ts->idle_jiffies;
396 /*
397 * We might be one off. Do not randomly account a huge number of ticks!
398 */
399 if (ticks && ticks < LONG_MAX) {
400 add_preempt_count(HARDIRQ_OFFSET);
401 account_system_time(current, HARDIRQ_OFFSET,
402 jiffies_to_cputime(ticks));
403 sub_preempt_count(HARDIRQ_OFFSET);
404 }
405
406 /*
407 * Cancel the scheduled timer and restore the tick
408 */
409 ts->tick_stopped = 0;
410 ts->idle_exittime = now;
411 hrtimer_cancel(&ts->sched_timer);
412 ts->sched_timer.expires = ts->idle_tick;
413
414 while (1) {
415 /* Forward the time to expire in the future */
416 hrtimer_forward(&ts->sched_timer, now, tick_period);
417
418 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
419 hrtimer_start(&ts->sched_timer,
420 ts->sched_timer.expires,
421 HRTIMER_MODE_ABS);
422 /* Check, if the timer was already in the past */
423 if (hrtimer_active(&ts->sched_timer))
424 break;
425 } else {
426 if (!tick_program_event(ts->sched_timer.expires, 0))
427 break;
428 }
429 /* Update jiffies and reread time */
430 tick_do_update_jiffies64(now);
431 now = ktime_get();
432 }
433 local_irq_enable();
434 }
435
436 static int tick_nohz_reprogram(struct tick_sched *ts, ktime_t now)
437 {
438 hrtimer_forward(&ts->sched_timer, now, tick_period);
439 return tick_program_event(ts->sched_timer.expires, 0);
440 }
441
442 /*
443 * The nohz low res interrupt handler
444 */
445 static void tick_nohz_handler(struct clock_event_device *dev)
446 {
447 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
448 struct pt_regs *regs = get_irq_regs();
449 int cpu = smp_processor_id();
450 ktime_t now = ktime_get();
451
452 dev->next_event.tv64 = KTIME_MAX;
453
454 /*
455 * Check if the do_timer duty was dropped. We don't care about
456 * concurrency: This happens only when the cpu in charge went
457 * into a long sleep. If two cpus happen to assign themself to
458 * this duty, then the jiffies update is still serialized by
459 * xtime_lock.
460 */
461 if (unlikely(tick_do_timer_cpu == -1))
462 tick_do_timer_cpu = cpu;
463
464 /* Check, if the jiffies need an update */
465 if (tick_do_timer_cpu == cpu)
466 tick_do_update_jiffies64(now);
467
468 /*
469 * When we are idle and the tick is stopped, we have to touch
470 * the watchdog as we might not schedule for a really long
471 * time. This happens on complete idle SMP systems while
472 * waiting on the login prompt. We also increment the "start
473 * of idle" jiffy stamp so the idle accounting adjustment we
474 * do when we go busy again does not account too much ticks.
475 */
476 if (ts->tick_stopped) {
477 touch_softlockup_watchdog();
478 ts->idle_jiffies++;
479 }
480
481 update_process_times(user_mode(regs));
482
483 /* Do not restart, when we are in the idle loop */
484 if (ts->tick_stopped)
485 return;
486
487 while (tick_nohz_reprogram(ts, now)) {
488 now = ktime_get();
489 tick_do_update_jiffies64(now);
490 }
491 }
492
493 /**
494 * tick_nohz_switch_to_nohz - switch to nohz mode
495 */
496 static void tick_nohz_switch_to_nohz(void)
497 {
498 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
499 ktime_t next;
500
501 if (!tick_nohz_enabled)
502 return;
503
504 local_irq_disable();
505 if (tick_switch_to_oneshot(tick_nohz_handler)) {
506 local_irq_enable();
507 return;
508 }
509
510 ts->nohz_mode = NOHZ_MODE_LOWRES;
511
512 /*
513 * Recycle the hrtimer in ts, so we can share the
514 * hrtimer_forward with the highres code.
515 */
516 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
517 /* Get the next period */
518 next = tick_init_jiffy_update();
519
520 for (;;) {
521 ts->sched_timer.expires = next;
522 if (!tick_program_event(next, 0))
523 break;
524 next = ktime_add(next, tick_period);
525 }
526 local_irq_enable();
527
528 printk(KERN_INFO "Switched to NOHz mode on CPU #%d\n",
529 smp_processor_id());
530 }
531
532 #else
533
534 static inline void tick_nohz_switch_to_nohz(void) { }
535
536 #endif /* NO_HZ */
537
538 /*
539 * High resolution timer specific code
540 */
541 #ifdef CONFIG_HIGH_RES_TIMERS
542 /*
543 * We rearm the timer until we get disabled by the idle code.
544 * Called with interrupts disabled and timer->base->cpu_base->lock held.
545 */
546 static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
547 {
548 struct tick_sched *ts =
549 container_of(timer, struct tick_sched, sched_timer);
550 struct pt_regs *regs = get_irq_regs();
551 ktime_t now = ktime_get();
552 int cpu = smp_processor_id();
553
554 #ifdef CONFIG_NO_HZ
555 /*
556 * Check if the do_timer duty was dropped. We don't care about
557 * concurrency: This happens only when the cpu in charge went
558 * into a long sleep. If two cpus happen to assign themself to
559 * this duty, then the jiffies update is still serialized by
560 * xtime_lock.
561 */
562 if (unlikely(tick_do_timer_cpu == -1))
563 tick_do_timer_cpu = cpu;
564 #endif
565
566 /* Check, if the jiffies need an update */
567 if (tick_do_timer_cpu == cpu)
568 tick_do_update_jiffies64(now);
569
570 /*
571 * Do not call, when we are not in irq context and have
572 * no valid regs pointer
573 */
574 if (regs) {
575 /*
576 * When we are idle and the tick is stopped, we have to touch
577 * the watchdog as we might not schedule for a really long
578 * time. This happens on complete idle SMP systems while
579 * waiting on the login prompt. We also increment the "start of
580 * idle" jiffy stamp so the idle accounting adjustment we do
581 * when we go busy again does not account too much ticks.
582 */
583 if (ts->tick_stopped) {
584 touch_softlockup_watchdog();
585 ts->idle_jiffies++;
586 }
587 update_process_times(user_mode(regs));
588 }
589
590 /* Do not restart, when we are in the idle loop */
591 if (ts->tick_stopped)
592 return HRTIMER_NORESTART;
593
594 hrtimer_forward(timer, now, tick_period);
595
596 return HRTIMER_RESTART;
597 }
598
599 /**
600 * tick_setup_sched_timer - setup the tick emulation timer
601 */
602 void tick_setup_sched_timer(void)
603 {
604 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
605 ktime_t now = ktime_get();
606 u64 offset;
607
608 /*
609 * Emulate tick processing via per-CPU hrtimers:
610 */
611 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
612 ts->sched_timer.function = tick_sched_timer;
613 ts->sched_timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ;
614
615 /* Get the next period (per cpu) */
616 ts->sched_timer.expires = tick_init_jiffy_update();
617 offset = ktime_to_ns(tick_period) >> 1;
618 do_div(offset, num_possible_cpus());
619 offset *= smp_processor_id();
620 ts->sched_timer.expires = ktime_add_ns(ts->sched_timer.expires, offset);
621
622 for (;;) {
623 hrtimer_forward(&ts->sched_timer, now, tick_period);
624 hrtimer_start(&ts->sched_timer, ts->sched_timer.expires,
625 HRTIMER_MODE_ABS);
626 /* Check, if the timer was already in the past */
627 if (hrtimer_active(&ts->sched_timer))
628 break;
629 now = ktime_get();
630 }
631
632 #ifdef CONFIG_NO_HZ
633 if (tick_nohz_enabled)
634 ts->nohz_mode = NOHZ_MODE_HIGHRES;
635 #endif
636 }
637
638 void tick_cancel_sched_timer(int cpu)
639 {
640 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
641
642 if (ts->sched_timer.base)
643 hrtimer_cancel(&ts->sched_timer);
644
645 ts->nohz_mode = NOHZ_MODE_INACTIVE;
646 }
647 #endif /* HIGH_RES_TIMERS */
648
649 /**
650 * Async notification about clocksource changes
651 */
652 void tick_clock_notify(void)
653 {
654 int cpu;
655
656 for_each_possible_cpu(cpu)
657 set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks);
658 }
659
660 /*
661 * Async notification about clock event changes
662 */
663 void tick_oneshot_notify(void)
664 {
665 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
666
667 set_bit(0, &ts->check_clocks);
668 }
669
670 /**
671 * Check, if a change happened, which makes oneshot possible.
672 *
673 * Called cyclic from the hrtimer softirq (driven by the timer
674 * softirq) allow_nohz signals, that we can switch into low-res nohz
675 * mode, because high resolution timers are disabled (either compile
676 * or runtime).
677 */
678 int tick_check_oneshot_change(int allow_nohz)
679 {
680 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
681
682 if (!test_and_clear_bit(0, &ts->check_clocks))
683 return 0;
684
685 if (ts->nohz_mode != NOHZ_MODE_INACTIVE)
686 return 0;
687
688 if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available())
689 return 0;
690
691 if (!allow_nohz)
692 return 1;
693
694 tick_nohz_switch_to_nohz();
695 return 0;
696 }
697
|
This page was automatically generated by the
LXR engine.
|