Chinaunix首页 | 论坛 | 博客
  • 博客访问: 1212749
  • 博文数量: 122
  • 博客积分: 0
  • 博客等级: 民兵
  • 技术积分: 4004
  • 用 户 组: 普通用户
  • 注册时间: 2014-02-20 08:27
文章分类
文章存档

2016年(1)

2015年(21)

2014年(100)

分类: LINUX

2014-04-18 16:07:38

softlockup(watchdog)用于检测系统调度是否正常,即软锁的情况,当发生softlockup时,内核不能调度,但还能响应中断,对用户的表现可能为:能ping通,但无法登陆系统,无法进行正常操作。
其基本原理为:为每个CPU启动一个内核线程(watchdog/x),此线程为优先级最高的实时线程,在该线程得到调度时,会更新相应的计数(时间戳),同时会启动定时器,当定时器到期时检查相应的时间戳,如果超过指定时间,都没有更新,则说明这段时间内都没有发生调度(因为此线程优先级最高),则打印相应告警或根据配置可以进入panic流程。
基本代码分析(2.6.32)
rest_init->kernel_init->lockup_detector_init->cpu_callback->watchdog_prepare_cpu(初始化watchdog定时器):

点击(此处)折叠或打开

  1. static int watchdog_prepare_cpu(int cpu)
  2. {
  3.     struct hrtimer *hrtimer = &per_cpu(watchdog_hrtimer, cpu);

  4.     WARN_ON(per_cpu(softlockup_watchdog, cpu));
  5.     hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);//初始化高精度定时器
  6.     hrtimer->function = watchdog_timer_fn;//设置定时器处理函数

  7.     return 0;
  8. }
看门狗定时器处理函数:

点击(此处)折叠或打开

  1. static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
  2. {
  3. //获取计数watchdog_touch_ts,该计数在watchdog内核线程被调度时更新
  4.     unsigned long touch_ts = __get_cpu_var(watchdog_touch_ts);
  5.     struct pt_regs *regs = get_irq_regs();
  6.     int duration;

  7.     /* kick the hardlockup detector */
  8. //增加中断计数,证明没有发生硬锁(关中断死锁)
  9.     watchdog_interrupt_count();

  10.     /* kick the softlockup detector */
  11. //唤醒wathdog内核线程
  12.     wake_up_process(__get_cpu_var(softlockup_watchdog));

  13.     /* .. and repeat */
  14. //重启定时器
  15.     hrtimer_forward_now(hrtimer, ns_to_ktime(get_sample_period()));
  16.     if (touch_ts == 0) {
  17.         if (unlikely(__get_cpu_var(softlockup_touch_sync))) {
  18.             /*
  19.              * If the time stamp was touched atomically
  20.              * make sure the scheduler tick is up to date.
  21.              */
  22.             __get_cpu_var(softlockup_touch_sync) = false;
  23.             sched_clock_tick();
  24.         }
  25.         __touch_watchdog();
  26.         return HRTIMER_RESTART;
  27.     }

  28.     /* check for a softlockup
  29.      * This is done by making sure a high priority task is
  30.      * being scheduled. The task touches the watchdog to
  31.      * indicate it is getting cpu time. If it hasn't then
  32.      * this is a good indication some task is hogging the cpu
  33.      */
  34. //判断是否发生了软锁,原理是判断touch_ts(时间戳)是否超过一定时间没有更新
  35.     duration = is_softlockup(touch_ts);
  36.     if (unlikely(duration)) {
  37.         /* only warn once */
  38.         if (__get_cpu_var(soft_watchdog_warn) == true)
  39.             return HRTIMER_RESTART;
  40. //发生了软锁后,进行一些列的信息记录和告警。
  41.         printk(KERN_EMERG "BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
  42.             smp_processor_id(), duration,
  43.             current->comm, task_pid_nr(current));
  44.         print_modules();
  45.         print_irqtrace_events(current);
  46.         if (regs)
  47.             show_regs(regs);
  48.         else
  49.             dump_stack();
  50. //如果配置了softlockup_panic(proc中配置),则panic
  51.         if (softlockup_panic)
  52.             panic("softlockup: hung tasks");
  53.         __get_cpu_var(soft_watchdog_warn) = true;
  54.     } else
  55.         __get_cpu_var(soft_watchdog_warn) = false;

  56.     return HRTIMER_RESTART;
  57. }

启动看门狗,即创建watchdog内核线程。

点击(此处)折叠或打开

  1. static int watchdog_enable(int cpu)
  2. {
  3.     struct task_struct *p = per_cpu(softlockup_watchdog, cpu);
  4.     int err = 0;

  5.     /* enable the perf event */
  6.     err = watchdog_nmi_enable(cpu);

  7.     /* Regardless of err above, fall through and start softlockup */

  8.     /* create the watchdog thread */
  9.     if (!p) {
  10. //创建watchdog内核线程
  11.         p = kthread_create(watchdog, (void *)(unsigned long)cpu, "watchdog/%d", cpu);
  12.         if (IS_ERR(p)) {
  13.             printk(KERN_ERR "softlockup watchdog for %i failed\n", cpu);
  14.             if (!err)
  15.                 /* if hardlockup hasn't already set this */
  16.                 err = PTR_ERR(p);
  17.             goto out;
  18.         }
  19.         kthread_bind(p, cpu);
  20.         per_cpu(watchdog_touch_ts, cpu) = 0;
  21.         per_cpu(softlockup_watchdog, cpu) = p;
  22.         wake_up_process(p);
  23.     }

  24. out:
  25.     return err;
  26. }

watchdog内核线程执行主函数,主要是要更新计数(时间戳)

点击(此处)折叠或打开

  1. static int watchdog(void *unused)
  2. {
  3. //设置为最高优先级
  4.     struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
  5.     struct hrtimer *hrtimer = &__raw_get_cpu_var(watchdog_hrtimer);
  6. //设置为实时线程
  7.     sched_setscheduler(current, SCHED_FIFO, &param);

  8.     /* initialize timestamp */
  9. //初始化计数(时间戳)
  10.     __touch_watchdog();

  11.     /* kick off the timer for the hardlockup detector */
  12.     /* done here because hrtimer_start can only pin to smp_processor_id() */
  13. //启动定时器,用于检测是否发生软锁
  14.     hrtimer_start(hrtimer, ns_to_ktime(get_sample_period()),
  15.          HRTIMER_MODE_REL_PINNED);
  16. //睡眠
  17.     set_current_state(TASK_INTERRUPTIBLE);
  18.     /*
  19.      * Run briefly once per second to reset the softlockup timestamp.
  20.      * If this gets delayed for more than 60 seconds then the
  21.      * debug-printout triggers in watchdog_timer_fn().
  22.      */
  23.     while (!kthread_should_stop()) {
  24. //更新计数
  25.         __touch_watchdog();
  26.         schedule();

  27.         if (kthread_should_stop())
  28.             break;

  29.         set_current_state(TASK_INTERRUPTIBLE);
  30.     }
  31.     __set_current_state(TASK_RUNNING);

  32.     return 0;
  33. }

判断是否发生软锁:is_softlockup

点击(此处)折叠或打开

  1. static int is_softlockup(unsigned long touch_ts)
  2. {
  3.     unsigned long now = get_timestamp();

  4.     /* Warn about unreasonable delays: */
  5.     /*检测多久没有更新相关计数了,默认阈值为10s,touch_us为上一次更新的时间*/
  6.     if (time_after(now, touch_ts + get_softlockup_thresh()))
  7.         return now - touch_ts;

  8.     return 0;
  9. }

阅读(6424) | 评论(3) | 转发(2) |
给主人留下些什么吧!~~

humjb_19832015-07-22 11:32:43

lantian728://检测计数多久没有更新了,如果超过了60s,则表示发生了软锁
    if (time_after(now, touch_ts + softlockup_thresh))

怎么算出来的60s呢,softlockup_thresh是内核参数配置的值,touch_ts 和now 怎么算出来的没看懂呢?

不好意思,代码贴多了,2.6.32中默认是60s,3.10版本中默认已经变成10s了,60s可能太长~,已经更新内容了。
另外,now就是当前的时间,其实就是读的墙上时间。touch_ts是上次计数的更新时间。

回复 | 举报

humjb_19832015-07-22 11:32:42

lantian728://检测计数多久没有更新了,如果超过了60s,则表示发生了软锁
    if (time_after(now, touch_ts + softlockup_thresh))

怎么算出来的60s呢,softlockup_thresh是内核参数配置的值,touch_ts 和now 怎么算出来的没看懂呢?

不好意思,代码贴多了,2.6.32中默认是60s,3.10版本中默认已经变成10s了,60s可能太长~,已经更新内容了。
另外,now就是当前的时间,其实就是读的墙上时间。touch_ts是上次计数的更新时间。

回复 | 举报

lantian7282015-07-21 16:17:05

//检测计数多久没有更新了,如果超过了60s,则表示发生了软锁
    if (time_after(now, touch_ts + softlockup_thresh))

怎么算出来的60s呢,softlockup_thresh是内核参数配置的值,touch_ts 和now 怎么算出来的没看懂呢?