SoftIrq 流程分析

------------------------- kernel/softirq.c -----------------------

static __init int spawn_ksoftirqd(void);
static __init int spawn_ksoftirqd(void)
{
void *cpu = (void *)(long)smp_processor_id();
int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);

BUG_ON(err == NOTIFY_BAD);
cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
register_cpu_notifier(&cpu_nfb);
return 0;
}

early_initcall(spawn_ksoftirqd);

spawn_ksoftirqd 在系统启动的时候会被执行，这个函数调用了两次cpu_callback，根据两次传入的CPU_UP_PREPARE和CPU_ONLINE 这两个状态来判断，一般稍微有点觉悟的好同志都会立刻想到这两个状态是彼此相互联系的。下面具体看cpu_allback的实现

static int __cpuinit cpu_callback(struct notifier_block *nfb,
unsigned long action,
void *hcpu)
{
int hotcpu = (unsigned long)hcpu;
struct task_struct *p;

switch (action) {
case CPU_UP_PREPARE:
case CPU_UP_PREPARE_FROZEN:
p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
if (IS_ERR(p)) {
printk("ksoftirqd for %i failed\n", hotcpu);
return NOTIFY_BAD;
}
kthread_bind(p, hotcpu);
per_cpu(ksoftirqd, hotcpu) = p;
break;
case CPU_ONLINE:
case CPU_ONLINE_FROZEN:
wake_up_process(per_cpu(ksoftirqd, hotcpu));
break;

}
return NOTIFY_OK;
}

当前CPU的状态为CPU_UP_PREPARE， CPU还没有完全开始调度线程之前，我们利用kthread_create来创建一个内核线程 ksoftirqd ；接下来当CPU的状态为CPU_ONLINE后， wake_up_process 会唤醒之前创建的ksoftirqd 线程。那现在问题就是ksoftirqd 这个线程被唤醒之后都干了哪些勾当？！ 

static int ksoftirqd(void * __bind_cpu)
{
set_current_state(TASK_INTERRUPTIBLE);

while (!kthread_should_stop()) {
preempt_disable();
if (!local_softirq_pending()) {
preempt_enable_no_resched();
schedule();
preempt_disable();
}

__set_current_state(TASK_RUNNING);

while (local_softirq_pending()) {  等待CPU状态从CPU_UP_PREPARE进入CPU_ONLINE后，被唤醒！
/* Preempt disable stops cpu going offline.
If already offline, we'll be on wrong CPU:
don't process */
if (cpu_is_offline((long)__bind_cpu))
goto wait_to_die;
do_softirq();  处理各种 soft irq；这个函数可以是kernel/softirq.c中的do_softirq，也可以在不同体系结构中自己实现；
preempt_enable_no_resched();
cond_resched();
preempt_disable();
rcu_qsctr_inc((long)__bind_cpu);
}
preempt_enable();
set_current_state(TASK_INTERRUPTIBLE);
}
__set_current_state(TASK_RUNNING);
return 0;

wait_to_die:
preempt_enable();
/* Wait for kthread_stop */
set_current_state(TASK_INTERRUPTIBLE);
while (!kthread_should_stop()) {
schedule();
set_current_state(TASK_INTERRUPTIBLE);
}
__set_current_state(TASK_RUNNING);
return 0;
}

------------------------- kernel/softirq.c -----------------------
asmlinkage void __do_softirq(void)
{
struct softirq_action *h;
__u32 pending;
int max_restart = MAX_SOFTIRQ_RESTART;
int cpu;

pending = local_softirq_pending(); Q1: 返回pending值的含义 ？
account_system_vtime(current);

__local_bh_disable((unsigned long)__builtin_return_address(0));
lockdep_softirq_enter();

cpu = smp_processor_id();
restart:
/* Reset the pending bitmask before enabling irqs */
set_softirq_pending(0);

local_irq_enable();

h = softirq_vec;   Q2:这个数组的含义？

do {
if (pending & 1) {
int prev_count = preempt_count();

trace_softirq_entry(h, softirq_vec);
h->action(h);     Q3:这里调用的具体是哪个函数？
trace_softirq_exit(h, softirq_vec);
if (unlikely(prev_count != preempt_count())) {
printk(KERN_ERR "huh, entered softirq %td %s %p"
"with preempt_count %08x,"
" exited with %08x?\n", h - softirq_vec,
softirq_to_name[h - softirq_vec],
h->action, prev_count, preempt_count());
preempt_count() = prev_count;
}

rcu_bh_qsctr_inc(cpu);
}
h++;
pending >>= 1;
} while (pending);

local_irq_disable();

pending = local_softirq_pending();
if (pending && --max_restart)
goto restart;

if (pending)
wakeup_softirqd();

lockdep_softirq_exit();

account_system_vtime(current);
_local_bh_enable();
}


#ifndef __ARCH_HAS_DO_SOFTIRQ

asmlinkage void do_softirq(void)
{
__u32 pending;
unsigned long flags;

if (in_interrupt())
return;

local_irq_save(flags);

pending = local_softirq_pending();

if (pending)
__do_softirq();

local_irq_restore(flags);
}

#endif

Q1:
-----------------arch/arm/include/asm/hardirq.h-----------------------
typedef struct {
unsigned int __softirq_pending;
unsigned int local_timer_irqs;
} ____cacheline_aligned irq_cpustat_t;

-----------------incude/linux/irq_cpustat.h-----------------------
#ifndef __ARCH_IRQ_STAT
extern irq_cpustat_t irq_stat[];        /* defined in asm/hardirq.h */
#define __IRQ_STAT(cpu, member)    (irq_stat[cpu].member)   __IRQ_STAT用于存取第cpu个CPU对应的irq_stat结构体中的member成员；确实有点绕口！
#endif

/* arch independent irq_stat fields */
#define local_softirq_pending() \
__IRQ_STAT(smp_processor_id(), __softirq_pending)

local_softirq_pending宏用去存取当前CPU对应的irq_cpustat_t结构体中的__softirq_pending成员；

#ifndef __ARCH_SET_SOFTIRQ_PENDING
#define set_softirq_pending(x) (local_softirq_pending() = (x))
#define or_softirq_pending(x)  (local_softirq_pending() |= (x))
#endif

#define __raise_softirq_irqoff(nr) do { or_softirq_pending(1UL << (nr)); } while (0)

/*
* This function must run with irqs disabled!
*/
inline void raise_softirq_irqoff(unsigned int nr)
{
__raise_softirq_irqoff(nr);

/*
* If we're in an interrupt or softirq, we're done
* (this also catches softirq-disabled code). We will
* actually run the softirq once we return from
* the irq or softirq.
*
* Otherwise we wake up ksoftirqd to make sure we
* schedule the softirq soon.
*/
if (!in_interrupt())
wakeup_softirqd();
}

void raise_softirq(unsigned int nr)
{
unsigned long flags;

local_irq_save(flags);
raise_softirq_irqoff(nr);
local_irq_restore(flags);
}

也就是说raise_softirq--->raise_softirq_irqoff--->__raise_softirq_irqoff--->or_softirq_pending 会将当前CPU对应的irq_cpustat_t结构体中的__softirq_pending成员的第nr个BIT置位。nr的具体的含义在Q2中解释。
提前先科普下，对于TIMER_SOFTIRQ来说，每次system clock产生中断时，即一个tick 到来时，在system clock的中断处理函数中会调用run_local_timers来设置TIMER_SOFTIRQ触发条件；也就是当前CPU对应的irq_cpustat_t结构体中的__softirq_pending成员的第TIMER_SOFTIRQ个BIT被置为1。 而当这个条件满足时，
ksoftirqd线程会被唤醒，然后按照下面的流程调用TIMER_SOFTIRQ在数组softirq_vec中注册的action，即run_timer_softirq。

ksoftirqd--->do_softirq--->__do_softirq--->softirq_vec[TIMER_SOFTIRQ].action

启示：
1：TIMER_SOFTIRQ的最小粒度为一个tick的时间间隔
2：之所以成为softirq，是因为这些中断是由硬件中断来间接触发的
3：__do_softirq会优先处理__softirq_pending中的BIT_0,即HI_SOFTIRQ，然后依次检查__softirq_pending中BIT_1 至BIT_7是否被置位，如果置位，执行对应的action。



/*
* Called by the local, per-CPU timer interrupt on SMP.
*/
void run_local_timers(void)
{
hrtimer_run_queues();
raise_softirq(TIMER_SOFTIRQ);
softlockup_tick();
}

Q2:
enum
{
HI_SOFTIRQ=0,
TIMER_SOFTIRQ,
NET_TX_SOFTIRQ,
NET_RX_SOFTIRQ,
BLOCK_SOFTIRQ,
TASKLET_SOFTIRQ,
SCHED_SOFTIRQ,
HRTIMER_SOFTIRQ,
RCU_SOFTIRQ,    /* Preferable RCU should always be the last softirq */

NR_SOFTIRQS
};

struct softirq_action
{
void    (*action)(struct softirq_action *);
};

static struct softirq_action softirq_vec[NR_SOFTIRQS] ;
内核中定义了用于不同场合中的各种软中断，例如网络收发、tasklet、高精度定时器和软中断等等应用场合；没一种应用场合都对应一个总的处理函数，这个函数就是struct softirq_action中的action成员。这写action都是在提前需要提前注册好的。我们以TIMER_SOFTIRQ为例：

Q3：
start_kernel--> init_timers
void __init init_timers(void)
{
int err = timer_cpu_notify(&timers_nb, (unsigned long)CPU_UP_PREPARE,
(void *)(long)smp_processor_id());

init_timer_stats();

BUG_ON(err == NOTIFY_BAD);
register_cpu_notifier(&timers_nb);
open_softirq(TIMER_SOFTIRQ, run_timer_softirq);
}

void open_softirq(int nr, void (*action)(struct softirq_action *))
{
softirq_vec[nr].action = action;
}
也就是说，当TIMER_SOFTIRQ 这个softirq发生时，run_timer_softirq 函数会被调用!
问题是谁回去触发这个TIMER_SOFTIRQ 信号呢？


总结：
ksoftirqd 是内核启动时创建的一个内核线程；
该线程会一次检查。。。。。。。。。。。
。。。。。。。。。。。。。。。。。。。
。。。。。。当时钟中断发生时。。。。。
。。。ksoftirqd 检测到。。。。。调用相应
的action。。。。。。。。。。。。。。。。
。。。。。。这就是为什么这个东西被称作
softirq:)