FreeBSD7.0远程调试内核时，回溯汇编栈帧出现异常的一种规避方法

双机远程串口调试FreeBSD7.0内核，建立调试连接之后，在vm_fault()函数处打断点，让目标机继续运行并遇到vm_fault()断点时，若用backtrace命令查看调用栈，会在进入汇编语言栈帧之后出现segmentation fault。

#
# kgdb -r /dev/cuad0 /sys/i386/compile/DEBUGKERNEL/kernel.debug
[GDB will not be able to debug user-mode threads: /usr/lib/libthread_db.so: Undefined symbol "ps_pglobal_lookup"]
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB.  Type "show warranty" for details.
This GDB was configured as "i386-marcel-freebsd".
Switching to remote protocol
kdb_enter (msg=0xc0aed472 "sysctl debug.kdb.enter") at ../../../kern/subr_kdb.c:310
310     }

Unread portion of the kernel message buffer:
KDB: enter: sysctl debug.kdb.enter

#0  kdb_enter (msg=0xc0aed472 "sysctl debug.kdb.enter") at ../../../kern/subr_kdb.c:310
310     }
(kgdb) b vm_fault
Breakpoint 1 at 0xc097a066: file ../../../vm/vm_fault.c, line 222.
(kgdb) c
Continuing.
[New Thread 100068]
[Switching to Thread 100068]

Breakpoint 1, vm_fault (map=0xc211cae0, vaddr=673189888, fault_type=3 '\003', fault_flags=1) at ../../../vm/vm_fault.c:222
222             PCPU_INC(cnt.v_vm_faults);
(kgdb) bt
#0  vm_fault (map=0xc211cae0, vaddr=673189888, fault_type=3 '\003', fault_flags=1) at ../../../vm/vm_fault.c:222
#1  0xc097c12a in vm_fault_wire (map=0xc211cae0, start=673189888, end=673193984, user_wire=0, fictitious=0)
    at ../../../vm/vm_fault.c:1025
#2  0xc0981576 in vm_map_wire (map=0xc211cae0, start=673189888, end=673193984, flags=0) at ../../../vm/vm_map.c:2068
#3  0xc097d1d5 in vslock (addr=0x28201040, len=4) at ../../../vm/vm_glue.c:226
#4  0xc076877e in sysctl_wire_old_buffer (req=0xcd0c9ba4, len=4) at ../../../kern/kern_sysctl.c:1179
#5  0xc0785c8f in kdb_sysctl_enter (oidp=0xc0b96460, arg1=0x0, arg2=0, req=0xcd0c9ba4) at ../../../kern/subr_kdb.c:157
#6  0xc0768277 in sysctl_root (oidp=Variable "oidp" is not available.
) at ../../../kern/kern_sysctl.c:1306
#7  0xc07683c4 in userland_sysctl (td=0xc251f210, name=0xcd0c9c14, namelen=3, old=0x28201040, oldlenp=0xbfbfe430, inkernel=0,
    new=0x0, newlen=0, retval=0xcd0c9c10, flags=0) at ../../../kern/kern_sysctl.c:1401
#8  0xc076915e in __sysctl (td=0xc251f210, uap=0xcd0c9cfc) at ../../../kern/kern_sysctl.c:1336
#9  0xc0a5aa85 in syscall (frame=0xcd0c9d38) at ../../../i386/i386/trap.c:1035
#10 0xc0a406b0 in Xint0x80_syscall () at ../../../i386/i386/exception.s:196
Segmentation fault (core dumped)
#

kgdb至出异常时的调用栈为：

Program received signal SIGSEGV, Segmentation fault.
0x2829ec87 in kvm_nlist () from /lib/libkvm.so.4
(gdb) bt
#0  0x2829ec87 in kvm_nlist () from /lib/libkvm.so.4
#1  0x0804d548 in kgdb_lookup (sym=0x81e70a4 "calltrap") at kthr.c:62
#2  0x0804e8a6 in kgdb_trgt_trapframe_prev_register (next_frame=0x285808ec, this_cache=0x28580d3c, regnum=8,
    optimizedp=0xbfbfe184, lvalp=0xbfbfe170, addrp=0xbfbfe178, realnump=0xbfbfe174, valuep=0xbfbfe1b0) at trgt_i386.c:303
#3  0x0811668e in frame_register_unwind (frame=0x28580d2c, regnum=8, optimizedp=0xbfbfe184, lvalp=0xbfbfe170, addrp=0xbfbfe178,
    realnump=0xbfbfe174, bufferp=0xbfbfe1b0) at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/frame.c:547
#4  0x08116a8b in frame_unwind_register (frame=0x28580d2c, regnum=8, buf=0xbfbfe1b0)
    at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/frame.c:626
#5  0x0817f40b in i386_unwind_pc (gdbarch=0x285a0008, next_frame=0x28580d2c)
    at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/i386-tdep.c:754
#6  0x081124b8 in gdbarch_unwind_pc (gdbarch=0x285a0008, next_frame=0x28580d2c)
    at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/gdbarch.c:4595
#7  0x0811627a in frame_pc_unwind (this_frame=0x28580d2c)
    at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/frame.c:399
#8  0x0811a1f1 in dummy_frame_sniffer (next_frame=0x28580d2c)
    at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/dummy-frame.c:413
#9  0x081195a6 in frame_unwind_find_by_frame (next_frame=0x28580d2c) at frame-unwind-kluge.c:90
#10 0x08118d88 in get_frame_type (frame=0x28580da0) at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/frame.c:2127
#11 0x0809e076 in print_frame_info (fi=0x28580da0, level=4, source=0, args=1)
    at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/stack.c:427
#12 0x0809f83b in backtrace_command_1 (count_exp=0x0, show_locals=0, from_tty=1)
    at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/stack.c:1223
#13 0x0809fa96 in backtrace_command (arg=0x0, from_tty=1)
    at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/stack.c:1289
#14 0x0817aa2b in do_cfunc (c=0x28567c10, args=0x0, from_tty=1)
    at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/cli/cli-decode.c:57
#15 0x0817d0a1 in cmd_func (cmd=0x28567c10, args=0x0, from_tty=1)
    at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/cli/cli-decode.c:1541
#16 0x0806cd8d in execute_command (p=0x28504082 "", from_tty=1)
    at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/top.c:743
#17 0x0806cf86 in command_loop () at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/top.c:822
#18 0x0804cc2c in kgdb_interp_command_loop (data=0x0) at main.c:275
#19 0x080f45b8 in current_interp_command_loop () at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/interps.c:277
#20 0x080e913b in captured_command_loop (data=0x0) at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/main.c:97
#21 0x0806c93c in do_catch_errors (uiout=0x28525800, data=0xbfbfe598)
    at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/top.c:523
#22 0x0806c6e0 in catcher (func=0x806c920 , func_uiout=0x28525800, func_args=0xbfbfe598, func_val=0xbfbfe5a4,
    func_caught=0xbfbfe5a0, errstring=0x8213b70 "", gdberrmsg=0x0, mask=6)
    at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/top.c:430
#23 0x0806c993 in catch_errors (func=0x80e9130 , func_args=0x0, errstring=0x8213b70 "", mask=6)
    at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/top.c:535
#24 0x080e9f27 in captured_main (data=0xbfbfe864) at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/main.c:805
#25 0x0806c93c in do_catch_errors (uiout=0x826d420, data=0xbfbfe7f8)
    at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/top.c:523
#26 0x0806c6e0 in catcher (func=0x806c920 , func_uiout=0x826d420, func_args=0xbfbfe7f8, func_val=0xbfbfe804,
    func_caught=0xbfbfe800, errstring=0x8213b70 "", gdberrmsg=0x0, mask=6)
    at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/top.c:430
#27 0x0806c993 in catch_errors (func=0x80e9180 , func_args=0xbfbfe864, errstring=0x8213b70 "", mask=6)
    at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/top.c:535
#28 0x080e9fa4 in gdb_main (args=0xbfbfe864) at /usr/src/gnu/usr.bin/gdb/libgdb/../../../../contrib/gdb/gdb/main.c:814
#29 0x0804d513 in main (argc=4, argv=0xbfbfed20) at main.c:495

从现象上看，故障原因是启动kgdb的时候指定了-r /dev/cuad0参数，kgdb因此不会去打开kvm文件操作符，参见/usr/src/gnu/usr.bin/gdb/kgdb/main.c的main()函数：

 if (remote == NULL) {
  kvm = kvm_openfiles(kernel, vmcore, NULL,
      writecore ? O_RDWR : O_RDONLY, kvm_err);
  if (kvm == NULL)
   errx(1, kvm_err);
  atexit(kgdb_atexit);
  kgdb_thr_init();
 }

因此，以这种方式启动时，kvm指针为空。而在后面的栈帧回溯过程中，kgdb调用到了kgdb_lookup()函数，参见/usr/src/gnu/usr.bin/gdb/kgdb/kthr.c：

uintptr_t
kgdb_lookup(const char *sym)
{
 struct nlist nl[2];

 nl[0].n_name = (char *)(uintptr_t)sym;
 nl[1].n_name = NULL;
 if (kvm_nlist(kvm, nl) != 0) {
  warnx("kvm_nlist(%s): %s", sym, kvm_geterr(kvm));
  return (0);
 }
 return (nl[0].n_value);
}

由于kvm为空，导致在kvm_nlist()函数中出现segmentation fault。如果不想去深究gdb栈帧回溯算法的问题，可以直接在kgdb_lookup()函数中加一个简单的保护来规避这个问题：

uintptr_t
kgdb_lookup(const char *sym)
{
        struct nlist nl[2];

        if (kvm == 0)
                return (0);

        nl[0].n_name = (char *)(uintptr_t)sym;
        nl[1].n_name = NULL;
        if (kvm_nlist(kvm, nl) != 0) {
                warnx("kvm_nlist(%s): %s", sym, kvm_geterr(kvm));
                return (0);
        }
        return (nl[0].n_value);
}

重新编译kgdb（如果之前没有在本地编译过gdb代码/usr/src/gnu/usr.bin中的gdb代码，则需要在/usr/src/gnu/usr.bin/gdb目录下make，并根据错误提示到上一级目录中对应的库目录下make。）这样修改之后，虽然在原故障位置之后的栈帧信息多少有些莫名其妙，但kgdb不会再因segmentation fault而退出了，我们可以继续跟踪感兴趣的内核代码：

#
# /usr/src/gnu/usr.bin/gdb/kgdb/kgdb -r /dev/cuad0 /sys/i386/compile/DEBUGKERNEL/kernel.debug
[GDB will not be able to debug user-mode threads: /usr/lib/libthread_db.so: Undefined symbol "ps_pglobal_lookup"]
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB.  Type "show warranty" for details.
This GDB was configured as "i386-marcel-freebsd".
Switching to remote protocol
vm_fault (map=0xc211cae0, vaddr=673189888, fault_type=3 '\003', fault_flags=1) at ../../../vm/vm_fault.c:222
222             PCPU_INC(cnt.v_vm_faults);

Unread portion of the kernel message buffer:
KDB: enter: sysctl debug.kdb.enter

#0  vm_fault (map=0xc211cae0, vaddr=673189888, fault_type=3 '\003', fault_flags=1) at ../../../vm/vm_fault.c:222
222             PCPU_INC(cnt.v_vm_faults);
(kgdb) bt
#0  vm_fault (map=0xc211cae0, vaddr=673189888, fault_type=3 '\003', fault_flags=1) at ../../../vm/vm_fault.c:222
#1  0xc097c12a in vm_fault_wire (map=0xc211cae0, start=673189888, end=673193984, user_wire=0, fictitious=0)
    at ../../../vm/vm_fault.c:1025
#2  0xc0981576 in vm_map_wire (map=0xc211cae0, start=673189888, end=673193984, flags=0) at ../../../vm/vm_map.c:2068
#3  0xc097d1d5 in vslock (addr=0x28201040, len=4) at ../../../vm/vm_glue.c:226
#4  0xc076877e in sysctl_wire_old_buffer (req=0xcd0c9ba4, len=4) at ../../../kern/kern_sysctl.c:1179
#5  0xc0785c8f in kdb_sysctl_enter (oidp=0xc0b96460, arg1=0x0, arg2=0, req=0xcd0c9ba4) at ../../../kern/subr_kdb.c:157
#6  0xc0768277 in sysctl_root (oidp=Variable "oidp" is not available.
) at ../../../kern/kern_sysctl.c:1306
#7  0xc07683c4 in userland_sysctl (td=0xc251f210, name=0xcd0c9c14, namelen=3, old=0x28201040, oldlenp=0xbfbfe430, inkernel=0,
    new=0x0, newlen=0, retval=0xcd0c9c10, flags=0) at ../../../kern/kern_sysctl.c:1401
#8  0xc076915e in __sysctl (td=0xc251f210, uap=0xcd0c9cfc) at ../../../kern/kern_sysctl.c:1336
#9  0xc0a5aa85 in syscall (frame=0xcd0c9d38) at ../../../i386/i386/trap.c:1035
#10 0xc0a406b0 in Xint0x80_syscall () at ../../../i386/i386/exception.s:196
#11 0x2815566f in ?? ()
Previous frame inner to this frame (corrupt stack?)
(kgdb)