Chinaunix首页 | 论坛 | 博客
  • 博客访问: 1811982
  • 博文数量: 159
  • 博客积分: 0
  • 博客等级: 民兵
  • 技术积分: 5639
  • 用 户 组: 普通用户
  • 注册时间: 2013-01-23 18:56
个人简介

将晦涩难懂的技术讲的通俗易懂

文章分类

全部博文(159)

文章存档

2019年(13)

2018年(19)

2017年(9)

2016年(26)

2015年(18)

2014年(54)

2013年(20)

分类: LINUX

2019-01-20 21:57:40

 dpdk pci设备初始化

(代码来自dpdk16.11)
——lvyilong316

dpdk pci设备的初始化主要由rte_eal_pci_init 函数完成,它是在rte_eal_init中被调用的。rte_eal_initàrte_eal_pci_init

l  rte_eal_pci_init

点击(此处)折叠或打开

  1. int
  2. rte_eal_pci_init(void)
  3. {
  4.      /* for debug purposes, PCI can be disabled */
  5.      if (internal_config.no_pci)
  6.         return 0;

  7.      if (rte_eal_pci_scan() < 0) {
  8.            RTE_LOG(ERR, EAL, "%s(): Cannot scan PCI bus\n", __func__);
  9.            return -1;

  10.      }
  11.      return 0;
  12. }

     如果配置了no_pci则直接返回,否则调用rte_eal_pci_scan

l  rte_eal_pci_scan

这个函数主要是遍历系统的/sys/bus/pci/devices目录中的每个子目录,/sys/bus/pci/devices目录中每个子目录都对应一个pci设备,如下图所示。

点击(此处)折叠或打开

  1. int
  2. rte_eal_pci_scan(void)
  3. {
  4.     struct dirent *e;
  5.     DIR *dir;
  6.     char dirname[PATH_MAX];
  7.     struct rte_pci_addr addr;
  8.     /*打开/sys/bus/pci/devices 目录*/
  9.     dir = opendir(pci_get_sysfs_path());
  10.     if (dir == NULL) {
  11.         RTE_LOG(ERR, EAL, "%s(): opendir failed: %s\n",
  12.             __func__, strerror(errno));
  13.         return -1;
  14.     }

  15.     while ((e = readdir(dir)) != NULL) {
  16.         if (e->d_name[0] == '.')
  17.             continue;
  18.         /*根据pci设备标识(地址)如:0000:01:11.4初始化addr中的domain,bus,devid,function*/
  19.         if (parse_pci_addr_format(e->d_name, sizeof(e->d_name), &addr) != 0)
  20.             continue;

  21.         snprintf(dirname, sizeof(dirname), "%s/%s",
  22.                 pci_get_sysfs_path(), e->d_name);
  23.         /*读取每个pci设备的目录,创建并初始化rte_pci_device结构,加入全局链表pci_device_list*/
  24.         if (pci_scan_one(dirname, &addr) < 0)
  25.             goto error;
  26.     }
  27.     closedir(dir);
  28.     return 0;

  29. error:
  30.     closedir(dir);
  31.     return -1;
  32. }

    pci_scan_one函数又会去读取每个子目录,每个子目录中包含这个pci设备的属性信息。如下图所示。

根据这些属性信息初始化一个pci设备,这个pci设备用rte_pci_device结构表示。

l  pci地址和pci id

这里说明一点,就是pci地址和pci id的区别,在/sys/bus/pci/devices目录下中看到的数字是pci地址,在dpdk中使用rte_pci_addr结构表示。

点击(此处)折叠或打开

  1. struct rte_pci_addr {
  2.     uint16_t domain; /**< Device domain */
  3.     uint8_t bus; /**< Device bus */
  4.     uint8_t devid; /**< Device ID */
  5.     uint8_t function; /**< Device function. */
  6. };

pci idclass_id vendor_id device_id 等组成。dpdk中用rte_pci_id结构表示。

点击(此处)折叠或打开

  1. struct rte_pci_id {
  2.     uint32_t class_id; /**< Class ID (class, subclass, pi) or RTE_CLASS_ANY_ID. */
  3.     uint16_t vendor_id; /**< Vendor ID or PCI_ANY_ID. */
  4.     uint16_t device_id; /**< Device ID or PCI_ANY_ID. */
  5.     uint16_t subsystem_vendor_id; /**< Subsystem vendor ID or PCI_ANY_ID. */
  6.     uint16_t subsystem_device_id; /**< Subsystem device ID or PCI_ANY_ID. */
  7. };

l  pci_scan_one

点击(此处)折叠或打开

  1. static int pci_scan_one(const char *dirname, const struct rte_pci_addr *addr)
  2. {
  3.     char filename[PATH_MAX];
  4.     unsigned long tmp;
  5.     struct rte_pci_device *dev;
  6.     char driver[PATH_MAX];
  7.     int ret;

  8.     dev = malloc(sizeof(*dev));
  9.     if (dev == NULL)
  10.         return -1;

  11.     memset(dev, 0, sizeof(*dev));
  12.     dev->addr = *addr;
  13.     /*根据设备属性,初始化设备的pci id*/
  14.     /* get vendor id */
  15.     snprintf(filename, sizeof(filename), "%s/vendor", dirname);
  16.     if (eal_parse_sysfs_value(filename, &tmp) < 0) {
  17.         free(dev);
  18.         return -1;
  19.     }
  20.     dev->id.vendor_id = (uint16_t)tmp;

  21.     /* get device id */
  22.     snprintf(filename, sizeof(filename), "%s/device", dirname);
  23.     if (eal_parse_sysfs_value(filename, &tmp) < 0) {
  24.         free(dev);
  25.         return -1;
  26.     }
  27.     dev->id.device_id = (uint16_t)tmp;

  28.     /* get subsystem_vendor id */
  29.     snprintf(filename, sizeof(filename), "%s/subsystem_vendor",
  30.          dirname);
  31.     if (eal_parse_sysfs_value(filename, &tmp) < 0) {
  32.         free(dev);
  33.         return -1;
  34.     }
  35.     dev->id.subsystem_vendor_id = (uint16_t)tmp;

  36.     /* get subsystem_device id */
  37.     snprintf(filename, sizeof(filename), "%s/subsystem_device",
  38.          dirname);
  39.     if (eal_parse_sysfs_value(filename, &tmp) < 0) {
  40.         free(dev);
  41.         return -1;
  42.     }
  43.     dev->id.subsystem_device_id = (uint16_t)tmp;

  44.     /* get class_id */
  45.     snprintf(filename, sizeof(filename), "%s/class",
  46.          dirname);
  47.     if (eal_parse_sysfs_value(filename, &tmp) < 0) {
  48.         free(dev);
  49.         return -1;
  50.     }
  51.     /* the least 24 bits are valid: class, subclass, program interface */
  52.     dev->id.class_id = (uint32_t)tmp & RTE_CLASS_ANY_ID;

  53.     /* get max_vfs */
  54.     /*获取设备的vf个数*/
  55.     dev->max_vfs = 0;
  56.     snprintf(filename, sizeof(filename), "%s/max_vfs", dirname);
  57.     if (!access(filename, F_OK) &&
  58.      eal_parse_sysfs_value(filename, &tmp) == 0)
  59.         dev->max_vfs = (uint16_t)tmp;
  60.     else {
  61.         /* for non igb_uio driver, need kernel version >= 3.8 */
  62.         snprintf(filename, sizeof(filename),
  63.              "%s/sriov_numvfs", dirname);
  64.         if (!access(filename, F_OK) &&
  65.          eal_parse_sysfs_value(filename, &tmp) == 0)
  66.             dev->max_vfs = (uint16_t)tmp;
  67.     }

  68.     /* get numa node */
  69.     /*如果设备开启numa,则在设备属性中会有numa_node的属性*/
  70.     snprintf(filename, sizeof(filename), "%s/numa_node",
  71.          dirname);
  72.     if (access(filename, R_OK) != 0) {
  73.         /* if no NUMA support, set default to 0 */
  74.         dev->device.numa_node = 0;
  75.     } else {
  76.         if (eal_parse_sysfs_value(filename, &tmp) < 0) {
  77.             free(dev);
  78.             return -1;
  79.         }
  80.         dev->device.numa_node = tmp;
  81.     }

  82.     /* parse resources */
  83.     /*获取pci设备映射的地址空间*/
  84.     snprintf(filename, sizeof(filename), "%s/resource", dirname);
  85.     if (pci_parse_sysfs_resource(filename, dev) < 0) {
  86.         RTE_LOG(ERR, EAL, "%s(): cannot parse resource\n", __func__);
  87.         free(dev);
  88.         return -1;
  89.     }

  90.     /* parse driver */
  91.     /*获取pci设备关联的驱动,如/sys/bus/pci/devices/0000\:01\:10.2/driver*/
  92.     snprintf(filename, sizeof(filename), "%s/driver", dirname);
  93.     /*使用readlink获取驱动的名称*/
  94.     ret = pci_get_kernel_driver_by_path(filename, driver);
  95.     if (ret < 0) {
  96.         RTE_LOG(ERR, EAL, "Fail to get kernel driver\n");
  97.         free(dev);
  98.         return -1;
  99.     }

  100.     if (!ret) { /*如果设备有关联驱动*/
  101.         if (!strcmp(driver, "vfio-pci"))
  102.             dev->kdrv = RTE_KDRV_VFIO; /*目前dpdk只支持三种驱动:vfio-pci,igb_uio,uio_pci_generic*/
  103.         else if (!strcmp(driver, "igb_uio"))
  104.             dev->kdrv = RTE_KDRV_IGB_UIO;
  105.         else if (!strcmp(driver, "uio_pci_generic"))
  106.             dev->kdrv = RTE_KDRV_UIO_GENERIC;
  107.         else
  108.             dev->kdrv = RTE_KDRV_UNKNOWN;
  109.     } else
  110.         dev->kdrv = RTE_KDRV_NONE;

  111.     /* device is valid, add in list (sorted) */
  112.     if (TAILQ_EMPTY(&pci_device_list)) { /*如果pci_device_list为空,直接将当前设备插入链表*/
  113.         rte_eal_device_insert(&dev->device);
  114.         TAILQ_INSERT_TAIL(&pci_device_list, dev, next);
  115.     } else {
  116.         struct rte_pci_device *dev2;
  117.         int ret;
  118.         /*将pci设备按照pci地址由大到小插入pci_device_list*/
  119.         TAILQ_FOREACH(dev2, &pci_device_list, next) {
  120.             ret = rte_eal_compare_pci_addr(&dev->addr, &dev2->addr);
  121.             if (ret > 0)
  122.                 continue;

  123.             if (ret < 0) {
  124.                 TAILQ_INSERT_BEFORE(dev2, dev, next);
  125.                 rte_eal_device_insert(&dev->device);
  126.             } else { /* already registered */
  127.                 dev2->kdrv = dev->kdrv;
  128.                 dev2->max_vfs = dev->max_vfs;
  129.                 memmove(dev2->mem_resource, dev->mem_resource,
  130.                     sizeof(dev->mem_resource));
  131.                 free(dev);
  132.             }
  133.             return 0;
  134.         }
  135.         rte_eal_device_insert(&dev->device);
  136.         TAILQ_INSERT_TAIL(&pci_device_list, dev, next);
  137.     }

  138.     return 0;
  139. }

其中一步需要提一下,就是打开pci设备目录下的resource文件,获取pci设备的地址空间(也就是pciBAR),并将其保存在dev->mem_resource中。这个pci地址在后面pci资源映射中会用到。

    通过readlink pci设备目录下的driver属性获取pci设备的驱动,目前dpdk只支持三种pci驱动:vfio-pci,igb_uio,uio_pci_generic

最后将创建初始化好的rte_pci_device结构按照pci地址由大到小插入全局链表pci_device_list中。相关数据结构关系如下图所示。

阅读(18281) | 评论(0) | 转发(0) |
给主人留下些什么吧!~~
评论热议
请登录后评论。

登录 注册