1、基本原理
如之前分析,kvm虚拟机实际运行于qemu-kvm的进程上下文中,因此,需要建立虚拟机的物理内存空间(GPA)与qemu-kvm进程的虚拟地址空间(HVA)的映射关系。
虚拟机的物理地址空间实际也是不连续的,分成不同的内存区域(slot),因为物理地址空间中通常还包括BIOS、MMIO、显存、ISA保留等部分。
qemu-kvm通过ioctl vm指令KVM_SET_USER_MEMORY_REGION来为虚拟机设置内存。主要建立guest物理地址空间中的内存区域与qemu-kvm虚拟地址空间中的内存区域的映射,从而建立其从GVA到HVA的对应关系,该对应关系主要通过kvm_mem_slot结构体保存,所以实质为设置kvm_mem_slot结构体。
本文简介ioctl vm指令KVM_SET_USER_MEMORY_REGION在内核中的执行流程,qemu-kvm用户态部分暂不包括。
2、基本流程
ioctl vm指令KVM_SET_USER_MEMORY_REGION在内核主要执行流程如下:
kvm_vm_ioctl()
kvm_vm_ioctl_set_memory_region()
kvm_set_memory_region()
__kvm_set_memory_region()
kvm_iommu_unmap_pages() // 原来的slot需要删除,所以需要unmap掉相应的内存区域
install_new_memslots() //将new分配的memslot写入kvm->memslots[]数组中
kvm_free_physmem_slot() // 释放旧内存区域相应的物理内存(HPA)
3、代码分析
kvm_mem_slot结构:
-
/*
-
* 由于GPA不能直接用于物理 MMU 进行寻址,所以需要将GPA转换为HVA,
-
* kvm中利用 kvm_memory_slot 数据结构来记录每一个地址区间(Guest中的物理
-
* 地址区间)中GPA与HVA的映射关系
-
*/
-
struct kvm_memory_slot {
-
// 虚拟机物理地址(即GPA)对应的页框号
-
gfn_t base_gfn;
-
// 当前slot中包含的page数
-
unsigned long npages;
-
// 脏页位图
-
unsigned long *dirty_bitmap;
-
// 架构相关的部分
-
struct kvm_arch_memory_slot arch;
-
/*
-
* GPA对应的Host虚拟地址(HVA),由于虚拟机都运行在qemu的地址空间中
-
* 而qemu是用户态程序,所以通常使用根模式下用户地址空间。
-
*/
-
unsigned long userspace_addr;
-
u32 flags;
-
short id;
-
};
kvm_vm_ioctl():
-
/*
-
* kvm ioctl vm指令的入口,传入的fd为KVM_CREATE_VM中返回的fd。
-
* 主要用于针对VM虚拟机进行控制,如:内存设置、创建VCPU等。
-
*/
-
static long kvm_vm_ioctl(struct file *filp,
-
unsigned int ioctl, unsigned long arg)
-
{
-
struct kvm *kvm = filp->private_data;
-
void __user *argp = (void __user *)arg;
-
int r;
-
-
if (kvm->mm != current->mm)
-
return -EIO;
-
switch (ioctl) {
-
// 创建VCPU
-
case KVM_CREATE_VCPU:
-
r = kvm_vm_ioctl_create_vcpu(kvm, arg);
-
break;
-
// 建立guest物理地址空间中的内存区域与qemu-kvm虚拟地址空间中的内存区域的映射
-
case KVM_SET_USER_MEMORY_REGION: {
-
// 存放内存区域信息的结构体,该内存区域从qemu-kvm进程的用户地址空间中分配
-
struct kvm_userspace_memory_region kvm_userspace_mem;
-
-
r = -EFAULT;
-
// 从用户态拷贝相应数据到内核态,入参argp指向用户态地址
-
if (copy_from_user(&kvm_userspace_mem, argp,
-
sizeof kvm_userspace_mem))
-
goto out;
-
// 进入实际处理流程
-
r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
-
break;
-
}
-
...
kvm_vm_ioctl()-->kvm_vm_ioctl_set_memory_region()-->kvm_set_memory_region()-->__kvm_set_memory_region()
-
/*
-
* 建立guest物理地址空间中的内存区域与qemu-kvm虚拟地址空间中的内存区域的映射
-
* 相应信息由uerspace_memory_region参数传入,而其源头来自于用户态qemu-kvm。每次
-
* 调用设置一个内存区间。内存区域可以不连续(实际的物理内存区域也经常不连
-
* 续,因为有可能有保留内存)
-
*/
-
int __kvm_set_memory_region(struct kvm *kvm,
-
struct kvm_userspace_memory_region *mem)
-
{
-
int r;
-
gfn_t base_gfn;
-
unsigned long npages;
-
struct kvm_memory_slot *slot;
-
struct kvm_memory_slot old, new;
-
struct kvm_memslots *slots = NULL, *old_memslots;
-
enum kvm_mr_change change;
-
-
// 标记检查
-
r = check_memory_region_flags(mem);
-
if (r)
-
goto out;
-
-
r = -EINVAL;
-
/* General sanity checks */
-
// 合规检查,防止用户态恶意传参,导致安全漏洞
-
if (mem->memory_size & (PAGE_SIZE - 1))
-
goto out;
-
if (mem->guest_phys_addr & (PAGE_SIZE - 1))
-
goto out;
-
/* We can read the guest memory with __xxx_user() later on. */
-
if ((mem->slot < KVM_USER_MEM_SLOTS) &&
-
((mem->userspace_addr & (PAGE_SIZE - 1)) ||
-
!access_ok(VERIFY_WRITE,
-
(void __user *)(unsigned long)mem->userspace_addr,
-
mem->memory_size)))
-
goto out;
-
if (mem->slot >= KVM_MEM_SLOTS_NUM)
-
goto out;
-
if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
-
goto out;
-
// 将kvm_userspace_memory_region->slot转换为kvm_mem_slot结构,该结构从kvm->memslots获取
-
slot = id_to_memslot(kvm->memslots, mem->slot);
-
// 内存区域起始位置在Guest物理地址空间中的页框号
-
base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
-
// 内存区域大小转换为page单位
-
npages = mem->memory_size >> PAGE_SHIFT;
-
-
r = -EINVAL;
-
if (npages > KVM_MEM_MAX_NR_PAGES)
-
goto out;
-
-
if (!npages)
-
mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
-
-
new = old = *slot;
-
-
new.id = mem->slot;
-
new.base_gfn = base_gfn;
-
new.npages = npages;
-
new.flags = mem->flags;
-
-
r = -EINVAL;
-
if (npages) {
-
// 判断是否需新创建内存区域
-
if (!old.npages)
-
change = KVM_MR_CREATE;
-
// 判断是否修改现有的内存区域
-
else { /* Modify an existing slot. */
-
// 修改的区域的HVA不同或者大小不同或者flag中的
-
// KVM_MEM_READONLY标记不同,直接退出。
-
if ((mem->userspace_addr != old.userspace_addr) ||
-
(npages != old.npages) ||
-
((new.flags ^ old.flags) & KVM_MEM_READONLY))
-
goto out;
-
/*
-
* 走到这,说明被修改的区域HVA和大小都是相同的
-
* 判断区域起始的GFN是否相同,如果是,则说明需
-
* 要在Guest物理地址空间中move这段区域,设置KVM_MR_MOVE标记
-
*/
-
if (base_gfn != old.base_gfn)
-
change = KVM_MR_MOVE;
-
// 如果仅仅是flag不同,则仅修改标记,设置KVM_MR_FLAGS_ONLY标记
-
else if (new.flags != old.flags)
-
change = KVM_MR_FLAGS_ONLY;
-
// 否则,啥也不干
-
else { /* Nothing to change. */
-
r = 0;
-
goto out;
-
}
-
}
-
} else if (old.npages) {/*如果新设置的区域大小为0,而老的区域大小不为0,则表示需要删除原有区域。*/
-
change = KVM_MR_DELETE;
-
} else /* Modify a non-existent slot: disallowed. */
-
goto out;
-
-
if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
-
/* Check for overlaps */
-
r = -EEXIST;
-
// 检查现有区域中是否重叠的
-
kvm_for_each_memslot(slot, kvm->memslots) {
-
if ((slot->id >= KVM_USER_MEM_SLOTS) ||
-
(slot->id == mem->slot))
-
continue;
-
if (!((base_gfn + npages <= slot->base_gfn) ||
-
(base_gfn >= slot->base_gfn + slot->npages)))
-
goto out;
-
}
-
}
-
-
/* Free page dirty bitmap if unneeded */
-
if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
-
new.dirty_bitmap = NULL;
-
-
r = -ENOMEM;
-
// 如果需要创建新区域
-
if (change == KVM_MR_CREATE) {
-
new.userspace_addr = mem->userspace_addr;
-
// 设置新的内存区域架构相关部分
-
if (kvm_arch_create_memslot(&new, npages))
-
goto out_free;
-
}
-
-
/* Allocate page dirty bitmap if needed */
-
if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
-
if (kvm_create_dirty_bitmap(&new) < 0)
-
goto out_free;
-
}
-
// 如果删除或move内存区域
-
if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
-
r = -ENOMEM;
-
// 复制kvm->memslots的副本
-
slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
-
GFP_KERNEL);
-
if (!slots)
-
goto out_free;
-
slot = id_to_memslot(slots, mem->slot);
-
slot->flags |= KVM_MEMSLOT_INVALID;
-
// 安装新memslots,返回旧的memslots
-
old_memslots = install_new_memslots(kvm, slots, NULL);
-
-
/* slot was deleted or moved, clear iommu mapping */
-
// 原来的slot需要删除,所以需要unmap掉相应的内存区域
-
kvm_iommu_unmap_pages(kvm, &old);
-
/* From this point no new shadow pages pointing to a deleted,
-
* or moved, memslot will be created.
-
*
-
* validation of sp->gfn happens in:
-
* - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
-
* - kvm_is_visible_gfn (mmu_check_roots)
-
*/
-
// flush影子页表中的条目
-
kvm_arch_flush_shadow_memslot(kvm, slot);
-
slots = old_memslots;
-
}
-
// 处理private memory slots,对其分配用户态地址,即HVA
-
r = kvm_arch_prepare_memory_region(kvm, &new, mem, change);
-
if (r)
-
goto out_slots;
-
-
r = -ENOMEM;
-
/*
-
* We can re-use the old_memslots from above, the only difference
-
* from the currently installed memslots is the invalid flag. This
-
* will get overwritten by update_memslots anyway.
-
*/
-
if (!slots) {
-
slots = kmemdup(kvm->memslots, sizeof(struct kvm_memslots),
-
GFP_KERNEL);
-
if (!slots)
-
goto out_free;
-
}
-
-
/*
-
* IOMMU mapping: New slots need to be mapped. Old slots need to be
-
* un-mapped and re-mapped if their base changes. Since base change
-
* unmapping is handled above with slot deletion, mapping alone is
-
* needed here. Anything else the iommu might care about for existing
-
* slots (size changes, userspace addr changes and read-only flag
-
* changes) is disallowed above, so any other attribute changes getting
-
* here can be skipped.
-
*/
-
if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
-
r = kvm_iommu_map_pages(kvm, &new);
-
if (r)
-
goto out_slots;
-
}
-
-
/* actual memory is freed via old in kvm_free_physmem_slot below */
-
if (change == KVM_MR_DELETE) {
-
new.dirty_bitmap = NULL;
-
memset(&new.arch, 0, sizeof(new.arch));
-
}
-
//将new分配的memslot写入kvm->memslots[]数组中
-
old_memslots = install_new_memslots(kvm, slots, &new);
-
-
kvm_arch_commit_memory_region(kvm, mem, &old, change);
-
// 释放旧内存区域相应的物理内存(HPA)
-
kvm_free_physmem_slot(&old, &new);
-
kfree(old_memslots);
-
-
return 0;
-
-
out_slots:
-
kfree(slots);
-
out_free:
-
kvm_free_physmem_slot(&new, &old);
-
out:
-
return r;
-
}
阅读(666) | 评论(0) | 转发(0) |