kernel 3.10内核源码分析--file read流程
1、基本原理
文件读取流程通常从用户态发起,总体流程为:
用户态进程先open指定文件,返回fd
用户态进程调用read库函数接口,fd作为入参
read库函数接口最终调用read对应的系统调用接口,sys_read,由此进入内核流程
--------用户态和内核态分界------------------------
内核态调用特定文件系统定义的对应相关操作接口,如ext2对应的read接口为ext2_file_operations->aio_read,由此进入mapping层(VFS层)
mapping层在进行相应处理后,构造并提交bio,进入通用块层。
--------mapping层和通用块层分界----------------
通用块层构造request,提交到IO调度层
--------通用块层和IO调度层分界----------------
IO调度层对request按特定的IO调度算法进行合并和排序后,将request放入对应块设备的request queue,由此进入块设备驱动层。常用的块设备驱动层为SCSI,即进入SCSI层。
--------IO调度层和块设备驱动层分界-------------
SCSI层又分3层:SCSI上层驱动、SCSI中间层和SCSI底层驱动。
request进入SCSI层后,先由SCSI上层驱动和中间层进行处理,主要是按SCSI协议规范构造SCSI命令,最终将请求提交到SCSI底层驱动。
SCSI底层驱动,如mptsas,对request queue中的request进行逐一处理,主要跟根据硬件特性,构造最终的命令,以及跟硬件的交互。构造的命令最终提交到磁盘硬件(准确的说,应该是固件)。
--------SCSI层和硬件(固件)层的分界--------------
硬件处理完IO请求后,产生中断通知SCSI底层驱动(初始化时预先注册了相应的中断)。
ISR中调用SCSI层的scsi_done接口,进行相应的处理。
scsi_done调用上层(块设备层)定义的blk_complete_request接口,并触发相应的软中断
软中断中进行后续的处理:包括错误处理、request的清理、定时器的清理、唤醒等待的进程等。
2、mapping层的read流程
mapping层的read流程(以ext2文件系统为例)如下:
ext2_file_operations->read->do_sync_read
ext2_file_operations->aio_read
generic_file_aio_read
do_generic_file_read
find_get_page //从page cache中查找page
address_space->a_ops->readpage // 如果page cache中没找到,或者找到的页不是uptodate的,就从磁盘中读取。
ext2_readpage
mpage_readpage
do_mpage_readpage
ext2_get_block // 获取文件逻辑块号和设备物理块号的对应关系,结果保存在buffer_head中(page->buffers),后续提交IO时,可
以根据该对应关系确定是读写整页,还是只读写其中的部分block。
mpage_bio_submit // 正常的IO提交流程
submit_bio
block_read_full_page //当读取的block不连续、文件中存在hole时,以buffer来读?即按block来读,不以page方式读,可能只读取
page中的部分block
submit_bh // 提交请求, 以buffer方式,按block读
file_read_actor //将相应page拷贝到用户态
3、代码分析
重要流程的代码分析如下。
generic_file_aio_read->do_generic_file_read()
-
/*
-
* 文件读取的主要处理函数,actor用于将数据从内核态拷贝到用户态。
-
*/
-
static void do_generic_file_read(struct file *filp, loff_t *ppos,
-
read_descriptor_t *desc, read_actor_t actor)
-
{
-
struct address_space *mapping = filp->f_mapping;
-
struct inode *inode = mapping->host;
-
struct file_ra_state *ra = &filp->f_ra;
-
pgoff_t index;
-
pgoff_t last_index;
-
pgoff_t prev_index;
-
unsigned long offset; /* offset into pagecache page */
-
unsigned int prev_offset;
-
int error;
-
/*文件中的位置信息转换*/
-
index = *ppos >> PAGE_CACHE_SHIFT;
-
prev_index = ra->prev_pos >> PAGE_CACHE_SHIFT;
-
prev_offset = ra->prev_pos & (PAGE_CACHE_SIZE-1);
-
last_index = (*ppos + desc->count + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
-
offset = *ppos & ~PAGE_CACHE_MASK;
-
-
for (;;) {
-
struct page *page;
-
pgoff_t end_index;
-
loff_t isize;
-
unsigned long nr, ret;
-
/*检测是否需要调度*/
-
cond_resched();
-
find_page:
-
// 从page cache中查找,即从address space中find指定index的page
-
page = find_get_page(mapping, index);
-
if (!page) {
-
// 当page cache中没有找到需要的page时,强制同步预读,阻塞等待预读完成。
-
page_cache_sync_readahead(mapping,
-
ra, filp,
-
index, last_index - index);
-
/*再次从page cache中查找*/
-
page = find_get_page(mapping, index);
-
if (unlikely(page == NULL))
-
goto no_cached_page;
-
}
-
if (PageReadahead(page)) {
-
/*当设置了预读相关标记时,预读,异步,在向下层提交读请求后返回,不会阻塞。*/
-
page_cache_async_readahead(mapping,
-
ra, filp, page,
-
index, last_index - index);
-
}
-
/*如果从page cache中获取的page不是最新的,需要writeback*/
-
if (!PageUptodate(page)) {
-
if (inode->i_blkbits == PAGE_CACHE_SHIFT ||
-
!mapping->a_ops->is_partially_uptodate)
-
goto page_not_up_to_date;
-
/*先获取锁,如果失败则需要在后面重新获取*/
-
if (!trylock_page(page))
-
goto page_not_up_to_date;
-
/* Did it get truncated before we got the lock? */
-
if (!page->mapping)
-
goto page_not_up_to_date_locked;
-
if (!mapping->a_ops->is_partially_uptodate(page,
-
desc, offset))
-
goto page_not_up_to_date_locked;
-
unlock_page(page);
-
}
-
// 从page cache中找到相应的页,且页的内容是uptodate的,即跟磁盘内容一致。
-
page_ok:
-
/*
-
* i_size must be checked after we know the page is Uptodate.
-
*
-
* Checking i_size after the check allows us to calculate
-
* the correct value for "nr", which means the zero-filled
-
* part of the page is not copied back to userspace (unless
-
* another truncate extends the file - this is desired though).
-
*/
-
-
// 更新相关计数
-
isize = i_size_read(inode);
-
end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
-
if (unlikely(!isize || index > end_index)) {
-
page_cache_release(page);
-
goto out;
-
}
-
-
/* nr is the maximum number of bytes to copy from this page */
-
nr = PAGE_CACHE_SIZE;
-
if (index == end_index) {
-
nr = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
-
if (nr <= offset) {
-
page_cache_release(page);
-
goto out;
-
}
-
}
-
nr = nr - offset;
-
-
/* If users can be writing to this page using arbitrary
-
* virtual addresses, take care about potential aliasing
-
* before reading the page on the kernel side.
-
*/
-
if (mapping_writably_mapped(mapping))
-
flush_dcache_page(page);
-
-
/*
-
* When a sequential read accesses a page several times,
-
* only mark it as accessed the first time.
-
*/
-
/*
-
* 设置page的accessed标记,在LRU页面回收时需要判断该标记,表示页面的active程度,second chance
-
* 当连续对同一page进行read时,只在第一次read时,设置一次。
-
*/
-
if (prev_index != index || offset != prev_offset)
-
mark_page_accessed(page);
-
prev_index = index;
-
-
/*
-
* Ok, we have the page, and it's up-to-date, so
-
* now we can copy it to user space...
-
*
-
* The actor routine returns how many bytes were actually used..
-
* This may not be the same as how much of a user buffer
-
* we filled up (we may be padding etc), so we can only update
-
* "pos" here (the actor routine has to update the user buffer
-
* pointers and the remaining count).
-
*/
-
// 调用actor(实际为file_read_actor)将page中的内容拷贝到用户态缓冲区中
-
ret = actor(desc, page, offset, nr);
-
/*更新位置信息*/
-
offset += ret;
-
index += offset >> PAGE_CACHE_SHIFT;
-
offset &= ~PAGE_CACHE_MASK;
-
prev_offset = offset;
-
-
page_cache_release(page);
-
if (ret == nr && desc->count)
-
continue;
-
goto out;
-
-
// page cache中找到相关的page,但不是uptodate的
-
page_not_up_to_date:
-
/* Get exclusive access to the page ... */
-
/*关键同步点。第一次read操作进入到这里时,先获取到锁,后面在发起实际readpage操作后,会重新获取该锁,由于此处已经获取了锁,就会阻塞等待。*/
-
error = lock_page_killable(page);
-
if (unlikely(error))
-
goto readpage_error;
-
/*前面已经获取过锁了,这里不用再获取了*/
-
page_not_up_to_date_locked:
-
/* Did it get truncated before we got the lock? */
-
if (!page->mapping) {
-
unlock_page(page);
-
page_cache_release(page);
-
continue;
-
}
-
-
/* Did somebody else fill it already? */
-
if (PageUptodate(page)) {
-
unlock_page(page);
-
goto page_ok;
-
}
-
// 从外部存储中读数据(1个page)
-
readpage:
-
/*
-
* A previous I/O error may have been due to temporary
-
* failures, eg. multipath errors.
-
* PG_error will be set again if readpage fails.
-
*/
-
ClearPageError(page);
-
/* Start the actual read. The read will unlock the page. */
-
/*
-
* 实际调用address_space中的readpage接口来执行实际的读操作
-
* 具体实现取决于具体的文件系统,如ext2对应的接口为ext2_readpage
-
*/
-
error = mapping->a_ops->readpage(filp, page);
-
-
if (unlikely(error)) {
-
if (error == AOP_TRUNCATED_PAGE) {
-
page_cache_release(page);
-
goto find_page;
-
}
-
goto readpage_error;
-
}
-
-
if (!PageUptodate(page)) {
-
/*
-
* 关键同步点。read操作是同步操作,由于前面已经获取过该锁。负责read的用户态进程在这里阻塞等待read操作完成,
-
* 即等待数据从磁盘读取完成后触发中断上来,从而触发bio_endio,在其中通过unlock_Page释放锁,最终唤醒该进程
-
*/
-
error = lock_page_killable(page);
-
if (unlikely(error))
-
goto readpage_error;
-
/*执行到这里,数据应该已经read完成了,如果期间没有新的脏数据的话,就应该是uptodate的数据了*/
-
if (!PageUptodate(page)) {
-
if (page->mapping == NULL) {
-
/*
-
* invalidate_mapping_pages got it
-
*/
-
/*释放锁*/
-
unlock_page(page);
-
page_cache_release(page);
-
goto find_page;
-
}
-
/*释放锁*/
-
unlock_page(page);
-
shrink_readahead_size_eio(filp, ra);
-
error = -EIO;
-
goto readpage_error;
-
}
-
/*释放锁*/
-
unlock_page(page);
-
}
-
// 跳转到page_ok, 调用actor将page拷贝到用户态
-
goto page_ok;
-
-
readpage_error:
-
/* A synchronous read error occurred. Report it */
-
desc->error = error;
-
page_cache_release(page);
-
goto out;
-
-
// 如果page cache中没有找到相应的page,那么就要在page cache中分配相应的page了。
-
no_cached_page:
-
/*
-
* Ok, it wasn't cached, so we need to create a new
-
* page..
-
*/
-
// 从cold缓存中分配新page
-
page = page_cache_alloc_cold(mapping);
-
if (!page) {
-
desc->error = -ENOMEM;
-
goto out;
-
}
-
// 将新分配的page加入到page cache中
-
error = add_to_page_cache_lru(page, mapping,
-
index, GFP_KERNEL);
-
if (error) {
-
page_cache_release(page);
-
if (error == -EEXIST)
-
goto find_page;
-
desc->error = error;
-
goto out;
-
}
-
// 跳转到readpage,从外部存储中读取page
-
goto readpage;
-
}
-
-
out:
-
ra->prev_pos = prev_index;
-
ra->prev_pos <<= PAGE_CACHE_SHIFT;
-
ra->prev_pos |= prev_offset;
-
// 更新当前文件中的位置
-
*ppos = ((loff_t)index << PAGE_CACHE_SHIFT) + offset;
-
/*更新文件的atime:访问时间*/
-
file_accessed(filp);
-
}
generic_file_aio_read->do_generic_file_read->ext2_readpage->mpage_readpages->do_mpage_readpage()
-
static struct bio *
-
do_mpage_readpage(struct bio *bio, struct page *page, unsigned nr_pages,
-
sector_t *last_block_in_bio, struct buffer_head *map_bh,
-
unsigned long *first_logical_block, get_block_t get_block)
-
{
-
struct inode *inode = page->mapping->host;
-
const unsigned blkbits = inode->i_blkbits;
-
const unsigned blocks_per_page = PAGE_CACHE_SIZE >> blkbits;
-
const unsigned blocksize = 1 << blkbits;
-
sector_t block_in_file;
-
sector_t last_block;
-
sector_t last_block_in_file;
-
sector_t blocks[MAX_BUF_PER_PAGE];
-
unsigned page_block;
-
unsigned first_hole = blocks_per_page;
-
struct block_device *bdev = NULL;
-
int length;
-
int fully_mapped = 1;
-
unsigned nblocks;
-
unsigned relative_block;
-
-
if (page_has_buffers(page))
-
goto confused;
-
-
block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
-
last_block = block_in_file + nr_pages * blocks_per_page;
-
last_block_in_file = (i_size_read(inode) + blocksize - 1) >> blkbits;
-
if (last_block > last_block_in_file)
-
last_block = last_block_in_file;
-
page_block = 0;
-
-
/*
-
* Map blocks using the result from the previous get_blocks call first.
-
*/
-
nblocks = map_bh->b_size >> blkbits;
-
if (buffer_mapped(map_bh) && block_in_file > *first_logical_block &&
-
block_in_file < (*first_logical_block + nblocks)) {
-
unsigned map_offset = block_in_file - *first_logical_block;
-
unsigned last = nblocks - map_offset;
-
-
for (relative_block = 0; ; relative_block++) {
-
if (relative_block == last) {
-
clear_buffer_mapped(map_bh);
-
break;
-
}
-
if (page_block == blocks_per_page)
-
break;
-
blocks[page_block] = map_bh->b_blocknr + map_offset +
-
relative_block;
-
page_block++;
-
block_in_file++;
-
}
-
bdev = map_bh->b_bdev;
-
}
-
-
/*
-
* Then do more get_blocks calls until we are done with this page.
-
*/
-
map_bh->b_page = page;
-
while (page_block < blocks_per_page) {
-
map_bh->b_state = 0;
-
map_bh->b_size = 0;
-
-
if (block_in_file < last_block) {
-
map_bh->b_size = (last_block-block_in_file) << blkbits;
-
/* 调用传入的get_block接口,ext2对应为ext2_get_block,用于获取
-
* 文件逻辑块和磁盘逻辑块之间的对应关系。get_block并不实际从磁盘
-
* 中读取文件数据,只是获取对应关系(通过直接或间接块信息),如果
-
* 相关的对应关系没有建立,则不创建相应的对应关系(比如创建间接块,
-
* 传入的最后一个参数为0,表示不需要创建),这个与write的流程不同。
-
* 对应关系需要在后续的write流程中创建。获取的对应关系保存于map_bh中
-
*/
-
if (get_block(inode, block_in_file, map_bh, 0))
-
goto confused;
-
*first_logical_block = block_in_file;
-
}
-
-
if (!buffer_mapped(map_bh)) {
-
fully_mapped = 0;
-
if (first_hole == blocks_per_page)
-
first_hole = page_block;
-
page_block++;
-
block_in_file++;
-
continue;
-
}
-
-
/* some filesystems will copy data into the page during
-
* the get_block call, in which case we don't want to
-
* read it again. map_buffer_to_page copies the data
-
* we just collected from get_block into the page's buffers
-
* so readpage doesn't have to repeat the get_block call
-
*/
-
if (buffer_uptodate(map_bh)) {
-
map_buffer_to_page(page, map_bh, page_block);
-
goto confused;
-
}
-
-
if (first_hole != blocks_per_page)
-
goto confused; /* hole -> non-hole */
-
-
/* Contiguous blocks? */
-
if (page_block && blocks[page_block-1] != map_bh->b_blocknr-1)
-
goto confused;
-
nblocks = map_bh->b_size >> blkbits;
-
for (relative_block = 0; ; relative_block++) {
-
if (relative_block == nblocks) {
-
clear_buffer_mapped(map_bh);
-
break;
-
} else if (page_block == blocks_per_page)
-
break;
-
blocks[page_block] = map_bh->b_blocknr+relative_block;
-
page_block++;
-
block_in_file++;
-
}
-
bdev = map_bh->b_bdev;
-
}
-
-
if (first_hole != blocks_per_page) {
-
zero_user_segment(page, first_hole << blkbits, PAGE_CACHE_SIZE);
-
if (first_hole == 0) {
-
SetPageUptodate(page);
-
unlock_page(page);
-
goto out;
-
}
-
} else if (fully_mapped) {
-
SetPageMappedToDisk(page);
-
}
-
-
if (fully_mapped && blocks_per_page == 1 && !PageUptodate(page) &&
-
cleancache_get_page(page) == 0) {
-
SetPageUptodate(page);
-
goto confused;
-
}
-
-
/*
-
* This page will go to BIO. Do we need to send this BIO off first?
-
*/
-
if (bio && (*last_block_in_bio != blocks[0] - 1))
-
bio = mpage_bio_submit(READ, bio);
-
// 通过bio,按page方式读,这也是file read的主要方式。
-
alloc_new:
-
if (bio == NULL) {
-
/* 分配bio,blocks[0] << (blkbits - 9)为第一个扇区号,
-
* min_t(int, nr_pages, bio_get_nr_vecs(bdev)为扇区数。
-
*/
-
bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9),
-
min_t(int, nr_pages, bio_get_nr_vecs(bdev)),
-
GFP_KERNEL);
-
if (bio == NULL)
-
goto confused;
-
}
-
-
length = first_hole << blkbits;
-
/*将需要读取的page加入到刚创建的bio中*/
-
if (bio_add_page(bio, page, length, 0) < length) {
-
/*Fixme:应该不会走到这里?*/
-
bio = mpage_bio_submit(READ, bio);
-
goto alloc_new;
-
}
-
-
relative_block = block_in_file - *first_logical_block;
-
nblocks = map_bh->b_size >> blkbits;
-
/*判断是否需要先提交bio*/
-
if ((buffer_boundary(map_bh) && relative_block == nblocks) ||
-
(first_hole != blocks_per_page))
-
/*提交成功后,返回null*/
-
bio = mpage_bio_submit(READ, bio);
-
else
-
*last_block_in_bio = blocks[blocks_per_page - 1];
-
out:
-
/*返回上层函数,由上层函数提交bio*/
-
return bio;
-
-
confused:
-
if (bio)
-
bio = mpage_bio_submit(READ, bio);
-
if (!PageUptodate(page))
-
/* 当读取的block不连续、文件中存在hole时,走到这里,正常的流程应该走
-
* mpage_bio_submit,通过bio,按page读?
-
* 这里通过buffer来读?即按block来读,不以page方式读,可能只读取page中
-
* 的部分block,最终通过submit_bh提交请求。
-
*/
-
block_read_full_page(page, get_block);
-
else
-
unlock_page(page);
-
goto out;
-
}
阅读(4473) | 评论(0) | 转发(4) |