摘要:
本文深入源码分析Coffee文件系统写入文件cfs_write技术细节,包括FD_WRITABLE、COFFEE_IO_SEMANTICS、merge_log、write_log_page、find_next_record、create_log、COFFEE_APPEND_ONLY、COFFEE_WRITE。
一、cfs_write
源代码如下:
- int cfs_write(int fd, const void *buf, unsigned size)
-
{
-
struct file_desc *fdp;
-
struct file *file;
-
-
#if COFFEE_MICRO_LOGS
-
int i;
-
struct log_param lp;
-
cfs_offset_t bytes_left;
-
const char dummy[1] =
-
{
-
0xff
-
};
-
#endif
-
-
if(!(FD_VALID(fd) && FD_WRITABLE(fd))) //fd有效性判断及检查写权限,详情见1.1
-
{
-
return - 1;
-
}
-
-
fdp = &coffee_fd_set[fd];
-
file = fdp->file;
-
-
-
/***若超过文件末尾写,则扩展文件(当没有设置CFS_COFFEE_IO_FIRM_SIZE时),见二***/
-
#if COFFEE_IO_SEMANTICS
-
if(!(fdp->io_flags &CFS_COFFEE_IO_FIRM_SIZE))
-
{
-
#endif
-
-
while(size + fdp->offset + sizeof(struct file_header) > (file->max_pages *COFFEE_PAGE_SIZE))
-
{
-
if(merge_log(file->page, 1) < 0)
-
{
-
return - 1;
-
} file = fdp->file;
-
PRINTF("Extended the file at page %u\n", (unsigned)file->page);
-
}
-
-
#if COFFEE_IO_SEMANTICS
-
}
-
#endif
-
-
/********写入文件概述,见1.2*************/
-
/**************实际写入文件***********************/
-
#if COFFEE_MICRO_LOGS
-
#if COFFEE_IO_SEMANTICS
-
if(!(fdp->io_flags&CFS_COFFEE_IO_FLASH_AWARE) && (FILE_MODIFIED(file) || fdp->offset<file->end))
-
#else
-
if(FILE_MODIFIED(file) || fdp->offset < file->end)
-
{
-
#endif
-
for(bytes_left = size; bytes_left > 0;)
-
{
-
lp.offset = fdp->offset;
-
lp.buf = buf;
-
lp.size = bytes_left;
-
i = write_log_page(file, &lp);
-
if(i < 0)
-
{
-
if(size == bytes_left)
-
{
-
return - 1;
-
}
-
break;
-
}
-
else if(i == 0)
-
{
-
file = fdp->file;
-
}
-
else
-
{
-
bytes_left -= i;
-
fdp->offset += i;
-
buf = (char*)buf + i;
-
-
if(fdp->offset > file->end)
-
{
-
file->end = fdp->offset;
-
}
-
}
-
}
-
if(fdp->offset > file->end)
-
{
-
COFFEE_WRITE(dummy, 1, absolute_offset(file->page, fdp->offset));
-
}
-
}
-
else
-
{
-
#endif /* COFFEE_MICRO_LOGS */
-
-
#if COFFEE_APPEND_ONLY
-
if(fdp->offset < file->end)
-
{
-
return - 1;
-
}
-
#endif
-
-
COFFEE_WRITE(buf, size, absolute_offset(file->page, fdp->offset));
-
fdp->offset += size;
-
-
#if COFFEE_MICRO_LOGS
-
}
-
#endif
-
-
if(fdp->offset > file->end)
-
{
-
file->end = fdp->offset;
-
}
-
-
return size;
-
}
1.1宏FD_VALID和FD_WRITABLE
(1) FD_VALID
FD_VALID宏用于判断cfs_read函数传进来的fd是否有效,不能小于0(如,-1是get_available_fd函数分配不到fd的返回值),也不能大于FD的上限(即COFFEE_FD_SET_SIZE),其对应的file_desc的标志flags不能为空闲(COFFEE_FD_FREE),源码如下:
- #define FD_VALID(fd) ((fd)>= 0 && (fd)<COFFEE_FD_SET_SIZE && coffee_fd_set[(fd)].flags!=COFFEE_FD_FREE)
(2) FD_WRITABLE
FD_WRITABLE判断该文件是否以CFS_WRITE打开,可见如果想从文件末尾写,则需同时指定CFS_WRITE和CFS_APPEND,源码如下:
- #define FD_WRITABLE(fd) (coffee_fd_set[(fd)].flags & CFS_WRITE)
注:尽管Coffee官方论文[1]声称指定了CFS_APPEND意味着指向CFS_WRITE,从源码分析,显然必须得指定CFS_WRITE才能写入。我觉得把FD_WRITABLE改下,会更符合编程习惯,如下:
- #define FD_WRITABLE(fd) (coffee_fd_set[(fd)].flags & CFS_WRITE & CFS_APPEND)
1.2 写入文件概述
扩展文件后(必要时,见二),就实际进入写阶段了,出现了很多预定义的宏,先理清下这些宏的层次关系,而后再详细分析各种情况,简化后源代码如下:
- /**************写入文件概述,见1.2***********************/
-
#if COFFEE_MICRO_LOGS
-
#if COFFEE_IO_SEMANTICS
-
if(!(fdp->io_flags &CFS_COFFEE_IO_FLASH_AWARE) && (FILE_MODIFIED(file)|| fdp->offset < file->end))
-
{
-
#else
-
if(FILE_MODIFIED(file) || fdp->offset < file->end)
-
{
-
#endif
-
/*************情型1,见三***********/
-
for(bytes_left = size; bytes_left > 0;)
-
{
-
}
-
}
-
else /*************情型2,见四***********/
-
{
-
#endif
-
-
#if COFFEE_MICRO_LOGS
-
}
-
#endif
-
-
/***情型3,见五***/
-
if(fdp->offset > file->end)
-
{
-
file->end = fdp->offset;
-
}
如果借助代码缩进(我用SourceFormat格式化代码,不是很智能,还得手动调)还没看清层次关系,那继续看下图:
图1 宏配置示意图
搞清了宏层次关系,就不难理解写入文件都有哪些情况:
(1)情形1
配置了COFFEE_MICRO_LOGS且文件要么被修改了(即微日志文件存在)或者文件偏移量小于end(即原始文件还有空间可以写入),如果Coffee还配置了COFFEE_IO_SEMANTICS,还需要求file_desc的io_flags中CFS_COFFEE_IO_FLASH_AWARE没有设置。
(2)情形2
配置了COFFEE_MICRO_LOGS,文件要么没有被修改(即微日志文件不存在)且文件偏移量大于等于end,如果Coffee还配置了COFFEE_IO_SEMANTICS,还需要求file_desc的io_flags中CFS_COFFEE_IO_FLASH_AWARE被设置。
若文件被扩展了(见二),此时file->end等于偏移量,file的flags中M位为0(即没有微日志文件),这种情况恰好满足这个条件(CFS_COFFEE_IO_FLASH_AWARE有设置的话)。
(3)情形3
没有配置COFFEE_MICRO_LOGS,就直接跳到情况三了。事实上,情型1和情型2都需要执行这段代码(也有可能中途就退出了)。
1.3 一个BUG?
假设系统配置了COFFEE_IO_SEMANTICS且io_flags也设置了CFS_COFFEE_IO_FIRM_SIZE,此时if(!(fdp->io_flags &CFS_COFFEE_IO_FIRM_SIZE))为假,直到跳到#if COFFEE_MICRO_LOGS。倘若系统没有配置COFFEE_MICRO_LOGS,再次跳过,执行以下语句,不论offset与end关系如何,直接返回size。也就是说,在这种最简单的模型下,传给cfs_write的size,原封不动返回。而这种情况是有可能存在的,这难道不是一个BUG?
- if(fdp->offset > file->end)
- {
-
file->end = fdp->offset;
- }
-
- return size;
二、COFFEE_IO_SEMANTICS与CFS_COFFEE_IO_FIRM_SIZE
如果没有设置CFS_COFFEE_IO_FIRM_SIZE,当现有的空间不足以数据写入时,则需扩展文件(merge_log函数),略去参数验证及与本节无关代码如下:
- //设置了COFFEE_IO_SEMANTICS
-
struct file_desc *fdp;
-
struct file *file;
-
-
fdp = &coffee_fd_set[fd];
-
file = fdp->file;
-
-
#if COFFEE_IO_SEMANTICS //见2.1
-
if(!(fdp->io_flags&CFS_COFFEE_IO_FIRM_SIZE))//若io_flags没设置CFS_COFFEE_IO_FIRM_SIZE则返回真,见2.1
-
{
- #endif
-
-
while(size + fdp->offset + sizeof(struct file_header) > (file->max_pages *COFFEE_PAGE_SIZE)) //若待写入的数据超过文件末尾,则合并日志
-
{
-
if(merge_log(file->page, 1) < 0) //见2.2
-
{
-
return - 1;
-
}
- file = fdp->file;
-
PRINTF("Extended the file at page %u\n", (unsigned)file->page);
-
}
- #if COFFEE_IO_SEMANTICS
-
}
-
#endif
2.1 COFFEE_IO_SEMANTICS
如果定义了COFFEE_IO_SEMANTICS,则在file_desc结构体会多一个成员变量io_flags,配置了COFFEE_IO_SEMANTICS可以优化某些存储设备的文件访问(optimize file access on certain storage types),系统默认没有配置COFFEE_IO_SEMANTICS。系统定义了io_flags两个值,即CFS_COFFEE_IO_FLASH_AWARE和CFS_COFFEE_IO_FIRM_SIZE。设置了CFS_COFFEE_IO_FIRM_SIZE,如果写入文件超过预留的大小,Coffee不会继续扩展文件。当文件有固定大小限制时,设置CFS_COFFEE_IO_FIRM_SIZE可以保护过度写。
在这里,如果io_flags设置了CFS_COFFEE_IO_FIRM_SIZE,就没必要扩展文件(即使是超过了)。如果连COFFEE_IO_SEMANTICS都没定义,也就是说file压根就没有io_flags,那就更省事了:-)
2.2 merge_log
merge_log用于合并日志,即当文件剩余空间不足以写入size字节时,需要扩展,具体做法是:将原始文件和微日志文件(如果有的话)拷贝到新文件。源代码如下:
- //merge_log(file->page, 1)
-
static int merge_log(coffee_page_t file_page, int extend)
-
{
-
struct file_header hdr, hdr2;
-
int fd, n;
-
cfs_offset_t offset;
-
coffee_page_t max_pages;
-
struct file *new_file;
-
int i;
-
-
read_header(&hdr, file_page);
-
-
fd = cfs_open(hdr.name, CFS_READ); //以只读方式找开文件
-
if(fd < 0)
-
{
-
return - 1;
-
}
-
-
/***创建新的文件***/
-
max_pages = hdr.max_pages << extend;
-
new_file = reserve(hdr.name, max_pages, 1, 0); //创建新文件的主体函数
-
if(new_file == NULL)
-
{
-
cfs_close(fd);
-
return - 1;
-
}
-
-
offset = 0;
-
-
/***将原文件的数据内容拷贝到新文件new_file***/
-
do
-
{
-
char buf[hdr.log_record_size == 0 ? COFFEE_PAGE_SIZE : hdr.log_record_size]; //数组表达式不是常量,编译出错
-
-
n = cfs_read(fd, buf, sizeof(buf));
-
if(n < 0)
-
{
-
remove_by_page(new_file->page, !REMOVE_LOG, !CLOSE_FDS, ALLOW_GC);
-
cfs_close(fd);
-
return - 1;
-
}
-
else if(n > 0)
-
{
-
COFFEE_WRITE(buf, n, absolute_offset(new_file->page, offset));
-
offset += n;
-
}
-
}while(n != 0);
-
-
/***利用原来那个file_desc***/
-
for(i = 0; i < COFFEE_FD_SET_SIZE; i++)
-
{
-
if(coffee_fd_set[i].flags != COFFEE_FD_FREE && coffee_fd_set[i].file->page == file_page)
-
{
-
coffee_fd_set[i].file = new_file;
-
new_file->references++;
-
}
-
}
-
-
/***删除原文件***/
-
if(remove_by_page(file_page, REMOVE_LOG, !CLOSE_FDS, !ALLOW_GC) < 0) //删除原始文件和微日志文件、不关闭FD、不进行垃圾回收
-
{
-
remove_by_page(new_file->page, !REMOVE_LOG, !CLOSE_FDS, !ALLOW_GC);
-
cfs_close(fd);
-
return - 1;
-
}
-
-
/***将原来file_header的log_record_size和log_records拷贝到新的file_header***/
-
read_header(&hdr2, new_file->page);
-
hdr2.log_record_size = hdr.log_record_size;
-
hdr2.log_records = hdr.log_records;
-
write_header(&hdr2, new_file->page);
-
-
/***设置新文件new_file的flags、end***/
-
new_file->flags &= ~COFFEE_FILE_MODIFIED;
-
new_file->end = offset;
-
-
cfs_close(fd);
-
-
return 0;
-
}
reserve是创建新文件的主体函数,首先进行参数验证,接着查看物理FLASH是否有连续pages空闲页,若有,则返回该文件即将占有页的第一页。否则,调用Coffee垃圾回收,再次查看物理FLASH是否有连续pages空闲页,如果还没有就返回NULL。创建成功后,加载文件(load_file),如果缓存失败也是返回NULL。详情见博文《Contiki学习笔记:Coffee文件系统创建文件》。
三、写入文件情形1
配置了COFFEE_MICRO_LOGS且文件要么被修改了(即微日志文件存在)或者文件偏移量小于end(即原始文件还有空间可以写入),如果Coffee还配置了COFFEE_IO_SEMANTICS,还需要求file_desc的io_flags中CFS_COFFEE_IO_FLASH_AWARE没有设置。略去参数验证及与本节无关代码如下:
- struct file_desc *fdp;
-
struct file *file;
-
-
#if COFFEE_MICRO_LOGS
-
int i;
-
struct log_param lp;
-
cfs_offset_t bytes_left;
-
const char dummy[1] =
-
{
-
0xff
-
};
-
#endif
-
-
fdp = &coffee_fd_set[fd];
-
file = fdp->file;
-
-
{
-
for(bytes_left = size; bytes_left > 0;)
-
{
-
/***初始化lp结构体***/
-
lp.offset = fdp->offset;
-
lp.buf = buf;
-
lp.size = bytes_left;
-
-
i = write_log_page(file, &lp); //见3.1
-
if(i < 0)
-
{
-
if(size == bytes_left)
-
{
-
return - 1;
-
}
-
break;
-
}
-
else if(i == 0)
-
{
-
file = fdp->file;
-
}
-
else
-
{
-
bytes_left -= i;
-
fdp->offset += i;
-
buf = (char*)buf + i;
-
-
if(fdp->offset > file->end)
-
{
-
file->end = fdp->offset;
-
}
-
}
-
}
-
if(fdp->offset > file->end)
-
{
-
COFFEE_WRITE(dummy, 1, absolute_offset(file->page, fdp->offset)); //向文件offset处写入字节"0xFF"
-
}
-
}
-
-
/*******************************/
-
if(fdp->offset > file->end)
-
{
-
file->end = fdp->offset;
-
}
-
-
return size;
3.1 write_log_page
write_log_page将内容写入日志,源代码如下:
- #if COFFEE_MICRO_LOGS
-
static int write_log_page(struct file *file, struct log_param *lp)
-
{
-
struct file_header hdr;
-
uint16_t region;
-
coffee_page_t log_page;
-
int16_t log_record;
-
uint16_t log_record_size;
-
uint16_t log_records;
-
cfs_offset_t offset;
-
struct log_param lp_out;
-
-
read_header(&hdr, file->page);
-
-
adjust_log_config(&hdr, &log_record_size, &log_records);
-
region = modify_log_buffer(log_record_size, &lp->offset, &lp->size);
-
-
log_page = 0;
-
if(HDR_MODIFIED(hdr))
-
//判断微日志文件是否存在
-
{
-
/* A log structure has already been created. */
-
log_page = hdr.log_page;
-
log_record = find_next_record(file, log_page, log_records); //见3.2
-
if(log_record >= log_records)
-
{
-
/* The log is full; merge the log. */
-
PRINTF("Coffee: Merging the file %s with its log\n", hdr.name);
-
return merge_log(file->page, 0);
-
}
-
}
-
else
-
{
-
/***创建微日志文件,见3.3***/
-
log_page = create_log(file, &hdr);
-
if(log_page == INVALID_PAGE)
-
{
-
return - 1;
-
}
-
PRINTF("Coffee: Created a log structure for file %s at page %u\n", hdr.name,(unsigned)log_page);
-
hdr.log_page = log_page;
-
log_record = 0;
-
}
-
-
{
-
char copy_buf[log_record_size];
-
-
lp_out.offset = offset = region * log_record_size;
-
lp_out.buf = copy_buf;
-
lp_out.size = log_record_size;
-
-
if((lp->offset>0 || lp->size != log_record_size) && read_log_page(&hdr,log_record, &lp_out) < 0)
-
{
-
COFFEE_READ(copy_buf, sizeof(copy_buf), absolute_offset(file->page, offset));
-
}
-
-
memcpy(©_buf[lp->offset], lp->buf, lp->size);
-
-
/*
-
* Write the region number in the region index table.
-
* The region number is incremented to avoid values of zero.
-
*/
-
offset = absolute_offset(log_page, 0);
-
++region;
-
COFFEE_WRITE(®ion, sizeof(region), offset + log_record * sizeof(region));
-
-
offset += log_records * sizeof(region);
-
COFFEE_WRITE(copy_buf, sizeof(copy_buf), offset + log_record * log_record_size);
-
file->record_count = + 1;
-
}
-
-
return lp->size;
-
}
-
#endif
3.2 find_next_record
find_next_record源代码如下:
- //log_record = find_next_record(file, log_page, log_records);
-
#if COFFEE_MICRO_LOGS
-
static int find_next_record(struct file *file, coffee_page_t log_page, int log_records)
-
{
-
int log_record, preferred_batch_size;
-
-
if(file->record_count >= 0)
-
{
-
return file->record_count;
-
}
-
-
preferred_batch_size = log_records > COFFEE_LOG_TABLE_LIMIT ? COFFEE_LOG_TABLE_LIMIT: log_records;
-
{
-
/* The next log record is unknown at this point; search for it. */
-
uint16_t indices[preferred_batch_size];
-
uint16_t processed;
-
uint16_t batch_size;
-
-
log_record = log_records;
-
for(processed = 0; processed < log_records; processed += batch_size)
-
{
-
batch_size = log_records - processed >= preferred_batch_size ? preferred_batch_size: log_records - processed;
-
-
COFFEE_READ(&indices, batch_size *sizeof(indices[0]), absolute_offset(log_page, processed *sizeof(indices[0])));
-
for(log_record = 0; log_record < batch_size; log_record++)
-
{
-
if(indices[log_record] == 0)
-
{
-
log_record += processed;
-
break;
-
}
-
}
-
}
-
}
-
-
return log_record;
-
}
-
#endif
3.3 创建微日志文件create_log
create_log首先调整日志记录大小和日志记录数量,reserve创建并加载微日志文件,若成功,对file_header及file相关成员变量进行一些设置,返回微日志文件的第一页,否则返回INVALID_PAGE。创建成功后的文件总体示意图如下:
图3 创建微日志文件示意图
创建微日志文件create_log源代码如下:
- //log_page = create_log(file, &hdr);
- #if COFFEE_MICRO_LOGS
- static coffee_page_t create_log(struct file *file, struct file_header *hdr)
- {
- uint16_t log_record_size, log_records;
- cfs_offset_t size;
- struct file *log_file;
- adjust_log_config(hdr, &log_record_size, &log_records); //调整微日志配置
- /************创建微日志文件***************/
- size = log_records *(sizeof(uint16_t) + log_record_size); //Log index size+log data size
- log_file = reserve(hdr->name, page_count(size), 1, HDR_FLAG_LOG);
- if (log_file == NULL)
- {
- return INVALID_PAGE;
- }
- hdr->flags |= HDR_FLAG_MODIFIED; //将file_header的flags中M位置1,表示文件已修改,微日志文件存在
- hdr->log_page = log_file->page; //将file_header的log_page指向微日志文件的第一页
- write_header(hdr, file->page); //写入file_header
- file->flags |= COFFEE_FILE_MODIFIED; //file_header的flag中M位为1(即物理文件被修改,日志存在),则file->flags设为COFFEE_FILE_MODIFIED
- return log_file->page;
- }
- #endif
adjust_log_config调整日志记录大小和日志记录数量,即如果log_record_size及log_records为0,则设成默认值,详情参见博文《Contiki学习笔记:Coffee文件系统读取文件cfs_read》。
reserve是创建微日志文件的主体函数,首先进行参数验证,接着查看物理FLASH是否有连续pages空闲页,若有,则返回该文件即将占有页的第一页。否则,调用Coffee垃圾回收,再次查看物理FLASH是否有连续pages空闲页,如果还没有就返回NULL。并加载文件load_file,若成功,返回file指针,否则返回NULL。详情参见博文《Contiki学习笔记:Coffee文件系统创建文件》三。
四、写入文件情形2
配置了COFFEE_MICRO_LOGS,文件要么没有被修改(即微日志文件不存在)且文件偏移量大于等于end,如果Coffee还配置了COFFEE_IO_SEMANTICS,还需要求file_desc的io_flags中CFS_COFFEE_IO_FLASH_AWARE被设置。
若文件被扩展了(见二),此时file->end等于偏移量,file的flags中M位为0(即没有微日志文件),这种情况恰好满足这个条件(CFS_COFFEE_IO_FLASH_AWARE有设置的话)。
当系统配置了COFFEE_APPEND_ONLY,即只允许文件末尾写入,确保偏移量在文件末尾之后,就直接写入。略去参数验证及与本节无关代码如下:
- struct file_desc *fdp;
-
struct file *file;
- #if COFFEE_MICRO_LOGS
- int i;
- struct log_param lp;
- cfs_offset_t bytes_left;
- const char dummy[1] =
- {
- 0xff
- };
- #endif
-
-
fdp = &coffee_fd_set[fd];
-
file = fdp->file;
-
-
#if COFFEE_APPEND_ONLY //见4.1
-
if(fdp->offset < file->end)
-
{
-
return - 1;
-
}
-
#endif
-
-
COFFEE_WRITE(buf, size, absolute_offset(file->page, fdp->offset)); //此时offset >= end,见4.2
-
fdp->offset += size;
/***************************/
- if(fdp->offset > file->end)
- {
- file->end = fdp->offset;
- }
- return size;
4.1 COFFEE_APPEND_ONLY
COFFEE_APPEND_ONLY用于指定文件写入只能从文件末尾写,系统默认是没有设置的,若设置了,可以节省代码空间( save some code space)。值得注意的是,COFFEE_APPEND_ONLY和COFFEE_MICRO_LOGS不能同时为1,否则编译错误,源码如下:
- #if COFFEE_MICRO_LOGS && COFFEE_APPEND_ONLY
-
#error "Cannot have COFFEE_APPEND_ONLY set when COFFEE_MICRO_LOGS is set."
-
#endif
4.2 COFFEE_WRITE
COFFEE_WRITE宏直接定位到硬件相关的读函数,移植Coffee文件的时候需要映射过去(在cfs-coffee-arch.h),源码如下:
- #define COFFEE_WRITE(buf, size, offset) stm32_flash_write(COFFEE_START + offset, buf, size)
COFFEE_WRITE与stm32_flash_write映射关系如下图,在原有的offset加上COFFEE_START,就得到了从FLASH_START处的偏移量,即实际物理FLASH位置。
图 COFFEE_WRITE与stm32_flash_write映射关系
我觉得好奇怪,既然已经超过了文件末尾,那应该是从file->end写入,否则会产生文件空洞。
五、写入文件情形3
没有配置COFFEE_MICRO_LOGS,就直接跳到情型3了,事实上,情型1和情型2都需要执行这段代码,
略去参数验证及与本节无关代码如下:
- struct file_desc *fdp;
-
struct file *file;
-
-
fdp = &coffee_fd_set[fd];
-
file = fdp->file;
-
-
if(fdp->offset > file->end)
-
{
-
file->end = fdp->offset;
-
}
-
return size;
更多Contiki学习笔记可通过博文《Contiki学习笔记:目录》索引访问。
本文图片1~2源文件如下:
COFFEE_WRITE与stm32_flash_write映射关系.rar
创建微日志示意图.rar
参考资料:
[1] Tsiftes Nicolas,Dunkels Adam,He Zhitao.Enabling large-scale storage in sensor networks with the coffee file system[J].International Conference on Information Processing in Sensor Networks.2009,349-360
[2]
[3] Contiki源代码
阅读(2724) | 评论(0) | 转发(1) |