linux写入磁盘过程经历VFS -> 页缓存（page cache） -> 具体的文件系统（ext2/3/4、XFS、ReiserFS等） -> Block IO ->设备驱动 -> SCSI指令（或者其他指令），总体来说linux文件写入磁盘过程比较复杂
1、VFS（虚拟文件系统）
Linux中采用了VFS的方式屏蔽了多个文件系统的差别， 当需要不同的设备或者其他文件系统时，采用挂载mount的方式访问其他设备或者其他文件系统（这里可以把文件系统理解为具体的设备）。正是因为使用了VFS，所以所有的文件系统设备使用统一的文件目录树视图访问，整个存储空间采用一个文件系统目录树来管理，屏蔽了底层多个文件系统之间的差别。当然，如果你需要把你自己编写的文件系统集成到Linux内核，采用VFS的方式进行访问，你需要采用模块加载的方式进行处理，相应的文件系统模块文件需要编入到系统目录/lib/modules/your-system-name/kernel/fs当中。当然VFS的作用远不止这些，通过VFS也进行访问设备，在Linux下所有的对象都是文件，简化了系统的访问。
1.1 正常情况下，所有的文件操作通过系统调用进入到VFS中，特殊的处理，直接操作原始设备。文件系统写入的系统调用为：
#include <unistd.h>
ssize_t write(int fd, const void * buffer, size_t count);
1.2 当采用系统调用进入VFS时，接下来的处理交给VFS层。处理过程比较中要的是vfs_write、generic_file_aio_write
1 ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
2 {
3 ssize_t ret;
4
5 if (!(file->f_mode & FMODE_WRITE))
6 return -EBADF;
7 if (!file->f_op || (!file->f_op->write && !file->f_op->aio_write))
8 return -EINVAL;
9 if (unlikely(!access_ok(VERIFY_READ, buf, count)))
10 return -EFAULT;
11
12 ret = rw_verify_area(WRITE, file, pos, count);
13 if (ret >= 0) {
14 count = ret;
15 if (file->f_op->write)
16 ret = file->f_op->write(file, buf, count, pos);
17 else
18 ret = do_sync_write(file, buf, count, pos);
19 if (ret > 0) {
20 fsnotify_modify(file->f_path.dentry);
21 add_wchar(current, ret);
22 }
23 inc_syscw(current);
24 }
25
26 return ret;
27 }
1 /**
2 * generic_file_aio_write - write data to a file
3 * @iocb: IO state structure
4 * @iov: vector with data to write
5 * @nr_segs: number of segments in the vector
6 * @pos: position in file where to write
7 *
8 * This is a wrapper around __generic_file_aio_write() to be used by most
9 * filesystems. It takes care of syncing the file in case of O_SYNC file
10 * and acquires i_mutex as needed.
11 */
12 ssize_t generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
13 unsigned long nr_segs, loff_t pos)
14 {
15 struct file *file = iocb->ki_filp;
16 struct inode *inode = file->f_mapping->host;
17 ssize_t ret;
18
19 BUG_ON(iocb->ki_pos != pos);
20
21 mutex_lock(&inode->i_mutex);
22 ret = __generic_file_aio_write(iocb, iov, nr_segs, &iocb->ki_pos);
23 mutex_unlock(&inode->i_mutex);
24
25 if (ret > 0 || ret == -EIOCBQUEUED) {
26 ssize_t err;
27
28 err = generic_write_sync(file, pos, ret);
29 if (err < 0 && ret > 0)
30 ret = err;
31 }
32 return ret;
33 }
2、 对于VFS层也有采用page cache和非page cache两种，下面重要介绍采用page cache的处理。
在VFS中， 每个打开操作的文件对应内核都有一个address_space 数据结构， 该数据结构是用来表示系统中打开的文件，并且一个打开的文件只有一个address_space数据结构。
如下：
1 struct address_space {
2 struct inode *host; /* owner: inode, block_device */
3 struct radix_tree_root page_tree; /* radix tree of all pages */
4 spinlock_t tree_lock; /* and lock protecting it */
5 unsigned int i_mmap_writable;/* count VM_SHARED mappings */
6 struct prio_tree_root i_mmap; /* tree of private and shared mappings */
7 struct list_head i_mmap_nonlinear;/*list VM_NONLINEAR mappings */
8 spinlock_t i_mmap_lock; /* protect tree, count, list */
9 unsigned int truncate_count; /* Cover race condition with truncate */
10 unsigned long nrpages; /* number of total pages */
11 pgoff_t writeback_index;/* writeback starts here */
12 const struct address_space_operations *a_ops; /* methods */
13 unsigned long flags; /* error bits/gfp mask */
14 struct backing_dev_info *backing_dev_info; /* device readahead, etc */
15 spinlock_t private_lock; /* for use by the address_space */
16 struct list_head private_list; /* ditto */
17 struct address_space *assoc_mapping; /* ditto */
18 struct mutex unmap_mutex; /* to protect unmapping */
19 } __attribute__((aligned(sizeof(long))));
对于文件中的文件内容缓存采用的是基数树的方式来保存的，在成员变量page_tree中，关于基数树的介绍参考[1]和[2]。 下面是关于page cache写处理的几个重要的函数
1 ssize_t
2 generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
3 unsigned long nr_segs, loff_t pos, loff_t *ppos,
4 size_t count, ssize_t written)
5 {
6 struct file *file = iocb->ki_filp;
7 struct address_space *mapping = file->f_mapping;
8 ssize_t status;
9 struct iov_iter i;
10
11 iov_iter_init(&i, iov, nr_segs, count, written);
12 status = generic_perform_write(file, &i, pos);
13
14 if (likely(status >= 0)) {
15 written += status;
16 *ppos = pos + status;
17 }
18
19 /*
20 * If we get here for O_DIRECT writes then we must have fallen through
21 * to buffered writes (block instantiation inside i_size). So we sync
22 * the file data here, to try to honour O_DIRECT expectations.
23 */
24 if (unlikely(file->f_flags & O_DIRECT) && written)
25 status = filemap_write_and_wait_range(mapping,
26 pos, pos + written - 1);
27
28 return written ? written : status;
29 }
调用page cache中的write_begin 和write_end
Note： 在进行VFS系统调用写入文件过程中，可以允许在文件中的任何位置写入，这其中就包括当写入的过程中写入的起始位置不是一个block的开始位置，这时需要特殊的处理，上述的过程都在write_begin这个函数调用过程中处理完毕。
3、ext2/3/4中文件的处理。
当在page cache中进行到write_begin时，需要ext4中的ext4_write_begin处理， 如下：
1 static int ext4_write_begin(struct file *file, struct address_space *mapping,
2 loff_t pos, unsigned len, unsigned flags,
3 struct page **pagep, void **fsdata)
4 {
5 struct inode *inode = mapping->host;
6 int ret, needed_blocks;
7 handle_t *handle;
8 int retries = 0;
9 struct page *page;
10 pgoff_t index;
11 unsigned from, to;
12 .........
13
14 index = pos >> PAGE_CACHE_SHIFT;
15 from = pos & (PAGE_CACHE_SIZE - 1);
16 to = from + len;
17
18 retry:
19 handle = ext4_journal_start(inode, needed_blocks);
20 if (IS_ERR(handle)) {
21 ret = PTR_ERR(handle);
22 goto out;
23 }
24
25 /* We cannot recurse into the filesystem as the transaction is already
26 * started */
27 flags |= AOP_FLAG_NOFS;
28
29 page = grab_cache_page_write_begin(mapping, index, flags);
30 if (!page) {
31 ext4_journal_stop(handle);
32 ret = -ENOMEM;
33 goto out;
34 }
35 *pagep = page;
36
37 ret = block_write_begin(file, mapping, pos, len, flags, pagep, fsdata,
38 ext4_get_block);
39
40 if (!ret && ext4_should_journal_data(inode)) {
41 ret = walk_page_buffers(handle, page_buffers(page),
42 from, to, NULL, do_journal_get_write_access);
43 }
44
45 if (ret) {
46 unlock_page(page);
47 page_cache_release(page);
48 /*
49 * block_write_begin may have instantiated a few blocks
50 * outside i_size. Trim these off again. Don‘t need
51 * i_size_read because we hold i_mutex.
52 *
53 * Add inode to orphan list in case we crash before
54 * truncate finishes
55 */
56 if (pos + len > inode->i_size && ext4_can_truncate(inode))
57 ext4_orphan_add(handle, inode);
58
59 ext4_journal_stop(handle);
60 if (pos + len > inode->i_size) {
61 ext4_truncate_failed_write(inode);
62 /*
63 * If truncate failed early the inode might
64 * still be on the orphan list; we need to
65 * make sure the inode is removed from the
66 * orphan list in that case.
67 */
68 if (inode->i_nlink)
69 ext4_orphan_del(NULL, inode);
70 }
71 }
72
73 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
74 goto retry;
75 out:
76 return ret;
77 }
其中在ext4_write_begin中包含了很多的处理功能，包括文件物理块的分配（假设ext4中的delay allocation特性没有开启）、文件块的部分写过程的处理等。下面是在ext_write_begin函数调用过程中比较重要的几个函数。
1 /*
2 * block_write_begin takes care of the basic task of block allocation and
3 * bringing partial write blocks uptodate first.
4 *
5 * If *pagep is not NULL, then block_write_begin uses the locked page
6 * at *pagep rather than allocating its own. In this case, the page will
7 * not be unlocked or deallocated on failure.
8 */
9 int block_write_begin(struct file *file, struct address_space *mapping,
10 loff_t pos, unsigned len, unsigned flags,
11 struct page **pagep, void **fsdata,
12 get_block_t *get_block)
13 {
14 struct inode *inode = mapping->host;
15 int status = 0;
16 struct page *page;
17 pgoff_t index;
18 unsigned start, end;
19 int ownpage = 0;
20
21 index = pos >> PAGE_CACHE_SHIFT;
22 start = pos & (PAGE_CACHE_SIZE - 1);
23 end = start + len;
24
25 page = *pagep;
26 if (page == NULL) {
27 ownpage = 1;
28 page = grab_cache_page_write_begin(mapping, index, flags);
29 if (!page) {
30 status = -ENOMEM;
31 goto out;
32 }
33 *pagep = page;
34 } else
35 BUG_ON(!PageLocked(page));
36
37 status = __block_prepare_write(inode, page, start, end, get_block);
38 if (unlikely(status)) {
39 ClearPageUptodate(page);
40
41 if (ownpage) {
42 unlock_page(page);
43 page_cache_release(page);
44 *pagep = NULL;
45
46 /*
47 * prepare_write() may have instantiated a few blocks
48 * outside i_size. Trim these off again. Don‘t need
49 * i_size_read because we hold i_mutex.
50 */
51 if (pos + len > inode->i_size)
52 vmtruncate(inode, inode->i_size);
53 }
54 }
55
56 out:
57 return status;
58 }
1 static int __block_prepare_write(struct inode *inode, struct page *page,
2 unsigned from, unsigned to, get_block_t *get_block)
3 {
4 unsigned block_start, block_end;
5 sector_t block;
6 int err = 0;
7 unsigned blocksize, bbits;
8 struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
9
10 BUG_ON(!PageLocked(page));
11 BUG_ON(from > PAGE_CACHE_SIZE);
12 BUG_ON(to > PAGE_CACHE_SIZE);
13 BUG_ON(from > to);
14
15 blocksize = 1 << inode->i_blkbits;
16 if (!page_has_buffers(page))
17 create_empty_buffers(page, blocksize, 0);
18 head = page_buffers(page);
19
20 bbits = inode->i_blkbits;
21 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
22
23 for(bh = head, block_start = 0; bh != head || !block_start;
24 block++, block_start=block_end, bh = bh->b_this_page) {
25 block_end = block_start + blocksize;
26 if (block_end <= from || block_start >= to) {
27 if (PageUptodate(page)) {
28 if (!buffer_uptodate(bh))
29 set_buffer_uptodate(bh);
30 }
31 continue;
32 }
33 if (buffer_new(bh))
34 clear_buffer_new(bh);
35 if (!buffer_mapped(bh)) {
36 WARN_ON(bh->b_size != blocksize);
37 err = get_block(inode, block, bh, 1);
38 if (err)
39 break;
40 if (buffer_new(bh)) {
41 unmap_underlying_metadata(bh->b_bdev,
42 bh->b_blocknr);
43 if (PageUptodate(page)) {
44 clear_buffer_new(bh);
45 set_buffer_uptodate(bh);
46 mark_buffer_dirty(bh);
47 continue;
48 }
49 if (block_end > to || block_start < from)
50 zero_user_segments(page,
51 to, block_end,
52 block_start, from);
53 continue;
54 }
55 }
56 if (PageUptodate(page)) {
57 if (!buffer_uptodate(bh))
58 set_buffer_uptodate(bh);
59 continue;
60 }
61 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
62 !buffer_unwritten(bh) &&
63 (block_start < from || block_end > to)) {
64 ll_rw_block(READ, 1, &bh);
65 *wait_bh++=bh;
66 }
67 }
68 /*
69 * If we issued read requests - let them complete.
70 */
71 while(wait_bh > wait) {
72 wait_on_buffer(*--wait_bh);
73 if (!buffer_uptodate(*wait_bh))
74 err = -EIO;
75 }
76 if (unlikely(err))
77 page_zero_new_buffers(page, from, to);
78 return err;
79 }
1 /**
2 * ll_rw_block: low-level access to block devices (DEPRECATED)
3 * @rw: whether to %READ or %WRITE or %SWRITE or maybe %READA (readahead)
4 * @nr: number of &struct buffer_heads in the array
5 * @bhs: array of pointers to &struct buffer_head
6 *
7 * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
8 * requests an I/O operation on them, either a %READ or a %WRITE. The third
9 * %SWRITE is like %WRITE only we make sure that the *current* data in buffers
10 * are sent to disk. The fourth %READA option is described in the documentation
11 * for generic_make_request() which ll_rw_block() calls.
12 *
13 * This function drops any buffer that it cannot get a lock on (with the
14 * BH_Lock state bit) unless SWRITE is required, any buffer that appears to be
15 * clean when doing a write request, and any buffer that appears to be
16 * up-to-date when doing read request. Further it marks as clean buffers that
17 * are processed for writing (the buffer cache won‘t assume that they are
18 * actually clean until the buffer gets unlocked).
19 *
20 * ll_rw_block sets b_end_io to simple completion handler that marks
21 * the buffer up-to-date (if approriate), unlocks the buffer and wakes
22 * any waiters.
23 *
24 * All of the buffers must be for the same device, and must also be a
25 * multiple of the current approved size for the device.
26 */
27 void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
28 {
29 int i;
30
31 for (i = 0; i < nr; i++) {
32 struct buffer_head *bh = bhs[i];
33
34 if (rw == SWRITE || rw == SWRITE_SYNC || rw == SWRITE_SYNC_PLUG)
35 lock_buffer(bh);
36 else if (!trylock_buffer(bh))
37 continue;
38
39 if (rw == WRITE || rw == SWRITE || rw == SWRITE_SYNC ||
40 rw == SWRITE_SYNC_PLUG) {
41 if (test_clear_buffer_dirty(bh)) {
42 bh->b_end_io = end_buffer_write_sync;
43 get_bh(bh);
44 if (rw == SWRITE_SYNC)
45 submit_bh(WRITE_SYNC, bh);
46 else
47 submit_bh(WRITE, bh);
48 continue;
49 }
50 } else {
51 if (!buffer_uptodate(bh)) {
52 bh->b_end_io = end_buffer_read_sync;
53 get_bh(bh);
54 submit_bh(rw, bh);
55 continue;
56 }
57 }
58 unlock_buffer(bh);
59 }
60 }
其中在ext4中块的分配过程中，管理块分配处理的函数实现在fs/ext4/balloc.c fs/ext4/mballoc.c
4、当page cache中的数据需要刷新到disk上的时候，这时处理的过程由Block IO接管。
在进行文件page cache刷新到disk上的过程中比较重要的数据结构有如下两个buffer_head 和 bio
1 struct buffer_head {
2 unsigned long b_state; /* buffer state bitmap (see above) */
3 struct buffer_head *b_this_page;/* circular list of page‘s buffers */
4 struct page *b_page; /* the page this bh is mapped to */
5
6 sector_t b_blocknr; /* start block number */
7 size_t b_size; /* size of mapping */
8 char *b_data; /* pointer to data within the page */
9
10 struct block_device *b_bdev;
11 bh_end_io_t *b_end_io; /* I/O completion */
12 void *b_private; /* reserved for b_end_io */
13 struct list_head b_assoc_buffers; /* associated with another mapping */
14 struct address_space *b_assoc_map; /* mapping this buffer is
15 associated with */
16 atomic_t b_count; /* users using this buffer_head */
17 };
1 /*
2 * main unit of I/O for the block layer and lower layers (ie drivers and
3 * stacking drivers)
4 */
5 struct bio {
6 sector_t bi_sector; /* device address in 512 byte
7 sectors */
8 struct bio *bi_next; /* request queue link */
9 struct block_device *bi_bdev;
10 unsigned long bi_flags; /* status, command, etc */
11 unsigned long bi_rw; /* bottom bits READ/WRITE,
12 * top bits priority
13 */
14
15 unsigned short bi_vcnt; /* how many bio_vec‘s */
16 unsigned short bi_idx; /* current index into bvl_vec */
17 ...............
18
19 /*
20 * We can inline a number of vecs at the end of the bio, to avoid
21 * double allocations for a small number of bio_vecs. This member
22 * MUST obviously be kept at the very end of the bio.
23 */
24 struct bio_vec bi_inline_vecs[0];
25 };
在Block IO层进行基本的IO request的合并和处理调度， 基本的层由elevator管理， 具体的调度算法有noop、deadline和anticipate等多种调度算法，现在默认的调度算法是deadline，当然调度算法可调，根据系统可以调成系统最有的处理。
[1] 基数树(radix tree). http://blog.csdn.net/joker0910/article/details/8250085
[2] Radix Tree. http://en.wikipedia.org/wiki/Radix_tree
时间： 2024-10-13 01:39:22