This adds lots of documentation to dev.c. This file is raised from
least documented to most documented status.
Signed-off-by: Miklos Szeredi <[email protected]>
dev.c | 246 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----
1 files changed, 232 insertions(+), 14 deletions(-)
diff -ru linux-2.6.12-rc1-mm2/fs/fuse/dev.c linux-fuse/fs/fuse/dev.c
--- linux-2.6.12-rc1-mm2/fs/fuse/dev.c 2005-03-24 19:13:14.000000000 +0100
+++ linux-fuse/fs/fuse/dev.c 2005-03-24 19:14:23.000000000 +0100
@@ -17,6 +17,138 @@
#include <linux/file.h>
#include <linux/slab.h>
+/*
+ * The following diagram shows how a filesystem operation (in this
+ * example unlink) is performed in FUSE.
+ *
+ * NOTE: everything in this description is greatly simplified
+ *
+ * | "rm /mnt/fuse/file" | FUSE filesystem daemon
+ * | |
+ * | | >sys_read()
+ * | | >fuse_dev_read()
+ * | | >request_wait()
+ * | | [sleep on fc->waitq]
+ * | |
+ * | >sys_unlink() |
+ * | >fuse_unlink() |
+ * | [get request from |
+ * | fc->unused_list] |
+ * | >request_send() |
+ * | [queue req on fc->pending] |
+ * | [wake up fc->waitq] | [woken up]
+ * | >request_wait_answer() |
+ * | [sleep on req->waitq] |
+ * | | <request_wait()
+ * | | [remove req from fc->pending]
+ * | | [copy req to read buffer]
+ * | | [add req to fc->processing]
+ * | | <fuse_dev_read()
+ * | | <sys_read()
+ * | |
+ * | | [perform unlink]
+ * | |
+ * | | >sys_write()
+ * | | >fuse_dev_write()
+ * | | [look up req in fc->processing]
+ * | | [remove from fc->processing]
+ * | | [copy write buffer to req]
+ * | [woken up] | [wake up req->waitq]
+ * | | <fuse_dev_write()
+ * | | <sys_write()
+ * | <request_wait_answer() |
+ * | <request_send() |
+ * | [add request to |
+ * | fc->unused_list] |
+ * | <fuse_unlink() |
+ * | <sys_unlink() |
+ *
+ * There are a couple of ways in which to deadlock a FUSE filesystem.
+ * Since we are talking about unprivileged userspace programs,
+ * something must be done about these.
+ *
+ * Scenario 1 - Simple deadlock
+ * -----------------------------
+ *
+ * | "rm /mnt/fuse/file" | FUSE filesystem daemon
+ * | |
+ * | >sys_unlink("/mnt/fuse/file") |
+ * | [acquire inode semaphore |
+ * | for "file"] |
+ * | >fuse_unlink() |
+ * | [sleep on req->waitq] |
+ * | | <sys_read()
+ * | | >sys_unlink("/mnt/fuse/file")
+ * | | [acquire inode semaphore
+ * | | for "file"]
+ * | | *DEADLOCK*
+ *
+ * The solution for this is to allow requests to be interrupted while
+ * they are in userspace:
+ *
+ * | [interrupted by signal] |
+ * | <fuse_unlink() |
+ * | [release semaphore] | [semaphore acquired]
+ * | <sys_unlink() |
+ * | | >fuse_unlink()
+ * | | [queue req on fc->pending]
+ * | | [wake up fc->waitq]
+ * | | [sleep on req->waitq]
+ *
+ * If the filesystem daemon was single threaded, this will stop here,
+ * since there's no other thread to dequeue and execute the request.
+ * In this case the solution is to kill the FUSE daemon as well. If
+ * there are multiple serving threads, you just have to kill them as
+ * long as any remain.
+ *
+ * Moral: a filesystem which deadlocks, can soon find itself dead.
+ *
+ * Scenario 2 - Tricky deadlock
+ * ----------------------------
+ *
+ * This one needs a carefully crafted filesystem. It's a variation on
+ * the above, only the call back to the filesystem is not explicit,
+ * but is caused by a pagefault.
+ *
+ * | Kamikaze filesystem thread 1 | Kamikaze filesystem thread 2
+ * | |
+ * | [fd = open("/mnt/fuse/file")] | [request served normally]
+ * | [mmap fd to 'addr'] |
+ * | [close fd] | [FLUSH triggers 'magic' flag]
+ * | [read a byte from addr] |
+ * | >do_page_fault() |
+ * | [find or create page] |
+ * | [lock page] |
+ * | >fuse_readpage() |
+ * | [queue READ request] |
+ * | [sleep on req->waitq] |
+ * | | [read request to buffer]
+ * | | [create reply header before addr]
+ * | | >sys_write(addr - headerlength)
+ * | | >fuse_dev_write()
+ * | | [look up req in fc->processing]
+ * | | [remove from fc->processing]
+ * | | [copy write buffer to req]
+ * | | >do_page_fault()
+ * | | [find or create page]
+ * | | [lock page]
+ * | | * DEADLOCK *
+ *
+ * Solution is again to let the the request be interrupted (not
+ * elaborated further).
+ *
+ * An additional problem is that while the write buffer is being
+ * copied to the request, the request must not be interrupted. This
+ * is because the destination address of the copy may not be valid
+ * after the request is interrupted.
+ *
+ * This is solved with doing the copy atomically, and allowing
+ * interruption while the page(s) belonging to the write buffer are
+ * faulted with get_user_pages(). The 'req->locked' flag indicates
+ * when the copy is taking place, and interruption is delayed until
+ * this flag is unset.
+ */
+
static kmem_cache_t *fuse_req_cachep;
static inline struct fuse_conn *fuse_get_conn(struct file *file)
@@ -108,6 +240,11 @@
return do_get_request(fc);
}
+/*
+ * Non-interruptible version of the above function is for operations
+ * which can't legally return -ERESTART{SYS,NOINTR}. This can still
+ * return NULL, but only in case the signal is SIGKILL.
+ */
struct fuse_req *fuse_get_request_nonint(struct fuse_conn *fc)
{
int intr;
@@ -127,6 +264,7 @@
else
fuse_request_free(req);
+ /* If we are in debt decrease that first */
if (fc->outstanding_debt)
fc->outstanding_debt--;
else
@@ -151,7 +289,17 @@
spin_unlock(&fuse_lock);
}
-/* Called with fuse_lock, unlocks it */
+/*
+ * This function is called when a request is finished. Either a reply
+ * has arrived or it was interrupted (and not yet sent) or some error
+ * occured during communication with userspace, or the device file was
+ * closed. It decreases the referece count for the request. In case
+ * of a background request the referece to the stored objects are
+ * released. The requester thread is woken up (if still waiting), and
+ * finally the request is either freed or put on the unused_list
+ *
+ * Called with fuse_lock, unlocks it
+ */
static void request_end(struct fuse_conn *fc, struct fuse_req *req)
{
int putback;
@@ -178,10 +326,37 @@
fuse_putback_request(fc, req);
}
+/*
+ * Unfortunately request interruption not just solves the deadlock
+ * problem, it causes problems too. These stem from the fact, that an
+ * interrupted request is continued to be processed in userspace,
+ * while all the locks and object references (inode and file) held
+ * during the operation are released.
+ *
+ * To release the locks is exactly why there's a need to interrupt the
+ * request, so there's not a lot that can be done about this, except
+ * introduce additional locking in userspace.
+ *
+ * More important is to keep inode and file references until userspace
+ * has replied, otherwise FORGET and RELEASE could be sent while the
+ * inode/file is still used by the filesystem.
+ *
+ * For this reason the concept of "background" request is introduced.
+ * An interrupted request is backgrounded if it has been already sent
+ * to userspace. Backgrounding involves getting an extra reference to
+ * inode(s) or file used in the request, and adding the request to
+ * fc->background list. When a reply is received for a background
+ * request, the object references are released, and the request is
+ * removed from the list. If the filesystem is unmounted while there
+ * are still background requests, the list is walked and references
+ * are released as if a reply was received.
+ *
+ * There's one more use for a background request. The RELEASE message is
+ * always sent as background, since it doesn't return an error or
+ * data.
+ */
static void background_request(struct fuse_conn *fc, struct fuse_req *req)
{
- /* Need to get hold of the inode(s) and/or file used in the
- request, so FORGET and RELEASE are not sent too early */
req->background = 1;
list_add(&req->bg_entry, &fc->background);
if (req->inode)
@@ -233,7 +408,7 @@
if (req->locked) {
/* This is uninterruptible sleep, because data is
being copied to/from the buffers of req. During
- locked state, there musn't be any filesystem
+ locked state, there mustn't be any filesystem
operation (e.g. page fault), since that could lead
to deadlock */
spin_unlock(&fuse_lock);
@@ -268,7 +443,12 @@
req->in.h.len = sizeof(struct fuse_in_header) +
len_args(req->in.numargs, (struct fuse_arg *) req->in.args);
if (!req->preallocated) {
- /* decrease outstanding_sem, but without blocking... */
+ /* If request is not preallocated (either FORGET or
+ RELEASE), then still decrease outstanding_sem, so
+ user can't open infinite number of files while not
+ processing the RELEASE requests. However for
+ efficiency do it without blocking, so if down()
+ would block, just increase the debt instead */
if (down_trylock(&fc->outstanding_sem))
fc->outstanding_debt++;
}
@@ -298,6 +478,11 @@
request_send_wait(fc, req, 1);
}
+/*
+ * Non-interruptible version of the above function is for operations
+ * which can't legally return -ERESTART{SYS,NOINTR}. This can still
+ * be interrupted but only with SIGKILL.
+ */
void request_send_nonint(struct fuse_conn *fc, struct fuse_req *req)
{
request_send_wait(fc, req, 0);
@@ -348,6 +533,11 @@
request_send_background(fc, req);
}
+/*
+ * Lock the request. Up to the next unlock_request() there mustn't be
+ * anything that could cause a page-fault. If the request was already
+ * interrupted bail out.
+ */
static inline int lock_request(struct fuse_req *req)
{
int err = 0;
@@ -362,6 +552,11 @@
return err;
}
+/*
+ * Unlock request. If it was interrupted during being locked, the
+ * requester thread is currently waiting for it to be unlocked, so
+ * wake it up.
+ */
static inline void unlock_request(struct fuse_req *req)
{
if (req) {
@@ -373,15 +568,6 @@
}
}
-/* Why all this complex one-page-at-a-time copying needed instead of
- just copy_to/from_user()? The reason is that blocking on a page
- fault must be avoided while the request is locked. This is because
- if servicing that pagefault happens to be done by this filesystem,
- an unbreakable deadlock can occur. So the code is careful to allow
- request interruption during get_user_pages(), and only lock the
- request while doing kmapped copying, which cannot block.
- */
-
struct fuse_copy_state {
int write;
struct fuse_req *req;
@@ -406,6 +592,7 @@
cs->nr_segs = nr_segs;
}
+/* Unmap and put previous page of userspace buffer */
static inline void fuse_copy_finish(struct fuse_copy_state *cs)
{
if (cs->mapaddr) {
@@ -419,6 +606,10 @@
}
}
+/*
+ * Get another pagefull of userspace buffer, and map it to kernel
+ * address space, and lock request
+ */
static int fuse_copy_fill(struct fuse_copy_state *cs)
{
unsigned long offset;
@@ -450,6 +641,7 @@
return lock_request(cs->req);
}
+/* Do as much copy to/from userspace buffer as we can */
static inline int fuse_copy_do(struct fuse_copy_state *cs, void **val,
unsigned *size)
{
@@ -467,6 +659,10 @@
return ncpy;
}
+/*
+ * Copy a page in the request to/from the userspace buffer. Must be
+ * done atomically
+ */
static inline int fuse_copy_page(struct fuse_copy_state *cs, struct page *page,
unsigned offset, unsigned count, int zeroing)
{
@@ -492,6 +688,7 @@
return 0;
}
+/* Copy pages in the request to/from userspace buffer */
static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes,
int zeroing)
{
@@ -513,6 +710,7 @@
return 0;
}
+/* Copy a single argument in the request to/from userspace buffer */
static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size)
{
while (size) {
@@ -524,6 +722,7 @@
return 0;
}
+/* Copy request arguments to/from userspace buffer */
static int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs,
unsigned argpages, struct fuse_arg *args,
int zeroing)
@@ -541,6 +740,7 @@
return err;
}
+/* Wait until a request is available on the pending list */
static void request_wait(struct fuse_conn *fc)
{
DECLARE_WAITQUEUE(wait, current);
@@ -559,6 +759,15 @@
remove_wait_queue(&fc->waitq, &wait);
}
+/*
+ * Read a single request into the userspace filesystem's buffer. This
+ * function waits until a request is available, then removes it from
+ * the pending list and copies request data to userspace buffer. If
+ * no reply is needed (FORGET) or request has been interrupted or
+ * there was an error during the copying then it's finished by calling
+ * request_end(). Otherwise add it to the processing list, and set
+ * the 'sent' flag.
+ */
static ssize_t fuse_dev_readv(struct file *file, const struct iovec *iov,
unsigned long nr_segs, loff_t *off)
{
@@ -631,6 +840,7 @@
return fuse_dev_readv(file, &iov, 1, off);
}
+/* Look up request on processing list by unique ID */
static struct fuse_req *request_find(struct fuse_conn *fc, unsigned unique)
{
struct list_head *entry;
@@ -667,6 +877,13 @@
out->page_zeroing);
}
+/*
+ * Write a single reply to a request. First the header is copied from
+ * the write buffer. The request is then searched on the processing
+ * list by the unique ID found in the header. If found, then remove
+ * it from the list and copy the rest of the buffer to the request.
+ * The request is finished by calling request_end()
+ */
static ssize_t fuse_dev_writev(struct file *file, const struct iovec *iov,
unsigned long nr_segs, loff_t *off)
{
@@ -756,6 +973,7 @@
return mask;
}
+/* Abort all requests on the given list (pending or processing) */
static void end_requests(struct fuse_conn *fc, struct list_head *head)
{
while (!list_empty(head)) {
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
[Index of Archives]
[Kernel Newbies]
[Netfilter]
[Bugtraq]
[Photo]
[Stuff]
[Gimp]
[Yosemite News]
[MIPS Linux]
[ARM Linux]
[Linux Security]
[Linux RAID]
[Video 4 Linux]
[Linux for the blind]
[Linux Resources]
