[8/9] Convert read into 2nd threading model

diff --git a/hw/virtio-9p.c b/hw/virtio-9p.c
index e08f284..82f0ed2 100644
--- a/hw/virtio-9p.c
+++ b/hw/virtio-9p.c
@@ -2001,158 +2001,112 @@  out:
     complete_pdu(s, pdu, err);
 }
 
-static void v9fs_read_post_readdir(V9fsState *, V9fsReadState *, ssize_t);
-
-static void v9fs_read_post_seekdir(V9fsState *s, V9fsReadState *vs, ssize_t err)
-{
-    if (err) {
-        goto out;
-    }
-    v9fs_stat_free(&vs->v9stat);
-    v9fs_string_free(&vs->name);
-    vs->offset += pdu_marshal(vs->pdu, vs->offset, "d", vs->count);
-    vs->offset += vs->count;
-    err = vs->offset;
-out:
-    complete_pdu(s, vs->pdu, err);
-    qemu_free(vs);
-    return;
-}
-
-static void v9fs_read_post_dir_lstat(V9fsState *s, V9fsReadState *vs,
-                                    ssize_t err)
-{
-    if (err) {
-        err = -errno;
-        goto out;
-    }
-    err = stat_to_v9stat(s, &vs->name, &vs->stbuf, &vs->v9stat);
-    if (err) {
-        goto out;
-    }
-
-    vs->len = pdu_marshal(vs->pdu, vs->offset + 4 + vs->count, "S",
-                            &vs->v9stat);
-    if ((vs->len != (vs->v9stat.size + 2)) ||
-            ((vs->count + vs->len) > vs->max_count)) {
-        v9fs_do_seekdir(s, vs->fidp->fs.dir, vs->dir_pos);
-        v9fs_read_post_seekdir(s, vs, err);
-        return;
-    }
-    vs->count += vs->len;
-    v9fs_stat_free(&vs->v9stat);
-    v9fs_string_free(&vs->name);
-    vs->dir_pos = vs->dent->d_off;
-    vs->dent = v9fs_do_readdir(s, vs->fidp->fs.dir);
-    v9fs_read_post_readdir(s, vs, err);
-    return;
-out:
-    v9fs_do_seekdir(s, vs->fidp->fs.dir, vs->dir_pos);
-    v9fs_read_post_seekdir(s, vs, err);
-    return;
-
-}
-
-static void v9fs_read_post_readdir(V9fsState *s, V9fsReadState *vs, ssize_t err)
+static void v9fs_read_do_complete(void *opaque)
 {
-    if (vs->dent) {
-        memset(&vs->v9stat, 0, sizeof(vs->v9stat));
-        v9fs_string_init(&vs->name);
-        v9fs_string_sprintf(&vs->name, "%s/%s", vs->fidp->path.data,
-                            vs->dent->d_name);
-        err = v9fs_do_lstat(s, &vs->name, &vs->stbuf);
-        v9fs_read_post_dir_lstat(s, vs, err);
-        return;
-    }
-
-    vs->offset += pdu_marshal(vs->pdu, vs->offset, "d", vs->count);
-    vs->offset += vs->count;
-    err = vs->offset;
-    complete_pdu(s, vs->pdu, err);
+    V9fsReadState *vs = (V9fsReadState *)opaque;
+    complete_pdu(vs->s, vs->pdu, vs->err);
     qemu_free(vs);
-    return;
-}
-
-static void v9fs_read_post_telldir(V9fsState *s, V9fsReadState *vs, ssize_t err)
-{
-    vs->dent = v9fs_do_readdir(s, vs->fidp->fs.dir);
-    v9fs_read_post_readdir(s, vs, err);
-    return;
-}
-
-static void v9fs_read_post_rewinddir(V9fsState *s, V9fsReadState *vs,
-                                       ssize_t err)
-{
-    vs->dir_pos = v9fs_do_telldir(s, vs->fidp->fs.dir);
-    v9fs_read_post_telldir(s, vs, err);
-    return;
 }
 
-static void v9fs_read_post_preadv(V9fsState *s, V9fsReadState *vs, ssize_t err)
+static void v9fs_read_worker(ThreadletWork *work)
 {
-    if (err  < 0) {
-        /* IO error return the error */
-        err = -errno;
+    V9fsReadState *vs = container_of(work, V9fsReadState, work);
+    vs->fidp = lookup_fid(vs->s, vs->fid);
+    if (vs->fidp == NULL) {
+        vs->err = -EINVAL;
         goto out;
     }
-    vs->total += vs->len;
-    vs->sg = adjust_sg(vs->sg, vs->len, &vs->cnt);
-    if (vs->total < vs->count && vs->len > 0) {
-        do {
-            if (0) {
-                print_sg(vs->sg, vs->cnt);
+    if (vs->fidp->fid_type == P9_FID_DIR) {
+        qemu_rwmutex_rdlock(&global_rename_lock);
+        vs->max_count = vs->count;
+        vs->count = 0;
+        if (vs->off == 0) {
+            v9fs_do_rewinddir(vs->s, vs->fidp->fs.dir);
+        }
+        vs->dir_pos = v9fs_do_telldir(vs->s, vs->fidp->fs.dir);
+        vs->dent = v9fs_do_readdir(vs->s, vs->fidp->fs.dir);
+        while (vs->dent) {
+            memset(&vs->v9stat, 0, sizeof(vs->v9stat));
+            v9fs_string_init(&vs->name);
+            v9fs_string_sprintf(&vs->name, "%s/%s", vs->fidp->path.data,
+                                                    vs->dent->d_name);
+            vs->err = v9fs_do_lstat(vs->s, &vs->name, &vs->stbuf);
+            if (vs->err) {
+                vs->err = -errno;
+                v9fs_do_seekdir(vs->s, vs->fidp->fs.dir, vs->dir_pos);
+/* FIXME: Is there memory leak here? */
+                qemu_rwmutex_unlock(&global_rename_lock);
+                goto out;
             }
+            vs->err = stat_to_v9stat(vs->s, &vs->name, &vs->stbuf, &vs->v9stat);
+            if (vs->err) {
+                v9fs_do_seekdir(vs->s, vs->fidp->fs.dir, vs->dir_pos);
+/* FIXME: Is there memory leak here? */
+                qemu_rwmutex_unlock(&global_rename_lock);
+                goto out;
+            }
+            vs->len = pdu_marshal(vs->pdu, vs->offset + 4 + vs->count, "S",
+                                                            &vs->v9stat);
+            if ((vs->len != (vs->v9stat.size + 2)) ||
+                ((vs->count + vs->len) > vs->max_count)) {
+                v9fs_do_seekdir(vs->s, vs->fidp->fs.dir, vs->dir_pos);
+                v9fs_stat_free(&vs->v9stat);
+                v9fs_string_free(&vs->name);
+                break;
+            }
+            vs->count += vs->len;
+            v9fs_stat_free(&vs->v9stat);
+            v9fs_string_free(&vs->name);
+            vs->dir_pos = vs->dent->d_off;
+            vs->dent = v9fs_do_readdir(vs->s, vs->fidp->fs.dir);
+        }
+        vs->offset += pdu_marshal(vs->pdu, vs->offset, "d", vs->count);
+        vs->offset += vs->count;
+        vs->err = vs->offset;
+        qemu_rwmutex_unlock(&global_rename_lock);
+    } else if (vs->fidp->fid_type == P9_FID_FILE) {
+        vs->sg = vs->iov;
+/* FIXME: Is it safe to call pdu_marshal from async thread context? */
+        pdu_marshal(vs->pdu, vs->offset + 4, "v", vs->sg, &vs->cnt);
+        vs->sg = cap_sg(vs->sg, vs->count, &vs->cnt);
+        if (vs->total <= vs->count) {
             vs->len = v9fs_do_preadv(s, vs->fidp->fs.fd, vs->sg, vs->cnt,
-                      vs->off);
+                                    vs->off);
             if (vs->len > 0) {
                 vs->off += vs->len;
             }
-        } while (vs->len == -1 && errno == EINTR);
-        if (vs->len == -1) {
-            err  = -errno;
+            err = vs->len;
         }
-        v9fs_read_post_preadv(s, vs, err);
-        return;
-    }
-    vs->offset += pdu_marshal(vs->pdu, vs->offset, "d", vs->total);
-    vs->offset += vs->count;
-    err = vs->offset;
-
-out:
-    complete_pdu(s, vs->pdu, err);
-    qemu_free(vs);
-}
-
-static void v9fs_xattr_read(V9fsState *s, V9fsReadState *vs)
-{
-    ssize_t err = 0;
-    int read_count;
-    int64_t xattr_len;
-
-    xattr_len = vs->fidp->fs.xattr.len;
-    read_count = xattr_len - vs->off;
-    if (read_count > vs->count) {
-        read_count = vs->count;
-    } else if (read_count < 0) {
+    } else if (vs->fidp->fid_type == P9_FID_XATTR) {
+        int read_count;
+        int64_t xattr_len;
+
+        xattr_len = vs->fidp->fs.xattr.len;
+        read_count = xattr_len - vs->off;
+        if (read_count > vs->count) {
+            read_count = vs->count;
+        } else if (read_count < 0) {
         /*
          * read beyond XATTR value
          */
         read_count = 0;
-    }
-    vs->offset += pdu_marshal(vs->pdu, vs->offset, "d", read_count);
-    vs->offset += pdu_pack(vs->pdu, vs->offset,
+        }
+        vs->offset += pdu_marshal(vs->pdu, vs->offset, "d", read_count);
+        vs->offset += pdu_pack(vs->pdu, vs->offset,
                            ((char *)vs->fidp->fs.xattr.value) + vs->off,
                            read_count);
-    err = vs->offset;
-    complete_pdu(s, vs->pdu, err);
-    qemu_free(vs);
+        vs->err = vs->offset;
+    } else {
+        vs->err = -EINVAL;
+    }
+out:
+    v9fs_async_helper_done(v9fs_read_do_complete, vs);
+    return;
 }
 
 static void v9fs_read(V9fsState *s, V9fsPDU *pdu)
 {
-    int32_t fid;
     V9fsReadState *vs;
-    ssize_t err = 0;
 
     vs = qemu_malloc(sizeof(*vs));
     vs->pdu = pdu;
@@ -2160,46 +2114,13 @@  static void v9fs_read(V9fsState *s, V9fsPDU *pdu)
     vs->total = 0;
     vs->len = 0;
     vs->count = 0;
+    vs->s = s;
 
-    pdu_unmarshal(vs->pdu, vs->offset, "dqd", &fid, &vs->off, &vs->count);
-
-    vs->fidp = lookup_fid(s, fid);
-    if (vs->fidp == NULL) {
-        err = -EINVAL;
-        goto out;
-    }
+    pdu_unmarshal(vs->pdu, vs->offset, "dqd", &vs->fid, &vs->off, &vs->count);
 
-    if (vs->fidp->fid_type == P9_FID_DIR) {
-        vs->max_count = vs->count;
-        vs->count = 0;
-        if (vs->off == 0) {
-            v9fs_do_rewinddir(s, vs->fidp->fs.dir);
-        }
-        v9fs_read_post_rewinddir(s, vs, err);
-        return;
-    } else if (vs->fidp->fid_type == P9_FID_FILE) {
-        vs->sg = vs->iov;
-        pdu_marshal(vs->pdu, vs->offset + 4, "v", vs->sg, &vs->cnt);
-        vs->sg = cap_sg(vs->sg, vs->count, &vs->cnt);
-        if (vs->total <= vs->count) {
-            vs->len = v9fs_do_preadv(s, vs->fidp->fs.fd, vs->sg, vs->cnt,
-                                    vs->off);
-            if (vs->len > 0) {
-                vs->off += vs->len;
-            }
-            err = vs->len;
-            v9fs_read_post_preadv(s, vs, err);
-        }
-        return;
-    } else if (vs->fidp->fid_type == P9_FID_XATTR) {
-        v9fs_xattr_read(s, vs);
-        return;
-    } else {
-        err = -EINVAL;
-    }
-out:
-    complete_pdu(s, pdu, err);
-    qemu_free(vs);
+    vs->work.func = v9fs_read_worker;
+    submit_threadlet(&vs->work);
+    return;
 }
 
 typedef struct V9fsReadDirState {
diff --git a/hw/virtio-9p.h b/hw/virtio-9p.h
index 45631dd..5c6bf74 100644
--- a/hw/virtio-9p.h
+++ b/hw/virtio-9p.h
@@ -334,6 +334,10 @@  typedef struct V9fsReadState {
     int32_t len;
     int32_t cnt;
     int32_t max_count;
+    V9fsState *s;
+    int32_t fid;
+    int32_t err;
+    ThreadletWork work;
 } V9fsReadState;
 
 typedef struct V9fsWriteState {