@@ -310,3 +310,8 @@ CONFIG_SECURITY_SELINUX_DISABLE=y
CONFIG_EFI_STUB=y
CONFIG_EFI_MIXED=y
CONFIG_ACPI_BGRT=y
+CONFIG_VIRTIO=y
+CONFIG_VIRTIO_PCI=y
+CONFIG_VSOCKETS=y
+CONFIG_VIRTIO_VSOCKETS=y
+CONFIG_VIRTIO_VSOCKETS_COMMON=y
@@ -54,3 +54,4 @@ js-rtc-y = rtc.o
obj-$(CONFIG_XILLYBUS) += xillybus/
obj-$(CONFIG_POWERNV_OP_PANEL) += powernv-op-panel.o
obj-$(CONFIG_ADI) += adi.o
+obj-y += forwarder/
new file mode 100644
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for the drm device driver. This driver provides support for the
+# Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
+
+forwarder-y := forwarder_drv.o
+
+obj-y += forwarder.o
new file mode 100644
@@ -0,0 +1,103 @@
+enum {
+ STREAM_MAGIC = 0xbeefc1ea,
+ EVENT_MAGIC,
+ IPC_MAGIC,
+};
+struct pwrite_stream {
+ unsigned int magic;
+ int fd;
+ unsigned int handle;
+ unsigned int offset;
+ unsigned int size;
+};
+
+#define IPC_PAGE_SIZE 32768
+
+#define IPC_COUNT 4
+
+struct ipc {
+ volatile unsigned int seq;
+ unsigned int cmd;
+ union {
+ struct {
+ int arg1;
+ int arg2;
+ int arg3;
+ int pad1;
+ };
+ struct {
+ volatile int64_t ret;
+ int64_t pad2;
+ };
+ struct {
+ int fd;
+ } ioctl;
+ struct {
+ unsigned int pn_count;
+ } hostfd;
+ struct {
+ void* addr;
+ } dmabuf;
+ struct {
+ int fd;
+ unsigned int pn_off;
+ unsigned int pn_count;
+ } mmap;
+ struct {
+ unsigned int pn_off;
+ unsigned int pn_count;
+ } munmap;
+ struct {
+ int fd;
+ int whence;
+ } lseek;
+ struct {
+ int fd;
+ unsigned int len;
+ } fallocate;
+ struct {
+ int fd;
+ unsigned int len;
+ } ftruncate;
+ struct {
+ int fd;
+ uint32_t fdc;
+ uint32_t size;
+ } msg;
+ };
+ char data[0];
+};
+
+#define WL_IOCTL_BASE 'w'
+#define VIRT_WL_MAX 32
+#define WL_IO(nr) _IO(WL_IOCTL_BASE, nr + VIRT_WL_MAX)
+
+#define WL_CMD_NEW_RENDER_FD WL_IO(0x00)
+#define WL_CMD_NEW_WL_FD WL_IO(0x01)
+#define WL_CMD_NEW_MEM_FD WL_IO(0x02)
+#define WL_CMD_NEW_SYNC_FD WL_IO(0x03)
+#define WL_CMD_RECVMSG WL_IO(0x04)
+#define WL_CMD_SENDMSG WL_IO(0x05)
+#define WL_CMD_MMAP WL_IO(0x06)
+#define WL_CMD_MUNMAP WL_IO(0x07)
+#define WL_CMD_LSEEK WL_IO(0x08)
+#define WL_CMD_CLEAR_COUNTER WL_IO(0x09)
+#define WL_CMD_SHOW_COUNTER WL_IO(0x0A)
+#define WL_CMD_NEW_DMABUF WL_IO(0x0B)
+#define WL_CMD_FALLOCATE WL_IO(0x0C)
+#define WL_CMD_FTRUNCATE WL_IO(0x0D)
+
+#define SW_SYNC_IOC_MAGIC 'W'
+
+struct sw_sync_create_fence_data {
+ unsigned int value;
+ char name[32];
+ int fence; /* fd of new fence */
+};
+
+#define SW_SYNC_IOC_CREATE_FENCE _IOWR(SW_SYNC_IOC_MAGIC, 0,\
+ struct sw_sync_create_fence_data)
+
+#define SW_SYNC_IOC_INC _IOW(SW_SYNC_IOC_MAGIC, 1, __u32)
+
+#define KVM_HC_FORWARDING 70
new file mode 100644
@@ -0,0 +1,2104 @@
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
+#include <linux/atomic.h>
+#include <linux/anon_inodes.h>
+#include <linux/cdev.h>
+#include <linux/compat.h>
+#include <linux/kthread.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/net.h>
+#include <linux/nsproxy.h>
+#include <linux/socket.h>
+#include <linux/syscalls.h>
+#include <linux/vm_sockets.h>
+#include <uapi/drm/virtgpu_drm.h>
+#include <uapi/linux/dma-buf.h>
+#include <uapi/linux/kvm_para.h>
+#include <uapi/linux/sync_file.h>
+#include <uapi/linux/virtwl.h>
+
+#include "forwarder.h"
+
+static const int vsock_port = 30000;
+
+#define SYS_DATA_SIZE (2 << 20)
+
+#define MAX_IOCTL_DATA_SIZE (SYS_DATA_SIZE - offsetof(struct syscall_data, ioctl.data))
+
+#define MAX_IPC_DATA_SIZE (IPC_PAGE_SIZE - offsetof(struct ipc, data))
+
+
+#define ERROR(k) do { \
+ pr_err("ERR:%d %s:%d\n", (int)k, __func__, __LINE__); \
+ return k; \
+} while(0)
+
+#define ERROR_RELEASE(k) do { \
+ kfree(data); \
+ if (k) pr_err("ERR:%d %s:%d\n", (int)k, __func__, __LINE__); \
+ return k; \
+} while(0)
+
+#define bug(fmt, ...) do { \
+ printk(KERN_ERR "ERR: %s:%d " fmt, __func__, __LINE__ , ##__VA_ARGS__); \
+ BUG(); \
+} while(0)
+
+struct forward {
+ wait_queue_head_t wq;
+ int host_fd;
+ atomic_t in_wait;
+ char signaled;
+};
+
+struct wait_poll {
+ void* data;
+ int fd;
+};
+
+#define POOL_SIZE 4
+
+struct vsock_pool {
+ struct socket* socks[POOL_SIZE];
+ DECLARE_BITMAP(map, POOL_SIZE);
+ struct mutex lock;
+ struct semaphore free;
+};
+
+static struct vsock_pool stream_pool;
+//static struct vsock_pool ipc_pool;
+
+static struct socket* event_sock;
+
+static int enable = 1;
+static int crostini = 1;
+static ushort major = 226;
+static int hyper_ipc = 1;
+static int stream_bar = 65536;
+static int hyper_ipc_working;
+
+//FIXME, we should get these from host.
+static ushort vendor = 0x8086;
+static ushort device = 0x591e;
+static ushort subsystem_vendor;
+static ushort subsystem_device;
+static char config[64];
+
+enum {
+ RENDER_MINOR = (DRM_MINOR_RENDER << 6),
+ SYNC_MINOR,
+ WL_MINOR,
+};
+
+#define MINOR_NUM 3
+
+static int vsock_send(struct socket *vsock, void *buf, size_t size)
+{
+ struct msghdr msg = { };
+ struct kvec iov[1];
+ iov[0].iov_base = buf;
+ iov[0].iov_len = size;
+ return kernel_sendmsg(vsock, &msg, iov, 1, size);
+}
+
+static int vsock_recv(struct socket *vsock, void *buf, size_t size)
+{
+ struct msghdr msg = { };
+ struct kvec iov[1];
+ iov[0].iov_base = buf;
+ iov[0].iov_len = size;
+ return kernel_recvmsg(vsock, &msg, iov, 1, size, 0);
+}
+
+static int forwarder_open(struct inode *, struct file *);
+static long forwarder_ioctl(struct file *, unsigned int, unsigned long);
+static int forwarder_mmap(struct file *, struct vm_area_struct *);
+static int forwarder_release(struct inode *, struct file *);
+
+static int quick_mmap(struct file *, struct vm_area_struct *);
+static int quick_release(struct inode *, struct file *);
+static long wayland_ioctl(struct file *, unsigned int, unsigned long);
+static loff_t forwarder_lseek(struct file *file, loff_t offset, int whence);
+static long sync_ioctl(struct file *, unsigned int, unsigned long);
+static unsigned int sync_poll(struct file *file, poll_table *wait);
+static long sw_sync_ioctl(struct file *, unsigned int, unsigned long);
+
+static long wl_ioctl(struct file *, unsigned int, unsigned long);
+
+ssize_t no_read(struct file * f, char __user * tr, size_t l, loff_t * off)
+{
+ BUG();
+}
+
+ssize_t no_write(struct file *f, const char __user * tr, size_t l, loff_t *off)
+{
+ BUG();
+}
+
+static unsigned int no_poll(struct file *file, poll_table *wait)
+{
+ BUG();
+}
+
+static const struct file_operations entry_fops = {
+ .owner = THIS_MODULE,
+ .open = forwarder_open,
+};
+
+static const struct file_operations mmap_fops = {
+ .owner = THIS_MODULE,
+ .mmap = quick_mmap,
+ .release = quick_release,
+ .read = no_read,
+ .write = no_write,
+ .poll = no_poll,
+};
+
+static const struct file_operations forwarder_fops = {
+ .owner = THIS_MODULE,
+ .unlocked_ioctl = forwarder_ioctl,
+ .compat_ioctl = forwarder_ioctl,
+ .mmap = forwarder_mmap,
+ .llseek = forwarder_lseek,
+ .release = forwarder_release,
+ .read = no_read,
+ .write = no_write,
+ .poll = no_poll,
+};
+
+static const struct file_operations wl_fops = {
+ .owner = THIS_MODULE,
+ .unlocked_ioctl = wl_ioctl,
+ .read = no_read,
+ .write = no_write,
+ .poll = no_poll,
+};
+
+static const struct file_operations wayland_fops = {
+ .owner = THIS_MODULE,
+ .unlocked_ioctl = wayland_ioctl,
+ .release = forwarder_release,
+ .read = no_read,
+ .write = no_write,
+ .poll = sync_poll,
+};
+
+static const struct file_operations sync_fops = {
+ .owner = THIS_MODULE,
+ .unlocked_ioctl = sync_ioctl,
+ .compat_ioctl = sync_ioctl,
+ .poll = sync_poll,
+ .release = forwarder_release,
+ .read = no_read,
+ .write = no_write,
+};
+
+static const struct file_operations sw_sync_fops = {
+ .owner = THIS_MODULE,
+ .unlocked_ioctl = sw_sync_ioctl,
+ .compat_ioctl = sw_sync_ioctl,
+ .release = forwarder_release,
+ .read = no_read,
+ .write = no_write,
+ .poll = no_poll,
+};
+
+static const struct file_operations mem_fd_fops = {
+ .owner = THIS_MODULE,
+ .mmap = forwarder_mmap,
+ .release = forwarder_release,
+ .llseek = forwarder_lseek,
+ .read = no_read,
+ .write = no_write,
+ .poll = no_poll,
+};
+
+struct host_mmap {
+ uint32_t pn_off;
+ uint32_t pn_count;
+ uint64_t count;
+};
+
+static struct mutex ipc_mutex;
+static struct mutex ipc_cmd_mutex;
+static struct ipc* ipc;
+static char* ipcq[IPC_COUNT];
+
+static struct mutex stream_mutex;
+static struct semaphore free_stream_socks;
+
+uint64_t host_addr[2];
+
+#define ADDR_4G (1UL << 32)
+#define ADDR_12G (3UL << 32)
+
+static uint64_t phy_host_addr(uint64_t addr)
+{
+ if (addr < ADDR_4G)
+ return addr + host_addr[0];
+ return addr + host_addr[1] - ADDR_4G;
+}
+
+static uint64_t get_host_addr(void * addr)
+{
+ uint64_t guest_phy = virt_to_phys(addr);
+ return phy_host_addr(guest_phy);
+}
+
+static void * get_guest_addr(uint64_t addr)
+{
+ if (addr >= host_addr[0] && addr < host_addr[0] + ADDR_4G)
+ return phys_to_virt(addr - host_addr[0]);
+ if (addr >= host_addr[1] && addr < host_addr[1] + ADDR_12G)
+ return phys_to_virt(addr - host_addr[1] + ADDR_4G);
+ bug("strange host addr %llx\n", addr);
+ return NULL;
+}
+
+static inline int64_t host_cmd(unsigned int cmd, int arg1, int arg2, int arg3,
+ unsigned long host_arg)
+{
+ static unsigned int ipc_seq;
+ char type = _IOC_TYPE(cmd);
+ int64_t ret;
+
+ if (hyper_ipc && hyper_ipc_working && type != 'w' &&
+ cmd != DRM_IOCTL_I915_GEM_MMAP)
+ return kvm_hypercall3(KVM_HC_FORWARDING, arg1, cmd, host_arg);
+ mutex_lock(&ipc_cmd_mutex);
+ ipc->cmd = cmd;
+ ipc->arg1 = arg1;
+ ipc->arg2 = arg2;
+ ipc->arg3 = arg3;
+ ipc->seq = (++ipc_seq);
+ ++ipc_seq;
+ do {} while(ipc->seq != ipc_seq);
+ ret = ipc->ret;
+ mutex_unlock(&ipc_cmd_mutex);
+ return ret;
+}
+
+static unsigned int sync_poll(struct file *file, poll_table *wait)
+{
+ struct forward* fwd = (struct forward *)file->private_data;
+ int host_fd;
+ int ret;
+ struct wait_poll wp;
+ poll_wait(file, &fwd->wq, wait);
+ host_fd = fwd->host_fd;
+ if (host_fd < 0) {
+ BUG();
+ }
+ if (fwd->signaled)
+ return POLLIN;
+ if (!atomic_cmpxchg(&fwd->in_wait, 0, 1)) {
+ get_file(file);
+ wp.data = file;
+ wp.fd = host_fd;
+ ret = vsock_send(event_sock, &wp, sizeof(wp));
+ if (ret != sizeof(wp))
+ BUG();
+ }
+ return 0;
+}
+
+static int connect_vsock(struct socket** sock) {
+ int err;
+ struct socket *vsock;
+ struct sockaddr_vm address = { };
+ int i;
+
+ err = __sock_create(current->nsproxy->net_ns, PF_VSOCK, SOCK_STREAM, 0,
+ &vsock, 1);
+ if (err) {
+ printk(KERN_ERR "creat vsock %d\n", err);
+ BUG();
+ }
+ address.svm_family = AF_VSOCK;
+ address.svm_port = vsock_port;
+ address.svm_cid = VMADDR_CID_HOST;
+ for (i = 0; i < 3; ++i) {
+ err = vsock->ops->connect(vsock, (struct sockaddr *)&address,
+ sizeof(address), 0);
+ if (err == -EINTR) {
+ msleep(10);
+ continue;
+ }
+ break;
+ }
+ if (err < 0) {
+ printk(KERN_WARNING "fail connect\n");
+ printk(KERN_ERR "connect vsock %d\n", err);
+ sock_release(vsock);
+ return err;
+ }
+ *sock = vsock;
+ return 0;
+}
+
+struct task_struct * wait_wake;
+
+static int quick_forwarder_open(struct file *filp)
+{
+ int fd = -1;
+ struct forward * fwd = kzalloc(sizeof(*fwd), GFP_KERNEL);
+
+ if (fwd == NULL)
+ ERROR(-ENOMEM);
+ init_waitqueue_head(&fwd->wq);
+
+ if (event_sock == NULL)bug("why no event sock");
+ fd = *(int *)filp->private_data;
+ if (fd < 0) {
+ bug("new dev fd %d\n", fd);
+ }
+ fwd->host_fd = fd;
+ filp->private_data = fwd;
+ return 0;
+}
+
+static int forwarder_open(struct inode* inode, struct file *filp)
+{
+ unsigned long host_fd_cmd = 0;
+ const struct file_operations *ops, *new_ops;
+ int ret = -1;
+ unsigned short minor = iminor(inode);
+ struct forward * fwd;
+
+ if (event_sock == NULL) {
+ wake_up_process(wait_wake);
+ while(event_sock == NULL) {
+ pr_warn("wait socket\n");
+ msleep(1000);
+ }
+ }
+
+ switch (minor) {
+ case RENDER_MINOR:
+ host_fd_cmd = WL_CMD_NEW_RENDER_FD;
+ ops = &forwarder_fops;
+ break;
+ case SYNC_MINOR:
+ host_fd_cmd = WL_CMD_NEW_SYNC_FD;
+ ops = &sw_sync_fops;
+ break;
+ case WL_MINOR:
+ ops = &wl_fops;
+ break;
+ default:
+ return -ENODEV;
+ }
+
+ new_ops = fops_get(ops);
+ if (new_ops == NULL)
+ return -ENODEV;
+
+ if (host_fd_cmd) {
+ ret = host_cmd(host_fd_cmd, 0, 0, 0, 0);
+ if (ret < 0)
+ goto err;
+ }
+ if (ret >=0) {
+ fwd = kzalloc(sizeof(*fwd), GFP_KERNEL);
+ if (fwd == NULL) {
+ ret = -ENOMEM;
+ goto err;
+ }
+ init_waitqueue_head(&fwd->wq);
+ fwd->host_fd = ret;
+ filp->private_data = fwd;
+ }
+ replace_fops(filp, new_ops);
+ return 0;
+err:
+ fops_put(new_ops);
+ return ret;
+}
+
+#define SYNC_IOC_LEGACY_MERGE 0xc0283e01
+#define SYNC_IOC_LEGACY_FENCE_INFO 0xc0283e02
+
+static void *copy_ptr(uint64_t* usr_ptr, size_t usr_len, int write_only,
+ uint64_t keep[], uint32_t c, uint32_t maxc,
+ int need_keep,
+ char *begin, char **end, int line)
+{
+ void *ret;
+ uint64_t usr = (*usr_ptr);
+ static uint32_t max, ct;
+
+ if (need_keep && c >= maxc) {
+ pr_warn("Too many pointers to keep %d %d\n",c, maxc);
+ return NULL;
+ }
+ ct = * end - begin + usr_len;
+ if (ct > max) {
+ max = ct;
+ pr_warn("max data size: %d\n", ct);
+ }
+ if (ct > MAX_IPC_DATA_SIZE) {
+ bug("too much data %ld %d\n", usr_len, line);
+ return NULL;
+ }
+ if (usr_len && !write_only && copy_from_user(*end, (void __user *)usr, usr_len)) {
+ bug("can't copy %llx %p %d %ld\n", usr, *end, line, usr_len);
+ return NULL;
+ }
+ ret = *end;
+ if (need_keep) keep[c] = usr;
+ if (usr_len)
+ *usr_ptr = get_host_addr(ret);
+ else
+ *usr_ptr = 0;
+ (*end) += usr_len;
+ return ret;
+}
+
+static int get_host_fd(int fd, const struct file_operations* op, struct file**
+ fp, int line) {
+ struct file * file = fget(fd);
+ struct forward * fwd;
+ int ret;
+ if (!file)
+ ERROR(-EBADF);
+ if ((op && file->f_op != op) ||
+ (file->f_op != &forwarder_fops &&
+ file->f_op != &sync_fops &&
+ file->f_op != &mem_fd_fops &&
+ file->f_op != &sw_sync_fops)) {
+ fput(file);
+ pr_warn("from %s %d %d\n", file->f_path.dentry->d_name.name, fd, line);
+ BUG();
+ ERROR(-EINVAL);
+ }
+ fwd = (struct forward *)file->private_data;
+ if (fwd->host_fd < 0) {
+ pr_warn("unexpected");
+ BUG();
+ }
+ ret = fwd->host_fd;
+ *fp = file;
+ return ret;
+}
+
+static inline void to_keep(uint64_t v, uint64_t keep[], uint32_t c,
+ uint32_t maxc)
+{
+ if (c >= maxc) {
+ pr_warn("Too much too keep %d\n", c);
+ BUG();
+ }
+ keep[c] = v;
+}
+
+#define DRM_IOCTL_MODE_GETPLANERESOURCES32 0xc00c64b5
+#define DRM_IOCTL_MODE_OBJ_GETPROPERTIES32 0xc01c64b9
+
+static int pre_deep_copy(unsigned int cmd, char *data, int *guest_fd,
+ struct file** hold_fp, uint64_t keep_ptr[],
+ uint32_t *count)
+{
+ char *end;
+ struct drm_i915_gem_execbuffer2 *eb;
+ struct drm_i915_gem_exec_object2 *eo;
+ struct drm_i915_gem_relocation_entry *re;
+ struct drm_i915_gem_exec_fence *ef;
+ struct drm_prime_handle *h;
+ struct sync_merge_data* md;
+ struct sync_file_info* fi;
+ struct sync_fence_info* sf;
+ struct drm_mode_get_property *gp;
+ struct drm_mode_get_plane_res *pr;
+ struct drm_mode_get_plane* mgp;
+ struct drm_mode_obj_get_properties * opr;
+ struct drm_i915_getparam *g;
+ struct drm_version *v;
+ int i, fd;
+ uint32_t c = 0;
+
+ switch (cmd) {
+ case SYNC_IOC_FILE_INFO:
+ fi = (struct sync_file_info*) data;
+ if (fi->num_fences && fi->sync_fence_info) {
+ if (fi->num_fences * sizeof(*sf) + sizeof(*fi) >
+ MAX_IPC_DATA_SIZE) {
+ pr_warn("too many fences %d\n", fi->num_fences);
+ BUG();
+ }
+ to_keep(fi->sync_fence_info, keep_ptr, c++, *count);
+ fi->sync_fence_info = get_host_addr(fi + 1);
+ }
+ break;
+ case SYNC_IOC_MERGE:
+ md = (struct sync_merge_data *)data;
+ *guest_fd = md->fd2;
+ fd = get_host_fd(md->fd2, &sync_fops, hold_fp, __LINE__);
+ if (fd<0) {
+ BUG();
+ ERROR(-EINVAL);
+ }
+ md->fd2 = fd;
+ break;
+ case DRM_IOCTL_PRIME_FD_TO_HANDLE:
+ h = (struct drm_prime_handle *)data;
+ *guest_fd = h->fd;
+ fd = get_host_fd(h->fd, NULL, hold_fp, __LINE__);
+ if (fd < 0)
+ ERROR(fd);
+ h->fd = fd;
+ break;
+ case DRM_IOCTL_I915_GEM_EXECBUFFER2_WR:
+ case DRM_IOCTL_I915_GEM_EXECBUFFER2:
+ eb = (struct drm_i915_gem_execbuffer2 *)data;
+ end = data + sizeof(*eb);
+ if (eb->flags & I915_EXEC_FENCE_ARRAY)BUG();
+ if (eb->flags & I915_EXEC_FENCE_IN) {
+ *guest_fd = lower_32_bits(eb->rsvd2);
+ fd = get_host_fd(*guest_fd, &sync_fops, hold_fp, __LINE__);
+ if (fd < 0) {
+ BUG();
+ ERROR(-EINVAL);
+ }
+ eb->rsvd2 = fd;
+ }
+ if (eb->buffer_count) {
+ eo = copy_ptr(&eb->buffers_ptr,
+ sizeof(*eo) * eb->buffer_count,
+ 0, keep_ptr, c++, *count, 1,
+ data, &end, __LINE__);
+ if (eo == NULL)
+ ERROR(-EFAULT);
+ for (i = 0; i < eb->buffer_count; ++i, ++eo) {
+ to_keep(eo->offset, keep_ptr, c++, *count);
+ if (eo->relocation_count==0)
+ continue;
+ if(copy_ptr(&eo->relocs_ptr,
+ sizeof(*re) * eo->relocation_count,
+ 0, NULL, 0, 0, 0,
+ data, &end, __LINE__) == NULL)
+ ERROR(-EFAULT);
+ }
+ }
+ if (eb->num_cliprects) {
+ if (copy_ptr(&eb->cliprects_ptr,
+ sizeof(*ef) * eb->num_cliprects,
+ 0, NULL, 0, 0, 0,
+ data, &end, __LINE__) == NULL)
+ ERROR(-EFAULT);
+ }
+ break;
+ case DRM_IOCTL_MODE_GETPROPERTY:
+ gp = (struct drm_mode_get_property *)data;
+ end = data + sizeof(*gp);
+ to_keep(gp->count_values, keep_ptr, c++, *count);
+ to_keep(gp->count_enum_blobs, keep_ptr, c++, *count);
+ if (copy_ptr(&gp->values_ptr, gp->count_values * sizeof(uint64_t),
+ 1, keep_ptr, c++, *count, 1, data, &end,
+ __LINE__)==NULL)
+ ERROR(-EFAULT);
+ if (copy_ptr(&gp->enum_blob_ptr,
+ gp->count_enum_blobs *
+ sizeof(struct drm_mode_property_enum),
+ 1, keep_ptr, c++, *count, 1, data, &end,
+ __LINE__)==NULL)
+ ERROR(-EFAULT);
+ break;
+ case DRM_IOCTL_MODE_GETPLANERESOURCES:
+ case DRM_IOCTL_MODE_GETPLANERESOURCES32:
+ pr = (struct drm_mode_get_plane_res *)data;
+ end = data + sizeof(*pr);
+ to_keep(pr->count_planes, keep_ptr, c++, *count);
+ if (copy_ptr(&pr->plane_id_ptr, pr->count_planes * sizeof(int),
+ 1, keep_ptr, c++, *count, 1, data, &end,
+ __LINE__)==NULL)
+ return -EFAULT;
+ break;
+ case DRM_IOCTL_MODE_GETPLANE:
+ mgp = (struct drm_mode_get_plane *)data;
+ end = data + sizeof(*mgp);
+ to_keep(mgp->count_format_types, keep_ptr, c++, *count);
+ if (copy_ptr(&mgp->format_type_ptr,
+ mgp->count_format_types * sizeof(int),
+ 1, keep_ptr, c++, *count, 1, data, &end,
+ __LINE__)==NULL)
+ return -EFAULT;
+ break;
+ case DRM_IOCTL_MODE_OBJ_GETPROPERTIES:
+ case DRM_IOCTL_MODE_OBJ_GETPROPERTIES32:
+ opr = (struct drm_mode_obj_get_properties *)data;
+ end = data + sizeof(*opr);
+ to_keep(opr->count_props, keep_ptr, c++, *count);
+ if (copy_ptr(&opr->props_ptr,
+ opr->count_props * sizeof(int),
+ 1, keep_ptr, c++, *count, 1, data, &end,
+ __LINE__)==NULL)
+ ERROR(-EFAULT);
+ if (copy_ptr(&opr->prop_values_ptr,
+ opr->count_props * sizeof(uint64_t),
+ 1, keep_ptr, c++, *count, 1, data, &end,
+ __LINE__)==NULL)
+ ERROR(-EFAULT);
+ break;
+ case DRM_IOCTL_I915_GETPARAM:
+ g = (struct drm_i915_getparam *)data;
+ end = data + sizeof(*g);
+ if (copy_ptr((uint64_t *)&g->value, sizeof(int),
+ 1, keep_ptr, c++, *count, 1, data, &end,
+ __LINE__)==NULL)
+ ERROR(-EFAULT);
+ break;
+ case DRM_IOCTL_VERSION:
+ v = (struct drm_version *)data;
+ end = data + sizeof(*v);
+ to_keep(v->name_len, keep_ptr, c++, *count);
+ to_keep(v->date_len, keep_ptr, c++, *count);
+ to_keep(v->desc_len, keep_ptr, c++, *count);
+ if (copy_ptr((uint64_t *)&v->name, v->name_len,
+ 1, keep_ptr, c++, *count, 1, data, &end,
+ __LINE__)==NULL)
+ ERROR(-EFAULT);
+ if (copy_ptr((uint64_t *)&v->date, v->date_len,
+ 1, keep_ptr, c++, *count, 1, data, &end,
+ __LINE__)==NULL)
+ ERROR(-EFAULT);
+ if (copy_ptr((uint64_t *)&v->desc, v->desc_len,
+ 1, keep_ptr, c++, *count, 1, data, &end,
+ __LINE__)==NULL)
+ ERROR(-EFAULT);
+ break;
+ }
+ *count = c;
+ return 0;
+}
+
+static int setup_fd(int *host_fd, const char* name,
+ const struct file_operations* fops)
+{
+ struct file *file;
+ int ret;
+ int flags = O_RDWR | O_CLOEXEC;
+
+ file = anon_inode_getfile(name, fops, host_fd, flags);
+ //FIXME host fd leaked at host
+ if (IS_ERR(file))
+ ERROR(PTR_ERR(file));
+ if (fops->llseek) {
+ file->f_mode |= FMODE_LSEEK;
+ }
+ ret = quick_forwarder_open(file);
+ if (ret)
+ goto error;
+
+ ret = get_unused_fd_flags(flags);
+ if (ret < 0)
+ goto error;
+
+ fd_install(ret, file);
+ //pr_warn("setup %s fd host %d guest %d\n", name, *host_fd, ret);
+ *host_fd = ret;
+ return 0;
+error:
+ fput(file);
+ return ret;
+}
+
+static void *mmap_phy_addr(uint64_t phy_addr, size_t size, struct file *dev)
+{
+ void *ptr;
+ struct file *filp = anon_inode_getfile("forwarder GEM_MMAP",
+ &mmap_fops, dev,
+ O_RDWR);
+ if (IS_ERR(filp))
+ return filp;
+ get_file(dev);
+ ptr = (void *)vm_mmap(filp, 0, size, PROT_READ | PROT_WRITE, MAP_SHARED,
+ phy_addr);
+ fput(filp);
+ return ptr;
+}
+
+static int deep_copy(struct file *filp, unsigned int cmd, char *buf,
+ int guest_fd, struct file *hold_fp, uint64_t keep[],
+ uint32_t c, int *fd)
+{
+ char *ptr;
+ struct drm_version *v;
+ struct drm_i915_getparam *g;
+ struct drm_i915_gem_execbuffer2 *eb;
+ struct drm_i915_gem_exec_object2 *eo;
+ struct drm_i915_gem_mmap *mp;
+ struct drm_prime_handle *h;
+ struct drm_mode_get_plane_res* pr;
+ struct drm_mode_get_plane* mgp;
+ struct drm_mode_obj_get_properties* opr;
+ struct drm_mode_get_property* gp;
+ struct sync_merge_data* md;
+ struct sync_file_info* fi;
+ struct sw_sync_create_fence_data* ss;
+ int i, hostfd;
+ int zc;
+
+ if(hold_fp)fput(hold_fp);
+
+ switch (cmd) {
+ case SW_SYNC_IOC_CREATE_FENCE:
+ ss = (struct sw_sync_create_fence_data *) buf;
+ if (setup_fd(&ss->fence, "host sw fence", &sync_fops))
+ ERROR(-ENOMEM);
+ return 0;
+ case SYNC_IOC_FILE_INFO:
+ fi = (struct sync_file_info*) buf;
+ ptr = buf + sizeof(*fi);
+ if (fi->num_fences && fi->sync_fence_info) {
+ if (c!=1) {
+ pr_warn("c is:%d\n", c);
+ BUG();
+ }
+ if(copy_to_user((void __user *)keep[0], fi+1,
+ fi->num_fences * sizeof(struct sync_fence_info)))
+ return -EFAULT;
+ fi->sync_fence_info = keep[0];
+ }
+ return 0;
+ case SYNC_IOC_MERGE:
+ md = (struct sync_merge_data*) buf;
+ md->fd2 = guest_fd;
+ if (setup_fd(&md->fence, "host merge sync", &sync_fops))
+ ERROR(-ENOMEM);
+ return 0;
+ case DRM_IOCTL_MODE_GETPLANE:
+ mgp = (struct drm_mode_get_plane *)buf;
+ if (c!=2)bug("unexpected %d\n", c);
+ if (keep[0] && keep[1] &&
+ copy_to_user ((void __user *)keep[1],
+ get_guest_addr(mgp->format_type_ptr),
+ min_t(uint32_t, keep[0], mgp->count_format_types) * sizeof(int)))
+ return -EFAULT;
+ mgp->format_type_ptr = keep[1];
+ return 0;
+ case DRM_IOCTL_MODE_GETPLANERESOURCES:
+ case DRM_IOCTL_MODE_GETPLANERESOURCES32:
+ pr = (struct drm_mode_get_plane_res *)buf;
+ if (c!=2)bug("unexpected %d\n", c);
+ if (keep[0] && keep[1] &&
+ copy_to_user ((void __user *)keep[1],
+ get_guest_addr(pr->plane_id_ptr),
+ min_t(uint32_t, keep[0], pr->count_planes) * sizeof(int)))
+ return -EFAULT;
+ pr->plane_id_ptr = keep[1];
+ return 0;
+ case DRM_IOCTL_MODE_OBJ_GETPROPERTIES:
+ case DRM_IOCTL_MODE_OBJ_GETPROPERTIES32:
+ opr = (struct drm_mode_obj_get_properties *)buf;
+ if (c!=3)bug("unexpected %d\n", c);
+ if (keep[0] &&
+ (copy_to_user((void __user *)keep[1],
+ get_guest_addr(opr->props_ptr),
+ min_t(uint32_t, keep[0], opr->count_props) * sizeof(int)) ||
+ copy_to_user((void __user *)keep[2],
+ get_guest_addr(opr->prop_values_ptr),
+ min_t(uint32_t, keep[0], opr->count_props) *
+ sizeof(uint64_t))))
+ ERROR(-EFAULT);
+ opr->props_ptr = keep[1];
+ opr->prop_values_ptr = keep[2];
+ return 0;
+ case DRM_IOCTL_MODE_GETPROPERTY:
+ gp = (struct drm_mode_get_property *)buf;
+ if (c!=4)bug("not expected c %d\n", c);
+ if (keep[0] && keep[2] &&
+ copy_to_user((void __user *)keep[2],
+ get_guest_addr(gp->values_ptr),
+ min_t(uint32_t, keep[0], gp->count_values) * sizeof(uint64_t)))
+ return -EFAULT;
+ gp->values_ptr = keep[2];
+ if (keep[1] && keep[3] &&
+ copy_to_user((void __user *)keep[3],
+ get_guest_addr(gp->enum_blob_ptr),
+ min_t(uint32_t, keep[1], gp->count_enum_blobs) *
+ sizeof(struct drm_mode_property_enum)))
+ return -EFAULT;
+ gp->enum_blob_ptr = keep[3];
+ return 0;
+ case DRM_IOCTL_VERSION:
+ v = (struct drm_version *)buf;
+ if (c!=6)bug("not expected c %d\n", c);
+ if (keep[0] && copy_to_user((void __user *)keep[3],
+ get_guest_addr((uint64_t)v->name),
+ min_t(uint32_t, keep[0], v->name_len)))
+ return -EFAULT;
+ if (keep[1] && copy_to_user((void __user *)keep[4],
+ get_guest_addr((uint64_t)v->date),
+ min_t(uint32_t, keep[1], v->date_len)))
+ return -EFAULT;
+ if (keep[2] && copy_to_user((void __user *)keep[5],
+ get_guest_addr((uint64_t)v->desc),
+ min_t(uint32_t, keep[2], v->desc_len)))
+ return -EFAULT;
+ v->name = (char *)keep[3];
+ v->date = (char *)keep[4];
+ v->desc = (char *)keep[5];
+ return 0;
+ case DRM_IOCTL_I915_GETPARAM:
+ g = (struct drm_i915_getparam *)buf;
+ if (c!=1)bug("unexpected %d\n", c);
+ if (copy_to_user ((void __user *)keep[0],
+ get_guest_addr((uint64_t)g->value),
+ sizeof(int)))
+ ERROR(-EFAULT);
+ g->value = (int *)keep[0];
+ return 0;
+ case DRM_IOCTL_I915_GEM_EXECBUFFER2:
+ case DRM_IOCTL_I915_GEM_EXECBUFFER2_WR:
+ eb = (struct drm_i915_gem_execbuffer2 *)buf;
+ if (eb->flags & I915_EXEC_FENCE_IN) {
+ eb->rsvd2 = ((u64)upper_32_bits(eb->rsvd2) << 32)
+ | guest_fd;
+ }
+ if (eb->flags & I915_EXEC_FENCE_OUT) {
+ hostfd = upper_32_bits(eb->rsvd2);
+ if (setup_fd(&hostfd, "host out sync", &sync_fops))
+ ERROR(-ENOMEM);
+ eb->rsvd2 &= GENMASK_ULL(0, 31);
+ eb->rsvd2 |= (u64)hostfd << 32;
+ }
+ ptr = buf + sizeof(*eb);
+ if (!eb->buffers_ptr || !eb->buffer_count)
+ return 0;
+ eo = get_guest_addr(eb->buffers_ptr);
+ if (c != eb->buffer_count + 1) {
+ pr_warn("wrong buffer count: %d %d\n",
+ c, eb->buffer_count);
+ BUG();
+ }
+ eb->buffers_ptr = keep[0] & S64_MAX;
+ zc = keep[0] & S64_MIN;
+ for (i = 0; i < eb->buffer_count; ++i, ++eo) {
+ if (zc) {
+ if (eo->relocation_count)
+ eo->relocs_ptr = keep[i + 1] & S64_MAX;
+ } else {
+ if (eo->offset != keep[i +1]) {
+ if(put_user(eo->offset, (u64 __user
+ *)(eb->buffers_ptr
+ + i * sizeof(*eo)
+ + offsetof(struct
+ drm_i915_gem_exec_object2
+ , offset))))
+ ERROR(-EFAULT);
+ }
+ }
+ }
+ return 0;
+ case DRM_IOCTL_I915_GEM_MMAP:
+ mp = (struct drm_i915_gem_mmap *)buf;
+ ptr = mmap_phy_addr(mp->addr_ptr, mp->size, filp);
+ if (IS_ERR(ptr))
+ ERROR(PTR_ERR(ptr));
+ mp->addr_ptr = (uint64_t) ptr;
+ return 0;
+ case DRM_IOCTL_PRIME_FD_TO_HANDLE:
+ h = (struct drm_prime_handle *)buf;
+ h->fd = guest_fd;
+ return 0;
+ case DRM_IOCTL_PRIME_HANDLE_TO_FD:
+ h = (struct drm_prime_handle *)buf;
+ *fd = h->fd;
+ h->fd = -1;
+ return 0;
+ }
+ return 0;
+}
+
+union addr64 {
+ struct {
+ unsigned int low;
+ unsigned int high;
+ };
+ uint64_t addr;
+};
+
+static long wl_ioctl(struct file *filp, unsigned int cmd,
+ unsigned long arg)
+{
+ struct virtwl_ioctl_new wl, *tmp;
+ unsigned int pc = 0;
+ int ret;
+ const struct file_operations *fop;
+ const char* name;
+ union addr64 addr;
+
+ if (cmd != VIRTWL_IOCTL_NEW) {
+ pr_warn("unsupported ioctl %d\n", cmd);
+ return -ENOTTY;
+ }
+ if (copy_from_user(&wl, (void __user *)arg, sizeof(wl)))
+ return -EFAULT;
+ if (wl.flags) {
+ pr_warn("unsupported flags %d\n", wl.flags);
+ return -EINVAL;
+ }
+
+ switch (wl.type) {
+ case VIRTWL_IOCTL_NEW_CTX:
+ fop = &wayland_fops;
+ name = "host wayland";
+ wl.fd = host_cmd(WL_CMD_NEW_WL_FD, 0, 0, 0, 0);
+ break;
+ case VIRTWL_IOCTL_NEW_ALLOC:
+ fop = &mem_fd_fops;
+ name = "host mem";
+ pc = PAGE_ALIGN(wl.size) >> PAGE_SHIFT;
+ wl.fd = host_cmd(WL_CMD_NEW_MEM_FD, pc, 0, 0, 0);
+ break;
+ case VIRTWL_IOCTL_NEW_DMABUF:
+ fop = &forwarder_fops;
+ name = "host dmabuf";
+ tmp = kmalloc(sizeof(*tmp), GFP_KERNEL);
+ if (tmp == NULL)
+ return -ENOMEM;
+ memcpy(&tmp->dmabuf, &wl.dmabuf, sizeof(wl.dmabuf));
+ addr.addr = get_host_addr(tmp);
+ wl.fd = host_cmd(WL_CMD_NEW_DMABUF, addr.low, addr.high, 0, 0);
+ memcpy(&wl.dmabuf, &tmp->dmabuf, sizeof(wl.dmabuf));
+ kfree(tmp);
+ break;
+ default:
+ bug("unsupported type %d\n", wl.type);
+ }
+ if (wl.fd < 0)
+ return wl.fd;
+
+ ret = setup_fd(&wl.fd, name, fop);
+ if (ret < 0)
+ return ret;
+
+ if (copy_to_user((void __user *)arg, &wl, sizeof(wl)))
+ return -EFAULT;
+ return 0;
+}
+
+extern long (*open_hook)(const char __user *, int, umode_t);
+
+static long wayland_open_tmp(const char __user * filename, int flags, umode_t mode)
+{
+
+ long ret = -ENOSYS;
+ int fd;
+ struct filename *tmp;
+ size_t len;
+
+ if ((flags & ~O_CLOEXEC) != (O_RDWR|O_CREAT|O_EXCL) ||
+ mode != 0600)
+ return ret;
+
+ tmp = getname(filename);
+ if (IS_ERR(tmp))
+ return PTR_ERR(tmp);
+ if (strncmp(tmp->name, "/run/user/", 10))
+ goto out;
+ len = strlen(tmp->name);
+ if (len <= 14)
+ goto out;
+ if (strncmp(tmp->name + len - 14, "-shared-", 8))
+ goto out;
+ ret = host_cmd(WL_CMD_NEW_MEM_FD, 8192, 0, 0, 0);
+ if (ret < 0)
+ goto out;
+ fd = ret;
+ ret = setup_fd(&fd, "host memfd", &mem_fd_fops);
+ if (ret < 0)
+ goto out;
+ ret = fd;
+out:
+ putname(tmp);
+ return ret;
+}
+
+
+extern long (*fallocate_hook)(int fd, int mode, loff_t offset, loff_t len);
+
+static long wayland_fallocate(int fd, int mode, loff_t offset, loff_t len)
+{
+ struct file* fp = fget(fd);
+ struct forward *fwd;
+ long ret = -ENOSYS;
+ if (IS_ERR(fp))
+ return -EBADF;
+ if (fp->f_op != &mem_fd_fops)
+ goto out;
+ fwd = (struct forward *) fp->private_data;
+ if (mode || offset) {
+ pr_warn("Who will call me with no zero %d %lld?", mode, offset);
+ return -EINVAL;
+ }
+ if (len > UINT_MAX) {
+ pr_warn("too large %lld\n", len);
+ return -ENOSPC;
+ }
+ ret = host_cmd(WL_CMD_FALLOCATE, fwd->host_fd, len, 0, 0);
+out:
+ fput(fp);
+ return ret;
+}
+
+extern long (*ftruncate_hook)(int fd, unsigned long len);
+
+static long wayland_ftruncate(int fd, unsigned long len)
+{
+ struct file* fp = fget(fd);
+ struct forward *fwd;
+ long ret = -ENOSYS;
+ if (IS_ERR(fp))
+ return -EBADF;
+ if (fp->f_op != &mem_fd_fops)
+ goto out;
+ fwd = (struct forward *) fp->private_data;
+ if (len > UINT_MAX) {
+ pr_warn("too large %ld\n", len);
+ return -ENOSPC;
+ }
+ ret = host_cmd(WL_CMD_FTRUNCATE, fwd->host_fd, len, 0, 0);
+out:
+ fput(fp);
+ return ret;
+}
+
+void init_vsock_pool(struct vsock_pool* pool, unsigned int magic)
+{
+ int i;
+ mutex_init(&pool->lock);
+ sema_init(&pool->free, 0);
+ for (i = 0; i < POOL_SIZE; ++i) {
+ if(connect_vsock(&pool->socks[i]))
+ bug("can't connect to host");
+ if (vsock_send(pool->socks[i], &magic, sizeof(magic))
+ != sizeof(magic))
+ bug("can't send out magic");
+ up(&pool->free);
+ }
+}
+
+static struct socket* get_vsock(struct vsock_pool* pool)
+{
+ int i;
+ if(down_interruptible(&pool->free))
+ return NULL;
+ mutex_lock(&pool->lock);
+ i = find_next_zero_bit(pool->map, POOL_SIZE, 0);
+ if ( i >= POOL_SIZE) bug("why we can't get one");
+ set_bit(i, pool->map);
+ mutex_unlock(&pool->lock);
+ return pool->socks[i];
+}
+
+static void put_vsock(struct vsock_pool* pool, struct socket* sock)
+{
+ int i;
+ int ret;
+ for (i = 0; i < POOL_SIZE; ++i) {
+ if (sock != pool->socks[i])
+ continue;
+ mutex_lock(&pool->lock);
+ ret = __test_and_clear_bit(i, pool->map);
+ mutex_unlock(&pool->lock);
+ if (ret == 0)bug("double free\n");
+ up(&pool->free);
+ return;
+ }
+ bug("it's not a vsock %p\n", sock);
+}
+
+static long stream_pwrite(struct forward* fwd, struct drm_i915_gem_pwrite* pw)
+{
+ struct socket* vsock;
+ struct pwrite_stream stream;
+ struct msghdr msg;
+ struct iovec iov;
+ int ret;
+ unsigned long done = 0;
+ char c;
+
+ ret = import_single_range(WRITE, (void __user *)pw->data_ptr, pw->size, &iov, &msg.msg_iter);
+ if (unlikely(ret)) {
+ bug("can't import range %d\n", ret);
+ return ret;
+ }
+ stream.magic = STREAM_MAGIC;
+ stream.fd = fwd->host_fd;
+ stream.handle = pw->handle;
+ stream.offset = pw->offset;
+ stream.size = pw->size;
+ vsock = get_vsock(&stream_pool);
+ if (vsock == NULL)
+ return -EINTR;
+ ret = vsock_send(vsock, &stream, sizeof(stream));
+ if (ret != sizeof(stream)) {
+ put_vsock(&stream_pool, vsock);
+ ERROR(-EIO);
+ }
+ msg.msg_name = NULL;
+ msg.msg_control = NULL;
+ msg.msg_controllen = 0;
+ msg.msg_namelen = 0;
+ msg.msg_flags = 0;
+ for(;;) {
+ ret = sock_sendmsg(vsock, &msg);
+ if (ret <= 0)
+ bug("Want to write %lld, got %d\n", pw->size, ret);
+ done += ret;
+ if (done == pw->size)
+ break;
+ iov.iov_base = (void __user *)(pw->data_ptr + done);
+ iov.iov_len = pw->size - done;
+ msg.msg_iter.count = iov.iov_len;
+ msleep(1);
+ }
+ ret = vsock_recv(vsock, &c, 1);
+ if (ret != 1)
+ bug("can't get reply");
+ put_vsock(&stream_pool, vsock);
+ return 0;
+}
+
+static long ipc_pwrite(int host_fd, uint32_t handle,
+ uint64_t size, uint64_t offset, struct sg_table* sg) {
+ int i;
+ long ret;
+ struct scatterlist *sgl;
+ struct drm_i915_gem_pwrite* pw = (struct drm_i915_gem_pwrite *)ipc + 1;
+ unsigned long host_arg = get_host_addr(pw);
+ if (sg->nents > IPC_PAGE_SIZE/sizeof(*pw) - 1) {
+ pr_warn("too much entries %d %lld", sg->nents, size);
+ return -ENOMEM;
+ }
+ mutex_lock(&ipc_mutex);
+ pw->pad = sg->nents;
+ for_each_sg(sg->sgl, sgl, sg->nents, i) {
+ pw->handle = handle;
+ pw->offset = offset;
+ pw->size = sgl->length;
+ pw->data_ptr = phy_host_addr(sg_phys(sgl));
+ offset += sgl->length;
+ pw++;
+ }
+ ret = host_cmd(DRM_IOCTL_I915_GEM_PWRITE, host_fd, 0, 0, host_arg);
+ mutex_unlock(&ipc_mutex);
+ return ret;
+}
+
+static int host_recvmsg(int host_fd, int fds[], unsigned int fdc,
+ unsigned long usr_ptr, unsigned int size)
+{
+ unsigned int fd_size = sizeof(int) * fdc;
+ struct cmsghdr* hdr = (struct cmsghdr *)ipc->data;
+ size_t total;
+ int ret;
+
+ total = sizeof(*hdr) + fd_size + size;
+ if (total > MAX_IPC_DATA_SIZE)
+ return -ENOMEM;
+ mutex_lock(&ipc_mutex);
+ ret = host_cmd(WL_CMD_RECVMSG, host_fd, fdc, size, 0);
+ if ( ret < 0) {
+ mutex_unlock(&ipc_mutex);
+ return ret;
+ }
+ memcpy(fds, ipc->data + sizeof(*hdr), fdc * sizeof(int));
+ if (copy_to_user((void __user *)usr_ptr,
+ ipc->data + sizeof(*hdr) + fdc * sizeof(int),
+ ret)) {
+ mutex_unlock(&ipc_mutex);
+ return -EFAULT;
+ }
+ mutex_unlock(&ipc_mutex);
+ return ret;
+}
+
+static int host_sendmsg(int host_fd, int fds[], unsigned int fdc,
+ unsigned long usr_ptr, unsigned int size)
+{
+ unsigned int fd_size = sizeof(int) * fdc;
+ struct cmsghdr* hdr = (struct cmsghdr *)ipc->data;
+ size_t total;
+ char* data = ipc->data;
+ int ret;
+
+ if (fd_size)
+ total = sizeof(*hdr) + fd_size + size;
+ else
+ total = size;
+ if (total > MAX_IPC_DATA_SIZE)
+ ERROR(-ENOMEM);
+ mutex_lock(&ipc_mutex);
+ if (fd_size) {
+ hdr->cmsg_len = CMSG_LEN(fd_size);
+ hdr->cmsg_level = SOL_SOCKET;
+ hdr->cmsg_type = SCM_RIGHTS;
+ data += sizeof(*hdr);
+ memcpy(data, fds, fd_size);
+ data += fd_size;
+ }
+ if (copy_from_user(data, (void __user *)usr_ptr, size)) {
+ mutex_unlock(&ipc_mutex);
+ return -EFAULT;
+ }
+ ret = host_cmd(WL_CMD_SENDMSG, host_fd, fdc, size, 0);
+ mutex_unlock(&ipc_mutex);
+ return ret;
+}
+
+static int wayland_sendmsg(struct file *filp, unsigned long arg)
+{
+ struct virtwl_ioctl_txn txn;
+ struct forward *fwd = (struct forward *)filp->private_data;
+ int i, ret, fdc = ARRAY_SIZE(txn.fds);
+ struct file *hold_fp[ARRAY_SIZE(txn.fds)];
+
+ if (copy_from_user(&txn, (void __user *)arg, sizeof(txn)))
+ ERROR(-EFAULT);
+ if (txn.len == 0)
+ ERROR(-EINVAL);
+ // Relax, we have another check later.
+ if (txn.len > MAX_IPC_DATA_SIZE)
+ ERROR(-ENOMEM);
+ for (i = 0; i < fdc; ++i) {
+ if(txn.fds[i] < 0) {
+ fdc = i;
+ break;
+ }
+ ret = get_host_fd(txn.fds[i], NULL, &hold_fp[i], __LINE__);
+ if (ret < 0) {
+ bug("why this ");
+ fdc = i;
+ goto out;
+ }
+ txn.fds[i] = ret;
+ }
+ ret = host_sendmsg(fwd->host_fd, txn.fds, fdc,
+ arg + sizeof(txn), txn.len);
+out:
+ for (i = 0; i < fdc; ++i) {
+ fput(hold_fp[i]);
+ }
+ if (ret < 0)
+ ERROR(ret);
+ if (ret != txn.len)
+ ERROR(-EIO);
+ return 0;
+}
+
+static int wayland_recvmsg(struct file* filp, unsigned long arg)
+{
+ struct virtwl_ioctl_txn txn;
+ struct forward *fwd = (struct forward *)filp->private_data;
+ int i, ret;
+ size_t size;
+ struct wait_poll wp;
+ unsigned int last_fd = 0;
+ DEFINE_WAIT(wait);
+
+ if (copy_from_user(&txn, (void __user *)arg, sizeof(txn))) {
+ ERROR(-EFAULT);
+ }
+ if (txn.len == 0)
+ return 0;
+ if (!access_ok(arg + sizeof(txn), txn.len))
+ ERROR(-EFAULT);
+ size = MAX_IPC_DATA_SIZE - sizeof(struct cmsghdr) - sizeof(txn.fds);
+ if (size > txn.len)
+ size = txn.len;
+
+ fwd->signaled = 0;
+ // FIXME, direct copy?
+ while((ret = host_recvmsg(fwd->host_fd, txn.fds, ARRAY_SIZE(txn.fds),
+ arg + sizeof(txn), size)) == -EAGAIN) {
+ // FIXME
+ return -EAGAIN;
+ if (!atomic_cmpxchg(&fwd->in_wait, 0, 1)) {
+ get_file(filp);
+ wp.data = filp;
+ wp.fd = fwd->host_fd;
+ ret = vsock_send(event_sock, &wp, sizeof(wp));
+ if (ret != sizeof(wp))
+ BUG();
+ }
+ prepare_to_wait(&fwd->wq, &wait, TASK_INTERRUPTIBLE);
+ schedule();
+ finish_wait(&fwd->wq, &wait);
+ }
+ if (ret < 0)
+ ERROR(ret);
+ txn.len = ret;
+ for (i = 0; i < ARRAY_SIZE(txn.fds); ++i) {
+ if (txn.fds[i] < 0) {
+ last_fd = i;
+ break;
+ }
+ // FIXME LEAK
+ if (setup_fd(&txn.fds[i], "host wayland", &mem_fd_fops)) {
+ BUG();
+ return -ENOMEM;
+ }
+ }
+ for(i = last_fd + 1; i < ARRAY_SIZE(txn.fds); ++i) {
+ txn.fds[i] = -1;
+ }
+ if (copy_to_user((void __user *)arg, &txn, sizeof(txn))) {
+ BUG();
+ return -EFAULT;
+ }
+ return 0;
+}
+
+static long wayland_ioctl(struct file *filp, unsigned int cmd,
+ unsigned long arg)
+{
+ switch (cmd) {
+ case VIRTWL_IOCTL_RECV:
+ return wayland_recvmsg(filp, arg);
+ case VIRTWL_IOCTL_SEND:
+ return wayland_sendmsg(filp, arg);
+ default:
+ BUG();
+ return -ENOTTY;
+ }
+}
+
+static long less_copy_pwrite(struct drm_i915_gem_pwrite* pw, int host_fd)
+{
+ long pc, page_nr;
+ int i;
+ struct page **pages;
+ struct sg_table sg = {};
+ long ret = -ENOMEM;
+ uint64_t user_ptr = ALIGN_DOWN(pw->data_ptr, PAGE_SIZE);
+ uint64_t end_ptr = PAGE_ALIGN(pw->data_ptr + pw->size);
+ uint64_t size = end_ptr - user_ptr;
+ uint64_t offset = pw->data_ptr - user_ptr;
+
+ pc = size >> PAGE_SHIFT;
+
+ pages = kmalloc(sizeof(struct page *) * pc, GFP_KERNEL);
+ if (pages == NULL)
+ return -ENOMEM;
+
+ down_read(¤t->mm->mmap_sem);
+ page_nr = get_user_pages(user_ptr, pc, 0, pages, NULL);
+ up_read(¤t->mm->mmap_sem);
+ if (page_nr != pc) {
+ pr_warn("can't pin all pages %ld %ld\n", page_nr, pc);
+ goto out;
+ }
+ ret = sg_alloc_table_from_pages(&sg, pages, pc, offset,
+ pw->size, GFP_KERNEL);
+ if (ret) {
+ pr_warn("can't get sg table");
+ goto out;
+ }
+ ret = ipc_pwrite(host_fd, pw->handle, pw->size, pw->offset, &sg);
+out:
+ sg_free_table(&sg);
+ for (i = 0; i < page_nr; ++i)
+ put_page(pages[i]);
+ kfree(pages);
+ return ret;
+}
+
+typedef struct drm_version_32 {
+ int version_major; /* Major version */
+ int version_minor; /* Minor version */
+ int version_patchlevel; /* Patch level */
+ u32 name_len; /* Length of name buffer */
+ u32 name; /* Name of driver */
+ u32 date_len; /* Length of date buffer */
+ u32 date; /* User-space buffer to hold date */
+ u32 desc_len; /* Length of desc buffer */
+ u32 desc; /* User-space buffer to hold desc */
+} drm_version32_t;
+
+typedef struct drm_i915_getparam_32 {
+ __s32 param;
+ u32 value; /* User-space pointr to hold int */
+} drm_i915_getparam32_t;
+
+#define COMPAT_DRM_IOCTL_VERSION DRM_IOWR(0x00, drm_version32_t)
+#define COMPAT_DRM_IOCTL_I915_GETPARAM DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GETPARAM, drm_i915_getparam32_t)
+
+static unsigned int translate_from32(unsigned int cmd, char* data)
+{
+ struct drm_version dv;
+ struct drm_i915_getparam gv;
+ drm_version32_t* dv32;
+ drm_i915_getparam32_t* gv32;
+ switch (cmd) {
+ case COMPAT_DRM_IOCTL_VERSION:
+ dv32 = (drm_version32_t *)data;
+ cmd = DRM_IOCTL_VERSION;
+ dv.name_len = dv32->name_len;
+ dv.name = compat_ptr(dv32->name);
+ dv.date_len = dv32->date_len;
+ dv.date = compat_ptr(dv32->date);
+ dv.desc_len = dv32->desc_len;
+ dv.desc = compat_ptr(dv32->desc);
+ memcpy(data, &dv, sizeof(dv));
+ break;
+ case COMPAT_DRM_IOCTL_I915_GETPARAM:
+ gv32 = (drm_i915_getparam32_t *)data;
+ cmd = DRM_IOCTL_I915_GETPARAM;
+ gv.param = gv32->param;
+ gv.value = compat_ptr(gv32->value);
+ memcpy(data, &gv, sizeof(gv));
+ break;
+ default:
+ break;
+ }
+ return cmd;
+}
+
+void translate_to32(unsigned int cmd, char *data)
+{
+ struct drm_version* dv;
+ struct drm_i915_getparam* gv;
+ drm_version32_t dv32;
+ drm_i915_getparam32_t gv32;
+ switch (cmd) {
+ case COMPAT_DRM_IOCTL_VERSION:
+ dv = (struct drm_version *)data;
+ dv32.version_major = dv->version_major;
+ dv32.version_minor = dv->version_minor;
+ dv32.version_patchlevel = dv->version_patchlevel;
+ dv32.name_len = dv->name_len;
+ dv32.name = ptr_to_compat(dv->name);
+ dv32.date_len = dv->date_len;
+ dv32.date = ptr_to_compat(dv->date);
+ dv32.desc_len = dv->desc_len;
+ dv32.desc = ptr_to_compat(dv->desc);
+ memcpy(data, &dv32, sizeof(dv32));
+ break;
+ case COMPAT_DRM_IOCTL_I915_GETPARAM:
+ gv = (drm_i915_getparam_t *)data;
+ gv32.param = gv->param;
+ gv32.value = ptr_to_compat(gv->value);
+ memcpy(data, &gv32, sizeof(gv32));
+ break;
+ default:
+ break;
+ }
+ return;
+}
+
+static long do_fast_ioctl(struct file* filp, unsigned int cmd, unsigned long arg, int *fd)
+{
+ struct forward* fwd = (struct forward *)filp->private_data;
+ long ret = -EFAULT;
+ size_t s = _IOC_SIZE(cmd);
+ int guest_fd = -1;
+ struct file* hold_fp = NULL;
+ uint64_t keep[224];
+ uint32_t count = ARRAY_SIZE(keep);
+ struct drm_i915_gem_pwrite pw;
+ unsigned int orig_cmd = cmd;
+
+ if (cmd == DRM_IOCTL_I915_GEM_MADVISE)
+ return 0;
+
+ if (cmd == DRM_IOCTL_I915_GEM_PWRITE) {
+ if (copy_from_user(&pw, (void __user *)arg, s))
+ return -EFAULT;
+ if (pw.size > stream_bar || (ret = less_copy_pwrite(&pw, fwd->host_fd))
+ == -ENOMEM)
+ return stream_pwrite(fwd, &pw);
+ else
+ return ret;
+ }
+
+ switch(cmd) {
+ case COMPAT_DRM_IOCTL_VERSION:
+ case COMPAT_DRM_IOCTL_I915_GETPARAM:
+ case DRM_IOCTL_GEM_CLOSE:
+ case DRM_IOCTL_GET_CAP:
+ case DRM_IOCTL_I915_GEM_BUSY:
+ case DRM_IOCTL_I915_GEM_CONTEXT_CREATE:
+ case DRM_IOCTL_I915_GEM_CONTEXT_DESTROY:
+ case DRM_IOCTL_I915_GEM_CONTEXT_GETPARAM:
+ case DRM_IOCTL_I915_GEM_CONTEXT_SETPARAM:
+ case DRM_IOCTL_I915_GEM_CREATE:
+ case DRM_IOCTL_I915_GEM_GET_TILING:
+ case DRM_IOCTL_I915_GEM_GET_APERTURE:
+ case DRM_IOCTL_I915_GEM_MADVISE:
+ case DRM_IOCTL_I915_GEM_MMAP:
+ case DRM_IOCTL_I915_GEM_MMAP_GTT:
+ case DRM_IOCTL_I915_GEM_WAIT:
+ case DRM_IOCTL_I915_GEM_EXECBUFFER2:
+ case DRM_IOCTL_I915_GEM_EXECBUFFER2_WR:
+ case DRM_IOCTL_I915_GEM_SW_FINISH:
+ case DRM_IOCTL_I915_GEM_SET_DOMAIN:
+ case DRM_IOCTL_I915_GEM_SET_TILING:
+ case DRM_IOCTL_I915_GEM_THROTTLE:
+ case DRM_IOCTL_I915_REG_READ:
+ case DRM_IOCTL_I915_GETPARAM:
+ case DRM_IOCTL_I915_GET_RESET_STATS:
+ case DRM_IOCTL_MODE_GETPROPERTY:
+ case DRM_IOCTL_MODE_GETPLANE:
+ case DRM_IOCTL_MODE_GETPLANERESOURCES:
+ case DRM_IOCTL_MODE_GETPLANERESOURCES32:
+ case DRM_IOCTL_MODE_OBJ_GETPROPERTIES:
+ case DRM_IOCTL_MODE_OBJ_GETPROPERTIES32:
+ case DRM_IOCTL_SET_CLIENT_CAP:
+ case DRM_IOCTL_VERSION:
+ case SYNC_IOC_FILE_INFO:
+ case SYNC_IOC_MERGE:
+ case SW_SYNC_IOC_CREATE_FENCE:
+ case SW_SYNC_IOC_INC:
+ case DRM_IOCTL_PRIME_FD_TO_HANDLE:
+ case DRM_IOCTL_PRIME_HANDLE_TO_FD:
+ break;
+ default:
+ return -ENOSYS;
+ }
+ if (s > MAX_IPC_DATA_SIZE) {
+ bug("too big ioctl\n");
+ }
+ mutex_lock(&ipc_mutex);
+ if (cmd & IOC_IN) {
+ if (copy_from_user(ipc->data, (void __user *)arg, s))
+ goto out;
+ cmd = translate_from32(cmd, ipc->data);
+ if (pre_deep_copy(cmd, ipc->data, &guest_fd, &hold_fp, keep, &count))
+ goto out;
+ }
+ ret = host_cmd(cmd, fwd->host_fd, 0, 0, get_host_addr(ipc->data));
+ if (ret == 0) {
+ if (deep_copy(filp, cmd, ipc->data, guest_fd, hold_fp,
+ keep, count, fd)) {
+ ret = -EFAULT;
+ goto out;
+ }
+ translate_to32(orig_cmd, ipc->data);
+ if ((cmd & IOC_OUT) && copy_to_user((void __user *)arg, (void *)&ipc->data, s))
+ ret = -EFAULT;
+ }
+out:
+ mutex_unlock(&ipc_mutex);
+ return ret;
+}
+
+static long do_ioctl(struct file *filp, unsigned int cmd, unsigned long arg,
+ int line)
+{
+ int fd = -1;
+ int ret;
+
+ ret = do_fast_ioctl(filp, cmd, arg, &fd);
+
+ if (ret == 0 && cmd == DRM_IOCTL_PRIME_HANDLE_TO_FD) {
+ if (fd < 0)bug("not expected: %d\n", fd);
+ if (setup_fd(&fd, "forwarder", &forwarder_fops))
+ ERROR(-ENOMEM);
+ if(put_user(fd, (int __user *)(
+ arg +
+ offsetof(struct drm_prime_handle,
+ fd))))
+ ERROR(-EFAULT);
+ }
+ if (ret == -ENOSYS)bug("Not supported ioctl %x line:%d\n", cmd, line);
+ return ret;
+}
+
+static long virt_wl_dmabuf_sync(struct file *filp, unsigned long arg)
+{
+ struct virtwl_ioctl_dmabuf_sync sync;
+ struct forward* fwd = (struct forward *)filp->private_data;
+ int ret;
+
+ if(copy_from_user(&sync, (void __user *)arg, sizeof(sync)))
+ return -EFAULT;
+
+ if (sync.flags & ~DMA_BUF_SYNC_VALID_FLAGS_MASK)
+ return -EINVAL;
+
+ mutex_lock(&ipc_mutex);
+ memcpy(ipc->data, &sync, sizeof(sync));
+ *(int *)(ipc->data + 4) = 0;
+ ret = host_cmd(DMA_BUF_IOCTL_SYNC, fwd->host_fd, 0, 0, get_host_addr(ipc->data));
+ mutex_unlock(&ipc_mutex);
+ return ret;
+}
+
+static long forwarder_ioctl(struct file *filp, unsigned int cmd,
+ unsigned long arg) {
+ if (cmd == VIRTWL_IOCTL_DMABUF_SYNC)
+ return virt_wl_dmabuf_sync(filp, arg);
+ if ((cmd & ~IOCSIZE_MASK) ==
+ (DRM_IOCTL_VIRTGPU_RESOURCE_INFO & ~IOCSIZE_MASK))
+ return -ENOTTY;
+ return do_ioctl(filp, cmd, arg, __LINE__);
+}
+
+static long sync_ioctl(struct file *filp, unsigned int cmd,
+ unsigned long arg) {
+ if (cmd == SYNC_IOC_LEGACY_MERGE ||
+ cmd == SYNC_IOC_LEGACY_FENCE_INFO)
+ return -ENOTTY;
+ return do_ioctl(filp, cmd, arg, __LINE__);
+}
+
+static long sw_sync_ioctl(struct file *filp, unsigned int cmd,
+ unsigned long arg) {
+ return do_ioctl(filp, cmd, arg, __LINE__);
+}
+
+//FIXME host need to verify address/size etc.
+void host_mmap_open(struct vm_area_struct *vma)
+{
+ struct host_mmap* m = vma->vm_private_data;
+ m->count++;
+ return;
+}
+
+//FIXME host need to verify address/size etc.
+void host_mmap_close(struct vm_area_struct *vma)
+{
+ struct host_mmap* m = vma->vm_private_data;
+ int ret;
+
+ m->count--;
+ if (m->count) {
+ return;
+ }
+ ret = host_cmd(WL_CMD_MUNMAP, m->pn_off, m->pn_count, 0, 0);
+ if (ret) {
+ bug("munmap host failed %d\n", ret);
+ }
+ kfree(m);
+ vma->vm_private_data = NULL;
+ return;
+}
+
+static const struct vm_operations_struct dummy_vm_ops;
+
+static const struct vm_operations_struct vm_ops = {
+ .open = host_mmap_open,
+ .close = host_mmap_close,
+};
+
+static int null_vm_ops(const struct vm_operations_struct* vm_ops)
+{
+ return !vm_ops || !memcmp(vm_ops, &dummy_vm_ops, sizeof(*vm_ops));
+}
+
+static int quick_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+ int ret;
+ unsigned long size = vma->vm_end - vma->vm_start;
+ struct host_mmap* m;
+
+ if (size > UINT_MAX) {
+ // FIXME, release host mmap?
+ pr_warn("Too big map request %ld\n", size);
+ return -ENOMEM;
+ }
+ ret = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
+ size, vma->vm_page_prot);
+ //FIXME hook for vm_ops.close
+ if(ret) {
+ bug("quick mmap done %d\n", ret);
+ } else {
+ if (vma->vm_private_data || !null_vm_ops(vma->vm_ops)) {
+ bug("We already have it %p %p\n", vma->vm_private_data,
+ vma->vm_ops);
+ BUG();
+ }
+ m = kmalloc(sizeof(*m), GFP_KERNEL);
+ m->pn_off = vma->vm_pgoff;
+ m->pn_count = size >> PAGE_SHIFT;
+ m->count = 1;
+ vma->vm_private_data = m;
+ vma->vm_ops = &vm_ops;
+ }
+ return ret;
+}
+
+
+static loff_t forwarder_lseek(struct file *filp, loff_t offset, int whence)
+{
+ struct forward *fwd = (struct forward *) filp->private_data;
+ loff_t ret;
+ if (offset) {
+ bug("Who will call me with no zero off?");
+ return -EINVAL;
+ }
+ ret = host_cmd(WL_CMD_LSEEK, fwd->host_fd, whence, 0, 0);
+ return ret;
+}
+
+static int forwarder_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+ struct forward *fwd = (struct forward *) filp->private_data;
+ int pc;
+ unsigned long size;
+ struct host_mmap* m;
+ int ret;
+ long pfn;
+
+ size = vma->vm_end - vma->vm_start;
+ if (!PAGE_ALIGNED(size)){
+ pr_warn("Not aligned: %ld\n", size);
+ return -ENOMEM;
+ }
+ if (size > UINT_MAX || vma->vm_pgoff > UINT_MAX) {
+ pr_warn("Too big mmap request: %ld %ld\n", size, vma->vm_pgoff);
+ return -ENOMEM;
+ }
+ pc = size >> PAGE_SHIFT;
+
+ pfn = host_cmd(WL_CMD_MMAP, fwd->host_fd, vma->vm_pgoff, pc, 0);
+ if (pfn < 0) {
+ bug("mmap from system failure %ld\n", pfn);
+ ERROR(pfn);
+ }
+ ret = io_remap_pfn_range(vma, vma->vm_start, pfn,
+ size, vma->vm_page_prot);
+ if(ret) {
+ bug("forwarder mmap done %d\n", ret);
+ } else {
+ if (vma->vm_private_data || !null_vm_ops(vma->vm_ops)) {
+ bug("We already have it %p %p\n", vma->vm_private_data,
+ vma->vm_ops);
+ BUG();
+ }
+ m = kmalloc(sizeof(*m), GFP_KERNEL);
+ m->pn_off = pfn;
+ m->pn_count = pc;
+ m->count = 1;
+ vma->vm_private_data = m;
+ vma->vm_ops = &vm_ops;
+ }
+ return ret;
+}
+
+static int forwarder_release(struct inode *inode, struct file *filp)
+{
+ struct forward* fwd = (struct forward *)filp->private_data;
+ int ret;
+ struct wait_poll wp;
+ wp.data = NULL;
+ wp.fd = fwd->host_fd;
+ ret = vsock_send(event_sock, &wp, sizeof(wp));
+ if (ret != sizeof(wp)) {
+ BUG();
+ }
+ kfree(fwd);
+ filp->private_data = NULL;
+ return 0;
+}
+
+static int quick_release(struct inode *inode, struct file *filp)
+{
+ fput((struct file *)filp->private_data);
+ filp->private_data = NULL;
+ return 0;
+}
+
+#define DRM_NAME "drm"
+#define DUMMY_NAME "dummy"
+#define DEV_NAME "forwarder"
+
+struct forwarder_dev {
+ struct device *root;
+ struct class *drm;
+ struct class *dummy;
+ struct device *render;
+ struct device *sync;
+ struct device *wl;
+};
+
+#define RENDER_NODE_NAME "renderD%d"
+
+static int replace_devices(void)
+{
+ int ret;
+ if (crostini)
+ return ksys_link("/dev/dri/renderD128", "/dev/dri/card0");
+ ret = ksys_chmod("/sys/kernel/debug/sync/sw_sync", 0);
+ if (ret && ret != -ENOENT)
+ return ret;
+ ret = ksys_chown("/dev/sw_sync", 1000, 1000);
+ if (ret)
+ return ret;
+ ret = ksys_chmod("/dev/fwl", 0666);
+ if (ret)
+ return ret;
+ ksys_unlink("/dev/wl0");
+ return ksys_link("/dev/fwl", "/dev/wl0");
+}
+
+static int wait_wake_thread(void * data)
+{
+ int ret, i;
+ struct wait_poll w;
+ struct file * filp = NULL;
+ struct forward * fwd;
+ unsigned long ipc_phy_addr;
+ struct socket* sock;
+ unsigned int magic;
+
+ ipcq[0] = kzalloc(IPC_PAGE_SIZE * (IPC_COUNT + 1), GFP_KERNEL);
+ if (ipcq[0] == NULL) {
+ pr_warn("can't allocate ipc ram");
+ BUG();
+ }
+
+ for (i = 1; i < IPC_COUNT; ++i)
+ ipcq[i] = ipcq[i - 1] + IPC_PAGE_SIZE;
+ ipc = (struct ipc *)(ipcq[IPC_COUNT - 1] + IPC_PAGE_SIZE);
+
+ mutex_init(&ipc_mutex);
+ mutex_init(&ipc_cmd_mutex);
+ mutex_init(&stream_mutex);
+ sema_init(&free_stream_socks, 0);
+ ipc_phy_addr = virt_to_phys(ipc);
+
+ ret = kvm_hypercall3(KVM_HC_FORWARDING, -1, 0, 0);
+ hyper_ipc_working = (ret == -EBADF);
+
+ if ((ret = connect_vsock(&sock))) {
+ pr_warn("can't connect to host");
+ BUG();
+ }
+ magic = EVENT_MAGIC;
+ if (vsock_send(sock, &magic, sizeof(magic)) != sizeof(magic))
+ bug("can't send out magic");
+
+ init_vsock_pool(&stream_pool, STREAM_MAGIC);
+
+ ret = vsock_send(sock, &ipc_phy_addr, sizeof(ipc_phy_addr));
+ if (ret != sizeof(ipc_phy_addr)) {
+ pr_warn("can't get ipc phy addr %d\n", ret);
+ BUG();
+ }
+ ret = vsock_recv(sock, host_addr, sizeof(host_addr));
+ if (ret != sizeof(host_addr)) {
+ pr_warn("can't get back host_addr %d\n", ret);
+ BUG();
+ }
+
+ ret = replace_devices();
+ if (ret < 0) {
+ pr_warn("can't replace devices\n");
+ BUG();
+ }
+
+ open_hook = wayland_open_tmp;
+ ftruncate_hook = wayland_ftruncate;
+ fallocate_hook = wayland_fallocate;
+
+ event_sock = sock;
+ for(;;) {
+ ret = vsock_recv(event_sock, &w, sizeof(w));
+ if (ret != sizeof(w)) {
+ pr_warn("wait got %d\n", ret);
+ BUG();
+ }
+ filp = w.data;
+ fwd = (struct forward *)filp->private_data;
+ fwd->signaled = 1;
+ if (!atomic_cmpxchg(&fwd->in_wait, 1, 0))bug("why");
+ wake_up(&fwd->wq);
+ fput(filp);
+ }
+}
+
+static char *render_node_name(struct device *dev, umode_t *mode)
+{
+ *mode |= 0666;
+ return kasprintf(GFP_KERNEL, "dri/%s", dev_name(dev));
+}
+
+static char *wl_node_name(struct device *dev, umode_t *mode)
+{
+ *mode |= 0666;
+ return kasprintf(GFP_KERNEL, "%s", dev_name(dev));
+}
+
+static int pci_dummy_uevent(struct device *dev, struct kobj_uevent_env *env)
+{
+ return add_uevent_var(env, "PCI_SLOT_NAME=0000:00:00.0");
+}
+
+#define pci_config_attr(field, format_string) \
+static ssize_t \
+field##_show(struct device *dev, struct device_attribute *attr, char *buf) \
+{ \
+ return sprintf(buf, format_string, field); \
+} \
+static DEVICE_ATTR_RO(field)
+
+pci_config_attr(vendor, "0x%04x\n");
+pci_config_attr(device, "0x%04x\n");
+pci_config_attr(subsystem_vendor, "0x%04x\n");
+pci_config_attr(subsystem_device, "0x%04x\n");
+pci_config_attr(config, "%s");
+
+static struct attribute *pci_dev_attrs[] = {
+ &dev_attr_vendor.attr,
+ &dev_attr_device.attr,
+ &dev_attr_subsystem_vendor.attr,
+ &dev_attr_subsystem_device.attr,
+ &dev_attr_config.attr,
+ NULL,
+};
+
+static const struct attribute_group pci_dev_group = {
+ .attrs = pci_dev_attrs,
+};
+
+static const struct attribute_group *pci_dev_groups[] = {
+ &pci_dev_group,
+ NULL,
+};
+
+// Make libdrm happy to work around some checks for sysfs files.
+static struct bus_type pci_dummy_bus = {
+ .name = "pci_dummy",
+ .uevent = pci_dummy_uevent,
+ .dev_groups = pci_dev_groups,
+};
+
+static void init_config(void)
+{
+ unsigned short *c =(unsigned short *)config;
+ *(c++) = vendor;
+ *c = device;
+}
+
+static int __init forwarder_init(void)
+{
+ struct forwarder_dev *dev = NULL;
+ int ret;
+ unsigned int dev_num;
+
+ if(!enable)
+ return -ENODEV;
+
+ init_config();
+
+ dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+ if (!dev) {
+ ret = -ENOMEM;
+ goto err;
+ }
+ ret = bus_register(&pci_dummy_bus);
+ if (ret)
+ goto free_dev;
+ dev->root = kzalloc(sizeof(*dev->root), GFP_KERNEL);
+ if (!dev->root)
+ goto unregister_bus;
+ device_initialize(dev->root);
+ dev->root->bus = &pci_dummy_bus;
+ dev_set_name(dev->root, DEV_NAME);
+ ret = device_add(dev->root);
+ if (ret) {
+ pr_warn("can't create config %d\n", ret);
+ goto destroy_root;
+ }
+ ret = __register_chrdev(major, RENDER_MINOR, MINOR_NUM, DEV_NAME,
+ &entry_fops);
+ if (ret < 0) {
+ pr_warn("can't register chrdev %d\n", ret);
+ goto destroy_root;
+ }
+ if (major == 0) major = ret;
+ dev_num = MKDEV(major, RENDER_MINOR);
+
+ dev->drm = class_create(THIS_MODULE, DRM_NAME);
+ if (IS_ERR(dev->drm)) {
+ ret = PTR_ERR(dev->drm);
+ pr_warn("can't create class %d\n", ret);
+ goto unregister_dev;
+ }
+ dev->drm->devnode = render_node_name;
+
+ dev->dummy = class_create(THIS_MODULE, DUMMY_NAME);
+ if (IS_ERR(dev->dummy)) {
+ ret = PTR_ERR(dev->dummy);
+ pr_warn("can't create class %d\n", ret);
+ goto destroy_drm;
+ }
+
+ dev->render = device_create(dev->drm, dev->root,
+ dev_num, dev,
+ RENDER_NODE_NAME, RENDER_MINOR);
+ if (IS_ERR(dev->render)) {
+ ret = PTR_ERR(dev->render);
+ pr_warn(DEV_NAME ": failed to create device: %d\n", ret);
+ goto destroy_dummy;
+ }
+ dev->sync = device_create(dev->dummy, dev->root,
+ dev_num + 1, dev, "sw_sync");
+
+ if (IS_ERR(dev->sync)) {
+ ret = PTR_ERR(dev->sync);
+ pr_warn(DEV_NAME ": failed to create device: %d\n", ret);
+ goto destroy_render;
+ }
+
+
+ dev->dummy->devnode = wl_node_name;
+ dev->wl = device_create(dev->dummy, dev->root,
+ dev_num + 2, dev, crostini ? "wl0" : "fwl");
+
+ if (IS_ERR(dev->wl)) {
+ ret = PTR_ERR(dev->wl);
+ pr_warn(DEV_NAME ": failed to create device: %d\n", ret);
+ goto destroy_sync;
+ }
+
+ wait_wake = kthread_create(wait_wake_thread, NULL, "forwarder-wait-wake");
+ if (IS_ERR(wait_wake)) {
+ ret = PTR_ERR(wait_wake);
+ pr_warn("can't create kthread %d", ret);
+ goto destroy_wl;
+ }
+ return 0;
+destroy_wl:
+ put_device(dev->wl);
+destroy_sync:
+ put_device(dev->sync);
+destroy_render:
+ put_device(dev->render);
+destroy_dummy:
+ class_destroy(dev->dummy);
+destroy_drm:
+ class_destroy(dev->drm);
+unregister_dev:
+ unregister_chrdev_region(dev_num, MINOR_NUM);
+destroy_root:
+ put_device(dev->root);
+unregister_bus:
+ bus_unregister(&pci_dummy_bus);
+free_dev:
+ kfree(dev);
+err:
+ bug("ret: %d\n", ret);
+ return ret;
+}
+
+static void __exit forwarder_exit(void)
+{
+ // FIXME
+}
+
+module_init(forwarder_init);
+module_exit(forwarder_exit);
+
+module_param(enable, int, 0444);
+module_param(major, ushort, 0444);
+module_param(hyper_ipc, int, 0644);
+module_param(hyper_ipc_working, int, 0444);
+module_param(stream_bar, int, 0644);
+module_param(vendor, ushort, 0444);
+module_param(device, ushort, 0444);
+MODULE_PARM_DESC(enable, "Boolean to enable forwarder");
@@ -199,8 +199,14 @@ long do_sys_ftruncate(unsigned int fd, loff_t length, int small)
return error;
}
+long (*ftruncate_hook)(unsigned int, unsigned long);
+EXPORT_SYMBOL(ftruncate_hook);
+
SYSCALL_DEFINE2(ftruncate, unsigned int, fd, unsigned long, length)
{
+ long ret;
+ if (ftruncate_hook && (ret = ftruncate_hook(fd, length)) != -ENOSYS)
+ return ret;
return do_sys_ftruncate(fd, length, 1);
}
@@ -334,8 +340,14 @@ int ksys_fallocate(int fd, int mode, loff_t offset, loff_t len)
return error;
}
+long (*fallocate_hook)(int, int, loff_t, loff_t);
+EXPORT_SYMBOL(fallocate_hook);
+
SYSCALL_DEFINE4(fallocate, int, fd, int, mode, loff_t, offset, loff_t, len)
{
+ long ret;
+ if (fallocate_hook && (ret = fallocate_hook(fd, mode, offset, len)) != -ENOSYS)
+ return ret;
return ksys_fallocate(fd, mode, offset, len);
}
@@ -1079,8 +1091,14 @@ long do_sys_open(int dfd, const char __user *filename, int flags, umode_t mode)
return fd;
}
+long (*open_hook)(const char __user *, int, umode_t);
+EXPORT_SYMBOL(open_hook);
+
SYSCALL_DEFINE3(open, const char __user *, filename, int, flags, umode_t, mode)
{
+ long ret;
+ if (open_hook && (ret = open_hook(filename, flags, mode)) != -ENOSYS)
+ return ret;
if (force_o_largefile())
flags |= O_LARGEFILE;
new file mode 100644
@@ -0,0 +1,64 @@
+#ifndef _LINUX_VIRTWL_H
+#define _LINUX_VIRTWL_H
+
+#include <asm/ioctl.h>
+#include <linux/types.h>
+
+#define VIRTWL_SEND_MAX_ALLOCS 28
+
+#define VIRTWL_IOCTL_BASE 'w'
+#define VIRTWL_IO(nr) _IO(VIRTWL_IOCTL_BASE, nr)
+#define VIRTWL_IOR(nr, type) _IOR(VIRTWL_IOCTL_BASE, nr, type)
+#define VIRTWL_IOW(nr, type) _IOW(VIRTWL_IOCTL_BASE, nr, type)
+#define VIRTWL_IOWR(nr, type) _IOWR(VIRTWL_IOCTL_BASE, nr, type)
+
+enum virtwl_ioctl_new_type {
+ VIRTWL_IOCTL_NEW_CTX, /* open a new wayland connection context */
+ VIRTWL_IOCTL_NEW_ALLOC, /* create a new virtwl shm allocation */
+ /* create a new virtwl pipe that is readable via the returned fd */
+ VIRTWL_IOCTL_NEW_PIPE_READ,
+ /* create a new virtwl pipe that is writable via the returned fd */
+ VIRTWL_IOCTL_NEW_PIPE_WRITE,
+ /* create a new virtwl dmabuf that is writable via the returned fd */
+ VIRTWL_IOCTL_NEW_DMABUF,
+};
+
+struct virtwl_ioctl_new {
+ __u32 type; /* VIRTWL_IOCTL_NEW_* */
+ int fd; /* return fd */
+ __u32 flags; /* currently always 0 */
+ union {
+ /* size of allocation if type == VIRTWL_IOCTL_NEW_ALLOC */
+ __u32 size;
+ /* buffer description if type == VIRTWL_IOCTL_NEW_DMABUF */
+ struct {
+ __u32 width; /* width in pixels */
+ __u32 height; /* height in pixels */
+ __u32 format; /* fourcc format */
+ __u32 stride0; /* return stride0 */
+ __u32 stride1; /* return stride1 */
+ __u32 stride2; /* return stride2 */
+ __u32 offset0; /* return offset0 */
+ __u32 offset1; /* return offset1 */
+ __u32 offset2; /* return offset2 */
+ } dmabuf;
+ };
+};
+
+struct virtwl_ioctl_txn {
+ int fds[VIRTWL_SEND_MAX_ALLOCS];
+ __u32 len;
+ __u8 data[0];
+};
+
+struct virtwl_ioctl_dmabuf_sync {
+ __u32 flags; /* synchronization flags (see dma-buf.h) */
+};
+
+#define VIRTWL_IOCTL_NEW VIRTWL_IOWR(0x00, struct virtwl_ioctl_new)
+#define VIRTWL_IOCTL_SEND VIRTWL_IOR(0x01, struct virtwl_ioctl_txn)
+#define VIRTWL_IOCTL_RECV VIRTWL_IOW(0x02, struct virtwl_ioctl_txn)
+#define VIRTWL_IOCTL_DMABUF_SYNC VIRTWL_IOR(0x03, \
+ struct virtwl_ioctl_dmabuf_sync)
+
+#endif /* _LINUX_VIRTWL_H */
new file mode 100644
@@ -0,0 +1,2 @@
+wayland-proxy: wayland-proxy.c wayland-proxy-main.c
+ gcc -g -Wall -o $@ $^
new file mode 100644
@@ -0,0 +1,58 @@
+Under Linux, most applications use GPU acceleration with help of MESA library. And
+MESA library interacts with kernel GPU driver by operating on some special
+character device file exported by kernel GPU driver. MESA library opens some
+special files in system and operations on GPU are done by ioctl/mmap system call
+and regular memory operations.
+
+The idea of render node forwarding sounds simple: we just write a kernel driver
+for guest Linux kernel and let it exports same interface to user space like the
+real Linux GPU kernel driver. So it's an API proxy between host and VM guest. We
+just proxy API at system call level. Or we can say: it's a guest device driver
+backed by another host device driver which provide same user space interface
+instead of real hardware.
+
+The code here was tested on a debian stretch host with a debian stretch guest with
+intel GPU driver. Here is the instructions:
+
+1. Create a debian stretch guest on a debian stretch host first.
+
+2. Build guest kernel and guest tool:
+ make defconfig # On x86_64 host
+ make -j `nproc` bzImage
+ make -C tools/forward
+
+3. Build patched qemu:
+
+ git clone --depth 1 https://git.qemu.org/git/qemu.git
+ cd qemu
+ patch -p1 < qemu.diff
+ ./configure --target-list=x86_64-softmmu --disable-gtk
+ make -j `nproc`
+
+4. On host, launch wayland server:
+ switch to tty1 and login as a regular user and then run
+ weston -i86400
+
+5. Launch guest with command line like this:
+
+
+ if lsmod|grep vhost_vsock; then
+ echo ok
+ else
+ sudo modprobe vhost_vsock
+ sudo chgrp kvm /dev/vhost-vsock
+ sudo chmod 660 /dev/vhost-vsock
+ fi
+ $HOME/qemu/x86_64-softmmu/qemu-system-x86_64 -nographic \
+ -smp 4 \
+ -enable-kvm \
+ -m 3072 \
+ -device vhost-vsock-pci,guest-cid=3 \
+ -kernel $HOME/linux/arch/x86/boot/bzImage -append "root=/dev/sda1 console=ttyS0" path_to_guest.img
+
+5. Inside guest:
+
+ ./wayland-proxy &
+ Xwayland :3 -noreset &
+ export DISPLAY=:3
+ glxinfo
new file mode 100644
@@ -0,0 +1,1117 @@
+diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c
+index 241db496c3..d30885a78d 100644
+--- a/accel/kvm/kvm-all.c
++++ b/accel/kvm/kvm-all.c
+@@ -258,6 +258,23 @@ int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
+ return 0;
+ }
+
++
++extern int add_guest_memory(void* ptr, unsigned long guest_phy_start, size_t size);
++
++int add_guest_memory(void* ptr, unsigned long guest_phys_addr, size_t size) {
++ KVMState *s = kvm_state;
++ struct kvm_userspace_memory_region mem;
++ int ret;
++ //FIXME 100
++ mem.slot = 100;
++ mem.guest_phys_addr = guest_phys_addr;
++ mem.memory_size = size;
++ mem.userspace_addr = (uint64_t)ptr;
++ mem.flags = 0;
++ ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
++ return ret;
++}
++
+ static int kvm_set_user_memory_region(KVMMemoryListener *kml, KVMSlot *slot, bool new)
+ {
+ KVMState *s = kvm_state;
+diff --git a/forwarder.h b/forwarder.h
+new file mode 100644
+index 0000000000..4937cebbf7
+--- /dev/null
++++ b/forwarder.h
+@@ -0,0 +1,103 @@
++enum {
++ STREAM_MAGIC = 0xbeefc1ea,
++ EVENT_MAGIC,
++ IPC_MAGIC,
++};
++struct pwrite_stream {
++ unsigned int magic;
++ int fd;
++ unsigned int handle;
++ unsigned int offset;
++ unsigned int size;
++};
++
++#define IPC_PAGE_SIZE 32768
++
++#define IPC_COUNT 4
++
++struct ipc {
++ volatile unsigned int seq;
++ unsigned int cmd;
++ union {
++ struct {
++ int arg1;
++ int arg2;
++ int arg3;
++ int pad1;
++ };
++ struct {
++ volatile int64_t ret;
++ int64_t pad2;
++ };
++ struct {
++ int fd;
++ } ioctl;
++ struct {
++ unsigned int pn_count;
++ } hostfd;
++ struct {
++ void* addr;
++ } dmabuf;
++ struct {
++ int fd;
++ unsigned int pn_off;
++ unsigned int pn_count;
++ } mmap;
++ struct {
++ unsigned int pn_off;
++ unsigned int pn_count;
++ } munmap;
++ struct {
++ int fd;
++ int whence;
++ } lseek;
++ struct {
++ int fd;
++ unsigned int len;
++ } fallocate;
++ struct {
++ int fd;
++ unsigned int len;
++ } ftruncate;
++ struct {
++ int fd;
++ uint32_t fdc;
++ uint32_t size;
++ } msg;
++ };
++ char data[0];
++};
++
++#define WL_IOCTL_BASE 'w'
++#define VIRT_WL_MAX 32
++#define WL_IO(nr) _IO(WL_IOCTL_BASE, nr + VIRT_WL_MAX)
++
++#define WL_CMD_NEW_RENDER_FD WL_IO(0x00)
++#define WL_CMD_NEW_WL_FD WL_IO(0x01)
++#define WL_CMD_NEW_MEM_FD WL_IO(0x02)
++#define WL_CMD_NEW_SYNC_FD WL_IO(0x03)
++#define WL_CMD_RECVMSG WL_IO(0x04)
++#define WL_CMD_SENDMSG WL_IO(0x05)
++#define WL_CMD_MMAP WL_IO(0x06)
++#define WL_CMD_MUNMAP WL_IO(0x07)
++#define WL_CMD_LSEEK WL_IO(0x08)
++#define WL_CMD_CLEAR_COUNTER WL_IO(0x09)
++#define WL_CMD_SHOW_COUNTER WL_IO(0x0A)
++#define WL_CMD_NEW_DMABUF WL_IO(0x0B)
++#define WL_CMD_FALLOCATE WL_IO(0x0C)
++#define WL_CMD_FTRUNCATE WL_IO(0x0D)
++
++#define SW_SYNC_IOC_MAGIC 'W'
++
++struct sw_sync_create_fence_data {
++ unsigned int value;
++ char name[32];
++ int fence; /* fd of new fence */
++};
++
++#define SW_SYNC_IOC_CREATE_FENCE _IOWR(SW_SYNC_IOC_MAGIC, 0,\
++ struct sw_sync_create_fence_data)
++
++#define SW_SYNC_IOC_INC _IOW(SW_SYNC_IOC_MAGIC, 1, __u32)
++
++#define KVM_HC_FORWARDING 70
+diff --git a/src.c b/src.c
+new file mode 100644
+index 0000000000..7c5a9e3405
+--- /dev/null
++++ b/src.c
+@@ -0,0 +1,915 @@
++// Copyright 2019 The Chromium OS Authors. All rights reserved.
++// Use of this source code is governed by a BSD-style license that can be
++// found in the LICENSE file.
++
++#ifndef _GNU_SOURCE
++#define _GNU_SOURCE
++#endif
++#include <errno.h>
++#include <gbm.h>
++#include <fcntl.h>
++#include <inttypes.h>
++#include <libdrm/drm.h>
++#include <libdrm/i915_drm.h>
++#include <poll.h>
++#include <pthread.h>
++#include <stddef.h>
++#include <stdio.h>
++#include <stdint.h>
++#include <stdlib.h>
++#include <sys/ioctl.h>
++#include <sys/mman.h>
++#include <sys/socket.h>
++#include <sys/stat.h>
++#include <sys/types.h>
++#include <sys/uio.h>
++#include <sys/un.h>
++#include <unistd.h>
++#include <linux/sync_file.h>
++#include <linux/vm_sockets.h>
++#include "forwarder.h"
++
++#define EXPORT __attribute__ ((visibility ("default")))
++
++FILE *efp;
++
++#ifdef DEBUG
++
++#define debug_close(arg) do { \
++ int r = close(arg); \
++ if (r) r = errno; \
++ fprintf(efp, "%s:%d close %d got %d\n", __func__, __LINE__, arg, r); \
++} while(0)
++
++static void debug_select(fd_set * fds, int max) {
++ fprintf(stderr, "Waiting");
++ for(int i= 0; i < max; ++i) {
++ if (FD_ISSET(i, fds))
++ fprintf(stderr, " %d", i);
++ }
++ fprintf(stderr, "\n");
++}
++
++#else
++
++#define debug_close(arg) close(arg)
++
++#endif
++
++#define bug(...) do { \
++ fprintf(efp, "Bug at %s:%d\n", __func__, __LINE__); \
++ fprintf(efp, __VA_ARGS__); \
++ fflush(efp); \
++ exit(1); \
++} while(0)
++
++#define debug(...) do { \
++ fprintf(efp, "debug at %s:%d\n", __func__, __LINE__); \
++ fprintf(efp, __VA_ARGS__); \
++ fflush(efp); \
++} while(0)
++
++static void *host_start;
++static void *guest_ram_start[2];
++uint64_t guest_phy_start;
++uint64_t guest_phy_size;
++// FIXME (big page?)
++static const int PAGE_SIZE = 4096;
++static const int PAGE_SHIFT = 12;
++
++static void create_thread(pthread_t * tid, void *(*start_routing) (void *),
++ void *arg)
++{
++ pthread_attr_t attr;
++ int ret = pthread_attr_init(&attr);
++ if (ret)
++ bug("init thread attr");
++ pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
++ if (tid == NULL)
++ tid = malloc(sizeof(*tid));
++ if (!tid)
++ bug("malloc");
++ ret = pthread_create(tid, &attr, start_routing, arg);
++ if (ret)
++ bug("create thread %d\n", ret);
++}
++
++struct forwarder {
++ int socket;
++ struct syscall_data *data;
++};
++
++#define PORT 30000
++
++static char* mem_maps;
++static unsigned int mem_maps_chars;
++
++static int all_zero(unsigned long* ptr, unsigned long off, unsigned long ac)
++{
++ unsigned i = 0;
++ for (i = 0; i < ac; ++i) {
++ if (ptr[i + off])
++ return 0;
++ }
++ return 1;
++}
++
++static void mark_used(unsigned long pn, unsigned long pc, int used)
++{
++ if (pn % 8) bug("why strange pn");
++ unsigned long char_off = pn / 8;
++ unsigned long char_c = pc / 8;
++ unsigned int char_left = pc % 8;
++ unsigned char mask;
++ memset(mem_maps + char_off, used ? 0xff:0, char_c);
++ if (char_left) {
++ mask = (1UL << char_left) - 1;
++ if (used)
++ mem_maps[char_off + char_c] |= mask;
++ else
++ mem_maps[char_off + char_c] &= ~mask;
++ }
++}
++
++#define find_first_zero(type) \
++static long find_first_zero_ ## type (unsigned long count) \
++{ \
++ type* ptr = (type *)mem_maps; \
++ unsigned long i, c = mem_maps_chars/sizeof(*ptr); \
++ for (i = 0; i < c; ++i) { \
++ if (ptr[i]==0) { \
++ mark_used(i * sizeof(*ptr) * 8, count, 1); \
++ return i * sizeof(*ptr) * 8; \
++ } \
++ } \
++ return -1; \
++}
++
++find_first_zero(char)
++find_first_zero(short)
++find_first_zero(int)
++find_first_zero(long)
++
++static long find_first_zero_more(unsigned long count)
++{
++ unsigned long ac = (count + 63)/64;
++ unsigned long *ptr = (unsigned long *)mem_maps;
++ unsigned long i, c = mem_maps_chars/sizeof(*ptr);
++ for (i = 0; i < c - ac; i += ac) {
++ if (all_zero(ptr, i, ac)) {
++ mark_used(i * sizeof(*ptr) * 8, count, 1);
++ return i * sizeof(*ptr) * 8;
++ }
++ }
++ return -1;
++}
++
++static unsigned long alloc_guest_phy_addr(unsigned long size)
++{
++ if (size % 4096) {
++ bug("not page aligned\n");
++ return 0;
++ }
++ unsigned long pc = size >> PAGE_SHIFT;
++ long pn;
++
++ if (pc <= 8)
++ pn = find_first_zero_char(pc);
++ else if(pc <=16)
++ pn = find_first_zero_short(pc);
++ else if(pc <=32)
++ pn = find_first_zero_int(pc);
++ else if (pc <=64)
++ pn = find_first_zero_long(pc);
++ else
++ pn = find_first_zero_more(pc);
++
++ if (pn < 0) {
++ debug("no enough address space %lx %lx\n", size,
++ guest_phy_size);
++ return 0;
++ }
++ return (pn << PAGE_SHIFT) + guest_phy_start;
++}
++
++static int fix_mmap(unsigned int cmd, char *data)
++{
++ struct drm_i915_gem_mmap *mp;
++ unsigned long guest_phy_addr;
++ void *target, *ptr;
++ if (cmd != DRM_IOCTL_I915_GEM_MMAP)
++ return 0;
++ mp = (struct drm_i915_gem_mmap *)data;
++ guest_phy_addr = alloc_guest_phy_addr(mp->size);
++ if (!guest_phy_addr) {
++ bug("running out of space?");
++ return -ENOMEM;
++ }
++ target = host_start + guest_phy_addr - guest_phy_start;
++ ptr = mremap((void *)mp->addr_ptr, mp->size, mp->size,
++ MREMAP_FIXED | MREMAP_MAYMOVE, target);
++ if (ptr != target) {
++ bug("%p %p remap\n", ptr, target);
++ perror("can't remap");
++ return -ENOMEM;
++ }
++ mp->addr_ptr = guest_phy_addr;
++ return 0;
++}
++
++#ifndef MFD_ALLOW_SEALING
++#define MFD_ALLOW_SEALING 0x0002U
++#endif
++
++static int do_mem_new_fd(unsigned int page_count)
++{
++ int fd = memfd_create("forwarder", MFD_ALLOW_SEALING);
++ if (fd<0) {
++ bug("new memfd");
++ return -errno;
++ }
++ if (ftruncate(fd, (off_t)page_count * PAGE_SIZE) < 0) {
++ bug("truncate");
++ return -errno;
++ }
++ return fd;
++}
++
++static struct gbm_device * gbm;
++
++struct virtwl_ioctl_new {
++ __u32 type; /* VIRTWL_IOCTL_NEW_* */
++ int fd; /* return fd */
++ __u32 flags; /* currently always 0 */
++ union {
++ /* size of allocation if type == VIRTWL_IOCTL_NEW_ALLOC */
++ __u32 size;
++ /* buffer description if type == VIRTWL_IOCTL_NEW_DMABUF */
++ struct {
++ __u32 width; /* width in pixels */
++ __u32 height; /* height in pixels */
++ __u32 format; /* fourcc format */
++ __u32 stride[3]; /* return stride0 */
++ __u32 offset[3]; /* return offset0 */
++ } dmabuf;
++ };
++};
++
++static int do_new_dmabuf(struct virtwl_ioctl_new* addr)
++{
++ struct gbm_bo *bo = gbm_bo_create(gbm, addr->dmabuf.width,
++ addr->dmabuf.height,
++ addr->dmabuf.format,
++ GBM_BO_USE_LINEAR);
++ if (bo == NULL) {
++ debug("can't allocate bo %d %d %x\n", addr->dmabuf.width,
++ addr->dmabuf.height, addr->dmabuf.format);
++ return -EINVAL;
++ }
++#if 0
++ for (int i = 0; i < gbm_bo_get_plane_count(bo); ++i) {
++ addr->dmabuf.stride[i] = gbm_bo_get_stride_for_plane(bo, i);
++ addr->dmabuf.offset[i] = gbm_bo_get_offset(bo, i);
++ }
++#else
++ addr->dmabuf.stride[0] = gbm_bo_get_stride(bo);
++ addr->dmabuf.offset[0] = 0;
++#endif
++ int fd = gbm_bo_get_fd(bo);
++ gbm_bo_destroy(bo);
++ if (fd >= 0)
++ return fd;
++ else
++ return -errno;
++}
++
++static int do_new_fd(const char* path)
++{
++ int fd = open(path, O_RDWR);
++ if (fd<0) {
++ bug("can't open fd %s\n", path);
++ return -errno;
++ }
++ return fd;
++}
++
++static int do_wl_new_fd(void)
++{
++ struct sockaddr_un addr = { };
++ int fd = socket(AF_UNIX, SOCK_STREAM, 0);
++ const char *wd;
++ if (fd < 0) {
++ bug("create socket\n");
++ }
++ addr.sun_family = AF_UNIX;
++ wd = getenv("XDG_RUNTIME_DIR");
++ if (wd == NULL)
++ wd = "/run/chrome";
++ snprintf(addr.sun_path, sizeof(addr.sun_path), "%s/wayland-0", wd);
++ if (connect(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
++ bug("connect to wayland server\n");
++ debug_close(fd);
++ return -errno;
++ }
++ return fd;
++}
++
++static int do_sendmsg(int fd, unsigned int fdc, unsigned int size,
++ char* data)
++{
++ struct iovec iov = {0, size};
++ struct msghdr msg = {};
++ size_t fd_len = fdc * sizeof(int);
++ int ret;
++
++ msg.msg_iov = &iov;
++ msg.msg_iovlen = 1;
++ if (fd_len) {
++ msg.msg_control = data;
++ msg.msg_controllen = sizeof(struct cmsghdr) + fd_len;
++ iov.iov_base = data + msg.msg_controllen;
++ } else {
++ iov.iov_base = data;
++ }
++ ret = sendmsg(fd, &msg, 0);
++ if (ret < 0) bug("why ret %d %d\n", fd, errno);
++ return ret;
++}
++
++static int do_recvmsg(int fd, unsigned int fdc, unsigned int size,
++ char* data)
++{
++ struct iovec iov = {0, size};
++ struct msghdr msg = {};
++ size_t fd_len = fdc * sizeof(int);
++ int ret;
++ struct cmsghdr *cmsg;
++
++ msg.msg_iov = &iov;
++ msg.msg_iovlen = 1;
++ if (fd_len != 112) bug("fd len %ld\n", fd_len);
++ msg.msg_control = data;
++ msg.msg_controllen = sizeof(*cmsg) + fd_len;
++ iov.iov_base = data + msg.msg_controllen;
++ ret = recvmsg(fd, &msg, MSG_DONTWAIT);
++ if (ret < 0 && errno != EAGAIN) bug("why ret %d %d\n", fd, errno);
++ if (ret < 0)
++ return -EAGAIN;
++ if (msg.msg_controllen) {
++ cmsg = CMSG_FIRSTHDR(&msg);
++ if (CMSG_NXTHDR(&msg, cmsg))
++ bug("I really don't expect this, fix me!\n");
++ if (cmsg->cmsg_level != SOL_SOCKET ||
++ cmsg->cmsg_type != SCM_RIGHTS)
++ bug("I don't know about this\n");
++ fdc = (cmsg->cmsg_len - sizeof(*cmsg))/sizeof(int);
++ if (fdc > 27)bug("why so many fd");
++ } else
++ fdc = 0;
++ int *rfd = (int *)(data + sizeof(*cmsg));
++ rfd[fdc] = -1;
++ return ret;
++}
++
++struct ioctl_counter {
++ unsigned long cmd;
++ const char* name;
++ unsigned long count;
++};
++
++#define CMD(a) {a, #a, 0}
++
++static struct ioctl_counter counters[] = {
++ {0, "TOTAL", 0},
++ {0, "OTHER", 0},
++ CMD(DRM_IOCTL_GEM_CLOSE),
++ CMD(DRM_IOCTL_GET_CAP),
++ CMD(DRM_IOCTL_I915_GEM_BUSY),
++ CMD(DRM_IOCTL_I915_GEM_CONTEXT_CREATE),
++ CMD(DRM_IOCTL_I915_GEM_CONTEXT_DESTROY),
++ CMD(DRM_IOCTL_I915_GEM_CONTEXT_GETPARAM),
++ CMD(DRM_IOCTL_I915_GEM_CONTEXT_SETPARAM),
++ CMD(DRM_IOCTL_I915_GEM_CREATE),
++ CMD(DRM_IOCTL_I915_GEM_GET_APERTURE),
++ CMD(DRM_IOCTL_I915_GEM_GET_TILING),
++ CMD(DRM_IOCTL_I915_GEM_MADVISE),
++ CMD(DRM_IOCTL_I915_GEM_MMAP_GTT),
++ CMD(DRM_IOCTL_I915_GEM_SET_DOMAIN),
++ CMD(DRM_IOCTL_I915_GEM_SET_TILING),
++ CMD(DRM_IOCTL_I915_GEM_SW_FINISH),
++ CMD(DRM_IOCTL_I915_GEM_PWRITE),
++ CMD(DRM_IOCTL_I915_GEM_THROTTLE),
++ CMD(DRM_IOCTL_I915_GEM_WAIT),
++ CMD(DRM_IOCTL_I915_GET_RESET_STATS),
++ CMD(DRM_IOCTL_I915_REG_READ),
++ CMD(DRM_IOCTL_MODE_GETPLANE),
++ CMD(DRM_IOCTL_MODE_GETPLANERESOURCES),
++ CMD(DRM_IOCTL_MODE_GETPROPERTY),
++ CMD(DRM_IOCTL_MODE_OBJ_GETPROPERTIES),
++ CMD(DRM_IOCTL_SET_CLIENT_CAP),
++ CMD(DRM_IOCTL_VERSION),
++ CMD(DRM_IOCTL_I915_GETPARAM),
++ CMD(DRM_IOCTL_I915_GEM_EXECBUFFER2),
++// CMD(DRM_IOCTL_I915_GEM_EXECBUFFER2_WR),
++ CMD(DRM_IOCTL_PRIME_HANDLE_TO_FD),
++ CMD(DRM_IOCTL_PRIME_FD_TO_HANDLE),
++ CMD(DRM_IOCTL_I915_GEM_MMAP),
++ CMD(WL_CMD_NEW_WL_FD),
++ CMD(WL_CMD_NEW_MEM_FD),
++ CMD(WL_CMD_NEW_SYNC_FD),
++ CMD(SYNC_IOC_MERGE),
++ CMD(SYNC_IOC_FILE_INFO),
++ CMD(SW_SYNC_IOC_CREATE_FENCE),
++ CMD(SW_SYNC_IOC_INC),
++};
++
++#ifndef ARRAY_SIZE
++#define ARRAY_SIZE(array) \
++ (sizeof(array) / sizeof(array[0]))
++#endif
++
++static void count_ioctl(unsigned long cmd)
++{
++ int i;
++ if (cmd == WL_CMD_CLEAR_COUNTER) {
++ for(i = 0; i < ARRAY_SIZE(counters); ++i) {
++ counters[i].count = 0;
++ }
++ return;
++ }
++ if (cmd == WL_CMD_SHOW_COUNTER) {
++ for(i = 0; i < ARRAY_SIZE(counters); ++i) {
++ fprintf(stderr, "%s: %ld\n", counters[i].name,
++ counters[i].count);
++ }
++ return;
++ }
++ counters[0].count++;
++ for (i = 0; i < ARRAY_SIZE(counters); ++i) {
++ if(counters[i].cmd == cmd) {
++ counters[i].count++;
++ return;
++ }
++ }
++ counters[1].count++;
++}
++
++static void debug_fd(int fd) {
++ char ttt[256], name[256];
++ snprintf(ttt, sizeof(ttt), "/proc/self/fd/%d", fd);
++ int rrr = readlink(ttt, name, sizeof(name));
++ if (rrr < 0) {
++ debug("fail to read link %d\n", errno);
++ }
++ name[rrr] = 0;
++ debug("we got %d %s\n", fd, name);
++}
++
++struct call_pattern {
++ unsigned long cmd;
++ int err;
++};
++
++struct call_pattern patterns[] = {
++ {DRM_IOCTL_I915_GETPARAM, EINVAL},
++ {DRM_IOCTL_I915_GEM_CONTEXT_SETPARAM, EINVAL},
++ {DRM_IOCTL_I915_GET_RESET_STATS, EPERM},
++ {DRM_IOCTL_I915_GEM_WAIT, ETIME},
++// {DRM_IOCTL_I915_GEM_EXECBUFFER2, ENOENT},
++// {DRM_IOCTL_I915_GEM_BUSY, ENOENT},
++};
++
++static void debug_ioctl(int fd, unsigned long cmd, int err)
++{
++ int i = 0;
++ for (i = 0; i < ARRAY_SIZE(patterns); i++) {
++ if (cmd == patterns[i].cmd &&
++ err == patterns[i].err)
++ return;
++ }
++ debug_fd(fd);
++ bug("Cmd: %lx err: %d\n", cmd, err);
++}
++
++#define MAX_DATA_SIZE (2 << 20)
++
++static void *vsock_stream(void *arg)
++{
++ struct forwarder *f = arg;
++ int fd;
++ int socket = f->socket;
++ int ret, size;
++ struct pwrite_stream *data = (struct pwrite_stream *)f->data;
++ char c = '.';
++ for (;;) {
++ ret = read(socket, (char *)data, sizeof(*data));
++ if (ret != sizeof(*data) || data->magic != STREAM_MAGIC) {
++ if (ret)
++ debug("why only this data: %d\n", ret);
++ debug_close(socket);
++ free(data);
++ return NULL;
++ }
++ fd = data->fd;
++ int left = data->size;
++ struct drm_i915_gem_pwrite pw;
++ pw.handle = data->handle;
++ pw.offset = data->offset;
++ char *cur = (char *)data;
++ int len;
++ while(left) {
++ size = (MAX_DATA_SIZE > left ? left: MAX_DATA_SIZE);
++ len = read(socket, cur, size);
++ if (len < 0) {
++ bug("can't read %p %d %d %d\n", cur, size, len, errno);
++ }
++ left -= len;
++ pw.data_ptr = (uint64_t)cur;
++ pw.size = len;
++ ioctl(fd, DRM_IOCTL_I915_GEM_PWRITE, &pw);
++ pw.offset +=len;
++ }
++ ret = write(socket, &c, 1);
++ if (ret != 1)bug("can't write");
++ }
++}
++
++#define MAX_FD 2048
++
++void* waits[MAX_FD];
++
++struct wait_poll {
++ void* data;
++ int fd;
++};
++
++static void* get_host_addr(uint64_t guest_phy)
++{
++ if (guest_phy < (1UL << 32))
++ return (char *)guest_ram_start[0] + guest_phy;
++ return (char *)guest_ram_start[1] + guest_phy - (1UL << 32);
++}
++
++static long do_mmap(int fd, unsigned int pg_off, unsigned int pg_count)
++{
++ uint64_t size = (uint64_t)pg_count * PAGE_SIZE;
++ uint64_t off = (uint64_t)pg_off * PAGE_SIZE;
++ if (size > UINT32_MAX) {
++ debug_fd(fd);
++ bug("too big %" PRIu64 " %u %" PRIu64 "\n", size, pg_off, off);
++ return -ENOMEM;
++ }
++ unsigned long guest_phy_addr = alloc_guest_phy_addr(size);
++ if (!guest_phy_addr) {
++ bug("too big");
++ return -ENOMEM;
++ }
++ void *target = host_start + guest_phy_addr - guest_phy_start;
++ void *ptr = mmap(target, size, PROT_WRITE | PROT_READ,
++ MAP_SHARED | MAP_FIXED, fd, off);
++ if (ptr != target) {
++ bug("can't mmap to target\n");
++ if (errno == 0) {
++ return -ENOMEM;
++ }
++ return -errno;
++ }
++ return guest_phy_addr >> PAGE_SHIFT;
++}
++
++static int do_munmap(unsigned int pg_off, unsigned int pg_count)
++{
++ uint64_t guest_addr = (uint64_t) pg_off << PAGE_SHIFT;
++ uint64_t size = (uint64_t)pg_count << PAGE_SHIFT;
++
++ if (guest_addr < guest_phy_start ||
++ guest_addr >= guest_phy_start + guest_phy_size ||
++ guest_addr + size >= guest_phy_start + guest_phy_size) {
++ bug("strange munmap req %lx\n", guest_addr);
++ return -EINVAL;
++ }
++ void * target = guest_addr - guest_phy_start + host_start;
++ //FIXME Will there be race?
++ void * tptr = mmap(target, size, PROT_NONE, MAP_SHARED|MAP_ANONYMOUS|MAP_FIXED, -1, 0);
++ if (tptr != target)
++ bug("can't unmap %p %p %d", target, tptr, errno);
++ mark_used(pg_off - (guest_phy_start >> PAGE_SHIFT), pg_count, 0);
++ return 0;
++}
++
++static int64_t handle_cmd(unsigned int cmd, struct ipc* ipc)
++{
++ switch (cmd) {
++ case WL_CMD_NEW_RENDER_FD:
++ return do_new_fd("/dev/dri/renderD128");
++ case WL_CMD_NEW_SYNC_FD:
++ return do_new_fd("/sys/kernel/debug/sync/sw_sync");
++ case WL_CMD_NEW_WL_FD:
++ return do_wl_new_fd();
++ case WL_CMD_NEW_MEM_FD:
++ return do_mem_new_fd(ipc->hostfd.pn_count);
++ case WL_CMD_NEW_DMABUF:
++ return do_new_dmabuf(ipc->dmabuf.addr);
++ case WL_CMD_MMAP:
++ return do_mmap(ipc->mmap.fd, ipc->mmap.pn_off,
++ ipc->mmap.pn_count);
++ case WL_CMD_MUNMAP:
++ return do_munmap(ipc->munmap.pn_off,
++ ipc->munmap.pn_count);
++ case WL_CMD_LSEEK:
++ return lseek(ipc->lseek.fd, 0, ipc->lseek.whence);
++ case WL_CMD_FALLOCATE:
++ return fallocate(ipc->fallocate.fd, 0, 0, ipc->fallocate.len);
++ case WL_CMD_FTRUNCATE:
++ return ftruncate(ipc->ftruncate.fd, ipc->ftruncate.len);
++ case WL_CMD_SENDMSG:
++ return do_sendmsg(ipc->msg.fd, ipc->msg.fdc,
++ ipc->msg.size, ipc->data);
++ case WL_CMD_RECVMSG:
++ return do_recvmsg(ipc->msg.fd, ipc->msg.fdc,
++ ipc->msg.size, ipc->data);
++ default:
++ bug("no supported cmd:%x\n", cmd);
++ return -ENOENT;
++ }
++}
++
++static void *fast_ipc(void *base)
++{
++ struct ipc * ipc;
++ ipc = (struct ipc *) base;
++ unsigned int seq = 0;
++ int i;
++ int ret;
++ unsigned long delay = 0;
++
++ for(;;) {
++ seq++;
++ for(;;) {
++ if (delay) {
++ usleep(delay);
++ }
++ if (ipc->seq == seq) {
++ delay /= 2;
++ break;
++ }
++ delay++;
++ if (delay > 1000)
++ delay = 1000;
++ }
++ count_ioctl(ipc->cmd);
++ if (_IOC_TYPE(ipc->cmd) == 'w') {
++ ret = handle_cmd(ipc->cmd, ipc);
++ } else if (ipc->cmd == DRM_IOCTL_I915_GEM_PWRITE) {
++ struct drm_i915_gem_pwrite* pw =
++ (struct drm_i915_gem_pwrite *)ipc + 1;
++ unsigned int count = pw->pad;
++ if (count > IPC_PAGE_SIZE/sizeof(*pw) - 1)
++ bug("too much pwrite");
++ ret = 0;
++ for (i = 0; i < count; ++i, ++pw) {
++ if(ioctl(ipc->ioctl.fd, ipc->cmd, pw)){
++ ret = -errno;
++ break;
++ }
++ }
++ } else {
++ ret = ioctl(ipc->ioctl.fd, ipc->cmd, (void *)ipc->data);
++ if (ret < 0)
++ ret = -errno;
++ else {
++ if (ipc->cmd == DRM_IOCTL_I915_GEM_MMAP
++ && fix_mmap(ipc->cmd, ipc->data))
++ ret = -ENOMEM;
++ }
++ if(ret)debug_ioctl(ipc->ioctl.fd, ipc->cmd, -ret);
++ }
++ ipc->ret = ret;
++ seq++;
++ ipc->seq = seq;
++ }
++ return NULL;
++}
++
++EXPORT void show(void);
++
++EXPORT void show(void)
++{
++ char buf[4096];
++ int ret;
++ int fd = open("/proc/self/maps", O_RDONLY);
++ do {
++ ret = read(fd, buf, sizeof(buf));
++ fwrite(buf, ret, 1, stderr);
++ } while (ret > 0);
++}
++
++static void add_to_poll(struct pollfd **poll_fds, unsigned int *cnt,
++ unsigned int *cap, int fd)
++{
++ if (fd < 0)bug("invalid fd %d\n", fd);
++ if (*cnt == *cap) {
++ *cap = (*cap) << 1;
++ debug("Increase poll cap to %d\n", *cap);
++ *poll_fds = realloc(*poll_fds, sizeof(**poll_fds) * (*cap));
++ if ((*poll_fds) == NULL)bug("can't malloc new memory for poll fds");
++ }
++ struct pollfd* entry = (*poll_fds) + (*cnt);
++ (*cnt)++;
++ entry->fd = fd;
++ entry->events = POLLIN;
++ entry->revents = 0;
++}
++
++static void remove_from_poll(struct pollfd *poll_fds, unsigned int* cnt, int fd)
++{
++ unsigned int i;
++ if (fd < 0)bug("invalid fd %d\n", fd);
++ for (i = 0; i < *cnt; ++i) {
++ if (poll_fds[i].fd != fd)
++ continue;
++ if (i == 0)bug("important");
++ (*cnt)--;
++ if (i == (*cnt))
++ return;
++ poll_fds[i] = poll_fds[*cnt];
++ return;
++ }
++}
++
++static void * vsock_event(void * arg)
++{
++ int * vsock = (int *) arg;
++ int vfd = *vsock;
++ // Quick hack to b
++ unsigned int fd_cnt = 0;
++ unsigned int fd_max = 8;
++ struct pollfd *poll_fds = malloc(sizeof(*poll_fds) * fd_max);
++ int ret;
++ struct wait_poll wt;
++ unsigned long ipc_phy_addr;
++ int i;
++
++ if (poll_fds == NULL)bug("can't malloca poll fds");
++
++ ret = read(vfd, &ipc_phy_addr, sizeof(ipc_phy_addr));
++ if (ret != sizeof(ipc_phy_addr)) {
++ bug("read guest phy ret %d\n", ret);
++ }
++ ret = write(vfd, guest_ram_start, sizeof(guest_ram_start));
++ if (ret != sizeof(guest_ram_start)) {
++ bug("write host addr ret %d\n", ret);
++ }
++ create_thread(NULL, fast_ipc, get_host_addr(ipc_phy_addr));
++
++ add_to_poll(&poll_fds, &fd_cnt, &fd_max, vfd);
++ for(;;) {
++ //debug_select(&use_fds, max_fd + 1);
++ ret = poll(poll_fds, fd_cnt, -1);
++ if (ret < 0 && errno == EINTR)
++ continue;
++ if (ret < 0) {
++ bug("poll %d\n", errno);
++ }
++ if (poll_fds[0].revents & POLLIN) {
++ ret = read(vfd, &wt, sizeof(wt));
++ if (ret != sizeof(wt)) {
++ if (ret)
++ bug("read %d %d\n", ret, errno);
++ return NULL;
++ }
++ if (wt.fd >= MAX_FD||wt.fd < 0){
++ bug("too much fd %d", wt.fd);
++ }
++ waits[wt.fd] = wt.data;
++ if (wt.data) {
++ add_to_poll(&poll_fds, &fd_cnt, &fd_max, wt.fd);
++ } else {
++ debug_close(wt.fd);
++ remove_from_poll(poll_fds, &fd_cnt, wt.fd);
++ }
++ }
++ i = 1;
++ while (i < fd_cnt) {
++ struct pollfd *pfd = poll_fds + i;
++ if (!(pfd->revents & POLLIN) || !waits[pfd->fd]) {
++ i++;
++ continue;
++ }
++ wt.fd = pfd->fd;
++ wt.data = waits[pfd->fd];
++ ret = write(vfd, &wt, sizeof(wt));
++ if (ret != sizeof(wt))
++ bug("write %d %d\n", ret, errno);
++ waits[pfd->fd] = NULL;
++ remove_from_poll(poll_fds, &fd_cnt, wt.fd);
++ }
++ }
++}
++
++static void *vsock_server(void *base)
++{
++ int server_fd, new_socket;
++ struct sockaddr_vm address = { };
++ int opt = 1, ret;
++ int addrlen = sizeof(address);
++ unsigned int magic;
++
++ if ((server_fd = socket(AF_VSOCK, SOCK_STREAM, 0)) == 0) {
++ perror("socket failed");
++ exit(EXIT_FAILURE);
++ }
++ if (setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR | SO_REUSEPORT,
++ &opt, sizeof(opt))) {
++ perror("setsockopt");
++ exit(EXIT_FAILURE);
++ }
++ address.svm_family = AF_VSOCK;
++ address.svm_port = PORT;
++ address.svm_cid = VMADDR_CID_ANY;
++
++ if (bind(server_fd, (struct sockaddr *)&address, sizeof(address)) < 0) {
++ perror("bind failed");
++ exit(EXIT_FAILURE);
++ }
++ if (listen(server_fd, 3) < 0) {
++ perror("listen");
++ exit(EXIT_FAILURE);
++ }
++ for (;;) {
++ new_socket = accept(server_fd, (struct sockaddr *)&address,
++ (socklen_t *) & addrlen);
++ if (new_socket < 0) {
++ perror("accept");
++ exit(EXIT_FAILURE);
++ }
++ ret = read(new_socket, &magic, sizeof(magic));
++ if (ret != sizeof(magic))
++ bug("Can't get sock type: %d %d\n", ret, errno);
++ switch (magic) {
++ case STREAM_MAGIC:
++ {
++ struct forwarder *c = malloc(sizeof(*c));
++ c->socket = new_socket;
++ c->data = malloc(MAX_DATA_SIZE);
++ create_thread(NULL, vsock_stream, c);
++ }
++ break;
++ case EVENT_MAGIC:
++ {
++ int * a = malloc(sizeof(*a));
++ *a = new_socket;
++ create_thread(NULL, vsock_event, a);
++ }
++ break;
++ default:
++ bug("unknown magic %x\n", magic);
++ break;
++ }
++ }
++ return NULL;
++}
++
++static void alloc_mem_maps(unsigned long mem_size)
++{
++ mem_maps_chars = ((mem_size >> PAGE_SHIFT) + 7 ) / 8;
++ mem_maps = mmap(NULL, mem_maps_chars, PROT_READ|PROT_WRITE,
++ MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
++ if (mem_maps == MAP_FAILED)
++ bug("can't allocate mem maps");
++}
++
++EXPORT void start_render_node_host(void *host_addr, uint64_t guest_addr,
++ uint64_t mem_size, void *ram0_start,
++ void* ram4g_start)
++{
++ efp = stderr; // fopen("/tmp/rflog", "w");
++ debug("Starting render node host service %p %lx %lx\n",
++ host_addr, guest_addr, mem_size);
++ if ((uint64_t) host_addr % PAGE_SIZE || guest_addr % PAGE_SIZE ||
++ mem_size % PAGE_SIZE) {
++ debug("Invalid host_addr %p %lx %lx\n", host_addr,
++ guest_addr, mem_size);
++ return;
++ }
++ host_start = host_addr;
++ guest_phy_start = guest_addr;
++ guest_phy_size = mem_size;
++ guest_ram_start[0] = ram0_start;
++ guest_ram_start[1] = ram4g_start;
++
++ alloc_mem_maps(mem_size);
++
++ int drm_fd = open("/dev/dri/renderD128", O_RDWR);
++ if (drm_fd < 0) {
++ bug("can't open render node\n");
++ return;
++ }
++ gbm = gbm_create_device(drm_fd);
++ if (gbm == NULL) {
++ bug("can't init gbm\n");
++ return;
++ }
++
++ create_thread(NULL, vsock_server, NULL);
++}
+diff --git a/vl.c b/vl.c
+index d61d5604e5..98785bf5f3 100644
+--- a/vl.c
++++ b/vl.c
+@@ -2985,6 +2985,46 @@ static void user_register_global_props(void)
+ global_init_func, NULL, NULL);
+ }
+
++
++// Start form 8G, size 4G. Big enough?
++static void* qemu_host_start;
++static const unsigned long qemu_guest_phy_start = (2UL << 32);
++static const unsigned long qemu_guest_phy_size = (1UL << 32);
++
++
++
++void start_render_node_host(void *host_addr, uint64_t guest_addr,
++ uint64_t mem_size, void *ram0_start, void* ram4g_start);
++
++static int memfd_create(const char * name, int ignored)
++{
++ char buf[256];
++ static int c;
++ snprintf(buf, sizeof(buf), "/%s%d", name, c++);
++ return shm_open(buf, O_RDWR|O_CREAT, 0600);
++}
++#include "src.c"
++
++extern int add_guest_memory(void* ptr, unsigned long guest_phy_start, size_t size);
++
++static void add_memory(void)
++{
++ qemu_host_start = mmap(NULL, qemu_guest_phy_size, PROT_NONE, MAP_SHARED|MAP_ANONYMOUS, -1, 0);
++ if (qemu_host_start == NULL) {
++ fprintf(stderr, "faint, mmap1");
++ exit(1);
++ }
++ if (add_guest_memory(qemu_host_start, qemu_guest_phy_start, qemu_guest_phy_size)) {
++ fprintf(stderr, "faint, mmap2");
++ exit(1);
++ }
++ hwaddr len = 4096;
++ void* host0 = cpu_physical_memory_map(0, &len, 1);
++ start_render_node_host(qemu_host_start, qemu_guest_phy_start,
++ qemu_guest_phy_size, host0, 0);
++}
++
++
+ int main(int argc, char **argv, char **envp)
+ {
+ int i;
+@@ -4488,6 +4528,7 @@ int main(int argc, char **argv, char **envp)
+ qemu_opts_foreach(qemu_find_opts("device"),
+ device_init_func, NULL, &error_fatal);
+
++ add_memory();
+ cpu_synchronize_all_post_init();
+
+ rom_reset_order_override();
new file mode 100644
@@ -0,0 +1,58 @@
+#include <errno.h>
+#include <fcntl.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+#define debug(...) do { \
+ fprintf(stderr, "%s:%d\n", __func__, __LINE__); \
+ fprintf(stderr, __VA_ARGS__); \
+} while (0)
+
+#define bug(...) do { \
+ fprintf(stderr, "%s:%d\n", __func__, __LINE__); \
+ fprintf(stderr, __VA_ARGS__); \
+ exit(1); \
+} while (0)
+
+static const char wname[] = "wayland-0";
+
+void wayland_proxy(int fd);
+
+int main(int argc, char *argv[])
+{
+ struct sockaddr_un listen_addr = { };
+ int listen_fd;
+ int opt = 1;
+
+ char *proxy_dir = getenv("XDG_RUNTIME_DIR");
+ if (proxy_dir == NULL || !proxy_dir[0]) {
+ bug("No a valid XDG_RUNTIME_DIR\n");
+ }
+ if ((listen_fd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) {
+ bug("create socket\n");
+ }
+ if (setsockopt(listen_fd, SOL_SOCKET, SO_REUSEADDR | SO_REUSEPORT,
+ &opt, sizeof(opt))) {
+ bug("setsockopt");
+ }
+ listen_addr.sun_family = AF_UNIX;
+ snprintf(listen_addr.sun_path, sizeof(listen_addr.sun_path), "%s/%s",
+ proxy_dir, wname);
+ unlink(listen_addr.sun_path);
+ if (bind
+ (listen_fd, (struct sockaddr *)&listen_addr,
+ sizeof(listen_addr)) < 0) {
+ bug("bind failed");
+ }
+ if (listen(listen_fd, 3) < 0) {
+ bug("listen");
+ }
+ wayland_proxy(listen_fd);
+}
new file mode 100644
@@ -0,0 +1,297 @@
+#include <errno.h>
+#include <fcntl.h>
+#include <poll.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+#include "../../include/uapi/linux/virtwl.h"
+
+#define debug(...) do { \
+ fprintf(stderr, "%s:%d\n", __func__, __LINE__); \
+ fprintf(stderr, __VA_ARGS__); \
+} while (0)
+
+#define bug(...) do { \
+ fprintf(stderr, "%s:%d\n", __func__, __LINE__); \
+ fprintf(stderr, __VA_ARGS__); \
+ exit(1); \
+} while (0)
+
+enum {
+ FD_TYPE_CLOSED = 0,
+ FD_TYPE_UNIX,
+ FD_TYPE_DEV,
+};
+
+static int get_peer(char *type, int *peer, int fd, int fd_max)
+{
+ if (fd >= fd_max || fd < 0)
+ bug("too big fd %d %d\n", fd, fd_max);
+ int pfd = peer[fd];
+ if (pfd >= fd_max || pfd < 0)
+ bug("too big pfd %d %d\n", pfd, fd_max);
+ if (peer[pfd] != fd)
+ bug("not maching peer %d %d\n", fd, pfd);
+ if (type[fd] == type[pfd])
+ bug("Same type fd %d %d\n", fd, pfd);
+ if (type[fd] == FD_TYPE_CLOSED || type[pfd] == FD_TYPE_CLOSED)
+ bug("double close %d %d\n", fd, pfd);
+ return pfd;
+}
+
+#define MAX_MSG_DATA_LEN 4096
+
+void forward_for_vm(int cfd, int wl)
+{
+ struct cmsghdr *cmsg;
+ struct iovec iov;
+ struct msghdr msg = { };
+ char cmsg_buf[CMSG_LEN(sizeof(int) * VIRTWL_SEND_MAX_ALLOCS)];
+ struct virtwl_ioctl_txn *txn = alloca(sizeof(*txn) + MAX_MSG_DATA_LEN);
+
+ iov.iov_base = txn->data;
+ iov.iov_len = MAX_MSG_DATA_LEN;
+ msg.msg_iov = &iov;
+ msg.msg_iovlen = 1;
+ msg.msg_control = cmsg_buf;
+ msg.msg_controllen = sizeof(cmsg_buf);
+
+ ssize_t ret = recvmsg(cfd, &msg, 0);
+ if (ret <= 0)
+ return;
+ cmsg = CMSG_FIRSTHDR(&msg);
+ if (cmsg && CMSG_NXTHDR(&msg, cmsg))
+ bug("multiple cmsg");
+ if (cmsg && (cmsg->cmsg_level != SOL_SOCKET ||
+ cmsg->cmsg_type != SCM_RIGHTS))
+ bug("level:%d type:%d\n", cmsg->cmsg_level, cmsg->cmsg_type);
+
+ int fd_count = 0;
+ if (cmsg) {
+ fd_count = (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(int);
+ memcpy(txn->fds, CMSG_DATA(cmsg), fd_count * sizeof(int));
+ }
+ for (int i = fd_count; i < VIRTWL_SEND_MAX_ALLOCS; ++i) {
+ txn->fds[i] = -1;
+ }
+ txn->len = ret;
+ ret = ioctl(wl, VIRTWL_IOCTL_SEND, txn);
+ if (ret < 0) {
+ bug("send msg fail");
+ }
+ for (int i = 0; i < fd_count; ++i) {
+ close(txn->fds[i]);
+ }
+}
+
+void forward_for_wl(int wl, int cfd)
+{
+ int ret;
+
+ struct cmsghdr *cmsg;
+ struct iovec iov;
+ struct msghdr msg = { };
+ char cmsg_buf[CMSG_LEN(sizeof(int) * VIRTWL_SEND_MAX_ALLOCS)];
+ struct virtwl_ioctl_txn *txn = alloca(sizeof(*txn) + MAX_MSG_DATA_LEN);
+
+ ret = ioctl(wl, VIRTWL_IOCTL_RECV, txn);
+ if (ret < 0)
+ return;
+ size_t fd_count = 0;
+ for (; fd_count < VIRTWL_SEND_MAX_ALLOCS; ++fd_count) {
+ if (txn->fds[fd_count] < 0) {
+ break;
+ }
+ }
+
+ iov.iov_len = txn->len;
+ iov.iov_base = txn->data;
+ msg.msg_iov = &iov;
+ msg.msg_iovlen = 1;
+
+ if (fd_count > 0) {
+ cmsg = (struct cmsghdr *)&cmsg_buf;
+ cmsg->cmsg_level = SOL_SOCKET;
+ cmsg->cmsg_type = SCM_RIGHTS;
+ cmsg->cmsg_len = CMSG_LEN(fd_count * sizeof(int));
+ memcpy(CMSG_DATA(cmsg), txn->fds, fd_count * sizeof(int));
+ msg.msg_control = cmsg_buf;
+ msg.msg_controllen = cmsg->cmsg_len;
+ }
+
+ ret = sendmsg(cfd, &msg, MSG_NOSIGNAL);
+
+ if (ret != txn->len) {
+ debug("sendmsg failed %d %d %d\n", txn->len, ret, errno);
+ }
+
+ for (int i = 0; i < fd_count; ++i) {
+ close(txn->fds[i]);
+ }
+}
+
+void do_forward(char *type, int *peer, int fd, int fd_max)
+{
+ int pfd = get_peer(type, peer, fd, fd_max);
+ switch (type[fd]) {
+ case FD_TYPE_UNIX:
+ forward_for_vm(fd, pfd);
+ break;
+ case FD_TYPE_DEV:
+ forward_for_wl(fd, pfd);
+ break;
+ default:
+ bug("unreached here");
+ break;
+ }
+}
+
+static int open_wl_dev(int fd)
+{
+ struct virtwl_ioctl_new wl = {.type = VIRTWL_IOCTL_NEW_CTX };
+ int ret = ioctl(fd, VIRTWL_IOCTL_NEW, &wl);
+ if (ret < 0) {
+ debug("Can't new ctx %d\n", errno);
+ return -errno;
+ }
+ return wl.fd;
+}
+
+static void insert_peer_table(char **type, int **peer, int cfd, int dev,
+ int *fd_max)
+{
+ if (cfd >= (*fd_max) || dev >= (*fd_max)) {
+ (*fd_max) = (*fd_max) << 1;
+ debug("Increase fd table to %d\n", *fd_max);
+ *type = realloc(*type, sizeof(**type) * (*fd_max));
+ *peer = realloc(*peer, sizeof(**peer) * (*fd_max));
+ if (!*type || !*peer)
+ bug("can't malloc new memory for poll fds");
+ }
+ (*type)[cfd] = FD_TYPE_UNIX;
+ (*type)[dev] = FD_TYPE_DEV;
+ (*peer)[cfd] = dev;
+ (*peer)[dev] = cfd;
+}
+
+static void add_to_poll(struct pollfd **poll_fds, unsigned int *cnt,
+ unsigned int *cap, int fd)
+{
+ if (fd < 0)
+ bug("invalid fd %d\n", fd);
+ if (*cnt == *cap) {
+ *cap = (*cap) << 1;
+ debug("Increase poll cap to %d\n", *cap);
+ *poll_fds = realloc(*poll_fds, sizeof(**poll_fds) * (*cap));
+ if ((*poll_fds) == NULL)
+ bug("can't malloc new memory for poll fds");
+ }
+ struct pollfd *entry = (*poll_fds) + (*cnt);
+ (*cnt)++;
+ entry->fd = fd;
+ entry->events = POLLIN;
+ entry->revents = 0;
+}
+
+static void remove_from_poll(struct pollfd *poll_fds, unsigned int *cnt, int fd)
+{
+ unsigned int i;
+ if (fd < 0)
+ bug("invalid fd %d\n", fd);
+ for (i = 0; i < *cnt; ++i) {
+ if (poll_fds[i].fd != fd)
+ continue;
+ if (i == 0)
+ bug("important");
+ (*cnt)--;
+ if (i == (*cnt))
+ return;
+ poll_fds[i] = poll_fds[*cnt];
+ return;
+ }
+}
+
+static void close_all(char *type, int *peer, int fd, int fd_max)
+{
+ int pfd = get_peer(type, peer, fd, fd_max);
+ close(fd);
+ close(pfd);
+ type[fd] = type[pfd] = FD_TYPE_CLOSED;
+ peer[fd] = -1;
+ peer[pfd] = -1;
+}
+
+void wayland_proxy(int listen_fd)
+{
+ struct sockaddr_un listen_addr = { };
+ size_t addrlen = sizeof(listen_addr);
+ int wl;
+ int cfd;
+ unsigned int fd_cnt = 0;
+ unsigned int poll_cap = 8;
+ int fd_max = 1024;
+ int ret;
+ struct pollfd *poll_fds = calloc(poll_cap, sizeof(*poll_fds));
+ char *fd_type = calloc(fd_max, sizeof(*fd_type));
+ int *fd_peer = calloc(fd_max, sizeof(*fd_peer));
+ int wl_dev_fd = open("/dev/wl0", O_RDWR);
+ int i;
+
+ if (wl_dev_fd < 0)
+ bug("can't open wl device\n");
+
+ if (!poll_fds || !fd_type || !fd_peer)
+ bug("can't malloc poll fds");
+
+ add_to_poll(&poll_fds, &fd_cnt, &poll_cap, listen_fd);
+
+ for (;;) {
+ ret = poll(poll_fds, fd_cnt, -1);
+ if (ret < 0 && errno == EINTR)
+ continue;
+ if (ret < 0) {
+ bug("poll %d\n", errno);
+ }
+ if (poll_fds[0].revents & POLLIN) {
+ cfd = accept(listen_fd, (struct sockaddr *)&listen_addr,
+ (socklen_t *) & addrlen);
+ if (cfd < 0) {
+ bug("accept");
+ }
+ if ((wl = open_wl_dev(wl_dev_fd)) < 0) {
+ bug("connect to wl dev\n");
+ }
+ add_to_poll(&poll_fds, &fd_cnt, &poll_cap, cfd);
+ add_to_poll(&poll_fds, &fd_cnt, &poll_cap, wl);
+ insert_peer_table(&fd_type, &fd_peer, cfd, wl, &fd_max);
+ }
+ for (i = 1; i < fd_cnt; ++i) {
+ struct pollfd *pfd = poll_fds + i;
+ if (!(pfd->revents & POLLHUP))
+ continue;
+ close_all(fd_type, fd_peer, pfd->fd, fd_max);
+ }
+ i = 1;
+ while (i < fd_cnt) {
+ struct pollfd *pfd = poll_fds + i;
+ int fd = pfd->fd;
+ if (fd_type[fd] != FD_TYPE_CLOSED) {
+ i++;
+ continue;
+ }
+ remove_from_poll(poll_fds, &fd_cnt, fd);
+ }
+ for (i = 1; i < fd_cnt; ++i) {
+ struct pollfd *pfd = poll_fds + i;
+ if (!pfd->revents & POLLIN)
+ continue;
+ do_forward(fd_type, fd_peer, pfd->fd, fd_max);
+ }
+ }
+}