marvell/linux/drivers/md/dm-user.c

// SPDX-License-Identifier: GPL-2.0+
/*
 * Copyright (C) 2020 Google, Inc
 * Copyright (C) 2020 Palmer Dabbelt <palmerdabbelt@google.com>
 */

#include <linux/device-mapper.h>
#include <uapi/linux/dm-user.h>

#include <linux/bio.h>
#include <linux/init.h>
#include <linux/mempool.h>
#include <linux/miscdevice.h>
#include <linux/module.h>
#include <linux/poll.h>
#include <linux/uio.h>
#include <linux/wait.h>
#include <linux/workqueue.h>

#define DM_MSG_PREFIX "user"

#define MAX_OUTSTANDING_MESSAGES 128

static unsigned int daemon_timeout_msec = 4000;
module_param_named(dm_user_daemon_timeout_msec, daemon_timeout_msec, uint,
		   0644);
MODULE_PARM_DESC(dm_user_daemon_timeout_msec,
		 "IO Timeout in msec if daemon does not process");

/*
 * dm-user uses four structures:
 *
 *  - "struct target", the outermost structure, corresponds to a single device
 *    mapper target.  This contains the set of outstanding BIOs that have been
 *    provided by DM and are not actively being processed by the user, along
 *    with a misc device that userspace can open to communicate with the
 *    kernel.  Each time userspaces opens the misc device a new channel is
 *    created.
 *  - "struct channel", which represents a single active communication channel
 *    with userspace.  Userspace may choose arbitrary read/write sizes to use
 *    when processing messages, channels form these into logical accesses.
 *    When userspace responds to a full message the channel completes the BIO
 *    and obtains a new message to process from the target.
 *  - "struct message", which wraps a BIO with the additional information
 *    required by the kernel to sort out what to do with BIOs when they return
 *    from userspace.
 *  - "struct dm_user_message", which is the exact message format that
 *    userspace sees.
 *
 * The hot path contains three distinct operations:
 *
 *  - user_map(), which is provided a BIO from device mapper that is queued
 *    into the target.  This allocates and enqueues a new message.
 *  - dev_read(), which dequeues a message, copies it to userspace.
 *  - dev_write(), which looks up a message (keyed by sequence number) and
 *    completes the corresponding BIO.
 *
 * Lock ordering (outer to inner)
 *
 * 1) miscdevice's global lock.  This is held around dev_open, so it has to be
 *    the outermost lock.
 * 2) target->lock
 * 3) channel->lock
 */

struct message {
	/*
	 * Messages themselves do not need a lock, they're protected by either
	 * the target or channel's lock, depending on which can reference them
	 * directly.
	 */
	struct dm_user_message msg;
	struct bio *bio;
	size_t posn_to_user;
	size_t total_to_user;
	size_t posn_from_user;
	size_t total_from_user;

	struct list_head from_user;
	struct list_head to_user;

	/*
	 * These are written back from the user.  They live in the same spot in
	 * the message, but we need to either keep the old values around or
	 * call a bunch more BIO helpers.  These are only valid after write has
	 * adopted the message.
	 */
	u64 return_type;
	u64 return_flags;

	struct delayed_work work;
	bool delayed;
	struct target *t;
};

struct target {
	/*
	 * A target has a single lock, which protects everything in the target
	 * (but does not protect the channels associated with a target).
	 */
	struct mutex lock;

	/*
	 * There is only one point at which anything blocks: userspace blocks
	 * reading a new message, which is woken up by device mapper providing
	 * a new BIO to process (or tearing down the target).  The
	 * corresponding write side doesn't block, instead we treat userspace's
	 * response containing a message that has yet to be mapped as an
	 * invalid operation.
	 */
	struct wait_queue_head wq;

	/*
	 * Messages are delivered to userspace in order, but may be returned
	 * out of order.  This allows userspace to schedule IO if it wants to.
	 */
	mempool_t message_pool;
	u64 next_seq_to_map;
	u64 next_seq_to_user;
	struct list_head to_user;

	/*
	 * There is a misc device per target.  The name is selected by
	 * userspace (via a DM create ioctl argument), and each ends up in
	 * /dev/dm-user/.  It looks like a better way to do this may be to have
	 * a filesystem to manage these, but this was more expedient.  The
	 * current mechanism is functional, but does result in an arbitrary
	 * number of dynamically created misc devices.
	 */
	struct miscdevice miscdev;

	/*
	 * Device mapper's target destructor triggers tearing this all down,
	 * but we can't actually free until every channel associated with this
	 * target has been destroyed.  Channels each have a reference to their
	 * target, and there is an additional single reference that corresponds
	 * to both DM and the misc device (both of which are destroyed by DM).
	 *
	 * In the common case userspace will be asleep waiting for a new
	 * message when device mapper decides to destroy the target, which
	 * means no new messages will appear.  The destroyed flag triggers a
	 * wakeup, which will end up removing the reference.
	 */
	struct kref references;
	int dm_destroyed;
	bool daemon_terminated;
};

struct channel {
	struct target *target;

	/*
	 * A channel has a single lock, which prevents multiple reads (or
	 * multiple writes) from conflicting with each other.
	 */
	struct mutex lock;

	struct message *cur_to_user;
	struct message *cur_from_user;
	ssize_t to_user_error;
	ssize_t from_user_error;

	/*
	 * Once a message has been forwarded to userspace on a channel it must
	 * be responded to on the same channel.  This allows us to error out
	 * the messages that have not yet been responded to by a channel when
	 * that channel closes, which makes handling errors more reasonable for
	 * fault-tolerant userspace daemons.  It also happens to make avoiding
	 * shared locks between user_map() and dev_read() a lot easier.
	 *
	 * This does preclude a multi-threaded work stealing userspace
	 * implementation (or at least, force a degree of head-of-line blocking
	 * on the response path).
	 */
	struct list_head from_user;

	/*
	 * Responses from userspace can arrive in arbitrarily small chunks.
	 * We need some place to buffer one up until we can find the
	 * corresponding kernel-side message to continue processing, so instead
	 * of allocating them we just keep one off to the side here.  This can
	 * only ever be pointer to by from_user_cur, and will never have a BIO.
	 */
	struct message scratch_message_from_user;
};

static void message_kill(struct message *m, mempool_t *pool)
{
	m->bio->bi_status = BLK_STS_IOERR;
	bio_endio(m->bio);
	bio_put(m->bio);
	mempool_free(m, pool);
}

static inline bool is_user_space_thread_present(struct target *t)
{
	lockdep_assert_held(&t->lock);
	return (kref_read(&t->references) > 1);
}

static void process_delayed_work(struct work_struct *work)
{
	struct delayed_work *del_work = to_delayed_work(work);
	struct message *msg = container_of(del_work, struct message, work);

	struct target *t = msg->t;

	mutex_lock(&t->lock);

	/*
	 * There is a atleast one thread to process the IO.
	 */
	if (is_user_space_thread_present(t)) {
		mutex_unlock(&t->lock);
		return;
	}

	/*
	 * Terminate the IO with an error
	 */
	list_del(&msg->to_user);
	pr_err("I/O error: sector %llu: no user-space daemon for %s target\n",
	       msg->bio->bi_iter.bi_sector,
	       t->miscdev.name);
	message_kill(msg, &t->message_pool);
	mutex_unlock(&t->lock);
}

static void enqueue_delayed_work(struct message *m, bool is_delay)
{
	unsigned long delay = 0;

	m->delayed = true;
	INIT_DELAYED_WORK(&m->work, process_delayed_work);

	/*
	 * Snapuserd daemon is the user-space process
	 * which processes IO request from dm-user
	 * when OTA is applied. Per the current design,
	 * when a dm-user target is created, daemon
	 * attaches to target and starts processing
	 * the IO's. Daemon is terminated only when
	 * dm-user target is destroyed.
	 *
	 * If for some reason, daemon crashes or terminates early,
	 * without destroying the dm-user target; then
	 * there is no mechanism to restart the daemon
	 * and start processing the IO's from the same target.
	 * Theoretically, it is possible but that infrastructure
	 * doesn't exist in the android ecosystem.
	 *
	 * Thus, when the daemon terminates, there is no way the IO's
	 * issued on that target will be processed. Hence,
	 * we set the delay to 0 and fail the IO's immediately.
	 *
	 * On the other hand, when a new dm-user target is created,
	 * we wait for the daemon to get attached for the first time.
	 * This primarily happens when init first stage spins up
	 * the daemon. At this point, since the snapshot device is mounted
	 * of a root filesystem, dm-user target may receive IO request
	 * even though daemon is not fully launched. We don't want
	 * to fail those IO requests immediately. Thus, we queue these
	 * requests with a timeout so that daemon is ready to process
	 * those IO requests. Again, if the daemon fails to launch within
	 * the timeout period, then IO's will be failed.
	 */
	if (is_delay)
		delay = msecs_to_jiffies(daemon_timeout_msec);

	queue_delayed_work(system_wq, &m->work, delay);
}

static inline struct target *target_from_target(struct dm_target *target)
{
	WARN_ON(target->private == NULL);
	return target->private;
}

static inline struct target *target_from_miscdev(struct miscdevice *miscdev)
{
	return container_of(miscdev, struct target, miscdev);
}

static inline struct channel *channel_from_file(struct file *file)
{
	WARN_ON(file->private_data == NULL);
	return file->private_data;
}

static inline struct target *target_from_channel(struct channel *c)
{
	WARN_ON(c->target == NULL);
	return c->target;
}

static inline size_t bio_size(struct bio *bio)
{
	struct bio_vec bvec;
	struct bvec_iter iter;
	size_t out = 0;

	bio_for_each_segment (bvec, bio, iter)
		out += bio_iter_len(bio, iter);
	return out;
}

static inline size_t bio_bytes_needed_to_user(struct bio *bio)
{
	switch (bio_op(bio)) {
	case REQ_OP_WRITE:
		return sizeof(struct dm_user_message) + bio_size(bio);
	case REQ_OP_READ:
	case REQ_OP_FLUSH:
	case REQ_OP_DISCARD:
	case REQ_OP_SECURE_ERASE:
	case REQ_OP_WRITE_SAME:
	case REQ_OP_WRITE_ZEROES:
		return sizeof(struct dm_user_message);

	/*
	 * These ops are not passed to userspace under the assumption that
	 * they're not going to be particularly useful in that context.
	 */
	default:
		return -EOPNOTSUPP;
	}
}

static inline size_t bio_bytes_needed_from_user(struct bio *bio)
{
	switch (bio_op(bio)) {
	case REQ_OP_READ:
		return sizeof(struct dm_user_message) + bio_size(bio);
	case REQ_OP_WRITE:
	case REQ_OP_FLUSH:
	case REQ_OP_DISCARD:
	case REQ_OP_SECURE_ERASE:
	case REQ_OP_WRITE_SAME:
	case REQ_OP_WRITE_ZEROES:
		return sizeof(struct dm_user_message);

	/*
	 * These ops are not passed to userspace under the assumption that
	 * they're not going to be particularly useful in that context.
	 */
	default:
		return -EOPNOTSUPP;
	}
}

static inline long bio_type_to_user_type(struct bio *bio)
{
	switch (bio_op(bio)) {
	case REQ_OP_READ:
		return DM_USER_REQ_MAP_READ;
	case REQ_OP_WRITE:
		return DM_USER_REQ_MAP_WRITE;
	case REQ_OP_FLUSH:
		return DM_USER_REQ_MAP_FLUSH;
	case REQ_OP_DISCARD:
		return DM_USER_REQ_MAP_DISCARD;
	case REQ_OP_SECURE_ERASE:
		return DM_USER_REQ_MAP_SECURE_ERASE;
	case REQ_OP_WRITE_SAME:
		return DM_USER_REQ_MAP_WRITE_SAME;
	case REQ_OP_WRITE_ZEROES:
		return DM_USER_REQ_MAP_WRITE_ZEROES;

	/*
	 * These ops are not passed to userspace under the assumption that
	 * they're not going to be particularly useful in that context.
	 */
	default:
		return -EOPNOTSUPP;
	}
}

static inline long bio_flags_to_user_flags(struct bio *bio)
{
	u64 out = 0;
	typeof(bio->bi_opf) opf = bio->bi_opf & ~REQ_OP_MASK;

	if (opf & REQ_FAILFAST_DEV) {
		opf &= ~REQ_FAILFAST_DEV;
		out |= DM_USER_REQ_MAP_FLAG_FAILFAST_DEV;
	}

	if (opf & REQ_FAILFAST_TRANSPORT) {
		opf &= ~REQ_FAILFAST_TRANSPORT;
		out |= DM_USER_REQ_MAP_FLAG_FAILFAST_TRANSPORT;
	}

	if (opf & REQ_FAILFAST_DRIVER) {
		opf &= ~REQ_FAILFAST_DRIVER;
		out |= DM_USER_REQ_MAP_FLAG_FAILFAST_DRIVER;
	}

	if (opf & REQ_SYNC) {
		opf &= ~REQ_SYNC;
		out |= DM_USER_REQ_MAP_FLAG_SYNC;
	}

	if (opf & REQ_META) {
		opf &= ~REQ_META;
		out |= DM_USER_REQ_MAP_FLAG_META;
	}

	if (opf & REQ_PRIO) {
		opf &= ~REQ_PRIO;
		out |= DM_USER_REQ_MAP_FLAG_PRIO;
	}

	if (opf & REQ_NOMERGE) {
		opf &= ~REQ_NOMERGE;
		out |= DM_USER_REQ_MAP_FLAG_NOMERGE;
	}

	if (opf & REQ_IDLE) {
		opf &= ~REQ_IDLE;
		out |= DM_USER_REQ_MAP_FLAG_IDLE;
	}

	if (opf & REQ_INTEGRITY) {
		opf &= ~REQ_INTEGRITY;
		out |= DM_USER_REQ_MAP_FLAG_INTEGRITY;
	}

	if (opf & REQ_FUA) {
		opf &= ~REQ_FUA;
		out |= DM_USER_REQ_MAP_FLAG_FUA;
	}

	if (opf & REQ_PREFLUSH) {
		opf &= ~REQ_PREFLUSH;
		out |= DM_USER_REQ_MAP_FLAG_PREFLUSH;
	}

	if (opf & REQ_RAHEAD) {
		opf &= ~REQ_RAHEAD;
		out |= DM_USER_REQ_MAP_FLAG_RAHEAD;
	}

	if (opf & REQ_BACKGROUND) {
		opf &= ~REQ_BACKGROUND;
		out |= DM_USER_REQ_MAP_FLAG_BACKGROUND;
	}

	if (opf & REQ_NOWAIT) {
		opf &= ~REQ_NOWAIT;
		out |= DM_USER_REQ_MAP_FLAG_NOWAIT;
	}

	if (opf & REQ_NOUNMAP) {
		opf &= ~REQ_NOUNMAP;
		out |= DM_USER_REQ_MAP_FLAG_NOUNMAP;
	}

	if (unlikely(opf)) {
		pr_warn("unsupported BIO type %x\n", opf);
		return -EOPNOTSUPP;
	}
	WARN_ON(out < 0);
	return out;
}

/*
 * Not quite what's in blk-map.c, but instead what I thought the functions in
 * blk-map did.  This one seems more generally useful and I think we could
 * write the blk-map version in terms of this one.  The differences are that
 * this has a return value that counts, and blk-map uses the BIO _all iters.
 * Neither  advance the BIO iter but don't advance the IOV iter, which is a bit
 * odd here.
 */
static ssize_t bio_copy_from_iter(struct bio *bio, struct iov_iter *iter)
{
	struct bio_vec bvec;
	struct bvec_iter biter;
	ssize_t out = 0;

	bio_for_each_segment (bvec, bio, biter) {
		ssize_t ret;

		ret = copy_page_from_iter(bvec.bv_page, bvec.bv_offset,
					  bvec.bv_len, iter);

		/*
		 * FIXME: I thought that IOV copies had a mechanism for
		 * terminating early, if for example a signal came in while
		 * sleeping waiting for a page to be mapped, but I don't see
		 * where that would happen.
		 */
		WARN_ON(ret < 0);
		out += ret;

		if (!iov_iter_count(iter))
			break;

		if (ret < bvec.bv_len)
			return ret;
	}

	return out;
}

static ssize_t bio_copy_to_iter(struct bio *bio, struct iov_iter *iter)
{
	struct bio_vec bvec;
	struct bvec_iter biter;
	ssize_t out = 0;

	bio_for_each_segment (bvec, bio, biter) {
		ssize_t ret;

		ret = copy_page_to_iter(bvec.bv_page, bvec.bv_offset,
					bvec.bv_len, iter);

		/* as above */
		WARN_ON(ret < 0);
		out += ret;

		if (!iov_iter_count(iter))
			break;

		if (ret < bvec.bv_len)
			return ret;
	}

	return out;
}

static ssize_t msg_copy_to_iov(struct message *msg, struct iov_iter *to)
{
	ssize_t copied = 0;

	if (!iov_iter_count(to))
		return 0;

	if (msg->posn_to_user < sizeof(msg->msg)) {
		copied = copy_to_iter((char *)(&msg->msg) + msg->posn_to_user,
				      sizeof(msg->msg) - msg->posn_to_user, to);
	} else {
		copied = bio_copy_to_iter(msg->bio, to);
		if (copied > 0)
			bio_advance(msg->bio, copied);
	}

	if (copied < 0)
		return copied;

	msg->posn_to_user += copied;
	return copied;
}

static ssize_t msg_copy_from_iov(struct message *msg, struct iov_iter *from)
{
	ssize_t copied = 0;

	if (!iov_iter_count(from))
		return 0;

	if (msg->posn_from_user < sizeof(msg->msg)) {
		copied = copy_from_iter(
			(char *)(&msg->msg) + msg->posn_from_user,
			sizeof(msg->msg) - msg->posn_from_user, from);
	} else {
		copied = bio_copy_from_iter(msg->bio, from);
		if (copied > 0)
			bio_advance(msg->bio, copied);
	}

	if (copied < 0)
		return copied;

	msg->posn_from_user += copied;
	return copied;
}

static struct message *msg_get_map(struct target *t)
{
	struct message *m;

	lockdep_assert_held(&t->lock);

	m = mempool_alloc(&t->message_pool, GFP_NOIO);
	m->msg.seq = t->next_seq_to_map++;
	INIT_LIST_HEAD(&m->to_user);
	INIT_LIST_HEAD(&m->from_user);
	return m;
}

static struct message *msg_get_to_user(struct target *t)
{
	struct message *m;

	lockdep_assert_held(&t->lock);

	if (list_empty(&t->to_user))
		return NULL;

	m = list_first_entry(&t->to_user, struct message, to_user);

	list_del(&m->to_user);

	/*
	 * If the IO was queued to workqueue since there
	 * was no daemon to service the IO, then we
	 * will have to cancel the delayed work as the
	 * IO will be processed by this user-space thread.
	 *
	 * If the delayed work was already picked up for
	 * processing, then wait for it to complete. Note
	 * that the IO will not be terminated by the work
	 * queue thread.
	 */
	if (unlikely(m->delayed)) {
		mutex_unlock(&t->lock);
		cancel_delayed_work_sync(&m->work);
		mutex_lock(&t->lock);
	}
	return m;
}

static struct message *msg_get_from_user(struct channel *c, u64 seq)
{
	struct message *m;
	struct list_head *cur, *tmp;

	lockdep_assert_held(&c->lock);

	list_for_each_safe (cur, tmp, &c->from_user) {
		m = list_entry(cur, struct message, from_user);
		if (m->msg.seq == seq) {
			list_del(&m->from_user);
			return m;
		}
	}

	return NULL;
}

/*
 * Returns 0 when there is no work left to do.  This must be callable without
 * holding the target lock, as it is part of the waitqueue's check expression.
 * When called without the lock it may spuriously indicate there is remaining
 * work, but when called with the lock it must be accurate.
 */
static int target_poll(struct target *t)
{
	return !list_empty(&t->to_user) || t->dm_destroyed;
}

static void target_release(struct kref *ref)
{
	struct target *t = container_of(ref, struct target, references);
	struct list_head *cur, *tmp;

	/*
	 * There may be outstanding BIOs that have not yet been given to
	 * userspace.  At this point there's nothing we can do about them, as
	 * there are and will never be any channels.
	 */
	list_for_each_safe (cur, tmp, &t->to_user) {
		struct message *m = list_entry(cur, struct message, to_user);

		if (unlikely(m->delayed)) {
			bool ret;

			mutex_unlock(&t->lock);
			ret = cancel_delayed_work_sync(&m->work);
			mutex_lock(&t->lock);
			if (!ret)
				continue;
		}
		message_kill(m, &t->message_pool);
	}

	mempool_exit(&t->message_pool);
	mutex_unlock(&t->lock);
	mutex_destroy(&t->lock);
	kfree(t);
}

static void target_put(struct target *t)
{
	/*
	 * This both releases a reference to the target and the lock.  We leave
	 * it up to the caller to hold the lock, as they probably needed it for
	 * something else.
	 */
	lockdep_assert_held(&t->lock);

	if (!kref_put(&t->references, target_release)) {
		/*
		 * User-space thread is getting terminated.
		 * We need to scan the list for all those
		 * pending IO's which were not processed yet
		 * and put them back to work-queue for delayed
		 * processing.
		 */
		if (!is_user_space_thread_present(t)) {
			struct list_head *cur, *tmp;

			list_for_each_safe(cur, tmp, &t->to_user) {
				struct message *m = list_entry(cur,
							       struct message,
							       to_user);
				if (!m->delayed)
					enqueue_delayed_work(m, false);
			}
			/*
			 * Daemon attached to this target is terminated.
			 */
			t->daemon_terminated = true;
		}
		mutex_unlock(&t->lock);
	}
}

static struct channel *channel_alloc(struct target *t)
{
	struct channel *c;

	lockdep_assert_held(&t->lock);

	c = kzalloc(sizeof(*c), GFP_KERNEL);
	if (c == NULL)
		return NULL;

	kref_get(&t->references);
	c->target = t;
	c->cur_from_user = &c->scratch_message_from_user;
	mutex_init(&c->lock);
	INIT_LIST_HEAD(&c->from_user);
	return c;
}

static void channel_free(struct channel *c)
{
	struct list_head *cur, *tmp;

	lockdep_assert_held(&c->lock);

	/*
	 * There may be outstanding BIOs that have been given to userspace but
	 * have not yet been completed.  The channel has been shut down so
	 * there's no way to process the rest of those messages, so we just go
	 * ahead and error out the BIOs.  Hopefully whatever's on the other end
	 * can handle the errors.  One could imagine splitting the BIOs and
	 * completing as much as we got, but that seems like overkill here.
	 *
	 * Our only other options would be to let the BIO hang around (which
	 * seems way worse) or to resubmit it to userspace in the hope there's
	 * another channel.  I don't really like the idea of submitting a
	 * message twice.
	 */
	if (c->cur_to_user != NULL)
		message_kill(c->cur_to_user, &c->target->message_pool);
	if (c->cur_from_user != &c->scratch_message_from_user)
		message_kill(c->cur_from_user, &c->target->message_pool);
	list_for_each_safe (cur, tmp, &c->from_user)
		message_kill(list_entry(cur, struct message, from_user),
			     &c->target->message_pool);

	mutex_lock(&c->target->lock);
	target_put(c->target);
	mutex_unlock(&c->lock);
	mutex_destroy(&c->lock);
	kfree(c);
}

static int dev_open(struct inode *inode, struct file *file)
{
	struct channel *c;
	struct target *t;

	/*
	 * This is called by miscdev, which sets private_data to point to the
	 * struct miscdevice that was opened.  The rest of our file operations
	 * want to refer to the channel that's been opened, so we swap that
	 * pointer out with a fresh channel.
	 *
	 * This is called with the miscdev lock held, which is also held while
	 * registering/unregistering the miscdev.  The miscdev must be
	 * registered for this to get called, which means there must be an
	 * outstanding reference to the target, which means it cannot be freed
	 * out from under us despite us not holding a reference yet.
	 */
	t = container_of(file->private_data, struct target, miscdev);
	mutex_lock(&t->lock);
	file->private_data = c = channel_alloc(t);

	if (c == NULL) {
		mutex_unlock(&t->lock);
		return -ENOMEM;
	}

	mutex_unlock(&t->lock);
	return 0;
}

static ssize_t dev_read(struct kiocb *iocb, struct iov_iter *to)
{
	struct channel *c = channel_from_file(iocb->ki_filp);
	ssize_t total_processed = 0;
	ssize_t processed;

	mutex_lock(&c->lock);

	if (unlikely(c->to_user_error)) {
		total_processed = c->to_user_error;
		goto cleanup_unlock;
	}

	if (c->cur_to_user == NULL) {
		struct target *t = target_from_channel(c);

		mutex_lock(&t->lock);

		while (!target_poll(t)) {
			int e;

			mutex_unlock(&t->lock);
			mutex_unlock(&c->lock);
			e = wait_event_interruptible(t->wq, target_poll(t));
			mutex_lock(&c->lock);
			mutex_lock(&t->lock);

			if (unlikely(e != 0)) {
				/*
				 * We haven't processed any bytes in either the
				 * BIO or the IOV, so we can just terminate
				 * right now.  Elsewhere in the kernel handles
				 * restarting the syscall when appropriate.
				 */
				total_processed = e;
				mutex_unlock(&t->lock);
				goto cleanup_unlock;
			}
		}

		if (unlikely(t->dm_destroyed)) {
			/*
			 * DM has destroyed this target, so just lock
			 * the user out.  There's really nothing else
			 * we can do here.  Note that we don't actually
			 * tear any thing down until userspace has
			 * closed the FD, as there may still be
			 * outstanding BIOs.
			 *
			 * This is kind of a wacky error code to
			 * return.  My goal was really just to try and
			 * find something that wasn't likely to be
			 * returned by anything else in the miscdev
			 * path.  The message "block device required"
			 * seems like a somewhat reasonable thing to
			 * say when the target has disappeared out from
			 * under us, but "not block" isn't sensible.
			 */
			c->to_user_error = total_processed = -ENOTBLK;
			mutex_unlock(&t->lock);
			goto cleanup_unlock;
		}

		/*
		 * Ensures that accesses to the message data are not ordered
		 * before the remote accesses that produce that message data.
		 *
		 * This pairs with the barrier in user_map(), via the
		 * conditional within the while loop above. Also see the lack
		 * of barrier in user_dtr(), which is why this can be after the
		 * destroyed check.
		 */
		smp_rmb();

		c->cur_to_user = msg_get_to_user(t);
		WARN_ON(c->cur_to_user == NULL);
		mutex_unlock(&t->lock);
	}

	processed = msg_copy_to_iov(c->cur_to_user, to);
	total_processed += processed;

	WARN_ON(c->cur_to_user->posn_to_user > c->cur_to_user->total_to_user);
	if (c->cur_to_user->posn_to_user == c->cur_to_user->total_to_user) {
		struct message *m = c->cur_to_user;

		c->cur_to_user = NULL;
		list_add_tail(&m->from_user, &c->from_user);
	}

cleanup_unlock:
	mutex_unlock(&c->lock);
	return total_processed;
}

static ssize_t dev_write(struct kiocb *iocb, struct iov_iter *from)
{
	struct channel *c = channel_from_file(iocb->ki_filp);
	ssize_t total_processed = 0;
	ssize_t processed;

	mutex_lock(&c->lock);

	if (unlikely(c->from_user_error)) {
		total_processed = c->from_user_error;
		goto cleanup_unlock;
	}

	/*
	 * cur_from_user can never be NULL.  If there's no real message it must
	 * point to the scratch space.
	 */
	WARN_ON(c->cur_from_user == NULL);
	if (c->cur_from_user->posn_from_user < sizeof(struct dm_user_message)) {
		struct message *msg, *old;

		processed = msg_copy_from_iov(c->cur_from_user, from);
		if (processed <= 0) {
			pr_warn("msg_copy_from_iov() returned %zu\n",
				processed);
			c->from_user_error = -EINVAL;
			goto cleanup_unlock;
		}
		total_processed += processed;

		/*
		 * In the unlikely event the user has provided us a very short
		 * write, not even big enough to fill a message, just succeed.
		 * We'll eventually build up enough bytes to do something.
		 */
		if (unlikely(c->cur_from_user->posn_from_user <
			     sizeof(struct dm_user_message)))
			goto cleanup_unlock;

		old = c->cur_from_user;
		mutex_lock(&c->target->lock);
		msg = msg_get_from_user(c, c->cur_from_user->msg.seq);
		if (msg == NULL) {
			pr_info("user provided an invalid messag seq of %llx\n",
				old->msg.seq);
			mutex_unlock(&c->target->lock);
			c->from_user_error = -EINVAL;
			goto cleanup_unlock;
		}
		mutex_unlock(&c->target->lock);

		WARN_ON(old->posn_from_user != sizeof(struct dm_user_message));
		msg->posn_from_user = sizeof(struct dm_user_message);
		msg->return_type = old->msg.type;
		msg->return_flags = old->msg.flags;
		WARN_ON(msg->posn_from_user > msg->total_from_user);
		c->cur_from_user = msg;
		WARN_ON(old != &c->scratch_message_from_user);
	}

	/*
	 * Userspace can signal an error for single requests by overwriting the
	 * seq field.
	 */
	switch (c->cur_from_user->return_type) {
	case DM_USER_RESP_SUCCESS:
		c->cur_from_user->bio->bi_status = BLK_STS_OK;
		break;
	case DM_USER_RESP_ERROR:
	case DM_USER_RESP_UNSUPPORTED:
	default:
		c->cur_from_user->bio->bi_status = BLK_STS_IOERR;
		goto finish_bio;
	}

	/*
	 * The op was a success as far as userspace is concerned, so process
	 * whatever data may come along with it.  The user may provide the BIO
	 * data in multiple chunks, in which case we don't need to finish the
	 * BIO.
	 */
	processed = msg_copy_from_iov(c->cur_from_user, from);
	total_processed += processed;

	if (c->cur_from_user->posn_from_user <
	    c->cur_from_user->total_from_user)
		goto cleanup_unlock;

finish_bio:
	/*
	 * When we set up this message the BIO's size matched the
	 * message size, if that's not still the case then something
	 * has gone off the rails.
	 */
	WARN_ON(bio_size(c->cur_from_user->bio) != 0);
	bio_endio(c->cur_from_user->bio);
	bio_put(c->cur_from_user->bio);

	/*
	 * We don't actually need to take the target lock here, as all
	 * we're doing is freeing the message and mempools have their
	 * own lock.  Each channel has its ows scratch message.
	 */
	WARN_ON(c->cur_from_user == &c->scratch_message_from_user);
	mempool_free(c->cur_from_user, &c->target->message_pool);
	c->scratch_message_from_user.posn_from_user = 0;
	c->cur_from_user = &c->scratch_message_from_user;

cleanup_unlock:
	mutex_unlock(&c->lock);
	return total_processed;
}

static int dev_release(struct inode *inode, struct file *file)
{
	struct channel *c;

	c = channel_from_file(file);
	mutex_lock(&c->lock);
	channel_free(c);

	return 0;
}

static const struct file_operations file_operations = {
	.owner = THIS_MODULE,
	.open = dev_open,
	.llseek = no_llseek,
	.read_iter = dev_read,
	.write_iter = dev_write,
	.release = dev_release,
};

static int user_ctr(struct dm_target *ti, unsigned int argc, char **argv)
{
	struct target *t;
	int r;

	if (argc != 3) {
		ti->error = "Invalid argument count";
		r = -EINVAL;
		goto cleanup_none;
	}

	t = kzalloc(sizeof(*t), GFP_KERNEL);
	if (t == NULL) {
		r = -ENOMEM;
		goto cleanup_none;
	}
	ti->private = t;

	/* Enable more BIO types. */
	ti->num_discard_bios = 1;
	ti->discards_supported = true;
	ti->num_flush_bios = 1;
	ti->flush_supported = true;

	/*
	 * We begin with a single reference to the target, which is miscdev's
	 * reference.  This ensures that the target won't be freed
	 * until after the miscdev has been unregistered and all extant
	 * channels have been closed.
	 */
	kref_init(&t->references);

	t->daemon_terminated = false;
	mutex_init(&t->lock);
	init_waitqueue_head(&t->wq);
	INIT_LIST_HEAD(&t->to_user);
	mempool_init_kmalloc_pool(&t->message_pool, MAX_OUTSTANDING_MESSAGES,
				  sizeof(struct message));

	t->miscdev.minor = MISC_DYNAMIC_MINOR;
	t->miscdev.fops = &file_operations;
	t->miscdev.name = kasprintf(GFP_KERNEL, "dm-user/%s", argv[2]);
	if (t->miscdev.name == NULL) {
		r = -ENOMEM;
		goto cleanup_message_pool;
	}

	/*
	 * Once the miscdev is registered it can be opened and therefor
	 * concurrent references to the channel can happen.  Holding the target
	 * lock during misc_register() could deadlock.  If registration
	 * succeeds then we will not access the target again so we just stick a
	 * barrier here, which pairs with taking the target lock everywhere
	 * else the target is accessed.
	 *
	 * I forgot where we ended up on the RCpc/RCsc locks.  IIU RCsc locks
	 * would mean that we could take the target lock earlier and release it
	 * here instead of the memory barrier.  I'm not sure that's any better,
	 * though, and this isn't on a hot path so it probably doesn't matter
	 * either way.
	 */
	smp_mb();

	r = misc_register(&t->miscdev);
	if (r) {
		DMERR("Unable to register miscdev %s for dm-user",
		      t->miscdev.name);
		r = -ENOMEM;
		goto cleanup_misc_name;
	}

	return 0;

cleanup_misc_name:
	kfree(t->miscdev.name);
cleanup_message_pool:
	mempool_exit(&t->message_pool);
	kfree(t);
cleanup_none:
	return r;
}

static void user_dtr(struct dm_target *ti)
{
	struct target *t = target_from_target(ti);

	/*
	 * Removes the miscdev.  This must be called without the target lock
	 * held to avoid a possible deadlock because our open implementation is
	 * called holding the miscdev lock and must later take the target lock.
	 *
	 * There is no race here because only DM can register/unregister the
	 * miscdev, and DM ensures that doesn't happen twice.  The internal
	 * miscdev lock is sufficient to ensure there are no races between
	 * deregistering the miscdev and open.
	 */
	misc_deregister(&t->miscdev);

	/*
	 * We are now free to take the target's lock and drop our reference to
	 * the target.  There are almost certainly tasks sleeping in read on at
	 * least one of the channels associated with this target, this
	 * explicitly wakes them up and terminates the read.
	 */
	mutex_lock(&t->lock);
	/*
	 * No barrier here, as wait/wake ensures that the flag visibility is
	 * correct WRT the wake/sleep state of the target tasks.
	 */
	t->dm_destroyed = true;
	wake_up_all(&t->wq);
	target_put(t);
}

/*
 * Consumes a BIO from device mapper, queueing it up for userspace.
 */
static int user_map(struct dm_target *ti, struct bio *bio)
{
	struct target *t;
	struct message *entry;

	t = target_from_target(ti);
	/*
	 * FIXME
	 *
	 * This seems like a bad idea.  Specifically, here we're
	 * directly on the IO path when we take the target lock, which may also
	 * be taken from a user context.  The user context doesn't actively
	 * trigger anything that may sleep while holding the lock, but this
	 * still seems like a bad idea.
	 *
	 * The obvious way to fix this would be to use a proper queue, which
	 * would result in no shared locks between the direct IO path and user
	 * tasks.  I had a version that did this, but the head-of-line blocking
	 * from the circular buffer resulted in us needing a fairly large
	 * allocation in order to avoid situations in which the queue fills up
	 * and everything goes off the rails.
	 *
	 * I could jump through a some hoops to avoid a shared lock while still
	 * allowing for a large queue, but I'm not actually sure that allowing
	 * for very large queues is the right thing to do here.  Intuitively it
	 * seems better to keep the queues small in here (essentially sized to
	 * the user latency for performance reasons only) and rely on returning
	 * DM_MAPIO_REQUEUE regularly, as that would give the rest of the
	 * kernel more information.
	 *
	 * I'll spend some time trying to figure out what's going on with
	 * DM_MAPIO_REQUEUE, but if someone has a better idea of how to fix
	 * this I'm all ears.
	 */
	mutex_lock(&t->lock);

	/*
	 * FIXME
	 *
	 * The assumption here is that there's no benefit to returning
	 * DM_MAPIO_KILL as opposed to just erroring out the BIO, but I'm not
	 * sure that's actually true -- for example, I could imagine users
	 * expecting that submitted BIOs are unlikely to fail and therefor
	 * relying on submission failure to indicate an unsupported type.
	 *
	 * There's two ways I can think of to fix this:
	 *   - Add DM arguments that are parsed during the constructor that
	 *     allow various dm_target flags to be set that indicate the op
	 *     types supported by this target.  This may make sense for things
	 *     like discard, where DM can already transform the BIOs to a form
	 *     that's likely to be supported.
	 *   - Some sort of pre-filter that allows userspace to hook in here
	 *     and kill BIOs before marking them as submitted.  My guess would
	 *     be that a userspace round trip is a bad idea here, but a BPF
	 *     call seems resonable.
	 *
	 * My guess is that we'd likely want to do both.  The first one is easy
	 * and gives DM the proper info, so it seems better.  The BPF call
	 * seems overly complex for just this, but one could imagine wanting to
	 * sometimes return _MAPPED and a BPF filter would be the way to do
	 * that.
	 *
	 * For example, in Android we have an in-kernel DM device called
	 * "dm-bow" that takes advange of some portion of the space that has
	 * been discarded on a device to provide opportunistic block-level
	 * backups.  While one could imagine just implementing this entirely in
	 * userspace, that would come with an appreciable performance penalty.
	 * Instead one could keep a BPF program that forwards most accesses
	 * directly to the backing block device while informing a userspace
	 * daemon of any discarded space and on writes to blocks that are to be
	 * backed up.
	 */
	if (unlikely((bio_type_to_user_type(bio) < 0) ||
		     (bio_flags_to_user_flags(bio) < 0))) {
		mutex_unlock(&t->lock);
		return DM_MAPIO_KILL;
	}

	entry = msg_get_map(t);
	if (unlikely(entry == NULL)) {
		mutex_unlock(&t->lock);
		return DM_MAPIO_REQUEUE;
	}

	bio_get(bio);
	entry->msg.type = bio_type_to_user_type(bio);
	entry->msg.flags = bio_flags_to_user_flags(bio);
	entry->msg.sector = bio->bi_iter.bi_sector;
	entry->msg.len = bio_size(bio);
	entry->bio = bio;
	entry->posn_to_user = 0;
	entry->total_to_user = bio_bytes_needed_to_user(bio);
	entry->posn_from_user = 0;
	entry->total_from_user = bio_bytes_needed_from_user(bio);
	entry->delayed = false;
	entry->t = t;
	/* Pairs with the barrier in dev_read() */
	smp_wmb();
	list_add_tail(&entry->to_user, &t->to_user);

	/*
	 * If there is no daemon to process the IO's,
	 * queue these messages into a workqueue with
	 * a timeout.
	 */
	if (!is_user_space_thread_present(t))
		enqueue_delayed_work(entry, !t->daemon_terminated);

	wake_up_interruptible(&t->wq);
	mutex_unlock(&t->lock);
	return DM_MAPIO_SUBMITTED;
}

static struct target_type user_target = {
	.name = "user",
	.version = { 1, 0, 0 },
	.module = THIS_MODULE,
	.ctr = user_ctr,
	.dtr = user_dtr,
	.map = user_map,
};

static int __init dm_user_init(void)
{
	int r;

	r = dm_register_target(&user_target);
	if (r) {
		DMERR("register failed %d", r);
		goto error;
	}

	return 0;

error:
	return r;
}

static void __exit dm_user_exit(void)
{
	dm_unregister_target(&user_target);
}

module_init(dm_user_init);
module_exit(dm_user_exit);
MODULE_AUTHOR("Palmer Dabbelt <palmerdabbelt@google.com>");
MODULE_DESCRIPTION(DM_NAME " target returning blocks from userspace");
MODULE_LICENSE("GPL");