[Feature]add MT2731_MP2_MR2_SVN388 baseline version
Change-Id: Ief04314834b31e27effab435d3ca8ba33b499059
diff --git a/src/bsp/lk/kernel/debug.c b/src/bsp/lk/kernel/debug.c
new file mode 100644
index 0000000..2003807
--- /dev/null
+++ b/src/bsp/lk/kernel/debug.c
@@ -0,0 +1,244 @@
+/*
+ * Copyright (c) 2008-2014 Travis Geiselbrecht
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/**
+ * @defgroup debug Debug
+ * @{
+ */
+
+/**
+ * @file
+ * @brief Debug console functions.
+ */
+
+#include <debug.h>
+#include <stdio.h>
+#include <kernel/thread.h>
+#include <kernel/timer.h>
+#include <kernel/debug.h>
+#include <kernel/mp.h>
+#include <err.h>
+#include <platform.h>
+
+#if WITH_LIB_CONSOLE
+#include <lib/console.h>
+
+static int cmd_threads(int argc, const cmd_args *argv);
+static int cmd_threadstats(int argc, const cmd_args *argv);
+static int cmd_threadload(int argc, const cmd_args *argv);
+static int cmd_kevlog(int argc, const cmd_args *argv);
+
+STATIC_COMMAND_START
+#if LK_DEBUGLEVEL > 1
+STATIC_COMMAND_MASKED("threads", "list kernel threads", &cmd_threads, CMD_AVAIL_ALWAYS)
+#endif
+#if THREAD_STATS
+STATIC_COMMAND("threadstats", "thread level statistics", &cmd_threadstats)
+STATIC_COMMAND("threadload", "toggle thread load display", &cmd_threadload)
+#endif
+#if WITH_KERNEL_EVLOG
+STATIC_COMMAND_MASKED("kevlog", "dump kernel event log", &cmd_kevlog, CMD_AVAIL_ALWAYS)
+#endif
+STATIC_COMMAND_END(kernel);
+
+#if LK_DEBUGLEVEL > 1
+static int cmd_threads(int argc, const cmd_args *argv)
+{
+ printf("thread list:\n");
+ dump_all_threads();
+
+ return 0;
+}
+#endif
+
+#if THREAD_STATS
+static int cmd_threadstats(int argc, const cmd_args *argv)
+{
+ for (uint i = 0; i < SMP_MAX_CPUS; i++) {
+ if (!mp_is_cpu_active(i))
+ continue;
+
+ printf("thread stats (cpu %d):\n", i);
+ printf("\ttotal idle time: %lld\n", thread_stats[i].idle_time);
+ printf("\ttotal busy time: %lld\n", current_time_hires() - thread_stats[i].idle_time);
+ printf("\treschedules: %lu\n", thread_stats[i].reschedules);
+#if WITH_SMP
+ printf("\treschedule_ipis: %lu\n", thread_stats[i].reschedule_ipis);
+#endif
+ printf("\tcontext_switches: %lu\n", thread_stats[i].context_switches);
+ printf("\tpreempts: %lu\n", thread_stats[i].preempts);
+ printf("\tyields: %lu\n", thread_stats[i].yields);
+ printf("\tinterrupts: %lu\n", thread_stats[i].interrupts);
+ printf("\ttimer interrupts: %lu\n", thread_stats[i].timer_ints);
+ printf("\ttimers: %lu\n", thread_stats[i].timers);
+ }
+
+ return 0;
+}
+
+static enum handler_return threadload(struct timer *t, lk_time_t now, void *arg)
+{
+ static struct thread_stats old_stats[SMP_MAX_CPUS];
+ static lk_bigtime_t last_idle_time[SMP_MAX_CPUS];
+
+ for (uint i = 0; i < SMP_MAX_CPUS; i++) {
+ /* dont display time for inactiv cpus */
+ if (!mp_is_cpu_active(i))
+ continue;
+
+ lk_bigtime_t idle_time = thread_stats[i].idle_time;
+
+ /* if the cpu is currently idle, add the time since it went idle up until now to the idle counter */
+ bool is_idle = !!mp_is_cpu_idle(i);
+ if (is_idle) {
+ idle_time += current_time_hires() - thread_stats[i].last_idle_timestamp;
+ }
+
+ lk_bigtime_t delta_time = idle_time - last_idle_time[i];
+ lk_bigtime_t busy_time = 1000000ULL - (delta_time > 1000000ULL ? 1000000ULL : delta_time);
+ uint busypercent = (busy_time * 10000) / (1000000);
+
+ printf("cpu %u LOAD: "
+ "%u.%02u%%, "
+ "cs %lu, "
+ "pmpts %lu, "
+#if WITH_SMP
+ "rs_ipis %lu, "
+#endif
+ "ints %lu, "
+ "tmr ints %lu, "
+ "tmrs %lu\n",
+ i,
+ busypercent / 100, busypercent % 100,
+ thread_stats[i].context_switches - old_stats[i].context_switches,
+ thread_stats[i].preempts - old_stats[i].preempts,
+#if WITH_SMP
+ thread_stats[i].reschedule_ipis - old_stats[i].reschedule_ipis,
+#endif
+ thread_stats[i].interrupts - old_stats[i].interrupts,
+ thread_stats[i].timer_ints - old_stats[i].timer_ints,
+ thread_stats[i].timers - old_stats[i].timers);
+
+ old_stats[i] = thread_stats[i];
+ last_idle_time[i] = idle_time;
+ }
+
+ return INT_NO_RESCHEDULE;
+}
+
+static int cmd_threadload(int argc, const cmd_args *argv)
+{
+ static bool showthreadload = false;
+ static timer_t tltimer;
+
+ if (showthreadload == false) {
+ // start the display
+ timer_initialize(&tltimer);
+ timer_set_periodic(&tltimer, 1000, &threadload, NULL);
+ showthreadload = true;
+ } else {
+ timer_cancel(&tltimer);
+ showthreadload = false;
+ }
+
+ return 0;
+}
+
+#endif // THREAD_STATS
+
+#endif // WITH_LIB_CONSOLE
+
+#if WITH_KERNEL_EVLOG
+
+#include <lib/evlog.h>
+
+static evlog_t kernel_evlog;
+volatile bool kernel_evlog_enable;
+
+void kernel_evlog_init(void)
+{
+ evlog_init(&kernel_evlog, KERNEL_EVLOG_LEN, 4);
+
+ kernel_evlog_enable = true;
+}
+
+void kernel_evlog_add(uintptr_t id, uintptr_t arg0, uintptr_t arg1)
+{
+ if (kernel_evlog_enable) {
+ uint index = evlog_bump_head(&kernel_evlog);
+
+ kernel_evlog.items[index] = (uintptr_t)current_time_hires();
+ kernel_evlog.items[index+1] = (arch_curr_cpu_num() << 16) | id;
+ kernel_evlog.items[index+2] = arg0;
+ kernel_evlog.items[index+3] = arg1;
+ }
+}
+
+#if WITH_LIB_CONSOLE
+
+static void kevdump_cb(const uintptr_t *i)
+{
+ switch (i[1] & 0xffff) {
+ case KERNEL_EVLOG_CONTEXT_SWITCH:
+ printf("%lu.%lu: context switch from %p to %p\n", i[0], i[1] >> 16, (void *)i[2], (void *)i[3]);
+ break;
+ case KERNEL_EVLOG_PREEMPT:
+ printf("%lu.%lu: preempt on thread %p\n", i[0], i[1] >> 16, (void *)i[2]);
+ break;
+ case KERNEL_EVLOG_TIMER_TICK:
+ printf("%lu.%lu: timer tick\n", i[0], i[1] >> 16);
+ break;
+ case KERNEL_EVLOG_TIMER_CALL:
+ printf("%lu.%lu: timer call %p, arg %p\n", i[0], i[1] >> 16, (void *)i[2], (void *)i[3]);
+ break;
+ case KERNEL_EVLOG_IRQ_ENTER:
+ printf("%lu.%lu: irq entry %lu\n", i[0], i[1] >> 16, i[2]);
+ break;
+ case KERNEL_EVLOG_IRQ_EXIT:
+ printf("%lu.%lu: irq exit %lu\n", i[0], i[1] >> 16, i[2]);
+ break;
+ default:
+ printf("%lu: unknown id 0x%lx 0x%lx 0x%lx\n", i[0], i[1], i[2], i[3]);
+ }
+}
+
+void kernel_evlog_dump(void)
+{
+ kernel_evlog_enable = false;
+ evlog_dump(&kernel_evlog, &kevdump_cb);
+ kernel_evlog_enable = true;
+}
+
+static int cmd_kevlog(int argc, const cmd_args *argv)
+{
+ printf("kernel event log:\n");
+ kernel_evlog_dump();
+
+ return NO_ERROR;
+}
+
+#endif
+
+#endif // WITH_KERNEL_EVLOG
+
+// vim: set noexpandtab:
diff --git a/src/bsp/lk/kernel/event.c b/src/bsp/lk/kernel/event.c
new file mode 100644
index 0000000..b289b75
--- /dev/null
+++ b/src/bsp/lk/kernel/event.c
@@ -0,0 +1,189 @@
+/*
+ * Copyright (c) 2008-2014 Travis Geiselbrecht
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/**
+ * @file
+ * @brief Event wait and signal functions for threads.
+ * @defgroup event Events
+ *
+ * An event is a subclass of a wait queue.
+ *
+ * Threads wait for events, with optional timeouts.
+ *
+ * Events are "signaled", releasing waiting threads to continue.
+ * Signals may be one-shot signals (EVENT_FLAG_AUTOUNSIGNAL), in which
+ * case one signal releases only one thread, at which point it is
+ * automatically cleared. Otherwise, signals release all waiting threads
+ * to continue immediately until the signal is manually cleared with
+ * event_unsignal().
+ *
+ * @{
+ */
+
+#include <kernel/event.h>
+#include <debug.h>
+#include <assert.h>
+#include <err.h>
+#include <kernel/thread.h>
+
+/**
+ * @brief Initialize an event object
+ *
+ * @param e Event object to initialize
+ * @param initial Initial value for "signaled" state
+ * @param flags 0 or EVENT_FLAG_AUTOUNSIGNAL
+ */
+void event_init(event_t *e, bool initial, uint flags)
+{
+ *e = (event_t)EVENT_INITIAL_VALUE(*e, initial, flags);
+}
+
+/**
+ * @brief Destroy an event object.
+ *
+ * Event's resources are freed and it may no longer be
+ * used until event_init() is called again. Any threads
+ * still waiting on the event will be resumed.
+ *
+ * @param e Event object to initialize
+ */
+void event_destroy(event_t *e)
+{
+ DEBUG_ASSERT(e->magic == EVENT_MAGIC);
+
+ THREAD_LOCK(state);
+
+ e->magic = 0;
+ e->signalled = false;
+ e->flags = 0;
+ wait_queue_destroy(&e->wait, true);
+
+ THREAD_UNLOCK(state);
+}
+
+/**
+ * @brief Wait for event to be signaled
+ *
+ * If the event has already been signaled, this function
+ * returns immediately. Otherwise, the current thread
+ * goes to sleep until the event object is signaled,
+ * the timeout is reached, or the event object is destroyed
+ * by another thread.
+ *
+ * @param e Event object
+ * @param timeout Timeout value, in ms
+ *
+ * @return 0 on success, ERR_TIMED_OUT on timeout,
+ * other values on other errors.
+ */
+status_t event_wait_timeout(event_t *e, lk_time_t timeout)
+{
+ status_t ret = NO_ERROR;
+
+ DEBUG_ASSERT(e->magic == EVENT_MAGIC);
+
+ THREAD_LOCK(state);
+
+ if (e->signalled) {
+ /* signalled, we're going to fall through */
+ if (e->flags & EVENT_FLAG_AUTOUNSIGNAL) {
+ /* autounsignal flag lets one thread fall through before unsignalling */
+ e->signalled = false;
+ }
+ } else {
+ /* unsignalled, block here */
+ ret = wait_queue_block(&e->wait, timeout);
+ }
+
+ THREAD_UNLOCK(state);
+
+ return ret;
+}
+
+/**
+ * @brief Signal an event
+ *
+ * Signals an event. If EVENT_FLAG_AUTOUNSIGNAL is set in the event
+ * object's flags, only one waiting thread is allowed to proceed. Otherwise,
+ * all waiting threads are allowed to proceed until such time as
+ * event_unsignal() is called.
+ *
+ * @param e Event object
+ * @param reschedule If true, waiting thread(s) are executed immediately,
+ * and the current thread resumes only after the
+ * waiting threads have been satisfied. If false,
+ * waiting threads are placed at the end of the run
+ * queue.
+ *
+ * @return Returns NO_ERROR on success.
+ */
+status_t event_signal(event_t *e, bool reschedule)
+{
+ DEBUG_ASSERT(e->magic == EVENT_MAGIC);
+
+ THREAD_LOCK(state);
+
+ if (!e->signalled) {
+ if (e->flags & EVENT_FLAG_AUTOUNSIGNAL) {
+ /* try to release one thread and leave unsignalled if successful */
+ if (wait_queue_wake_one(&e->wait, reschedule, NO_ERROR) <= 0) {
+ /*
+ * if we didn't actually find a thread to wake up, go to
+ * signalled state and let the next call to event_wait
+ * unsignal the event.
+ */
+ e->signalled = true;
+ }
+ } else {
+ /* release all threads and remain signalled */
+ e->signalled = true;
+ wait_queue_wake_all(&e->wait, reschedule, NO_ERROR);
+ }
+ }
+
+ THREAD_UNLOCK(state);
+
+ return NO_ERROR;
+}
+
+/**
+ * @brief Clear the "signaled" property of an event
+ *
+ * Used mainly for event objects without the EVENT_FLAG_AUTOUNSIGNAL
+ * flag. Once this function is called, threads that call event_wait()
+ * functions will once again need to wait until the event object
+ * is signaled.
+ *
+ * @param e Event object
+ *
+ * @return Returns NO_ERROR on success.
+ */
+status_t event_unsignal(event_t *e)
+{
+ DEBUG_ASSERT(e->magic == EVENT_MAGIC);
+
+ e->signalled = false;
+
+ return NO_ERROR;
+}
+
diff --git a/src/bsp/lk/kernel/init.c b/src/bsp/lk/kernel/init.c
new file mode 100644
index 0000000..8bc4f3f
--- /dev/null
+++ b/src/bsp/lk/kernel/init.c
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2013 Travis Geiselbrecht
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include <compiler.h>
+#include <debug.h>
+#include <kernel/debug.h>
+#include <kernel/thread.h>
+#include <kernel/timer.h>
+#include <kernel/mp.h>
+#include <kernel/port.h>
+
+void kernel_init(void)
+{
+ // if enabled, configure the kernel's event log
+ kernel_evlog_init();
+
+ // initialize the threading system
+ dprintf(SPEW, "initializing mp\n");
+ mp_init();
+
+ // initialize the threading system
+ dprintf(SPEW, "initializing threads\n");
+ thread_init();
+
+ // initialize kernel timers
+ dprintf(SPEW, "initializing timers\n");
+ timer_init();
+
+ // initialize ports
+ dprintf(SPEW, "initializing ports\n");
+ port_init();
+}
+
diff --git a/src/bsp/lk/kernel/mp.c b/src/bsp/lk/kernel/mp.c
new file mode 100644
index 0000000..3b800c2
--- /dev/null
+++ b/src/bsp/lk/kernel/mp.c
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2014 Travis Geiselbrecht
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <kernel/mp.h>
+
+#include <stdlib.h>
+#include <debug.h>
+#include <assert.h>
+#include <trace.h>
+#include <arch/mp.h>
+#include <kernel/spinlock.h>
+
+#define LOCAL_TRACE 0
+
+#if WITH_SMP
+/* a global state structure, aligned on cpu cache line to minimize aliasing */
+struct mp_state mp __CPU_ALIGN;
+
+void mp_init(void)
+{
+}
+
+void mp_reschedule(mp_cpu_mask_t target, uint flags)
+{
+ uint local_cpu = arch_curr_cpu_num();
+
+ LTRACEF("local %d, target 0x%x\n", local_cpu, target);
+
+ /* mask out cpus that are not active and the local cpu */
+ target &= mp.active_cpus;
+
+ /* mask out cpus that are currently running realtime code */
+ if ((flags & MP_RESCHEDULE_FLAG_REALTIME) == 0) {
+ target &= ~mp.realtime_cpus;
+ }
+ target &= ~(1U << local_cpu);
+
+ LTRACEF("local %d, post mask target now 0x%x\n", local_cpu, target);
+
+ arch_mp_send_ipi(target, MP_IPI_RESCHEDULE);
+}
+
+void mp_set_curr_cpu_active(bool active)
+{
+ if (active)
+ atomic_or((volatile int *)&mp.active_cpus, 1U << arch_curr_cpu_num());
+ else
+ atomic_and((volatile int *)&mp.active_cpus, ~(1U << arch_curr_cpu_num()));
+}
+
+enum handler_return mp_mbx_reschedule_irq(void)
+{
+ uint cpu = arch_curr_cpu_num();
+
+ LTRACEF("cpu %u\n", cpu);
+
+ THREAD_STATS_INC(reschedule_ipis);
+
+ return (mp.active_cpus & (1U << cpu)) ? INT_RESCHEDULE : INT_NO_RESCHEDULE;
+}
+#endif
+
+// vim: set noexpandtab:
+
diff --git a/src/bsp/lk/kernel/mutex.c b/src/bsp/lk/kernel/mutex.c
new file mode 100644
index 0000000..69b7f5d
--- /dev/null
+++ b/src/bsp/lk/kernel/mutex.c
@@ -0,0 +1,145 @@
+/*
+ * Copyright (c) 2008-2014 Travis Geiselbrecht
+ * Copyright (c) 2012-2012 Shantanu Gupta
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/**
+ * @file
+ * @brief Mutex functions
+ *
+ * @defgroup mutex Mutex
+ * @{
+ */
+
+#include <kernel/mutex.h>
+#include <debug.h>
+#include <assert.h>
+#include <err.h>
+#include <kernel/thread.h>
+
+/**
+ * @brief Initialize a mutex_t
+ */
+void mutex_init(mutex_t *m)
+{
+ *m = (mutex_t)MUTEX_INITIAL_VALUE(*m);
+}
+
+/**
+ * @brief Destroy a mutex_t
+ *
+ * This function frees any resources that were allocated
+ * in mutex_init(). The mutex_t object itself is not freed.
+ */
+void mutex_destroy(mutex_t *m)
+{
+ DEBUG_ASSERT(m->magic == MUTEX_MAGIC);
+
+#if LK_DEBUGLEVEL > 0
+ if (unlikely(m->holder != 0 && get_current_thread() != m->holder))
+ panic("mutex_destroy: thread %p (%s) tried to release mutex %p it doesn't own. owned by %p (%s)\n",
+ get_current_thread(), get_current_thread()->name, m, m->holder, m->holder->name);
+#endif
+
+ THREAD_LOCK(state);
+ m->magic = 0;
+ m->count = 0;
+ wait_queue_destroy(&m->wait, true);
+ THREAD_UNLOCK(state);
+}
+
+/**
+ * @brief Mutex wait with timeout
+ *
+ * This function waits up to \a timeout ms for the mutex to become available.
+ * Timeout may be zero, in which case this function returns immediately if
+ * the mutex is not free.
+ *
+ * @return NO_ERROR on success, ERR_TIMED_OUT on timeout,
+ * other values on error
+ */
+status_t mutex_acquire_timeout(mutex_t *m, lk_time_t timeout)
+{
+ DEBUG_ASSERT(m->magic == MUTEX_MAGIC);
+
+#if LK_DEBUGLEVEL > 0
+ if (unlikely(get_current_thread() == m->holder))
+ panic("mutex_acquire_timeout: thread %p (%s) tried to acquire mutex %p it already owns.\n",
+ get_current_thread(), get_current_thread()->name, m);
+#endif
+
+ THREAD_LOCK(state);
+
+ status_t ret = NO_ERROR;
+ if (unlikely(++m->count > 1)) {
+ ret = wait_queue_block(&m->wait, timeout);
+ if (unlikely(ret < NO_ERROR)) {
+ /* if the acquisition timed out, back out the acquire and exit */
+ if (likely(ret == ERR_TIMED_OUT)) {
+ /*
+ * race: the mutex may have been destroyed after the timeout,
+ * but before we got scheduled again which makes messing with the
+ * count variable dangerous.
+ */
+ m->count--;
+ }
+ /* if there was a general error, it may have been destroyed out from
+ * underneath us, so just exit (which is really an invalid state anyway)
+ */
+ goto err;
+ }
+ }
+
+ m->holder = get_current_thread();
+
+err:
+ THREAD_UNLOCK(state);
+ return ret;
+}
+
+/**
+ * @brief Release mutex
+ */
+status_t mutex_release(mutex_t *m)
+{
+ DEBUG_ASSERT(m->magic == MUTEX_MAGIC);
+
+#if LK_DEBUGLEVEL > 0
+ if (unlikely(get_current_thread() != m->holder)) {
+ panic("mutex_release: thread %p (%s) tried to release mutex %p it doesn't own. owned by %p (%s)\n",
+ get_current_thread(), get_current_thread()->name, m, m->holder, m->holder ? m->holder->name : "none");
+ }
+#endif
+
+ THREAD_LOCK(state);
+
+ m->holder = 0;
+
+ if (unlikely(--m->count >= 1)) {
+ /* release a thread */
+ wait_queue_wake_one(&m->wait, true, NO_ERROR);
+ }
+
+ THREAD_UNLOCK(state);
+ return NO_ERROR;
+}
+
diff --git a/src/bsp/lk/kernel/novm/novm.c b/src/bsp/lk/kernel/novm/novm.c
new file mode 100644
index 0000000..5e6b22e
--- /dev/null
+++ b/src/bsp/lk/kernel/novm/novm.c
@@ -0,0 +1,329 @@
+/*
+ * Copyright (c) 2015 Google, Inc. All rights reserved
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "kernel/novm.h"
+
+#include <err.h>
+#include <assert.h>
+#include <trace.h>
+#include <stdlib.h>
+#include <string.h>
+#include <lk/init.h>
+#include <kernel/mutex.h>
+
+#define LOCAL_TRACE 0
+
+struct novm_arena {
+ mutex_t lock;
+ const char *name;
+ size_t pages;
+ char *map;
+ char *base;
+ size_t size;
+
+ // We divide the memory up into pages. If there is memory we can use before
+ // the first aligned page address, then we record it here and the heap will use
+ // it.
+#define MINIMUM_USEFUL_UNALIGNED_SIZE 64
+ void* unaligned_area;
+ size_t unaligned_size;
+};
+
+
+/* not a static vm, not using the kernel vm */
+extern int _end;
+extern int _end_of_ram;
+
+#define MEM_START ((uintptr_t)&_end)
+#define MEM_SIZE ((MEMBASE + MEMSIZE) - MEM_START)
+#define DEFAULT_MAP_SIZE (MEMSIZE >> PAGE_SIZE_SHIFT)
+
+/* a static list of arenas */
+#ifndef NOVM_MAX_ARENAS
+#define NOVM_MAX_ARENAS 1
+#endif
+struct novm_arena arena[NOVM_MAX_ARENAS];
+
+void *novm_alloc_unaligned(size_t *size_return)
+{
+ /* only do the unaligned thing in the first arena */
+ if (arena[0].unaligned_area != NULL) {
+ *size_return = arena[0].unaligned_size;
+ void *result = arena[0].unaligned_area;
+ arena[0].unaligned_area = NULL;
+ arena[0].unaligned_size = 0;
+ return result;
+ }
+ *size_return = PAGE_SIZE;
+ return novm_alloc_pages(1, NOVM_ARENA_ANY);
+}
+
+static bool in_arena(struct novm_arena *n, void* p)
+{
+ if (n->size == 0)
+ return false;
+
+ char *ptr = (char *)p;
+ char *base = n->base;
+ return ptr >= base && ptr < base + n->size;
+}
+
+static void novm_init_helper(struct novm_arena* n, const char *name,
+ uintptr_t arena_start, uintptr_t arena_size,
+ char* default_map, size_t default_map_size)
+{
+ uintptr_t start = ROUNDUP(arena_start, PAGE_SIZE);
+ uintptr_t size = ROUNDDOWN(arena_start + arena_size, PAGE_SIZE) - start;
+
+ mutex_init(&n->lock);
+
+ size_t map_size = size >> PAGE_SIZE_SHIFT;
+ char* map = default_map;
+ if (map == NULL || default_map_size < map_size) {
+ // allocate the map out of the arena itself
+ map = (char *)arena_start;
+
+ // Grab enough map for 16Mbyte of arena each time around the loop.
+ while (start - arena_start < map_size) {
+ start += PAGE_SIZE;
+ size -= PAGE_SIZE;
+ map_size--;
+ }
+
+ if ((char *)start - (map + ROUNDUP(map_size, 4)) >= MINIMUM_USEFUL_UNALIGNED_SIZE) {
+ n->unaligned_area = map + ROUNDUP(map_size, 4);
+ n->unaligned_size = (char *)start - (map + ROUNDUP(map_size, 4));
+ }
+ } else if (start - arena_start >= MINIMUM_USEFUL_UNALIGNED_SIZE) {
+ n->unaligned_area = (char *)arena_start;
+ n->unaligned_size = start - arena_start;
+ }
+ n->name = name;
+ n->map = map;
+ memset(n->map, 0, map_size);
+ n->pages = map_size;
+ n->base = (char *)start;
+ n->size = size;
+}
+
+void novm_add_arena(const char *name, uintptr_t arena_start, uintptr_t arena_size)
+{
+ for (uint i = 0; i < NOVM_MAX_ARENAS; i++) {
+ if (arena[i].pages == 0) {
+ novm_init_helper(&arena[i], name, arena_start, arena_size, NULL, 0);
+ return;
+ }
+ }
+ panic("novm_add_arena: too many arenas added, bump NOVM_MAX_ARENAS!\n");
+}
+
+static void novm_init(uint level)
+{
+ static char mem_allocation_map[DEFAULT_MAP_SIZE];
+ novm_init_helper(&arena[0], "main", MEM_START, MEM_SIZE, mem_allocation_map, DEFAULT_MAP_SIZE);
+}
+
+LK_INIT_HOOK(novm, &novm_init, LK_INIT_LEVEL_PLATFORM_EARLY - 1);
+
+void *novm_alloc_helper(struct novm_arena *n, size_t pages)
+{
+ if (pages == 0 || pages > n->pages)
+ return NULL;
+
+ mutex_acquire(&n->lock);
+ for (size_t i = 0; i <= n->pages - pages; i++) {
+ bool found = true;
+ for (size_t j = 0; j < pages; j++) {
+ if (n->map[i + j] != 0) {
+ i += j;
+ found = false;
+ break;
+ }
+ }
+ if (found) {
+ memset(n->map + i, 1, pages);
+ mutex_release(&n->lock);
+ return n->base + (i << PAGE_SIZE_SHIFT);
+ }
+ }
+ mutex_release(&n->lock);
+
+ return NULL;
+}
+
+void* novm_alloc_pages(size_t pages, uint32_t arena_bitmap)
+{
+ LTRACEF("pages %zu\n", pages);
+
+ /* allocate from any arena */
+ for (uint i = 0; i < NOVM_MAX_ARENAS; i++) {
+ if (arena_bitmap & (1U << i)) {
+ void *result = novm_alloc_helper(&arena[i], pages);
+ if (result)
+ return result;
+ }
+ }
+
+ return NULL;
+}
+
+void novm_free_pages(void* address, size_t pages)
+{
+ LTRACEF("address %p, pages %zu\n", address, pages);
+
+ struct novm_arena *n = NULL;
+ for (uint i = 0; i < NOVM_MAX_ARENAS; i++) {
+ if (in_arena(&arena[i], address)) {
+ n = &arena[i];
+ break;
+ }
+ }
+ if (!n)
+ return;
+
+ DEBUG_ASSERT(in_arena(n, address));
+
+ size_t index = ((char *)address - (char*)(n->base)) >> PAGE_SIZE_SHIFT;
+ char *map = n->map;
+
+ mutex_acquire(&n->lock);
+ for (size_t i = 0; i < pages; i++) map[index + i] = 0;
+ mutex_release(&n->lock);
+}
+
+status_t novm_alloc_specific_pages(void *address, size_t pages)
+{
+ LTRACEF("address %p, pages %zu\n", address, pages);
+
+ struct novm_arena *n = NULL;
+ for (uint i = 0; i < NOVM_MAX_ARENAS; i++) {
+ if (in_arena(&arena[i], address)) {
+ n = &arena[i];
+ break;
+ }
+ }
+ if (!n)
+ return ERR_NOT_FOUND;
+
+ size_t index = ((char *)address - (char*)(n->base)) >> PAGE_SIZE_SHIFT;
+ char *map = n->map;
+
+ status_t err = NO_ERROR;
+
+ mutex_acquire(&n->lock);
+ for (size_t i = 0; i < pages; i++) {
+ if (map[index + i] != 0) {
+ err = ERR_NO_MEMORY;
+ break;
+ }
+ map[index + i] = 1;
+ }
+ mutex_release(&n->lock);
+
+ return err;
+}
+
+
+#if LK_DEBUGLEVEL > 1
+#if WITH_LIB_CONSOLE
+
+#include <lib/console.h>
+
+static int cmd_novm(int argc, const cmd_args *argv);
+static void novm_dump(void);
+
+STATIC_COMMAND_START
+STATIC_COMMAND("novm", "page allocator (for devices without VM support) debug commands", &cmd_novm)
+STATIC_COMMAND_END(novm);
+
+static int cmd_novm(int argc, const cmd_args *argv)
+{
+ if (argc < 2) {
+notenoughargs:
+ printf("not enough arguments\n");
+usage:
+ printf("usage:\n");
+ printf("\t%s info\n", argv[0].str);
+ printf("\t%s alloc <numberofpages> [arena bitmap]\n", argv[0].str);
+ printf("\t%s free <address> [numberofpages]\n", argv[0].str);
+ return -1;
+ }
+
+ if (strcmp(argv[1].str, "info") == 0) {
+ novm_dump();
+ } else if (strcmp(argv[1].str, "alloc") == 0) {
+ if (argc < 3) goto notenoughargs;
+
+ uint32_t arena_bitmap = (argc >= 4) ? argv[3].u : NOVM_ARENA_ANY;
+ void *ptr = novm_alloc_pages(argv[2].u, arena_bitmap);
+ printf("novm_alloc_pages returns %p\n", ptr);
+ } else if (strcmp(argv[1].str, "free") == 0) {
+ if (argc < 3) goto notenoughargs;
+ size_t pages = (argc >= 4) ? argv[3].u : 1;
+ novm_free_pages(argv[2].p, pages);
+ printf("novm_free_pages: %zd pages at %p\n", pages, argv[2].p);
+ } else {
+ printf("unrecognized command\n");
+ goto usage;
+ }
+
+ return 0;
+}
+
+static void novm_dump_arena(struct novm_arena *n)
+{
+ if (n->pages == 0) {
+ return;
+ }
+
+ mutex_acquire(&n->lock);
+ printf("name '%s', %d pages, each %zdk (%zdk in all)\n", n->name, n->pages, PAGE_SIZE >> 10, (PAGE_SIZE * n->pages) >> 10);
+ printf(" range: %p-%p\n", (void *)n->base, (char *)n->base + n->size);
+ printf(" unaligned range: %p-%p\n", n->unaligned_area, n->unaligned_area + n->unaligned_size);
+ unsigned i;
+ size_t in_use = 0;
+ for (i = 0; i < n->pages; i++) if (n->map[i] != 0) in_use++;
+ printf(" %zd/%zd in use\n", in_use, n->pages);
+#define MAX_PRINT 1024u
+ for (i = 0; i < MAX_PRINT && i < n->pages; i++) {
+ if ((i & 63) == 0) printf(" ");
+ printf("%c", n->map[i] ? '*' : '.');
+ if ((i & 63) == 63) printf("\n");
+ }
+ if (i == MAX_PRINT && n->pages > MAX_PRINT) {
+ printf(" etc., %zd more pages.", n->pages - MAX_PRINT);
+ }
+ printf("\n");
+ mutex_release(&n->lock);
+}
+
+static void novm_dump(void)
+{
+ for (uint i = 0; i < NOVM_MAX_ARENAS; i++) {
+ novm_dump_arena(&arena[i]);
+ }
+}
+
+#endif
+#endif
+
diff --git a/src/bsp/lk/kernel/novm/rules.mk b/src/bsp/lk/kernel/novm/rules.mk
new file mode 100644
index 0000000..73e4db8
--- /dev/null
+++ b/src/bsp/lk/kernel/novm/rules.mk
@@ -0,0 +1,8 @@
+LOCAL_DIR := $(GET_LOCAL_DIR)
+
+MODULE := $(LOCAL_DIR)
+
+MODULE_SRCS += \
+ $(LOCAL_DIR)/novm.c
+
+include make/module.mk
diff --git a/src/bsp/lk/kernel/port.c b/src/bsp/lk/kernel/port.c
new file mode 100644
index 0000000..6ed67b9
--- /dev/null
+++ b/src/bsp/lk/kernel/port.c
@@ -0,0 +1,519 @@
+/*
+ * Copyright (c) 2015 Carlos Pizano-Uribe cpu@chromium.org
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/**
+ * @file
+ * @brief Port object functions
+ * @defgroup event Events
+ *
+ */
+
+#include <debug.h>
+#include <list.h>
+#include <malloc.h>
+#include <string.h>
+#include <pow2.h>
+#include <err.h>
+#include <kernel/thread.h>
+#include <kernel/port.h>
+
+// write ports can be in two states, open and closed, which have a
+// different magic number.
+
+#define WRITEPORT_MAGIC_W 'prtw'
+#define WRITEPORT_MAGIC_X 'prtx'
+
+#define READPORT_MAGIC 'prtr'
+#define PORTGROUP_MAGIC 'prtg'
+
+#define PORT_BUFF_SIZE 8
+#define PORT_BUFF_SIZE_BIG 64
+
+#define RESCHEDULE_POLICY 1
+
+#define MAX_PORT_GROUP_COUNT 256
+
+typedef struct {
+ uint log2;
+ uint avail;
+ uint head;
+ uint tail;
+ port_packet_t packet[1];
+} port_buf_t;
+
+typedef struct {
+ int magic;
+ struct list_node node;
+ port_buf_t* buf;
+ struct list_node rp_list;
+ port_mode_t mode;
+ char name[PORT_NAME_LEN];
+} write_port_t;
+
+typedef struct {
+ int magic;
+ wait_queue_t wait;
+ struct list_node rp_list;
+} port_group_t;
+
+typedef struct {
+ int magic;
+ struct list_node w_node;
+ struct list_node g_node;
+ port_buf_t* buf;
+ void* ctx;
+ wait_queue_t wait;
+ write_port_t* wport;
+ port_group_t* gport;
+} read_port_t;
+
+
+static struct list_node write_port_list;
+
+
+static port_buf_t* make_buf(uint pk_count)
+{
+ uint size = sizeof(port_buf_t) + ((pk_count - 1) * sizeof(port_packet_t));
+ port_buf_t* buf = (port_buf_t*) malloc(size);
+ if (!buf)
+ return NULL;
+ buf->log2 = log2_uint(pk_count);
+ buf->head = buf->tail = 0;
+ buf->avail = pk_count;
+ return buf;
+}
+
+static status_t buf_write(port_buf_t* buf, const port_packet_t* packets, size_t count)
+{
+ if (buf->avail < count)
+ return ERR_NOT_ENOUGH_BUFFER;
+
+ for (size_t ix = 0; ix != count; ix++) {
+ buf->packet[buf->tail] = packets[ix];
+ buf->tail = modpow2(++buf->tail, buf->log2);
+ }
+ buf->avail -= count;
+ return NO_ERROR;
+}
+
+static status_t buf_read(port_buf_t* buf, port_result_t* pr)
+{
+ if (buf->avail == valpow2(buf->log2))
+ return ERR_NO_MSG;
+ pr->packet = buf->packet[buf->head];
+ buf->head = modpow2(++buf->head, buf->log2);
+ ++buf->avail;
+ return NO_ERROR;
+}
+
+// must be called before any use of ports.
+void port_init(void)
+{
+ list_initialize(&write_port_list);
+}
+
+status_t port_create(const char* name, port_mode_t mode, port_t* port)
+{
+ if (!name || !port)
+ return ERR_INVALID_ARGS;
+
+ // only unicast ports can have a large buffer.
+ if (mode & PORT_MODE_BROADCAST) {
+ if (mode & PORT_MODE_BIG_BUFFER)
+ return ERR_INVALID_ARGS;
+ }
+
+ if (strlen(name) >= PORT_NAME_LEN)
+ return ERR_INVALID_ARGS;
+
+ // lookup for existing port, return that if found.
+ write_port_t* wp = NULL;
+ THREAD_LOCK(state1);
+ list_for_every_entry(&write_port_list, wp, write_port_t, node) {
+ if (strcmp(wp->name, name) == 0) {
+ // can't return closed ports.
+ if (wp->magic == WRITEPORT_MAGIC_X)
+ wp = NULL;
+ THREAD_UNLOCK(state1);
+ if (wp) {
+ *port = (void*) wp;
+ return ERR_ALREADY_EXISTS;
+ } else {
+ return ERR_BUSY;
+ }
+ }
+ }
+ THREAD_UNLOCK(state1);
+
+ // not found, create the write port and the circular buffer.
+ wp = calloc(1, sizeof(write_port_t));
+ if (!wp)
+ return ERR_NO_MEMORY;
+
+ wp->magic = WRITEPORT_MAGIC_W;
+ wp->mode = mode;
+ strlcpy(wp->name, name, sizeof(wp->name));
+ list_initialize(&wp->rp_list);
+
+ uint size = (mode & PORT_MODE_BIG_BUFFER) ? PORT_BUFF_SIZE_BIG : PORT_BUFF_SIZE;
+ wp->buf = make_buf(size);
+ if (!wp->buf) {
+ free(wp);
+ return ERR_NO_MEMORY;
+ }
+
+ // todo: race condtion! a port with the same name could have been created
+ // by another thread at is point.
+ THREAD_LOCK(state2);
+ list_add_tail(&write_port_list, &wp->node);
+ THREAD_UNLOCK(state2);
+
+ *port = (void*)wp;
+ return NO_ERROR;
+}
+
+status_t port_open(const char* name, void* ctx, port_t* port)
+{
+ if (!name || !port)
+ return ERR_INVALID_ARGS;
+
+ // assume success; create the read port and buffer now.
+ read_port_t* rp = calloc(1, sizeof(read_port_t));
+ if (!rp)
+ return ERR_NO_MEMORY;
+
+ rp->magic = READPORT_MAGIC;
+ wait_queue_init(&rp->wait);
+ rp->ctx = ctx;
+
+ // |buf| might not be needed, but we always allocate outside the lock.
+ // this buffer is only needed for broadcast ports, but we don't know
+ // that here.
+ port_buf_t* buf = make_buf(PORT_BUFF_SIZE);
+ if (!buf) {
+ free(rp);
+ return ERR_NO_MEMORY;
+ }
+
+ // find the named write port and associate it with read port.
+ status_t rc = ERR_NOT_FOUND;
+
+ THREAD_LOCK(state);
+ write_port_t* wp = NULL;
+ list_for_every_entry(&write_port_list, wp, write_port_t, node) {
+ if (strcmp(wp->name, name) == 0) {
+ // found; add read port to write port list.
+ rp->wport = wp;
+ if (wp->buf) {
+ // this is the first read port; transfer the circular buffer.
+ list_add_tail(&wp->rp_list, &rp->w_node);
+ rp->buf = wp->buf;
+ wp->buf = NULL;
+ rc = NO_ERROR;
+ } else if (buf) {
+ // not first read port.
+ if (wp->mode & PORT_MODE_UNICAST) {
+ // cannot add a second listener.
+ rc = ERR_NOT_ALLOWED;
+ break;
+ }
+ // use the new (small) circular buffer.
+ list_add_tail(&wp->rp_list, &rp->w_node);
+ rp->buf = buf;
+ buf = NULL;
+ rc = NO_ERROR;
+ } else {
+ // |buf| allocation failed and the buffer was needed.
+ rc = ERR_NO_MEMORY;
+ }
+ break;
+ }
+ }
+ THREAD_UNLOCK(state);
+
+ if (buf)
+ free(buf);
+
+ if (rc == NO_ERROR) {
+ *port = (void*)rp;
+ } else {
+ free(rp);
+ }
+ return rc;
+}
+
+status_t port_group(port_t* ports, size_t count, port_t* group)
+{
+ if (count > MAX_PORT_GROUP_COUNT)
+ return ERR_TOO_BIG;
+
+ if (!ports || !group)
+ return ERR_INVALID_ARGS;
+
+ // assume success; create port group now.
+ port_group_t* pg = calloc(1, sizeof(port_group_t));
+ if (!pg)
+ return ERR_NO_MEMORY;
+
+ pg->magic = PORTGROUP_MAGIC;
+ wait_queue_init(&pg->wait);
+ list_initialize(&pg->rp_list);
+
+ status_t rc = NO_ERROR;
+
+ THREAD_LOCK(state);
+ for (size_t ix = 0; ix != count; ix++) {
+ read_port_t* rp = (read_port_t*)ports[ix];
+ if ((rp->magic != READPORT_MAGIC) || rp->gport) {
+ // wrong type of port, or port already part of a group,
+ // in any case, undo the changes to the previous read ports.
+ for (size_t jx = 0; jx != ix; jx++) {
+ ((read_port_t*)ports[jx])->gport = NULL;
+ }
+ rc = ERR_BAD_HANDLE;
+ break;
+ }
+ // link port group and read port.
+ rp->gport = pg;
+ list_add_tail(&pg->rp_list, &rp->g_node);
+ }
+ THREAD_UNLOCK(state);
+
+ if (rc == NO_ERROR) {
+ *group = (port_t*)pg;
+ } else {
+ free(pg);
+ }
+ return rc;
+}
+
+status_t port_write(port_t port, const port_packet_t* pk, size_t count)
+{
+ if (!port || !pk)
+ return ERR_INVALID_ARGS;
+
+ write_port_t* wp = (write_port_t*)port;
+ THREAD_LOCK(state);
+ if (wp->magic != WRITEPORT_MAGIC_W) {
+ // wrong port type.
+ THREAD_UNLOCK(state);
+ return ERR_BAD_HANDLE;
+ }
+
+ status_t status = NO_ERROR;
+ int awake_count = 0;
+
+ if (wp->buf) {
+ // there are no read ports, just write to the buffer.
+ status = buf_write(wp->buf, pk, count);
+ } else {
+ // there are read ports. for each, write and attempt to wake a thread
+ // from the port group or from the read port itself.
+ read_port_t* rp;
+ list_for_every_entry(&wp->rp_list, rp, read_port_t, w_node) {
+ if (buf_write(rp->buf, pk, count) < 0) {
+ // buffer full.
+ status = ERR_PARTIAL_WRITE;
+ continue;
+ }
+
+ int awaken = 0;
+ if (rp->gport) {
+ awaken = wait_queue_wake_one(&rp->gport->wait, false, NO_ERROR);
+ }
+ if (!awaken) {
+ awaken = wait_queue_wake_one(&rp->wait, false, NO_ERROR);
+ }
+
+ awake_count += awaken;
+ }
+ }
+
+ THREAD_UNLOCK(state);
+
+#if RESCHEDULE_POLICY
+ if (awake_count)
+ thread_yield();
+#endif
+
+ return status;
+}
+
+static inline status_t read_no_lock(read_port_t* rp, lk_time_t timeout, port_result_t* result)
+{
+ status_t status = buf_read(rp->buf, result);
+ result->ctx = rp->ctx;
+
+ if (status != ERR_NO_MSG)
+ return status;
+
+ // early return allows compiler to elide the rest for the group read case.
+ if (!timeout)
+ return ERR_TIMED_OUT;
+
+ status_t wr = wait_queue_block(&rp->wait, timeout);
+ if (wr != NO_ERROR)
+ return wr;
+ // recursive tail call is usually optimized away with a goto.
+ return read_no_lock(rp, timeout, result);
+}
+
+status_t port_read(port_t port, lk_time_t timeout, port_result_t* result)
+{
+ if (!port || !result)
+ return ERR_INVALID_ARGS;
+
+ status_t rc = ERR_GENERIC;
+ read_port_t* rp = (read_port_t*)port;
+
+ THREAD_LOCK(state);
+ if (rp->magic == READPORT_MAGIC) {
+ // dealing with a single port.
+ rc = read_no_lock(rp, timeout, result);
+ } else if (rp->magic == PORTGROUP_MAGIC) {
+ // dealing with a port group.
+ port_group_t* pg = (port_group_t*)port;
+ do {
+ // read each port with no timeout.
+ // todo: this order is fixed, probably a bad thing.
+ list_for_every_entry(&pg->rp_list, rp, read_port_t, g_node) {
+ rc = read_no_lock(rp, 0, result);
+ if (rc != ERR_TIMED_OUT)
+ goto read_exit;
+ }
+ // no data, block on the group waitqueue.
+ rc = wait_queue_block(&pg->wait, timeout);
+ } while (rc == NO_ERROR);
+ } else {
+ // wrong port type.
+ rc = ERR_BAD_HANDLE;
+ }
+
+read_exit:
+ THREAD_UNLOCK(state);
+ return rc;
+}
+
+status_t port_destroy(port_t port)
+{
+ if (!port)
+ return ERR_INVALID_ARGS;
+
+ write_port_t* wp = (write_port_t*) port;
+ port_buf_t* buf = NULL;
+
+ THREAD_LOCK(state);
+ if (wp->magic != WRITEPORT_MAGIC_X) {
+ // wrong port type.
+ THREAD_UNLOCK(state);
+ return ERR_BAD_HANDLE;
+ }
+ // remove self from global named ports list.
+ list_delete(&wp->node);
+
+ if (wp->buf) {
+ // we have no readers.
+ buf = wp->buf;
+ } else {
+ // for each reader:
+ read_port_t* rp;
+ list_for_every_entry(&wp->rp_list, rp, read_port_t, w_node) {
+ // wake the read and group ports.
+ wait_queue_wake_all(&rp->wait, false, ERR_CANCELLED);
+ if (rp->gport) {
+ wait_queue_wake_all(&rp->gport->wait, false, ERR_CANCELLED);
+ }
+ // remove self from reader ports.
+ rp->wport = NULL;
+ }
+ }
+
+ wp->magic = 0;
+ THREAD_UNLOCK(state);
+
+ free(buf);
+ free(wp);
+ return NO_ERROR;
+}
+
+status_t port_close(port_t port)
+{
+ if (!port)
+ return ERR_INVALID_ARGS;
+
+ read_port_t* rp = (read_port_t*) port;
+ port_buf_t* buf = NULL;
+
+ THREAD_LOCK(state);
+ if (rp->magic == READPORT_MAGIC) {
+ // dealing with a read port.
+ if (rp->wport) {
+ // remove self from write port list and reassign the bufer if last.
+ list_delete(&rp->w_node);
+ if (list_is_empty(&rp->wport->rp_list)) {
+ rp->wport->buf = rp->buf;
+ rp->buf = NULL;
+ } else {
+ buf = rp->buf;
+ }
+ }
+ if (rp->gport) {
+ // remove self from port group list.
+ list_delete(&rp->g_node);
+ }
+ // wake up waiters, the return code is ERR_OBJECT_DESTROYED.
+ wait_queue_destroy(&rp->wait, true);
+ rp->magic = 0;
+
+ } else if (rp->magic == PORTGROUP_MAGIC) {
+ // dealing with a port group.
+ port_group_t* pg = (port_group_t*) port;
+ // wake up waiters.
+ wait_queue_destroy(&pg->wait, true);
+ // remove self from reader ports.
+ rp = NULL;
+ list_for_every_entry(&pg->rp_list, rp, read_port_t, g_node) {
+ rp->gport = NULL;
+ }
+ pg->magic = 0;
+
+ } else if (rp->magic == WRITEPORT_MAGIC_W) {
+ // dealing with a write port.
+ write_port_t* wp = (write_port_t*) port;
+ // mark it as closed. Now it can be read but not written to.
+ wp->magic = WRITEPORT_MAGIC_X;
+ THREAD_UNLOCK(state);
+ return NO_ERROR;
+
+ } else {
+ THREAD_UNLOCK(state);
+ return ERR_BAD_HANDLE;
+ }
+
+ THREAD_UNLOCK(state);
+
+ free(buf);
+ free(port);
+ return NO_ERROR;
+}
+
diff --git a/src/bsp/lk/kernel/rules.mk b/src/bsp/lk/kernel/rules.mk
new file mode 100644
index 0000000..fc7cd16
--- /dev/null
+++ b/src/bsp/lk/kernel/rules.mk
@@ -0,0 +1,27 @@
+LOCAL_DIR := $(GET_LOCAL_DIR)
+
+MODULE := $(LOCAL_DIR)
+
+MODULE_DEPS := \
+ lib/libc \
+ lib/debug \
+ lib/heap
+
+MODULE_SRCS := \
+ $(LOCAL_DIR)/debug.c \
+ $(LOCAL_DIR)/event.c \
+ $(LOCAL_DIR)/init.c \
+ $(LOCAL_DIR)/mutex.c \
+ $(LOCAL_DIR)/thread.c \
+ $(LOCAL_DIR)/timer.c \
+ $(LOCAL_DIR)/semaphore.c \
+ $(LOCAL_DIR)/mp.c \
+ $(LOCAL_DIR)/port.c
+
+ifeq ($(WITH_KERNEL_VM),1)
+MODULE_DEPS += kernel/vm
+else
+MODULE_DEPS += kernel/novm
+endif
+
+include make/module.mk
diff --git a/src/bsp/lk/kernel/semaphore.c b/src/bsp/lk/kernel/semaphore.c
new file mode 100644
index 0000000..219360a
--- /dev/null
+++ b/src/bsp/lk/kernel/semaphore.c
@@ -0,0 +1,101 @@
+/* semaphore.c
+ *
+ * Copyright 2012 Christopher Anderson <chris@nullcode.org>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <debug.h>
+#include <err.h>
+#include <kernel/semaphore.h>
+#include <kernel/thread.h>
+
+void sem_init(semaphore_t *sem, unsigned int value)
+{
+ *sem = (semaphore_t)SEMAPHORE_INITIAL_VALUE(*sem, value);
+}
+
+void sem_destroy(semaphore_t *sem)
+{
+ THREAD_LOCK(state);
+ sem->count = 0;
+ wait_queue_destroy(&sem->wait, true);
+ THREAD_UNLOCK(state);
+}
+
+int sem_post(semaphore_t *sem, bool resched)
+{
+ int ret = 0;
+
+ THREAD_LOCK(state);
+
+ /*
+ * If the count is or was negative then a thread is waiting for a resource, otherwise
+ * it's safe to just increase the count available with no downsides
+ */
+ if (unlikely(++sem->count <= 0))
+ ret = wait_queue_wake_one(&sem->wait, resched, NO_ERROR);
+
+ THREAD_UNLOCK(state);
+
+ return ret;
+}
+
+status_t sem_wait(semaphore_t *sem)
+{
+ status_t ret = NO_ERROR;
+ THREAD_LOCK(state);
+
+ /*
+ * If there are no resources available then we need to
+ * sit in the wait queue until sem_post adds some.
+ */
+ if (unlikely(--sem->count < 0))
+ ret = wait_queue_block(&sem->wait, INFINITE_TIME);
+
+ THREAD_UNLOCK(state);
+ return ret;
+}
+
+status_t sem_trywait(semaphore_t *sem)
+{
+ status_t ret = NO_ERROR;
+ THREAD_LOCK(state);
+
+ if (unlikely(sem->count <= 0))
+ ret = ERR_NOT_READY;
+ else
+ sem->count--;
+
+ THREAD_UNLOCK(state);
+ return ret;
+}
+
+status_t sem_timedwait(semaphore_t *sem, lk_time_t timeout)
+{
+ status_t ret = NO_ERROR;
+ THREAD_LOCK(state);
+
+ if (unlikely(--sem->count < 0)) {
+ ret = wait_queue_block(&sem->wait, timeout);
+ if (ret < NO_ERROR) {
+ if (ret == ERR_TIMED_OUT) {
+ sem->count++;
+ }
+ }
+ }
+
+ THREAD_UNLOCK(state);
+ return ret;
+}
+
+/* vim: set noexpandtab: */
diff --git a/src/bsp/lk/kernel/thread.c b/src/bsp/lk/kernel/thread.c
new file mode 100644
index 0000000..688c1d4
--- /dev/null
+++ b/src/bsp/lk/kernel/thread.c
@@ -0,0 +1,1267 @@
+/*
+ * Copyright (c) 2008-2015 Travis Geiselbrecht
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/**
+ * @file
+ * @brief Kernel threading
+ *
+ * This file is the core kernel threading interface.
+ *
+ * @defgroup thread Threads
+ * @{
+ */
+#include <debug.h>
+#include <assert.h>
+#include <list.h>
+#include <malloc.h>
+#include <string.h>
+#include <printf.h>
+#include <err.h>
+#include <lib/dpc.h>
+#include <kernel/thread.h>
+#include <kernel/timer.h>
+#include <kernel/debug.h>
+#include <kernel/mp.h>
+#include <platform.h>
+#include <target.h>
+#include <lib/heap.h>
+
+#if THREAD_STATS
+struct thread_stats thread_stats[SMP_MAX_CPUS];
+#endif
+
+#define STACK_DEBUG_BYTE (0x99)
+#define STACK_DEBUG_WORD (0x99999999)
+
+#define DEBUG_THREAD_CONTEXT_SWITCH 0
+
+/* global thread list */
+static struct list_node thread_list;
+
+/* master thread spinlock */
+spin_lock_t thread_lock = SPIN_LOCK_INITIAL_VALUE;
+
+/* the run queue */
+static struct list_node run_queue[NUM_PRIORITIES];
+static uint32_t run_queue_bitmap;
+
+/* make sure the bitmap is large enough to cover our number of priorities */
+STATIC_ASSERT(NUM_PRIORITIES <= sizeof(run_queue_bitmap) * 8);
+
+/* the idle thread(s) (statically allocated) */
+#if WITH_SMP
+static thread_t _idle_threads[SMP_MAX_CPUS];
+#define idle_thread(cpu) (&_idle_threads[cpu])
+#else
+static thread_t _idle_thread;
+#define idle_thread(cpu) (&_idle_thread)
+#endif
+
+/* local routines */
+static void thread_resched(void);
+static void idle_thread_routine(void) __NO_RETURN;
+
+#if PLATFORM_HAS_DYNAMIC_TIMER
+/* preemption timer */
+static timer_t preempt_timer[SMP_MAX_CPUS];
+#endif
+
+/* run queue manipulation */
+static void insert_in_run_queue_head(thread_t *t)
+{
+ DEBUG_ASSERT(t->magic == THREAD_MAGIC);
+ DEBUG_ASSERT(t->state == THREAD_READY);
+ DEBUG_ASSERT(!list_in_list(&t->queue_node));
+ DEBUG_ASSERT(arch_ints_disabled());
+ DEBUG_ASSERT(spin_lock_held(&thread_lock));
+
+ list_add_head(&run_queue[t->priority], &t->queue_node);
+ run_queue_bitmap |= (1<<t->priority);
+}
+
+static void insert_in_run_queue_tail(thread_t *t)
+{
+ DEBUG_ASSERT(t->magic == THREAD_MAGIC);
+ DEBUG_ASSERT(t->state == THREAD_READY);
+ DEBUG_ASSERT(!list_in_list(&t->queue_node));
+ DEBUG_ASSERT(arch_ints_disabled());
+ DEBUG_ASSERT(spin_lock_held(&thread_lock));
+
+ list_add_tail(&run_queue[t->priority], &t->queue_node);
+ run_queue_bitmap |= (1<<t->priority);
+}
+
+static void init_thread_struct(thread_t *t, const char *name)
+{
+ memset(t, 0, sizeof(thread_t));
+ t->magic = THREAD_MAGIC;
+ thread_set_pinned_cpu(t, -1);
+ strlcpy(t->name, name, sizeof(t->name));
+}
+
+/**
+ * @brief Create a new thread
+ *
+ * This function creates a new thread. The thread is initially suspended, so you
+ * need to call thread_resume() to execute it.
+ *
+ * @param name Name of thread
+ * @param entry Entry point of thread
+ * @param arg Arbitrary argument passed to entry()
+ * @param priority Execution priority for the thread.
+ * @param stack_size Stack size for the thread.
+ *
+ * Thread priority is an integer from 0 (lowest) to 31 (highest). Some standard
+ * prioritys are defined in <kernel/thread.h>:
+ *
+ * HIGHEST_PRIORITY
+ * DPC_PRIORITY
+ * HIGH_PRIORITY
+ * DEFAULT_PRIORITY
+ * LOW_PRIORITY
+ * IDLE_PRIORITY
+ * LOWEST_PRIORITY
+ *
+ * Stack size is typically set to DEFAULT_STACK_SIZE
+ *
+ * @return Pointer to thread object, or NULL on failure.
+ */
+thread_t *thread_create_etc(thread_t *t, const char *name, thread_start_routine entry, void *arg, int priority, void *stack, size_t stack_size)
+{
+ unsigned int flags = 0;
+
+ if (!t) {
+ t = malloc(sizeof(thread_t));
+ if (!t)
+ return NULL;
+ flags |= THREAD_FLAG_FREE_STRUCT;
+ }
+
+ init_thread_struct(t, name);
+
+ t->entry = entry;
+ t->arg = arg;
+ t->priority = priority;
+ t->state = THREAD_SUSPENDED;
+ t->blocking_wait_queue = NULL;
+ t->wait_queue_block_ret = NO_ERROR;
+ thread_set_curr_cpu(t, -1);
+
+ t->retcode = 0;
+ wait_queue_init(&t->retcode_wait_queue);
+
+ /* create the stack */
+ if (!stack) {
+#if THREAD_STACK_BOUNDS_CHECK
+ stack_size += THREAD_STACK_PADDING_SIZE;
+ flags |= THREAD_FLAG_DEBUG_STACK_BOUNDS_CHECK;
+#endif
+ t->stack = malloc(stack_size);
+ if (!t->stack) {
+ if (flags & THREAD_FLAG_FREE_STRUCT)
+ free(t);
+ return NULL;
+ }
+ flags |= THREAD_FLAG_FREE_STACK;
+#if THREAD_STACK_BOUNDS_CHECK
+ memset(t->stack, STACK_DEBUG_BYTE, THREAD_STACK_PADDING_SIZE);
+#endif
+ } else {
+ t->stack = stack;
+ }
+
+ t->stack_size = stack_size;
+
+ /* save whether or not we need to free the thread struct and/or stack */
+ t->flags = flags;
+
+ /* inheirit thread local storage from the parent */
+ thread_t *current_thread = get_current_thread();
+ int i;
+ for (i=0; i < MAX_TLS_ENTRY; i++)
+ t->tls[i] = current_thread->tls[i];
+
+ /* set up the initial stack frame */
+ arch_thread_initialize(t);
+
+ /* add it to the global thread list */
+ THREAD_LOCK(state);
+ list_add_head(&thread_list, &t->thread_list_node);
+ THREAD_UNLOCK(state);
+
+ return t;
+}
+
+thread_t *thread_create(const char *name, thread_start_routine entry, void *arg, int priority, size_t stack_size)
+{
+ return thread_create_etc(NULL, name, entry, arg, priority, NULL, stack_size);
+}
+
+/**
+ * @brief Flag a thread as real time
+ *
+ * @param t Thread to flag
+ *
+ * @return NO_ERROR on success
+ */
+status_t thread_set_real_time(thread_t *t)
+{
+ if (!t)
+ return ERR_INVALID_ARGS;
+
+ DEBUG_ASSERT(t->magic == THREAD_MAGIC);
+
+ THREAD_LOCK(state);
+#if PLATFORM_HAS_DYNAMIC_TIMER
+ if (t == get_current_thread()) {
+ /* if we're currently running, cancel the preemption timer. */
+ timer_cancel(&preempt_timer[arch_curr_cpu_num()]);
+ }
+#endif
+ t->flags |= THREAD_FLAG_REAL_TIME;
+ THREAD_UNLOCK(state);
+
+ return NO_ERROR;
+}
+
+static bool thread_is_realtime(thread_t *t)
+{
+ return (t->flags & THREAD_FLAG_REAL_TIME) && t->priority > DEFAULT_PRIORITY;
+}
+
+static bool thread_is_idle(thread_t *t)
+{
+ return !!(t->flags & THREAD_FLAG_IDLE);
+}
+
+static bool thread_is_real_time_or_idle(thread_t *t)
+{
+ return !!(t->flags & (THREAD_FLAG_REAL_TIME | THREAD_FLAG_IDLE));
+}
+
+/**
+ * @brief Make a suspended thread executable.
+ *
+ * This function is typically called to start a thread which has just been
+ * created with thread_create()
+ *
+ * @param t Thread to resume
+ *
+ * @return NO_ERROR on success, ERR_NOT_SUSPENDED if thread was not suspended.
+ */
+status_t thread_resume(thread_t *t)
+{
+ DEBUG_ASSERT(t->magic == THREAD_MAGIC);
+ DEBUG_ASSERT(t->state != THREAD_DEATH);
+
+ bool resched = false;
+ bool ints_disabled = arch_ints_disabled();
+ THREAD_LOCK(state);
+ if (t->state == THREAD_SUSPENDED) {
+ t->state = THREAD_READY;
+ insert_in_run_queue_head(t);
+ if (!ints_disabled) /* HACK, don't resced into bootstrap thread before idle thread is set up */
+ resched = true;
+ }
+
+ mp_reschedule(MP_CPU_ALL_BUT_LOCAL, 0);
+
+ THREAD_UNLOCK(state);
+
+ if (resched)
+ thread_yield();
+
+ return NO_ERROR;
+}
+
+status_t thread_detach_and_resume(thread_t *t)
+{
+ status_t err;
+ err = thread_detach(t);
+ if (err < 0)
+ return err;
+ return thread_resume(t);
+}
+
+status_t thread_join(thread_t *t, int *retcode, lk_time_t timeout)
+{
+ DEBUG_ASSERT(t->magic == THREAD_MAGIC);
+
+ THREAD_LOCK(state);
+
+ if (t->flags & THREAD_FLAG_DETACHED) {
+ /* the thread is detached, go ahead and exit */
+ THREAD_UNLOCK(state);
+ return ERR_THREAD_DETACHED;
+ }
+
+ /* wait for the thread to die */
+ if (t->state != THREAD_DEATH) {
+ status_t err = wait_queue_block(&t->retcode_wait_queue, timeout);
+ if (err < 0) {
+ THREAD_UNLOCK(state);
+ return err;
+ }
+ }
+
+ DEBUG_ASSERT(t->magic == THREAD_MAGIC);
+ DEBUG_ASSERT(t->state == THREAD_DEATH);
+ DEBUG_ASSERT(t->blocking_wait_queue == NULL);
+ DEBUG_ASSERT(!list_in_list(&t->queue_node));
+
+ /* save the return code */
+ if (retcode)
+ *retcode = t->retcode;
+
+ /* remove it from the master thread list */
+ list_delete(&t->thread_list_node);
+
+ /* clear the structure's magic */
+ t->magic = 0;
+
+ THREAD_UNLOCK(state);
+
+ /* free its stack and the thread structure itself */
+ if (t->flags & THREAD_FLAG_FREE_STACK && t->stack)
+ free(t->stack);
+
+ if (t->flags & THREAD_FLAG_FREE_STRUCT)
+ free(t);
+
+ return NO_ERROR;
+}
+
+status_t thread_detach(thread_t *t)
+{
+ DEBUG_ASSERT(t->magic == THREAD_MAGIC);
+
+ THREAD_LOCK(state);
+
+ /* if another thread is blocked inside thread_join() on this thread,
+ * wake them up with a specific return code */
+ wait_queue_wake_all(&t->retcode_wait_queue, false, ERR_THREAD_DETACHED);
+
+ /* if it's already dead, then just do what join would have and exit */
+ if (t->state == THREAD_DEATH) {
+ t->flags &= ~THREAD_FLAG_DETACHED; /* makes sure thread_join continues */
+ THREAD_UNLOCK(state);
+ return thread_join(t, NULL, 0);
+ } else {
+ t->flags |= THREAD_FLAG_DETACHED;
+ THREAD_UNLOCK(state);
+ return NO_ERROR;
+ }
+}
+
+/**
+ * @brief Terminate the current thread
+ *
+ * Current thread exits with the specified return code.
+ *
+ * This function does not return.
+ */
+void thread_exit(int retcode)
+{
+ thread_t *current_thread = get_current_thread();
+
+ DEBUG_ASSERT(current_thread->magic == THREAD_MAGIC);
+ DEBUG_ASSERT(current_thread->state == THREAD_RUNNING);
+ DEBUG_ASSERT(!thread_is_idle(current_thread));
+
+// dprintf("thread_exit: current %p\n", current_thread);
+
+ THREAD_LOCK(state);
+
+ /* enter the dead state */
+ current_thread->state = THREAD_DEATH;
+ current_thread->retcode = retcode;
+
+ /* if we're detached, then do our teardown here */
+ if (current_thread->flags & THREAD_FLAG_DETACHED) {
+ /* remove it from the master thread list */
+ list_delete(¤t_thread->thread_list_node);
+
+ /* clear the structure's magic */
+ current_thread->magic = 0;
+
+ /* free its stack and the thread structure itself */
+ if (current_thread->flags & THREAD_FLAG_FREE_STACK && current_thread->stack) {
+ heap_delayed_free(current_thread->stack);
+
+ /* make sure its not going to get a bounds check performed on the half-freed stack */
+ current_thread->flags &= ~THREAD_FLAG_DEBUG_STACK_BOUNDS_CHECK;
+ }
+
+ if (current_thread->flags & THREAD_FLAG_FREE_STRUCT)
+ heap_delayed_free(current_thread);
+ } else {
+ /* signal if anyone is waiting */
+ wait_queue_wake_all(¤t_thread->retcode_wait_queue, false, 0);
+ }
+
+ /* reschedule */
+ thread_resched();
+
+ panic("somehow fell through thread_exit()\n");
+}
+
+static void idle_thread_routine(void)
+{
+ for (;;)
+ arch_idle();
+}
+
+static thread_t *get_top_thread(int cpu)
+{
+ thread_t *newthread;
+ uint32_t local_run_queue_bitmap = run_queue_bitmap;
+
+ while (local_run_queue_bitmap) {
+ /* find the first (remaining) queue with a thread in it */
+ uint next_queue = HIGHEST_PRIORITY - __builtin_clz(local_run_queue_bitmap)
+ - (sizeof(run_queue_bitmap) * 8 - NUM_PRIORITIES);
+
+ list_for_every_entry(&run_queue[next_queue], newthread, thread_t, queue_node) {
+#if WITH_SMP
+ if (newthread->pinned_cpu < 0 || newthread->pinned_cpu == cpu)
+#endif
+ {
+ list_delete(&newthread->queue_node);
+
+ if (list_is_empty(&run_queue[next_queue]))
+ run_queue_bitmap &= ~(1<<next_queue);
+
+ return newthread;
+ }
+ }
+
+ local_run_queue_bitmap &= ~(1<<next_queue);
+ }
+ /* no threads to run, select the idle thread for this cpu */
+ return idle_thread(cpu);
+}
+
+/**
+ * @brief Cause another thread to be executed.
+ *
+ * Internal reschedule routine. The current thread needs to already be in whatever
+ * state and queues it needs to be in. This routine simply picks the next thread and
+ * switches to it.
+ *
+ * This is probably not the function you're looking for. See
+ * thread_yield() instead.
+ */
+void thread_resched(void)
+{
+ thread_t *oldthread;
+ thread_t *newthread;
+
+ thread_t *current_thread = get_current_thread();
+ uint cpu = arch_curr_cpu_num();
+
+ DEBUG_ASSERT(arch_ints_disabled());
+ DEBUG_ASSERT(spin_lock_held(&thread_lock));
+ DEBUG_ASSERT(current_thread->state != THREAD_RUNNING);
+
+ THREAD_STATS_INC(reschedules);
+
+ newthread = get_top_thread(cpu);
+
+ DEBUG_ASSERT(newthread);
+
+ newthread->state = THREAD_RUNNING;
+
+ oldthread = current_thread;
+
+ if (newthread == oldthread)
+ return;
+
+ /* set up quantum for the new thread if it was consumed */
+ if (newthread->remaining_quantum <= 0) {
+ newthread->remaining_quantum = 5; // XXX make this smarter
+ }
+
+ /* mark the cpu ownership of the threads */
+ thread_set_curr_cpu(oldthread, -1);
+ thread_set_curr_cpu(newthread, cpu);
+
+#if WITH_SMP
+ if (thread_is_idle(newthread)) {
+ mp_set_cpu_idle(cpu);
+ } else {
+ mp_set_cpu_busy(cpu);
+ }
+
+ if (thread_is_realtime(newthread)) {
+ mp_set_cpu_realtime(cpu);
+ } else {
+ mp_set_cpu_non_realtime(cpu);
+ }
+#endif
+
+#if THREAD_STATS
+ THREAD_STATS_INC(context_switches);
+
+ if (thread_is_idle(oldthread)) {
+ lk_bigtime_t now = current_time_hires();
+ thread_stats[cpu].idle_time += now - thread_stats[cpu].last_idle_timestamp;
+ }
+ if (thread_is_idle(newthread)) {
+ thread_stats[cpu].last_idle_timestamp = current_time_hires();
+ }
+#endif
+
+ KEVLOG_THREAD_SWITCH(oldthread, newthread);
+
+#if PLATFORM_HAS_DYNAMIC_TIMER
+ if (thread_is_real_time_or_idle(newthread)) {
+ if (!thread_is_real_time_or_idle(oldthread)) {
+ /* if we're switching from a non real time to a real time, cancel
+ * the preemption timer. */
+#if DEBUG_THREAD_CONTEXT_SWITCH
+ dprintf(ALWAYS, "arch_context_switch: stop preempt, cpu %d, old %p (%s), new %p (%s)\n",
+ cpu, oldthread, oldthread->name, newthread, newthread->name);
+#endif
+ timer_cancel(&preempt_timer[cpu]);
+ }
+ } else if (thread_is_real_time_or_idle(oldthread)) {
+ /* if we're switching from a real time (or idle thread) to a regular one,
+ * set up a periodic timer to run our preemption tick. */
+#if DEBUG_THREAD_CONTEXT_SWITCH
+ dprintf(ALWAYS, "arch_context_switch: start preempt, cpu %d, old %p (%s), new %p (%s)\n",
+ cpu, oldthread, oldthread->name, newthread, newthread->name);
+#endif
+ timer_set_periodic(&preempt_timer[cpu], 10, thread_timer_tick, NULL);
+ }
+#endif
+
+ /* set some optional target debug leds */
+ target_set_debug_led(0, !thread_is_idle(newthread));
+
+ /* do the switch */
+ set_current_thread(newthread);
+
+#if DEBUG_THREAD_CONTEXT_SWITCH
+ dprintf(ALWAYS, "arch_context_switch: cpu %d, old %p (%s, pri %d, flags 0x%x), new %p (%s, pri %d, flags 0x%x)\n",
+ cpu, oldthread, oldthread->name, oldthread->priority,
+ oldthread->flags, newthread, newthread->name,
+ newthread->priority, newthread->flags);
+#endif
+
+#if THREAD_STACK_BOUNDS_CHECK
+ /* check that the old thread has not blown its stack just before pushing its context */
+ if (oldthread->flags & THREAD_FLAG_DEBUG_STACK_BOUNDS_CHECK) {
+ STATIC_ASSERT((THREAD_STACK_PADDING_SIZE % sizeof(uint32_t)) == 0);
+ uint32_t *s = (uint32_t *)oldthread->stack;
+ for (size_t i = 0; i < THREAD_STACK_PADDING_SIZE / sizeof(uint32_t); i++) {
+ if (unlikely(s[i] != STACK_DEBUG_WORD)) {
+ /* NOTE: will probably blow the stack harder here, but hopefully enough
+ * state exists to at least get some sort of debugging done.
+ */
+ panic("stack overrun at %p: thread %p (%s), stack %p\n", &s[i],
+ oldthread, oldthread->name, oldthread->stack);
+ }
+ }
+ }
+#endif
+
+#ifdef WITH_LIB_UTHREAD
+ uthread_context_switch(oldthread, newthread);
+#endif
+ arch_context_switch(oldthread, newthread);
+}
+
+/**
+ * @brief Yield the cpu to another thread
+ *
+ * This function places the current thread at the end of the run queue
+ * and yields the cpu to another waiting thread (if any.)
+ *
+ * This function will return at some later time. Possibly immediately if
+ * no other threads are waiting to execute.
+ */
+void thread_yield(void)
+{
+ thread_t *current_thread = get_current_thread();
+
+ DEBUG_ASSERT(current_thread->magic == THREAD_MAGIC);
+ DEBUG_ASSERT(current_thread->state == THREAD_RUNNING);
+
+ THREAD_LOCK(state);
+
+ THREAD_STATS_INC(yields);
+
+ /* we are yielding the cpu, so stick ourselves into the tail of the run queue and reschedule */
+ current_thread->state = THREAD_READY;
+ current_thread->remaining_quantum = 0;
+ if (likely(!thread_is_idle(current_thread))) { /* idle thread doesn't go in the run queue */
+ insert_in_run_queue_tail(current_thread);
+ }
+ thread_resched();
+
+ THREAD_UNLOCK(state);
+}
+
+/**
+ * @brief Briefly yield cpu to another thread
+ *
+ * This function is similar to thread_yield(), except that it will
+ * restart more quickly.
+ *
+ * This function places the current thread at the head of the run
+ * queue and then yields the cpu to another thread.
+ *
+ * Exception: If the time slice for this thread has expired, then
+ * the thread goes to the end of the run queue.
+ *
+ * This function will return at some later time. Possibly immediately if
+ * no other threads are waiting to execute.
+ */
+void thread_preempt(void)
+{
+ thread_t *current_thread = get_current_thread();
+
+ DEBUG_ASSERT(current_thread->magic == THREAD_MAGIC);
+ DEBUG_ASSERT(current_thread->state == THREAD_RUNNING);
+
+#if THREAD_STATS
+ if (!thread_is_idle(current_thread))
+ THREAD_STATS_INC(preempts); /* only track when a meaningful preempt happens */
+#endif
+
+ KEVLOG_THREAD_PREEMPT(current_thread);
+
+ THREAD_LOCK(state);
+
+ /* we are being preempted, so we get to go back into the front of the run queue if we have quantum left */
+ current_thread->state = THREAD_READY;
+ if (likely(!thread_is_idle(current_thread))) { /* idle thread doesn't go in the run queue */
+ if (current_thread->remaining_quantum > 0)
+ insert_in_run_queue_head(current_thread);
+ else
+ insert_in_run_queue_tail(current_thread); /* if we're out of quantum, go to the tail of the queue */
+ }
+ thread_resched();
+
+ THREAD_UNLOCK(state);
+}
+
+/**
+ * @brief Suspend thread until woken.
+ *
+ * This function schedules another thread to execute. This function does not
+ * return until the thread is made runable again by some other module.
+ *
+ * You probably don't want to call this function directly; it's meant to be called
+ * from other modules, such as mutex, which will presumably set the thread's
+ * state to blocked and add it to some queue or another.
+ */
+void thread_block(void)
+{
+ __UNUSED thread_t *current_thread = get_current_thread();
+
+ DEBUG_ASSERT(current_thread->magic == THREAD_MAGIC);
+ DEBUG_ASSERT(current_thread->state == THREAD_BLOCKED);
+ DEBUG_ASSERT(spin_lock_held(&thread_lock));
+ DEBUG_ASSERT(!thread_is_idle(current_thread));
+
+ /* we are blocking on something. the blocking code should have already stuck us on a queue */
+ thread_resched();
+}
+
+void thread_unblock(thread_t *t, bool resched)
+{
+ DEBUG_ASSERT(t->magic == THREAD_MAGIC);
+ DEBUG_ASSERT(t->state == THREAD_BLOCKED);
+ DEBUG_ASSERT(spin_lock_held(&thread_lock));
+ DEBUG_ASSERT(!thread_is_idle(t));
+
+ t->state = THREAD_READY;
+ insert_in_run_queue_head(t);
+ mp_reschedule(MP_CPU_ALL_BUT_LOCAL, 0);
+ if (resched)
+ thread_resched();
+}
+
+enum handler_return thread_timer_tick(struct timer *t, lk_time_t now, void *arg)
+{
+ thread_t *current_thread = get_current_thread();
+
+ if (thread_is_real_time_or_idle(current_thread))
+ return INT_NO_RESCHEDULE;
+
+ current_thread->remaining_quantum--;
+ if (current_thread->remaining_quantum <= 0) {
+ return INT_RESCHEDULE;
+ } else {
+ return INT_NO_RESCHEDULE;
+ }
+}
+
+/* timer callback to wake up a sleeping thread */
+static enum handler_return thread_sleep_handler(timer_t *timer, lk_time_t now, void *arg)
+{
+ thread_t *t = (thread_t *)arg;
+
+ DEBUG_ASSERT(t->magic == THREAD_MAGIC);
+ DEBUG_ASSERT(t->state == THREAD_SLEEPING);
+
+ THREAD_LOCK(state);
+
+ t->state = THREAD_READY;
+ insert_in_run_queue_head(t);
+
+ THREAD_UNLOCK(state);
+
+ return INT_RESCHEDULE;
+}
+
+/**
+ * @brief Put thread to sleep; delay specified in ms
+ *
+ * This function puts the current thread to sleep until the specified
+ * delay in ms has expired.
+ *
+ * Note that this function could sleep for longer than the specified delay if
+ * other threads are running. When the timer expires, this thread will
+ * be placed at the head of the run queue.
+ */
+void thread_sleep(lk_time_t delay)
+{
+ timer_t timer;
+
+ thread_t *current_thread = get_current_thread();
+
+ DEBUG_ASSERT(current_thread->magic == THREAD_MAGIC);
+ DEBUG_ASSERT(current_thread->state == THREAD_RUNNING);
+ DEBUG_ASSERT(!thread_is_idle(current_thread));
+
+ timer_initialize(&timer);
+
+ THREAD_LOCK(state);
+ timer_set_oneshot(&timer, delay, thread_sleep_handler, (void *)current_thread);
+ current_thread->state = THREAD_SLEEPING;
+ thread_resched();
+ THREAD_UNLOCK(state);
+}
+
+/**
+ * @brief Initialize threading system
+ *
+ * This function is called once, from kmain()
+ */
+void thread_init_early(void)
+{
+ int i;
+
+ DEBUG_ASSERT(arch_curr_cpu_num() == 0);
+
+ /* initialize the run queues */
+ for (i=0; i < NUM_PRIORITIES; i++)
+ list_initialize(&run_queue[i]);
+
+ /* initialize the thread list */
+ list_initialize(&thread_list);
+
+ /* create a thread to cover the current running state */
+ thread_t *t = idle_thread(0);
+ init_thread_struct(t, "bootstrap");
+
+ /* half construct this thread, since we're already running */
+ t->priority = HIGHEST_PRIORITY;
+ t->state = THREAD_RUNNING;
+ t->flags = THREAD_FLAG_DETACHED;
+ thread_set_curr_cpu(t, 0);
+ thread_set_pinned_cpu(t, 0);
+ wait_queue_init(&t->retcode_wait_queue);
+ list_add_head(&thread_list, &t->thread_list_node);
+ set_current_thread(t);
+}
+
+/**
+ * @brief Complete thread initialization
+ *
+ * This function is called once at boot time
+ */
+void thread_init(void)
+{
+#if PLATFORM_HAS_DYNAMIC_TIMER
+ for (uint i = 0; i < SMP_MAX_CPUS; i++) {
+ timer_initialize(&preempt_timer[i]);
+ }
+#endif
+}
+
+/**
+ * @brief Change name of current thread
+ */
+void thread_set_name(const char *name)
+{
+ thread_t *current_thread = get_current_thread();
+ strlcpy(current_thread->name, name, sizeof(current_thread->name));
+}
+
+/**
+ * @brief Change priority of current thread
+ *
+ * See thread_create() for a discussion of priority values.
+ */
+void thread_set_priority(int priority)
+{
+ thread_t *current_thread = get_current_thread();
+
+ THREAD_LOCK(state);
+
+ if (priority <= IDLE_PRIORITY)
+ priority = IDLE_PRIORITY + 1;
+ if (priority > HIGHEST_PRIORITY)
+ priority = HIGHEST_PRIORITY;
+ current_thread->priority = priority;
+
+ current_thread->state = THREAD_READY;
+ insert_in_run_queue_head(current_thread);
+ thread_resched();
+
+ THREAD_UNLOCK(state);
+}
+
+/**
+ * @brief Become an idle thread
+ *
+ * This function marks the current thread as the idle thread -- the one which
+ * executes when there is nothing else to do. This function does not return.
+ * This function is called once at boot time.
+ */
+void thread_become_idle(void)
+{
+ DEBUG_ASSERT(arch_ints_disabled());
+
+ thread_t *t = get_current_thread();
+
+#if WITH_SMP
+ char name[16];
+ snprintf(name, sizeof(name), "idle %d", arch_curr_cpu_num());
+ thread_set_name(name);
+#else
+ thread_set_name("idle");
+#endif
+
+ /* mark ourself as idle */
+ t->priority = IDLE_PRIORITY;
+ t->flags |= THREAD_FLAG_IDLE;
+ thread_set_pinned_cpu(t, arch_curr_cpu_num());
+
+ mp_set_curr_cpu_active(true);
+ mp_set_cpu_idle(arch_curr_cpu_num());
+
+ /* enable interrupts and start the scheduler */
+ arch_enable_ints();
+ thread_yield();
+
+ idle_thread_routine();
+}
+
+/* create an idle thread for the cpu we're on, and start scheduling */
+
+void thread_secondary_cpu_init_early(void)
+{
+ DEBUG_ASSERT(arch_ints_disabled());
+
+ /* construct an idle thread to cover our cpu */
+ uint cpu = arch_curr_cpu_num();
+ thread_t *t = idle_thread(cpu);
+
+ char name[16];
+ snprintf(name, sizeof(name), "idle %u", cpu);
+ init_thread_struct(t, name);
+ thread_set_pinned_cpu(t, cpu);
+
+ /* half construct this thread, since we're already running */
+ t->priority = HIGHEST_PRIORITY;
+ t->state = THREAD_RUNNING;
+ t->flags = THREAD_FLAG_DETACHED | THREAD_FLAG_IDLE;
+ thread_set_curr_cpu(t, cpu);
+ thread_set_pinned_cpu(t, cpu);
+ wait_queue_init(&t->retcode_wait_queue);
+
+ THREAD_LOCK(state);
+
+ list_add_head(&thread_list, &t->thread_list_node);
+ set_current_thread(t);
+
+ THREAD_UNLOCK(state);
+}
+
+void thread_secondary_cpu_entry(void)
+{
+ uint cpu = arch_curr_cpu_num();
+ thread_t *t = get_current_thread();
+ t->priority = IDLE_PRIORITY;
+
+ mp_set_curr_cpu_active(true);
+ mp_set_cpu_idle(cpu);
+
+ /* enable interrupts and start the scheduler on this cpu */
+ arch_enable_ints();
+ thread_yield();
+
+ idle_thread_routine();
+}
+
+static const char *thread_state_to_str(enum thread_state state)
+{
+ switch (state) {
+ case THREAD_SUSPENDED: return "susp";
+ case THREAD_READY: return "rdy";
+ case THREAD_RUNNING: return "run";
+ case THREAD_BLOCKED: return "blok";
+ case THREAD_SLEEPING: return "slep";
+ case THREAD_DEATH: return "deth";
+ default: return "unkn";
+ }
+}
+
+/**
+ * @brief Dump debugging info about the specified thread.
+ */
+void dump_thread(thread_t *t)
+{
+ dprintf(INFO, "dump_thread: t %p (%s)\n", t, t->name);
+#if WITH_SMP
+ dprintf(INFO, "\tstate %s, curr_cpu %d, pinned_cpu %d, priority %d, remaining quantum %d\n",
+ thread_state_to_str(t->state), t->curr_cpu, t->pinned_cpu, t->priority, t->remaining_quantum);
+#else
+ dprintf(INFO, "\tstate %s, priority %d, remaining quantum %d\n",
+ thread_state_to_str(t->state), t->priority, t->remaining_quantum);
+#endif
+ dprintf(INFO, "\tstack %p, stack_size %zd\n", t->stack, t->stack_size);
+ dprintf(INFO, "\tentry %p, arg %p, flags 0x%x\n", t->entry, t->arg, t->flags);
+ dprintf(INFO, "\twait queue %p, wait queue ret %d\n", t->blocking_wait_queue, t->wait_queue_block_ret);
+#if (MAX_TLS_ENTRY > 0)
+ dprintf(INFO, "\ttls:");
+ int i;
+ for (i=0; i < MAX_TLS_ENTRY; i++) {
+ dprintf(INFO, " 0x%lx", t->tls[i]);
+ }
+ dprintf(INFO, "\n");
+#endif
+ arch_dump_thread(t);
+}
+
+/**
+ * @brief Dump debugging info about all threads
+ */
+void dump_all_threads(void)
+{
+ thread_t *t;
+
+ THREAD_LOCK(state);
+ list_for_every_entry(&thread_list, t, thread_t, thread_list_node) {
+ if (t->magic != THREAD_MAGIC) {
+ dprintf(INFO, "bad magic on thread struct %p, aborting.\n", t);
+ hexdump(t, sizeof(thread_t));
+ break;
+ }
+ dump_thread(t);
+ }
+ THREAD_UNLOCK(state);
+}
+
+/** @} */
+
+
+/**
+ * @defgroup wait Wait Queue
+ * @{
+ */
+void wait_queue_init(wait_queue_t *wait)
+{
+ *wait = (wait_queue_t)WAIT_QUEUE_INITIAL_VALUE(*wait);
+}
+
+static enum handler_return wait_queue_timeout_handler(timer_t *timer, lk_time_t now, void *arg)
+{
+ thread_t *thread = (thread_t *)arg;
+
+ DEBUG_ASSERT(thread->magic == THREAD_MAGIC);
+
+ spin_lock(&thread_lock);
+
+ enum handler_return ret = INT_NO_RESCHEDULE;
+ if (thread_unblock_from_wait_queue(thread, ERR_TIMED_OUT) >= NO_ERROR) {
+ ret = INT_RESCHEDULE;
+ }
+
+ spin_unlock(&thread_lock);
+
+ return ret;
+}
+
+/**
+ * @brief Block until a wait queue is notified.
+ *
+ * This function puts the current thread at the end of a wait
+ * queue and then blocks until some other thread wakes the queue
+ * up again.
+ *
+ * @param wait The wait queue to enter
+ * @param timeout The maximum time, in ms, to wait
+ *
+ * If the timeout is zero, this function returns immediately with
+ * ERR_TIMED_OUT. If the timeout is INFINITE_TIME, this function
+ * waits indefinitely. Otherwise, this function returns with
+ * ERR_TIMED_OUT at the end of the timeout period.
+ *
+ * @return ERR_TIMED_OUT on timeout, else returns the return
+ * value specified when the queue was woken by wait_queue_wake_one().
+ */
+status_t wait_queue_block(wait_queue_t *wait, lk_time_t timeout)
+{
+ timer_t timer;
+
+ thread_t *current_thread = get_current_thread();
+
+ DEBUG_ASSERT(wait->magic == WAIT_QUEUE_MAGIC);
+ DEBUG_ASSERT(current_thread->state == THREAD_RUNNING);
+ DEBUG_ASSERT(arch_ints_disabled());
+ DEBUG_ASSERT(spin_lock_held(&thread_lock));
+
+ if (timeout == 0)
+ return ERR_TIMED_OUT;
+
+ list_add_tail(&wait->list, ¤t_thread->queue_node);
+ wait->count++;
+ current_thread->state = THREAD_BLOCKED;
+ current_thread->blocking_wait_queue = wait;
+ current_thread->wait_queue_block_ret = NO_ERROR;
+
+ /* if the timeout is nonzero or noninfinite, set a callback to yank us out of the queue */
+ if (timeout != INFINITE_TIME) {
+ timer_initialize(&timer);
+ timer_set_oneshot(&timer, timeout, wait_queue_timeout_handler, (void *)current_thread);
+ }
+
+ thread_resched();
+
+ /* we don't really know if the timer fired or not, so it's better safe to try to cancel it */
+ if (timeout != INFINITE_TIME) {
+ timer_cancel(&timer);
+ }
+
+ return current_thread->wait_queue_block_ret;
+}
+
+/**
+ * @brief Wake up one thread sleeping on a wait queue
+ *
+ * This function removes one thread (if any) from the head of the wait queue and
+ * makes it executable. The new thread will be placed at the head of the
+ * run queue.
+ *
+ * @param wait The wait queue to wake
+ * @param reschedule If true, the newly-woken thread will run immediately.
+ * @param wait_queue_error The return value which the new thread will receive
+ * from wait_queue_block().
+ *
+ * @return The number of threads woken (zero or one)
+ */
+int wait_queue_wake_one(wait_queue_t *wait, bool reschedule, status_t wait_queue_error)
+{
+ thread_t *t;
+ int ret = 0;
+
+ thread_t *current_thread = get_current_thread();
+
+ DEBUG_ASSERT(wait->magic == WAIT_QUEUE_MAGIC);
+ DEBUG_ASSERT(arch_ints_disabled());
+ DEBUG_ASSERT(spin_lock_held(&thread_lock));
+
+ t = list_remove_head_type(&wait->list, thread_t, queue_node);
+ if (t) {
+ wait->count--;
+ DEBUG_ASSERT(t->state == THREAD_BLOCKED);
+ t->state = THREAD_READY;
+ t->wait_queue_block_ret = wait_queue_error;
+ t->blocking_wait_queue = NULL;
+
+ /* if we're instructed to reschedule, stick the current thread on the head
+ * of the run queue first, so that the newly awakened thread gets a chance to run
+ * before the current one, but the current one doesn't get unnecessarilly punished.
+ */
+ if (reschedule) {
+ current_thread->state = THREAD_READY;
+ insert_in_run_queue_head(current_thread);
+ }
+ insert_in_run_queue_head(t);
+ mp_reschedule(MP_CPU_ALL_BUT_LOCAL, 0);
+ if (reschedule) {
+ thread_resched();
+ }
+ ret = 1;
+
+ }
+
+ return ret;
+}
+
+
+/**
+ * @brief Wake all threads sleeping on a wait queue
+ *
+ * This function removes all threads (if any) from the wait queue and
+ * makes them executable. The new threads will be placed at the head of the
+ * run queue.
+ *
+ * @param wait The wait queue to wake
+ * @param reschedule If true, the newly-woken threads will run immediately.
+ * @param wait_queue_error The return value which the new thread will receive
+ * from wait_queue_block().
+ *
+ * @return The number of threads woken (zero or one)
+ */
+int wait_queue_wake_all(wait_queue_t *wait, bool reschedule, status_t wait_queue_error)
+{
+ thread_t *t;
+ int ret = 0;
+
+ thread_t *current_thread = get_current_thread();
+
+ DEBUG_ASSERT(wait->magic == WAIT_QUEUE_MAGIC);
+ DEBUG_ASSERT(arch_ints_disabled());
+ DEBUG_ASSERT(spin_lock_held(&thread_lock));
+
+ if (reschedule && wait->count > 0) {
+ /* if we're instructed to reschedule, stick the current thread on the head
+ * of the run queue first, so that the newly awakened threads get a chance to run
+ * before the current one, but the current one doesn't get unnecessarilly punished.
+ */
+ current_thread->state = THREAD_READY;
+ insert_in_run_queue_head(current_thread);
+ }
+
+ /* pop all the threads off the wait queue into the run queue */
+ while ((t = list_remove_head_type(&wait->list, thread_t, queue_node))) {
+ wait->count--;
+ DEBUG_ASSERT(t->state == THREAD_BLOCKED);
+ t->state = THREAD_READY;
+ t->wait_queue_block_ret = wait_queue_error;
+ t->blocking_wait_queue = NULL;
+
+ insert_in_run_queue_head(t);
+ ret++;
+ }
+
+ DEBUG_ASSERT(wait->count == 0);
+
+ if (ret > 0) {
+ mp_reschedule(MP_CPU_ALL_BUT_LOCAL, 0);
+ if (reschedule) {
+ thread_resched();
+ }
+ }
+
+ return ret;
+}
+
+/**
+ * @brief Free all resources allocated in wait_queue_init()
+ *
+ * If any threads were waiting on this queue, they are all woken.
+ */
+void wait_queue_destroy(wait_queue_t *wait, bool reschedule)
+{
+ DEBUG_ASSERT(wait->magic == WAIT_QUEUE_MAGIC);
+ DEBUG_ASSERT(arch_ints_disabled());
+ DEBUG_ASSERT(spin_lock_held(&thread_lock));
+
+ wait_queue_wake_all(wait, reschedule, ERR_OBJECT_DESTROYED);
+ wait->magic = 0;
+}
+
+/**
+ * @brief Wake a specific thread in a wait queue
+ *
+ * This function extracts a specific thread from a wait queue, wakes it, and
+ * puts it at the head of the run queue.
+ *
+ * @param t The thread to wake
+ * @param wait_queue_error The return value which the new thread will receive
+ * from wait_queue_block().
+ *
+ * @return ERR_NOT_BLOCKED if thread was not in any wait queue.
+ */
+status_t thread_unblock_from_wait_queue(thread_t *t, status_t wait_queue_error)
+{
+ DEBUG_ASSERT(t->magic == THREAD_MAGIC);
+ DEBUG_ASSERT(arch_ints_disabled());
+ DEBUG_ASSERT(spin_lock_held(&thread_lock));
+
+ if (t->state != THREAD_BLOCKED)
+ return ERR_NOT_BLOCKED;
+
+ DEBUG_ASSERT(t->blocking_wait_queue != NULL);
+ DEBUG_ASSERT(t->blocking_wait_queue->magic == WAIT_QUEUE_MAGIC);
+ DEBUG_ASSERT(list_in_list(&t->queue_node));
+
+ list_delete(&t->queue_node);
+ t->blocking_wait_queue->count--;
+ t->blocking_wait_queue = NULL;
+ t->state = THREAD_READY;
+ t->wait_queue_block_ret = wait_queue_error;
+ insert_in_run_queue_head(t);
+ mp_reschedule(MP_CPU_ALL_BUT_LOCAL, 0);
+
+ return NO_ERROR;
+}
+
+#if defined(WITH_DEBUGGER_INFO)
+// This is, by necessity, arch-specific, and arm-m specific right now,
+// but lives here due to thread_list being static.
+//
+// It contains sufficient information for a remote debugger to walk
+// the thread list without needing the symbols and debug sections in
+// the elf binary for lk or the ability to parse them.
+const struct __debugger_info__ {
+ u32 version; // flags:16 major:8 minor:8
+ void *thread_list_ptr;
+ void *current_thread_ptr;
+ u8 off_list_node;
+ u8 off_state;
+ u8 off_saved_sp;
+ u8 off_was_preempted;
+ u8 off_name;
+ u8 off_waitq;
+} _debugger_info = {
+ .version = 0x0100,
+ .thread_list_ptr = &thread_list,
+ .current_thread_ptr = &_current_thread,
+ .off_list_node = __builtin_offsetof(thread_t, thread_list_node),
+ .off_state = __builtin_offsetof(thread_t, state),
+ .off_saved_sp = __builtin_offsetof(thread_t, arch.sp),
+ .off_was_preempted = __builtin_offsetof(thread_t, arch.was_preempted),
+ .off_name = __builtin_offsetof(thread_t, name),
+ .off_waitq = __builtin_offsetof(thread_t, blocking_wait_queue),
+};
+#endif
+
+/* vim: set ts=4 sw=4 noexpandtab: */
diff --git a/src/bsp/lk/kernel/timer.c b/src/bsp/lk/kernel/timer.c
new file mode 100644
index 0000000..b01f6ab
--- /dev/null
+++ b/src/bsp/lk/kernel/timer.c
@@ -0,0 +1,314 @@
+/*
+ * Copyright (c) 2008-2014 Travis Geiselbrecht
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/**
+ * @file
+ * @brief Kernel timer subsystem
+ * @defgroup timer Timers
+ *
+ * The timer subsystem allows functions to be scheduled for later
+ * execution. Each timer object is used to cause one function to
+ * be executed at a later time.
+ *
+ * Timer callback functions are called in interrupt context.
+ *
+ * @{
+ */
+#include <debug.h>
+#include <trace.h>
+#include <assert.h>
+#include <list.h>
+#include <kernel/thread.h>
+#include <kernel/timer.h>
+#include <kernel/debug.h>
+#include <kernel/spinlock.h>
+#include <platform/timer.h>
+#include <platform.h>
+
+#define LOCAL_TRACE 0
+
+spin_lock_t timer_lock;
+
+struct timer_state {
+ struct list_node timer_queue;
+} __CPU_ALIGN;
+
+static struct timer_state timers[SMP_MAX_CPUS];
+
+static enum handler_return timer_tick(void *arg, lk_time_t now);
+
+/**
+ * @brief Initialize a timer object
+ */
+void timer_initialize(timer_t *timer)
+{
+ *timer = (timer_t)TIMER_INITIAL_VALUE(*timer);
+}
+
+static void insert_timer_in_queue(uint cpu, timer_t *timer)
+{
+ timer_t *entry;
+
+ DEBUG_ASSERT(arch_ints_disabled());
+
+ LTRACEF("timer %p, cpu %u, scheduled %u, periodic %u\n", timer, cpu, timer->scheduled_time, timer->periodic_time);
+
+ list_for_every_entry(&timers[cpu].timer_queue, entry, timer_t, node) {
+ if (TIME_GT(entry->scheduled_time, timer->scheduled_time)) {
+ list_add_before(&entry->node, &timer->node);
+ return;
+ }
+ }
+
+ /* walked off the end of the list */
+ list_add_tail(&timers[cpu].timer_queue, &timer->node);
+}
+
+static void timer_set(timer_t *timer, lk_time_t delay, lk_time_t period, timer_callback callback, void *arg)
+{
+ lk_time_t now;
+
+ LTRACEF("timer %p, delay %u, period %u, callback %p, arg %p\n", timer, delay, period, callback, arg);
+
+ DEBUG_ASSERT(timer->magic == TIMER_MAGIC);
+
+ if (list_in_list(&timer->node)) {
+ panic("timer %p already in list\n", timer);
+ }
+
+ now = current_time();
+ timer->scheduled_time = now + delay;
+ timer->periodic_time = period;
+ timer->callback = callback;
+ timer->arg = arg;
+
+ LTRACEF("scheduled time %u\n", timer->scheduled_time);
+
+ spin_lock_saved_state_t state;
+ spin_lock_irqsave(&timer_lock, state);
+
+ uint cpu = arch_curr_cpu_num();
+ insert_timer_in_queue(cpu, timer);
+
+#if PLATFORM_HAS_DYNAMIC_TIMER
+ if (list_peek_head_type(&timers[cpu].timer_queue, timer_t, node) == timer) {
+ /* we just modified the head of the timer queue */
+ LTRACEF("setting new timer for %u msecs\n", delay);
+ platform_set_oneshot_timer(timer_tick, NULL, delay);
+ }
+#endif
+
+ spin_unlock_irqrestore(&timer_lock, state);
+}
+
+/**
+ * @brief Set up a timer that executes once
+ *
+ * This function specifies a callback function to be called after a specified
+ * delay. The function will be called one time.
+ *
+ * @param timer The timer to use
+ * @param delay The delay, in ms, before the timer is executed
+ * @param callback The function to call when the timer expires
+ * @param arg The argument to pass to the callback
+ *
+ * The timer function is declared as:
+ * enum handler_return callback(struct timer *, lk_time_t now, void *arg) { ... }
+ */
+void timer_set_oneshot(timer_t *timer, lk_time_t delay, timer_callback callback, void *arg)
+{
+ if (delay == 0)
+ delay = 1;
+ timer_set(timer, delay, 0, callback, arg);
+}
+
+/**
+ * @brief Set up a timer that executes repeatedly
+ *
+ * This function specifies a callback function to be called after a specified
+ * delay. The function will be called repeatedly.
+ *
+ * @param timer The timer to use
+ * @param delay The delay, in ms, before the timer is executed
+ * @param callback The function to call when the timer expires
+ * @param arg The argument to pass to the callback
+ *
+ * The timer function is declared as:
+ * enum handler_return callback(struct timer *, lk_time_t now, void *arg) { ... }
+ */
+void timer_set_periodic(timer_t *timer, lk_time_t period, timer_callback callback, void *arg)
+{
+ if (period == 0)
+ period = 1;
+ timer_set(timer, period, period, callback, arg);
+}
+
+/**
+ * @brief Cancel a pending timer
+ */
+void timer_cancel(timer_t *timer)
+{
+ DEBUG_ASSERT(timer->magic == TIMER_MAGIC);
+
+ spin_lock_saved_state_t state;
+ spin_lock_irqsave(&timer_lock, state);
+
+#if PLATFORM_HAS_DYNAMIC_TIMER
+ uint cpu = arch_curr_cpu_num();
+
+ timer_t *oldhead = list_peek_head_type(&timers[cpu].timer_queue, timer_t, node);
+#endif
+
+ if (list_in_list(&timer->node))
+ list_delete(&timer->node);
+
+ /* to keep it from being reinserted into the queue if called from
+ * periodic timer callback.
+ */
+ timer->periodic_time = 0;
+ timer->callback = NULL;
+ timer->arg = NULL;
+
+#if PLATFORM_HAS_DYNAMIC_TIMER
+ /* see if we've just modified the head of the timer queue */
+ timer_t *newhead = list_peek_head_type(&timers[cpu].timer_queue, timer_t, node);
+ if (newhead == NULL) {
+ LTRACEF("clearing old hw timer, nothing in the queue\n");
+ platform_stop_timer();
+ } else if (newhead != oldhead) {
+ lk_time_t delay;
+ lk_time_t now = current_time();
+
+ if (TIME_LT(newhead->scheduled_time, now))
+ delay = 0;
+ else
+ delay = newhead->scheduled_time - now;
+
+ LTRACEF("setting new timer to %u\n", (uint) delay);
+ platform_set_oneshot_timer(timer_tick, NULL, delay);
+ }
+#endif
+
+ spin_unlock_irqrestore(&timer_lock, state);
+}
+
+/* called at interrupt time to process any pending timers */
+static enum handler_return timer_tick(void *arg, lk_time_t now)
+{
+ timer_t *timer;
+ enum handler_return ret = INT_NO_RESCHEDULE;
+
+ DEBUG_ASSERT(arch_ints_disabled());
+
+ THREAD_STATS_INC(timer_ints);
+// KEVLOG_TIMER_TICK(); // enable only if necessary
+
+ uint cpu = arch_curr_cpu_num();
+
+ LTRACEF("cpu %u now %u, sp %p\n", cpu, now, __GET_FRAME());
+
+ spin_lock(&timer_lock);
+
+ for (;;) {
+ /* see if there's an event to process */
+ timer = list_peek_head_type(&timers[cpu].timer_queue, timer_t, node);
+ if (likely(timer == 0))
+ break;
+ LTRACEF("next item on timer queue %p at %u now %u (%p, arg %p)\n", timer, timer->scheduled_time, now, timer->callback, timer->arg);
+ if (likely(TIME_LT(now, timer->scheduled_time)))
+ break;
+
+ /* process it */
+ LTRACEF("timer %p\n", timer);
+ DEBUG_ASSERT(timer && timer->magic == TIMER_MAGIC);
+ list_delete(&timer->node);
+
+ /* we pulled it off the list, release the list lock to handle it */
+ spin_unlock(&timer_lock);
+
+ LTRACEF("dequeued timer %p, scheduled %u periodic %u\n", timer, timer->scheduled_time, timer->periodic_time);
+
+ THREAD_STATS_INC(timers);
+
+ bool periodic = timer->periodic_time > 0;
+
+ LTRACEF("timer %p firing callback %p, arg %p\n", timer, timer->callback, timer->arg);
+ KEVLOG_TIMER_CALL(timer->callback, timer->arg);
+ if (timer->callback(timer, now, timer->arg) == INT_RESCHEDULE)
+ ret = INT_RESCHEDULE;
+
+ /* it may have been requeued or periodic, grab the lock so we can safely inspect it */
+ spin_lock(&timer_lock);
+
+ /* if it was a periodic timer and it hasn't been requeued
+ * by the callback put it back in the list
+ */
+ if (periodic && !list_in_list(&timer->node) && timer->periodic_time > 0) {
+ LTRACEF("periodic timer, period %u\n", timer->periodic_time);
+ timer->scheduled_time = now + timer->periodic_time;
+ insert_timer_in_queue(cpu, timer);
+ }
+ }
+
+#if PLATFORM_HAS_DYNAMIC_TIMER
+ /* reset the timer to the next event */
+ timer = list_peek_head_type(&timers[cpu].timer_queue, timer_t, node);
+ if (timer) {
+ /* has to be the case or it would have fired already */
+ DEBUG_ASSERT(TIME_GT(timer->scheduled_time, now));
+
+ lk_time_t delay = timer->scheduled_time - now;
+
+ LTRACEF("setting new timer for %u msecs for event %p\n", (uint)delay, timer);
+ platform_set_oneshot_timer(timer_tick, NULL, delay);
+ }
+
+ /* we're done manipulating the timer queue */
+ spin_unlock(&timer_lock);
+#else
+ /* release the timer lock before calling the tick handler */
+ spin_unlock(&timer_lock);
+
+ /* let the scheduler have a shot to do quantum expiration, etc */
+ /* in case of dynamic timer, the scheduler will set up a periodic timer */
+ if (thread_timer_tick(NULL, now, NULL) == INT_RESCHEDULE)
+ ret = INT_RESCHEDULE;
+#endif
+
+ return ret;
+}
+
+void timer_init(void)
+{
+ timer_lock = SPIN_LOCK_INITIAL_VALUE;
+ for (uint i = 0; i < SMP_MAX_CPUS; i++) {
+ list_initialize(&timers[i].timer_queue);
+ }
+#if !PLATFORM_HAS_DYNAMIC_TIMER
+ /* register for a periodic timer tick */
+ platform_set_periodic_timer(timer_tick, NULL, 10); /* 10ms */
+#endif
+}
+
+/* vim: set noexpandtab */
+
diff --git a/src/bsp/lk/kernel/vm/bootalloc.c b/src/bsp/lk/kernel/vm/bootalloc.c
new file mode 100644
index 0000000..8847bca
--- /dev/null
+++ b/src/bsp/lk/kernel/vm/bootalloc.c
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2014 Travis Geiselbrecht
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include <kernel/vm.h>
+#include "vm_priv.h"
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <trace.h>
+
+#define LOCAL_TRACE 0
+
+/* cheezy allocator that chews up space just after the end of the kernel mapping */
+
+/* track how much memory we've used */
+extern int _end;
+
+uintptr_t boot_alloc_start = (uintptr_t)&_end;
+uintptr_t boot_alloc_end = (uintptr_t)&_end;
+
+void *boot_alloc_mem(size_t len)
+{
+ uintptr_t ptr;
+
+ ptr = ALIGN(boot_alloc_end, 8);
+ boot_alloc_end = (ptr + ALIGN(len, 8));
+
+ LTRACEF("len %zu, ptr %p\n", len, (void *)ptr);
+
+ return (void *)ptr;
+}
+
diff --git a/src/bsp/lk/kernel/vm/pmm.c b/src/bsp/lk/kernel/vm/pmm.c
new file mode 100644
index 0000000..3bcdc77
--- /dev/null
+++ b/src/bsp/lk/kernel/vm/pmm.c
@@ -0,0 +1,518 @@
+/*
+ * Copyright (c) 2014 Travis Geiselbrecht
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include <kernel/vm.h>
+#include "vm_priv.h"
+
+#include <trace.h>
+#include <assert.h>
+#include <list.h>
+#include <stdlib.h>
+#include <err.h>
+#include <string.h>
+#include <pow2.h>
+#include <lib/console.h>
+#include <kernel/mutex.h>
+
+#define LOCAL_TRACE 0
+
+static struct list_node arena_list = LIST_INITIAL_VALUE(arena_list);
+static mutex_t lock = MUTEX_INITIAL_VALUE(lock);
+
+#define PAGE_BELONGS_TO_ARENA(page, arena) \
+ (((uintptr_t)(page) >= (uintptr_t)(arena)->page_array) && \
+ ((uintptr_t)(page) < ((uintptr_t)(arena)->page_array + (arena)->size / PAGE_SIZE * sizeof(vm_page_t))))
+
+#define PAGE_ADDRESS_FROM_ARENA(page, arena) \
+ (paddr_t)(((uintptr_t)page - (uintptr_t)a->page_array) / sizeof(vm_page_t)) * PAGE_SIZE + a->base;
+
+#define ADDRESS_IN_ARENA(address, arena) \
+ ((address) >= (arena)->base && (address) <= (arena)->base + (arena)->size - 1)
+
+static inline bool page_is_free(const vm_page_t *page)
+{
+ DEBUG_ASSERT(page);
+
+ return !(page->flags & VM_PAGE_FLAG_NONFREE);
+}
+
+paddr_t page_to_address(const vm_page_t *page)
+{
+ DEBUG_ASSERT(page);
+
+ pmm_arena_t *a;
+ list_for_every_entry(&arena_list, a, pmm_arena_t, node) {
+ if (PAGE_BELONGS_TO_ARENA(page, a)) {
+ return PAGE_ADDRESS_FROM_ARENA(page, a);
+ }
+ }
+ return -1;
+}
+
+vm_page_t *address_to_page(paddr_t addr)
+{
+ pmm_arena_t *a;
+ list_for_every_entry(&arena_list, a, pmm_arena_t, node) {
+ if (addr >= a->base && addr <= a->base + a->size - 1) {
+ size_t index = (addr - a->base) / PAGE_SIZE;
+ return &a->page_array[index];
+ }
+ }
+ return NULL;
+}
+
+status_t pmm_add_arena(pmm_arena_t *arena)
+{
+ LTRACEF("arena %p name '%s' base 0x%lx size 0x%zx\n", arena, arena->name, arena->base, arena->size);
+
+ DEBUG_ASSERT(arena);
+ DEBUG_ASSERT(IS_PAGE_ALIGNED(arena->base));
+ DEBUG_ASSERT(IS_PAGE_ALIGNED(arena->size));
+ DEBUG_ASSERT(arena->size > 0);
+
+ /* walk the arena list and add arena based on priority order */
+ pmm_arena_t *a;
+ list_for_every_entry(&arena_list, a, pmm_arena_t, node) {
+ if (a->priority > arena->priority) {
+ list_add_before(&a->node, &arena->node);
+ goto done_add;
+ }
+ }
+
+ /* walked off the end, add it to the end of the list */
+ list_add_tail(&arena_list, &arena->node);
+
+done_add:
+
+ /* zero out some of the structure */
+ arena->free_count = 0;
+ list_initialize(&arena->free_list);
+
+ /* allocate an array of pages to back this one */
+ size_t page_count = arena->size / PAGE_SIZE;
+ arena->page_array = boot_alloc_mem(page_count * sizeof(vm_page_t));
+
+ /* initialize all of the pages */
+ memset(arena->page_array, 0, page_count * sizeof(vm_page_t));
+
+ /* add them to the free list */
+ for (size_t i = 0; i < page_count; i++) {
+ vm_page_t *p = &arena->page_array[i];
+
+ list_add_tail(&arena->free_list, &p->node);
+
+ arena->free_count++;
+ }
+
+ return NO_ERROR;
+}
+
+size_t pmm_alloc_pages(uint count, struct list_node *list)
+{
+ LTRACEF("count %u\n", count);
+
+ uint allocated = 0;
+ if (count == 0)
+ return 0;
+
+ mutex_acquire(&lock);
+
+ /* walk the arenas in order, allocating as many pages as we can from each */
+ pmm_arena_t *a;
+ list_for_every_entry(&arena_list, a, pmm_arena_t, node) {
+ while (allocated < count) {
+ vm_page_t *page = list_remove_head_type(&a->free_list, vm_page_t, node);
+ if (!page)
+ goto done;
+
+ a->free_count--;
+
+ page->flags |= VM_PAGE_FLAG_NONFREE;
+ list_add_tail(list, &page->node);
+
+ allocated++;
+ }
+ }
+
+done:
+ mutex_release(&lock);
+ return allocated;
+}
+
+size_t pmm_alloc_range(paddr_t address, uint count, struct list_node *list)
+{
+ LTRACEF("address 0x%lx, count %u\n", address, count);
+
+ uint allocated = 0;
+ if (count == 0)
+ return 0;
+
+ address = ROUNDDOWN(address, PAGE_SIZE);
+
+ mutex_acquire(&lock);
+
+ /* walk through the arenas, looking to see if the physical page belongs to it */
+ pmm_arena_t *a;
+ list_for_every_entry(&arena_list, a, pmm_arena_t, node) {
+ while (allocated < count && ADDRESS_IN_ARENA(address, a)) {
+ size_t index = (address - a->base) / PAGE_SIZE;
+
+ DEBUG_ASSERT(index < a->size / PAGE_SIZE);
+
+ vm_page_t *page = &a->page_array[index];
+ if (page->flags & VM_PAGE_FLAG_NONFREE) {
+ /* we hit an allocated page */
+ break;
+ }
+
+ DEBUG_ASSERT(list_in_list(&page->node));
+
+ list_delete(&page->node);
+ page->flags |= VM_PAGE_FLAG_NONFREE;
+ list_add_tail(list, &page->node);
+
+ a->free_count--;
+ allocated++;
+ address += PAGE_SIZE;
+ }
+
+ if (allocated == count)
+ break;
+ }
+
+ mutex_release(&lock);
+ return allocated;
+}
+
+size_t pmm_free(struct list_node *list)
+{
+ LTRACEF("list %p\n", list);
+
+ mutex_acquire(&lock);
+
+ uint count = 0;
+ while (!list_is_empty(list)) {
+ vm_page_t *page = list_remove_head_type(list, vm_page_t, node);
+
+ DEBUG_ASSERT(!list_in_list(&page->node));
+ DEBUG_ASSERT(page->flags & VM_PAGE_FLAG_NONFREE);
+
+ /* see which arena this page belongs to and add it */
+ pmm_arena_t *a;
+ list_for_every_entry(&arena_list, a, pmm_arena_t, node) {
+ if (PAGE_BELONGS_TO_ARENA(page, a)) {
+ page->flags &= ~VM_PAGE_FLAG_NONFREE;
+
+ list_add_head(&a->free_list, &page->node);
+ a->free_count++;
+ count++;
+ break;
+ }
+ }
+ }
+
+ mutex_release(&lock);
+ return count;
+}
+
+size_t pmm_free_page(vm_page_t *page)
+{
+ DEBUG_ASSERT(page);
+
+ struct list_node list;
+ list_initialize(&list);
+
+ list_add_head(&list, &page->node);
+
+ return pmm_free(&list);
+}
+
+/* physically allocate a run from arenas marked as KMAP */
+void *pmm_alloc_kpages(uint count, struct list_node *list)
+{
+ LTRACEF("count %u\n", count);
+
+ // XXX do fast path for single page
+
+
+ paddr_t pa;
+ size_t alloc_count = pmm_alloc_contiguous(count, PAGE_SIZE_SHIFT, &pa, list);
+ if (alloc_count == 0)
+ return NULL;
+
+ return paddr_to_kvaddr(pa);
+}
+
+size_t pmm_free_kpages(void *_ptr, uint count)
+{
+ LTRACEF("ptr %p, count %u\n", _ptr, count);
+
+ uint8_t *ptr = (uint8_t *)_ptr;
+
+ struct list_node list;
+ list_initialize(&list);
+
+ while (count > 0) {
+ vm_page_t *p = address_to_page(kvaddr_to_paddr(ptr));
+ if (p) {
+ list_add_tail(&list, &p->node);
+ }
+
+ ptr += PAGE_SIZE;
+ count--;
+ }
+
+ return pmm_free(&list);
+}
+
+size_t pmm_alloc_contiguous(uint count, uint8_t alignment_log2, paddr_t *pa, struct list_node *list)
+{
+ LTRACEF("count %u, align %u\n", count, alignment_log2);
+
+ if (count == 0)
+ return 0;
+ if (alignment_log2 < PAGE_SIZE_SHIFT)
+ alignment_log2 = PAGE_SIZE_SHIFT;
+
+ mutex_acquire(&lock);
+
+ pmm_arena_t *a;
+ list_for_every_entry(&arena_list, a, pmm_arena_t, node) {
+ // XXX make this a flag to only search kmap?
+ if (a->flags & PMM_ARENA_FLAG_KMAP) {
+ /* walk the list starting at alignment boundaries.
+ * calculate the starting offset into this arena, based on the
+ * base address of the arena to handle the case where the arena
+ * is not aligned on the same boundary requested.
+ */
+ paddr_t rounded_base = ROUNDUP(a->base, 1UL << alignment_log2);
+ if (rounded_base < a->base || rounded_base > a->base + a->size - 1)
+ continue;
+
+ uint aligned_offset = (rounded_base - a->base) / PAGE_SIZE;
+ uint start = aligned_offset;
+ LTRACEF("starting search at aligned offset %u\n", start);
+ LTRACEF("arena base 0x%lx size %zu\n", a->base, a->size);
+
+retry:
+ /* search while we're still within the arena and have a chance of finding a slot
+ (start + count < end of arena) */
+ while ((start < a->size / PAGE_SIZE) &&
+ ((start + count) <= a->size / PAGE_SIZE)) {
+ vm_page_t *p = &a->page_array[start];
+ for (uint i = 0; i < count; i++) {
+ if (p->flags & VM_PAGE_FLAG_NONFREE) {
+ /* this run is broken, break out of the inner loop.
+ * start over at the next alignment boundary
+ */
+ start = ROUNDUP(start - aligned_offset + i + 1, 1UL << (alignment_log2 - PAGE_SIZE_SHIFT)) + aligned_offset;
+ goto retry;
+ }
+ p++;
+ }
+
+ /* we found a run */
+ LTRACEF("found run from pn %u to %u\n", start, start + count);
+
+ /* remove the pages from the run out of the free list */
+ for (uint i = start; i < start + count; i++) {
+ p = &a->page_array[i];
+ DEBUG_ASSERT(!(p->flags & VM_PAGE_FLAG_NONFREE));
+ DEBUG_ASSERT(list_in_list(&p->node));
+
+ list_delete(&p->node);
+ p->flags |= VM_PAGE_FLAG_NONFREE;
+ a->free_count--;
+
+ if (list)
+ list_add_tail(list, &p->node);
+ }
+
+ if (pa)
+ *pa = a->base + start * PAGE_SIZE;
+
+ mutex_release(&lock);
+
+ return count;
+ }
+ }
+ }
+
+ mutex_release(&lock);
+
+ LTRACEF("couldn't find run\n");
+ return 0;
+}
+
+static void dump_page(const vm_page_t *page)
+{
+ DEBUG_ASSERT(page);
+
+ printf("page %p: address 0x%lx flags 0x%x\n", page, page_to_address(page), page->flags);
+}
+
+static void dump_arena(const pmm_arena_t *arena, bool dump_pages)
+{
+ DEBUG_ASSERT(arena);
+
+ printf("arena %p: name '%s' base 0x%lx size 0x%zx priority %u flags 0x%x\n",
+ arena, arena->name, arena->base, arena->size, arena->priority, arena->flags);
+ printf("\tpage_array %p, free_count %zu\n",
+ arena->page_array, arena->free_count);
+
+ /* dump all of the pages */
+ if (dump_pages) {
+ for (size_t i = 0; i < arena->size / PAGE_SIZE; i++) {
+ dump_page(&arena->page_array[i]);
+ }
+ }
+
+ /* dump the free pages */
+ printf("\tfree ranges:\n");
+ ssize_t last = -1;
+ for (size_t i = 0; i < arena->size / PAGE_SIZE; i++) {
+ if (page_is_free(&arena->page_array[i])) {
+ if (last == -1) {
+ last = i;
+ }
+ } else {
+ if (last != -1) {
+ printf("\t\t0x%lx - 0x%lx\n", arena->base + last * PAGE_SIZE, arena->base + i * PAGE_SIZE);
+ }
+ last = -1;
+ }
+ }
+
+ if (last != -1) {
+ printf("\t\t0x%lx - 0x%lx\n", arena->base + last * PAGE_SIZE, arena->base + arena->size);
+ }
+}
+
+static int cmd_pmm(int argc, const cmd_args *argv)
+{
+ if (argc < 2) {
+notenoughargs:
+ printf("not enough arguments\n");
+usage:
+ printf("usage:\n");
+ printf("%s arenas\n", argv[0].str);
+ printf("%s alloc <count>\n", argv[0].str);
+ printf("%s alloc_range <address> <count>\n", argv[0].str);
+ printf("%s alloc_kpages <count>\n", argv[0].str);
+ printf("%s alloc_contig <count> <alignment>\n", argv[0].str);
+ printf("%s dump_alloced\n", argv[0].str);
+ printf("%s free_alloced\n", argv[0].str);
+ return ERR_GENERIC;
+ }
+
+ static struct list_node allocated = LIST_INITIAL_VALUE(allocated);
+
+ if (!strcmp(argv[1].str, "arenas")) {
+ pmm_arena_t *a;
+ list_for_every_entry(&arena_list, a, pmm_arena_t, node) {
+ dump_arena(a, false);
+ }
+ } else if (!strcmp(argv[1].str, "alloc")) {
+ if (argc < 3) goto notenoughargs;
+
+ struct list_node list;
+ list_initialize(&list);
+
+ uint count = pmm_alloc_pages(argv[2].u, &list);
+ printf("alloc returns %u\n", count);
+
+ vm_page_t *p;
+ list_for_every_entry(&list, p, vm_page_t, node) {
+ printf("\tpage %p, address 0x%lx\n", p, page_to_address(p));
+ }
+
+ /* add the pages to the local allocated list */
+ struct list_node *node;
+ while ((node = list_remove_head(&list))) {
+ list_add_tail(&allocated, node);
+ }
+ } else if (!strcmp(argv[1].str, "dump_alloced")) {
+ vm_page_t *page;
+
+ list_for_every_entry(&allocated, page, vm_page_t, node) {
+ dump_page(page);
+ }
+ } else if (!strcmp(argv[1].str, "alloc_range")) {
+ if (argc < 4) goto notenoughargs;
+
+ struct list_node list;
+ list_initialize(&list);
+
+ uint count = pmm_alloc_range(argv[2].u, argv[3].u, &list);
+ printf("alloc returns %u\n", count);
+
+ vm_page_t *p;
+ list_for_every_entry(&list, p, vm_page_t, node) {
+ printf("\tpage %p, address 0x%lx\n", p, page_to_address(p));
+ }
+
+ /* add the pages to the local allocated list */
+ struct list_node *node;
+ while ((node = list_remove_head(&list))) {
+ list_add_tail(&allocated, node);
+ }
+ } else if (!strcmp(argv[1].str, "alloc_kpages")) {
+ if (argc < 3) goto notenoughargs;
+
+ void *ptr = pmm_alloc_kpages(argv[2].u, NULL);
+ printf("pmm_alloc_kpages returns %p\n", ptr);
+ } else if (!strcmp(argv[1].str, "alloc_contig")) {
+ if (argc < 4) goto notenoughargs;
+
+ struct list_node list;
+ list_initialize(&list);
+
+ paddr_t pa;
+ size_t ret = pmm_alloc_contiguous(argv[2].u, argv[3].u, &pa, &list);
+ printf("pmm_alloc_contiguous returns %zu, address 0x%lx\n", ret, pa);
+ printf("address %% align = 0x%lx\n", pa % argv[3].u);
+
+ /* add the pages to the local allocated list */
+ struct list_node *node;
+ while ((node = list_remove_head(&list))) {
+ list_add_tail(&allocated, node);
+ }
+ } else if (!strcmp(argv[1].str, "free_alloced")) {
+ size_t err = pmm_free(&allocated);
+ printf("pmm_free returns %zu\n", err);
+ } else {
+ printf("unknown command\n");
+ goto usage;
+ }
+
+ return NO_ERROR;
+}
+
+STATIC_COMMAND_START
+#if LK_DEBUGLEVEL > 0
+STATIC_COMMAND("pmm", "physical memory manager", &cmd_pmm)
+#endif
+STATIC_COMMAND_END(pmm);
+
+
+
+
diff --git a/src/bsp/lk/kernel/vm/rules.mk b/src/bsp/lk/kernel/vm/rules.mk
new file mode 100644
index 0000000..d511c1c
--- /dev/null
+++ b/src/bsp/lk/kernel/vm/rules.mk
@@ -0,0 +1,11 @@
+LOCAL_DIR := $(GET_LOCAL_DIR)
+
+MODULE := $(LOCAL_DIR)
+
+MODULE_SRCS += \
+ $(LOCAL_DIR)/bootalloc.c \
+ $(LOCAL_DIR)/pmm.c \
+ $(LOCAL_DIR)/vm.c \
+ $(LOCAL_DIR)/vmm.c \
+
+include make/module.mk
diff --git a/src/bsp/lk/kernel/vm/vm.c b/src/bsp/lk/kernel/vm/vm.c
new file mode 100644
index 0000000..a5c7aba
--- /dev/null
+++ b/src/bsp/lk/kernel/vm/vm.c
@@ -0,0 +1,183 @@
+/*
+ * Copyright (c) 2014 Travis Geiselbrecht
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include <kernel/vm.h>
+#include "vm_priv.h"
+
+#include <trace.h>
+#include <err.h>
+#include <string.h>
+#include <lk/init.h>
+#include <lib/console.h>
+#include <arch/mmu.h>
+#include <debug.h>
+
+#define LOCAL_TRACE 0
+
+extern int _start;
+extern int _end;
+
+/* mark the physical pages backing a range of virtual as in use.
+ * allocate the physical pages and throw them away */
+static void mark_pages_in_use(vaddr_t va, size_t len)
+{
+ LTRACEF("va 0x%lx, len 0x%zx\n", va, len);
+
+ struct list_node list;
+ list_initialize(&list);
+
+ /* make sure we are inclusive of all of the pages in the address range */
+ len = PAGE_ALIGN(len + (va & (PAGE_SIZE - 1)));
+ va = ROUNDDOWN(va, PAGE_SIZE);
+
+ LTRACEF("aligned va 0x%lx, len 0x%zx\n", va, len);
+
+ for (size_t offset = 0; offset < len; offset += PAGE_SIZE) {
+ uint flags;
+ paddr_t pa;
+
+ status_t err = arch_mmu_query(va + offset, &pa, &flags);
+ if (err >= 0) {
+ //LTRACEF("va 0x%x, pa 0x%x, flags 0x%x, err %d\n", va + offset, pa, flags, err);
+
+ /* alloate the range, throw the results away */
+ pmm_alloc_range(pa, 1, &list);
+ } else {
+ panic("Could not find pa for va 0x%lx\n", va);
+ }
+ }
+}
+
+static void vm_init_preheap(uint level)
+{
+ LTRACE_ENTRY;
+
+ /* mark all of the kernel pages in use */
+ LTRACEF("marking all kernel pages as used\n");
+ mark_pages_in_use((vaddr_t)&_start, ((uintptr_t)&_end - (uintptr_t)&_start));
+
+ /* mark the physical pages used by the boot time allocator */
+ if (boot_alloc_end != boot_alloc_start) {
+ LTRACEF("marking boot alloc used from 0x%lx to 0x%lx\n", boot_alloc_start, boot_alloc_end);
+
+ mark_pages_in_use(boot_alloc_start, boot_alloc_end - boot_alloc_start);
+ }
+}
+
+static void vm_init_postheap(uint level)
+{
+ LTRACE_ENTRY;
+
+ vmm_init();
+
+ /* create vmm regions to cover what is already there from the initial mapping table */
+ struct mmu_initial_mapping *map = mmu_initial_mappings;
+ while (map->size > 0) {
+ if (!(map->flags & MMU_INITIAL_MAPPING_TEMPORARY)) {
+ vmm_reserve_space(vmm_get_kernel_aspace(), map->name, map->size, map->virt);
+ }
+
+ map++;
+ }
+}
+
+void *paddr_to_kvaddr(paddr_t pa)
+{
+ /* slow path to do reverse lookup */
+ struct mmu_initial_mapping *map = mmu_initial_mappings;
+ while (map->size > 0) {
+ if (!(map->flags & MMU_INITIAL_MAPPING_TEMPORARY) &&
+ pa >= map->phys &&
+ pa <= map->phys + map->size - 1) {
+ return (void *)(map->virt + (pa - map->phys));
+ }
+ map++;
+ }
+ return NULL;
+}
+
+paddr_t kvaddr_to_paddr(void *ptr)
+{
+ status_t rc;
+ paddr_t pa;
+
+ rc = arch_mmu_query((vaddr_t)ptr, &pa, NULL);
+ if (rc)
+ return (paddr_t) NULL;
+ return pa;
+}
+
+static int cmd_vm(int argc, const cmd_args *argv)
+{
+ if (argc < 2) {
+notenoughargs:
+ printf("not enough arguments\n");
+usage:
+ printf("usage:\n");
+ printf("%s phys2virt <address>\n", argv[0].str);
+ printf("%s virt2phys <address>\n", argv[0].str);
+ printf("%s map <phys> <virt> <count> <flags>\n", argv[0].str);
+ printf("%s unmap <virt> <count>\n", argv[0].str);
+ return ERR_GENERIC;
+ }
+
+ if (!strcmp(argv[1].str, "phys2virt")) {
+ if (argc < 3) goto notenoughargs;
+
+ void *ptr = paddr_to_kvaddr(argv[2].u);
+ printf("paddr_to_kvaddr returns %p\n", ptr);
+ } else if (!strcmp(argv[1].str, "virt2phys")) {
+ if (argc < 3) goto notenoughargs;
+
+ paddr_t pa;
+ uint flags;
+ status_t err = arch_mmu_query(argv[2].u, &pa, &flags);
+ printf("arch_mmu_query returns %d\n", err);
+ if (err >= 0) {
+ printf("\tpa 0x%lx, flags 0x%x\n", pa, flags);
+ }
+ } else if (!strcmp(argv[1].str, "map")) {
+ if (argc < 6) goto notenoughargs;
+
+ int err = arch_mmu_map(argv[3].u, argv[2].u, argv[4].u, argv[5].u);
+ printf("arch_mmu_map returns %d\n", err);
+ } else if (!strcmp(argv[1].str, "unmap")) {
+ if (argc < 4) goto notenoughargs;
+
+ int err = arch_mmu_unmap(argv[2].u, argv[3].u);
+ printf("arch_mmu_unmap returns %d\n", err);
+ } else {
+ printf("unknown command\n");
+ goto usage;
+ }
+
+ return NO_ERROR;
+}
+
+STATIC_COMMAND_START
+#if LK_DEBUGLEVEL > 0
+STATIC_COMMAND("vm", "vm commands", &cmd_vm)
+#endif
+STATIC_COMMAND_END(vm);
+
+LK_INIT_HOOK(vm_preheap, &vm_init_preheap, LK_INIT_LEVEL_HEAP - 1);
+LK_INIT_HOOK(vm, &vm_init_postheap, LK_INIT_LEVEL_VM);
diff --git a/src/bsp/lk/kernel/vm/vm_priv.h b/src/bsp/lk/kernel/vm/vm_priv.h
new file mode 100644
index 0000000..939a5bf
--- /dev/null
+++ b/src/bsp/lk/kernel/vm/vm_priv.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2014 Travis Geiselbrecht
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#pragma once
+
+#include <stdint.h>
+#include <sys/types.h>
+#include <kernel/vm.h>
+
+/* simple boot time allocator */
+void *boot_alloc_mem(size_t len) __MALLOC;
+extern uintptr_t boot_alloc_start;
+extern uintptr_t boot_alloc_end;
+
+paddr_t page_to_address(const vm_page_t *page);
+vm_page_t *address_to_page(paddr_t addr);
+
+void vmm_init(void);
+
diff --git a/src/bsp/lk/kernel/vm/vmm.c b/src/bsp/lk/kernel/vm/vmm.c
new file mode 100644
index 0000000..978d0f3
--- /dev/null
+++ b/src/bsp/lk/kernel/vm/vmm.c
@@ -0,0 +1,756 @@
+/*
+ * Copyright (c) 2014 Travis Geiselbrecht
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include <trace.h>
+#include <assert.h>
+#include <err.h>
+#include <string.h>
+#include <lib/console.h>
+#include <kernel/vm.h>
+#include <kernel/mutex.h>
+#include "vm_priv.h"
+
+#define LOCAL_TRACE 0
+
+static struct list_node aspace_list = LIST_INITIAL_VALUE(aspace_list);
+static mutex_t vmm_lock = MUTEX_INITIAL_VALUE(vmm_lock);
+
+vmm_aspace_t _kernel_aspace;
+
+static void dump_aspace(const vmm_aspace_t *a);
+static void dump_region(const vmm_region_t *r);
+
+void vmm_init(void)
+{
+ /* initialize the kernel address space */
+ strlcpy(_kernel_aspace.name, "kernel", sizeof(_kernel_aspace.name));
+ _kernel_aspace.base = KERNEL_ASPACE_BASE,
+ _kernel_aspace.size = KERNEL_ASPACE_SIZE,
+ _kernel_aspace.flags = VMM_FLAG_ASPACE_KERNEL;
+ list_initialize(&_kernel_aspace.region_list);
+
+ list_add_head(&aspace_list, &_kernel_aspace.node);
+}
+
+static inline bool is_inside_aspace(const vmm_aspace_t *aspace, vaddr_t vaddr)
+{
+ DEBUG_ASSERT(aspace);
+
+ return (vaddr >= aspace->base && vaddr <= aspace->base + aspace->size - 1);
+}
+
+static bool is_region_inside_aspace(const vmm_aspace_t *aspace, vaddr_t vaddr, size_t size)
+{
+ DEBUG_ASSERT(aspace);
+
+ /* is the starting address within the address space*/
+ if (!is_inside_aspace(aspace, vaddr))
+ return false;
+
+ if (size == 0)
+ return true;
+
+ /* see if the size is enough to wrap the integer */
+ if (vaddr + size - 1 < vaddr)
+ return false;
+
+ /* test to see if the end address is within the address space's */
+ if (vaddr + size - 1 > aspace->base + aspace->size - 1)
+ return false;
+
+ return true;
+}
+
+static size_t trim_to_aspace(const vmm_aspace_t *aspace, vaddr_t vaddr, size_t size)
+{
+ DEBUG_ASSERT(aspace);
+ DEBUG_ASSERT(is_inside_aspace(aspace, vaddr));
+
+ if (size == 0)
+ return size;
+
+ size_t offset = vaddr - aspace->base;
+
+ //LTRACEF("vaddr 0x%lx size 0x%zx offset 0x%zx aspace base 0x%lx aspace size 0x%zx\n",
+ // vaddr, size, offset, aspace->base, aspace->size);
+
+ if (offset + size < offset)
+ size = ULONG_MAX - offset - 1;
+
+ //LTRACEF("size now 0x%zx\n", size);
+
+ if (offset + size >= aspace->size - 1)
+ size = aspace->size - offset;
+
+ //LTRACEF("size now 0x%zx\n", size);
+
+ return size;
+}
+
+static vmm_region_t *alloc_region_struct(const char *name, vaddr_t base, size_t size, uint flags, uint arch_mmu_flags)
+{
+ DEBUG_ASSERT(name);
+
+ vmm_region_t *r = malloc(sizeof(vmm_region_t));
+ if (!r)
+ return NULL;
+
+ strlcpy(r->name, name, sizeof(r->name));
+ r->base = base;
+ r->size = size;
+ r->flags = flags;
+ r->arch_mmu_flags = arch_mmu_flags;
+ list_initialize(&r->page_list);
+
+ return r;
+}
+
+/* add a region to the appropriate spot in the address space list,
+ * testing to see if there's a space */
+static status_t add_region_to_aspace(vmm_aspace_t *aspace, vmm_region_t *r)
+{
+ DEBUG_ASSERT(aspace);
+ DEBUG_ASSERT(r);
+
+ LTRACEF("aspace %p base 0x%lx size 0x%zx r %p base 0x%lx size 0x%zx\n",
+ aspace, aspace->base, aspace->size, r, r->base, r->size);
+
+ /* only try if the region will at least fit in the address space */
+ if (r->size == 0 || !is_region_inside_aspace(aspace, r->base, r->size)) {
+ LTRACEF("region was out of range\n");
+ return ERR_OUT_OF_RANGE;
+ }
+
+ vaddr_t r_end = r->base + r->size - 1;
+
+ /* does it fit in front */
+ vmm_region_t *last;
+ last = list_peek_head_type(&aspace->region_list, vmm_region_t, node);
+ if (!last || r_end < last->base) {
+ /* empty list or not empty and fits before the first element */
+ list_add_head(&aspace->region_list, &r->node);
+ return NO_ERROR;
+ }
+
+ /* walk the list, finding the right spot to put it */
+ list_for_every_entry(&aspace->region_list, last, vmm_region_t, node) {
+ /* does it go after last? */
+ if (r->base > last->base + last->size - 1) {
+ /* get the next element in the list */
+ vmm_region_t *next = list_next_type(&aspace->region_list, &last->node, vmm_region_t, node);
+ if (!next || (r_end < next->base)) {
+ /* end of the list or next exists and it goes between them */
+ list_add_after(&last->node, &r->node);
+ return NO_ERROR;
+ }
+ }
+ }
+
+ LTRACEF("couldn't find spot\n");
+ return ERR_NO_MEMORY;
+}
+
+/*
+ * Try to pick the spot within specified gap
+ *
+ * Arch can override this to impose it's own restrictions.
+ */
+__WEAK vaddr_t arch_mmu_pick_spot(vaddr_t base, uint prev_region_arch_mmu_flags,
+ vaddr_t end, uint next_region_arch_mmu_flags,
+ vaddr_t align, size_t size, uint arch_mmu_flags)
+{
+ /* just align it by default */
+ return ALIGN(base, align);
+}
+
+/*
+ * Returns true if the caller has to stop search
+ */
+static inline bool check_gap(vmm_aspace_t *aspace,
+ vmm_region_t *prev, vmm_region_t *next,
+ vaddr_t *pva, vaddr_t align, size_t size,
+ uint arch_mmu_flags)
+{
+ vaddr_t gap_beg; /* first byte of a gap */
+ vaddr_t gap_end; /* last byte of a gap */
+
+ DEBUG_ASSERT(aspace);
+ DEBUG_ASSERT(pva);
+
+ if (prev)
+ gap_beg = prev->base + prev->size;
+ else
+ gap_beg = aspace->base;
+
+ if (next) {
+ if (gap_beg == next->base)
+ goto next_gap; /* no gap between regions */
+ gap_end = next->base - 1;
+ } else {
+ if (gap_beg == (aspace->base + aspace->size))
+ goto not_found; /* no gap at the end of address space. Stop search */
+ gap_end = aspace->base + aspace->size - 1;
+ }
+
+ *pva = arch_mmu_pick_spot(gap_beg, prev ? prev->arch_mmu_flags : ARCH_MMU_FLAG_INVALID,
+ gap_end, next ? next->arch_mmu_flags : ARCH_MMU_FLAG_INVALID,
+ align, size, arch_mmu_flags);
+ if (*pva < gap_beg)
+ goto not_found; /* address wrapped around */
+
+ if (*pva < gap_end && ((gap_end - *pva + 1) >= size)) {
+ /* we have enough room */
+ return true; /* found spot, stop search */
+ }
+
+next_gap:
+ return false; /* continue search */
+
+not_found:
+ *pva = -1;
+ return true; /* not_found: stop search */
+}
+
+static vaddr_t alloc_spot(vmm_aspace_t *aspace, size_t size, uint8_t align_pow2,
+ uint arch_mmu_flags, struct list_node **before)
+{
+ DEBUG_ASSERT(aspace);
+ DEBUG_ASSERT(size > 0 && IS_PAGE_ALIGNED(size));
+
+ LTRACEF("aspace %p size 0x%zx align %hhu\n", aspace, size, align_pow2);
+
+ if (align_pow2 < PAGE_SIZE_SHIFT)
+ align_pow2 = PAGE_SIZE_SHIFT;
+ vaddr_t align = 1UL << align_pow2;
+
+ vaddr_t spot;
+ vmm_region_t *r = NULL;
+
+ /* try to pick spot at the beginning of address space */
+ if (check_gap(aspace, NULL,
+ list_peek_head_type(&aspace->region_list, vmm_region_t, node),
+ &spot, align, size, arch_mmu_flags))
+ goto done;
+
+ /* search the middle of the list */
+ list_for_every_entry(&aspace->region_list, r, vmm_region_t, node) {
+ if (check_gap(aspace, r,
+ list_next_type(&aspace->region_list, &r->node, vmm_region_t, node),
+ &spot, align, size, arch_mmu_flags))
+ goto done;
+ }
+
+ /* couldn't find anything */
+ return -1;
+
+done:
+ if (before)
+ *before = r ? &r->node : &aspace->region_list;
+ return spot;
+}
+
+/* allocate a region structure and stick it in the address space */
+static vmm_region_t *alloc_region(vmm_aspace_t *aspace, const char *name, size_t size,
+ vaddr_t vaddr, uint8_t align_pow2,
+ uint vmm_flags, uint region_flags, uint arch_mmu_flags)
+{
+ /* make a region struct for it and stick it in the list */
+ vmm_region_t *r = alloc_region_struct(name, vaddr, size, region_flags, arch_mmu_flags);
+ if (!r)
+ return NULL;
+
+ /* if they ask us for a specific spot, put it there */
+ if (vmm_flags & VMM_FLAG_VALLOC_SPECIFIC) {
+ /* stick it in the list, checking to see if it fits */
+ if (add_region_to_aspace(aspace, r) < 0) {
+ /* didn't fit */
+ free(r);
+ return NULL;
+ }
+ } else {
+ /* allocate a virtual slot for it */
+ struct list_node *before = NULL;
+
+ vaddr = alloc_spot(aspace, size, align_pow2, arch_mmu_flags, &before);
+ LTRACEF("alloc_spot returns 0x%lx, before %p\n", vaddr, before);
+
+ if (vaddr == (vaddr_t)-1) {
+ LTRACEF("failed to find spot\n");
+ free(r);
+ return NULL;
+ }
+
+ DEBUG_ASSERT(before != NULL);
+
+ r->base = (vaddr_t)vaddr;
+
+ /* add it to the region list */
+ list_add_after(before, &r->node);
+ }
+
+ return r;
+}
+
+status_t vmm_reserve_space(vmm_aspace_t *aspace, const char *name, size_t size, vaddr_t vaddr)
+{
+ LTRACEF("aspace %p name '%s' size 0x%zx vaddr 0x%lx\n", aspace, name, size, vaddr);
+
+ DEBUG_ASSERT(IS_PAGE_ALIGNED(vaddr));
+ DEBUG_ASSERT(IS_PAGE_ALIGNED(size));
+
+ if (!name)
+ name = "";
+
+ if (size == 0)
+ return NO_ERROR;
+ if (!IS_PAGE_ALIGNED(vaddr) || !IS_PAGE_ALIGNED(size))
+ return ERR_INVALID_ARGS;
+
+ if (!is_inside_aspace(aspace, vaddr))
+ return ERR_OUT_OF_RANGE;
+
+ /* trim the size */
+ size = trim_to_aspace(aspace, vaddr, size);
+
+ mutex_acquire(&vmm_lock);
+
+ /* lookup how it's already mapped */
+ uint arch_mmu_flags = 0;
+ arch_mmu_query(vaddr, NULL, &arch_mmu_flags);
+
+ /* build a new region structure */
+ vmm_region_t *r = alloc_region(aspace, name, size, vaddr, 0, VMM_FLAG_VALLOC_SPECIFIC, VMM_REGION_FLAG_RESERVED, arch_mmu_flags);
+
+ mutex_release(&vmm_lock);
+ return r ? NO_ERROR : ERR_NO_MEMORY;
+}
+
+status_t vmm_alloc_physical(vmm_aspace_t *aspace, const char *name, size_t size, void **ptr, uint8_t align_log2, paddr_t paddr, uint vmm_flags, uint arch_mmu_flags)
+{
+ status_t ret;
+
+ LTRACEF("aspace %p name '%s' size 0x%zx ptr %p paddr 0x%lx vmm_flags 0x%x arch_mmu_flags 0x%x\n",
+ aspace, name, size, ptr ? *ptr : 0, paddr, vmm_flags, arch_mmu_flags);
+
+ DEBUG_ASSERT(IS_PAGE_ALIGNED(paddr));
+ DEBUG_ASSERT(IS_PAGE_ALIGNED(size));
+
+ if (!name)
+ name = "";
+
+ if (size == 0)
+ return NO_ERROR;
+ if (!IS_PAGE_ALIGNED(paddr) || !IS_PAGE_ALIGNED(size))
+ return ERR_INVALID_ARGS;
+
+ vaddr_t vaddr = 0;
+
+ /* if they're asking for a specific spot, copy the address */
+ if (vmm_flags & VMM_FLAG_VALLOC_SPECIFIC) {
+ /* can't ask for a specific spot and then not provide one */
+ if (!ptr) {
+ return ERR_INVALID_ARGS;
+ }
+ vaddr = (vaddr_t)*ptr;
+ }
+
+ mutex_acquire(&vmm_lock);
+
+ /* allocate a region and put it in the aspace list */
+ vmm_region_t *r = alloc_region(aspace, name, size, vaddr, align_log2, vmm_flags,
+ VMM_REGION_FLAG_PHYSICAL, arch_mmu_flags);
+ if (!r) {
+ ret = ERR_NO_MEMORY;
+ goto err_alloc_region;
+ }
+
+ /* return the vaddr if requested */
+ if (ptr)
+ *ptr = (void *)r->base;
+
+ /* map all of the pages */
+ int err = arch_mmu_map(r->base, paddr, size / PAGE_SIZE, arch_mmu_flags);
+ LTRACEF("arch_mmu_map returns %d\n", err);
+
+ ret = NO_ERROR;
+
+err_alloc_region:
+ mutex_release(&vmm_lock);
+ return ret;
+}
+
+status_t vmm_alloc_contiguous(vmm_aspace_t *aspace, const char *name, size_t size, void **ptr,
+ uint8_t align_pow2, uint vmm_flags, uint arch_mmu_flags)
+{
+ status_t err = NO_ERROR;
+
+ LTRACEF("aspace %p name '%s' size 0x%zx ptr %p align %hhu vmm_flags 0x%x arch_mmu_flags 0x%x\n",
+ aspace, name, size, ptr ? *ptr : 0, align_pow2, vmm_flags, arch_mmu_flags);
+
+ size = ROUNDUP(size, PAGE_SIZE);
+ if (size == 0)
+ return ERR_INVALID_ARGS;
+
+ if (!name)
+ name = "";
+
+ vaddr_t vaddr = 0;
+
+ /* if they're asking for a specific spot, copy the address */
+ if (vmm_flags & VMM_FLAG_VALLOC_SPECIFIC) {
+ /* can't ask for a specific spot and then not provide one */
+ if (!ptr) {
+ err = ERR_INVALID_ARGS;
+ goto err;
+ }
+ vaddr = (vaddr_t)*ptr;
+ }
+
+ /* allocate physical memory up front, in case it cant be satisfied */
+ struct list_node page_list;
+ list_initialize(&page_list);
+
+ paddr_t pa = 0;
+ /* allocate a run of physical pages */
+ size_t count = pmm_alloc_contiguous(size / PAGE_SIZE, align_pow2, &pa, &page_list);
+ if (count < size / PAGE_SIZE) {
+ DEBUG_ASSERT(count == 0); /* check that the pmm didn't allocate a partial run */
+ err = ERR_NO_MEMORY;
+ goto err;
+ }
+
+ mutex_acquire(&vmm_lock);
+
+ /* allocate a region and put it in the aspace list */
+ vmm_region_t *r = alloc_region(aspace, name, size, vaddr, align_pow2, vmm_flags,
+ VMM_REGION_FLAG_PHYSICAL, arch_mmu_flags);
+ if (!r) {
+ err = ERR_NO_MEMORY;
+ goto err1;
+ }
+
+ /* return the vaddr if requested */
+ if (ptr)
+ *ptr = (void *)r->base;
+
+ /* map all of the pages */
+ arch_mmu_map(r->base, pa, size / PAGE_SIZE, arch_mmu_flags);
+ // XXX deal with error mapping here
+
+ vm_page_t *p;
+ while ((p = list_remove_head_type(&page_list, vm_page_t, node))) {
+ list_add_tail(&r->page_list, &p->node);
+ }
+
+ mutex_release(&vmm_lock);
+ return NO_ERROR;
+
+err1:
+ mutex_release(&vmm_lock);
+ pmm_free(&page_list);
+err:
+ return err;
+}
+
+status_t vmm_alloc(vmm_aspace_t *aspace, const char *name, size_t size, void **ptr,
+ uint8_t align_pow2, uint vmm_flags, uint arch_mmu_flags)
+{
+ status_t err = NO_ERROR;
+
+ LTRACEF("aspace %p name '%s' size 0x%zx ptr %p align %hhu vmm_flags 0x%x arch_mmu_flags 0x%x\n",
+ aspace, name, size, ptr ? *ptr : 0, align_pow2, vmm_flags, arch_mmu_flags);
+
+ size = ROUNDUP(size, PAGE_SIZE);
+ if (size == 0)
+ return ERR_INVALID_ARGS;
+
+ if (!name)
+ name = "";
+
+ vaddr_t vaddr = 0;
+
+ /* if they're asking for a specific spot, copy the address */
+ if (vmm_flags & VMM_FLAG_VALLOC_SPECIFIC) {
+ /* can't ask for a specific spot and then not provide one */
+ if (!ptr) {
+ err = ERR_INVALID_ARGS;
+ goto err;
+ }
+ vaddr = (vaddr_t)*ptr;
+ }
+
+ /* allocate physical memory up front, in case it cant be satisfied */
+
+ /* allocate a random pile of pages */
+ struct list_node page_list;
+ list_initialize(&page_list);
+
+ size_t count = pmm_alloc_pages(size / PAGE_SIZE, &page_list);
+ DEBUG_ASSERT(count <= size);
+ if (count < size / PAGE_SIZE) {
+ LTRACEF("failed to allocate enough pages (asked for %zu, got %zu)\n", size / PAGE_SIZE, count);
+ pmm_free(&page_list);
+ err = ERR_NO_MEMORY;
+ goto err;
+ }
+
+ mutex_acquire(&vmm_lock);
+
+ /* allocate a region and put it in the aspace list */
+ vmm_region_t *r = alloc_region(aspace, name, size, vaddr, align_pow2, vmm_flags,
+ VMM_REGION_FLAG_PHYSICAL, arch_mmu_flags);
+ if (!r) {
+ err = ERR_NO_MEMORY;
+ goto err1;
+ }
+
+ /* return the vaddr if requested */
+ if (ptr)
+ *ptr = (void *)r->base;
+
+ /* map all of the pages */
+ /* XXX use smarter algorithm that tries to build runs */
+ vm_page_t *p;
+ vaddr_t va = r->base;
+ DEBUG_ASSERT(IS_PAGE_ALIGNED(va));
+ while ((p = list_remove_head_type(&page_list, vm_page_t, node))) {
+ DEBUG_ASSERT(va <= r->base + r->size - 1);
+
+ paddr_t pa = page_to_address(p);
+ DEBUG_ASSERT(IS_PAGE_ALIGNED(pa));
+
+ arch_mmu_map(va, pa, 1, arch_mmu_flags);
+ // XXX deal with error mapping here
+
+ list_add_tail(&r->page_list, &p->node);
+
+ va += PAGE_SIZE;
+ }
+
+ mutex_release(&vmm_lock);
+ return NO_ERROR;
+
+err1:
+ mutex_release(&vmm_lock);
+ pmm_free(&page_list);
+err:
+ return err;
+}
+
+static vmm_region_t *vmm_find_region(const vmm_aspace_t *aspace, vaddr_t vaddr)
+{
+ vmm_region_t *r;
+
+ DEBUG_ASSERT(aspace);
+
+ if (!aspace)
+ return NULL;
+
+ /* search the region list */
+ list_for_every_entry(&aspace->region_list, r, vmm_region_t, node) {
+ if ((vaddr >= r->base) && (vaddr <= r->base + r->size - 1))
+ return r;
+ }
+
+ return NULL;
+}
+
+status_t vmm_free_region(vmm_aspace_t *aspace, vaddr_t vaddr)
+{
+ DEBUG_ASSERT(aspace);
+
+ mutex_acquire(&vmm_lock);
+
+ vmm_region_t *r = vmm_find_region (aspace, vaddr);
+ if (!r) {
+ mutex_release(&vmm_lock);
+ return ERR_NOT_FOUND;
+ }
+
+ /* remove it from aspace */
+ list_delete(&r->node);
+
+ /* unmap it */
+ arch_mmu_unmap(r->base, r->size / PAGE_SIZE);
+
+ mutex_release(&vmm_lock);
+
+ /* return physical pages if any */
+ pmm_free(&r->page_list);
+
+ /* free it */
+ free(r);
+
+ return NO_ERROR;
+}
+
+status_t vmm_create_aspace(vmm_aspace_t **_aspace, const char *name, uint flags)
+{
+ DEBUG_ASSERT(_aspace);
+
+ vmm_aspace_t *aspace = malloc(sizeof(vmm_aspace_t));
+ if (!aspace)
+ return ERR_NO_MEMORY;
+
+ if (name)
+ strlcpy(aspace->name, name, sizeof(aspace->name));
+ else
+ strlcpy(aspace->name, "unnamed", sizeof(aspace->name));
+
+ if (flags & VMM_FLAG_ASPACE_KERNEL) {
+ aspace->base = KERNEL_ASPACE_BASE;
+ aspace->size = KERNEL_ASPACE_SIZE;
+ } else {
+ aspace->base = USER_ASPACE_BASE;
+ aspace->size = USER_ASPACE_SIZE;
+ }
+
+ list_clear_node(&aspace->node);
+ list_initialize(&aspace->region_list);
+
+ mutex_acquire(&vmm_lock);
+ list_add_head(&aspace_list, &aspace->node);
+ mutex_release(&vmm_lock);
+
+ *_aspace = aspace;
+
+ return NO_ERROR;
+}
+
+status_t vmm_free_aspace(vmm_aspace_t *aspace)
+{
+ DEBUG_ASSERT(aspace);
+
+ /* pop it out of the global aspace list */
+ mutex_acquire(&vmm_lock);
+ if (!list_in_list(&aspace->node)) {
+ mutex_release(&vmm_lock);
+ return ERR_INVALID_ARGS;
+ }
+ list_delete(&aspace->node);
+
+ /* free all of the regions */
+ struct list_node region_list = LIST_INITIAL_VALUE(region_list);
+
+ vmm_region_t *r;
+ while ((r = list_remove_head_type(&aspace->region_list, vmm_region_t, node))) {
+ /* add it to our tempoary list */
+ list_add_tail(®ion_list, &r->node);
+
+ /* unmap it */
+ arch_mmu_unmap(r->base, r->size / PAGE_SIZE);
+ }
+ mutex_release(&vmm_lock);
+
+ /* without the vmm lock held, free all of the pmm pages and the structure */
+ while ((r = list_remove_head_type(®ion_list, vmm_region_t, node))) {
+ /* return physical pages if any */
+ pmm_free(&r->page_list);
+
+ /* free it */
+ free(r);
+ }
+
+ /* free the aspace */
+ free(aspace);
+
+ return NO_ERROR;
+}
+
+static void dump_region(const vmm_region_t *r)
+{
+ DEBUG_ASSERT(r);
+
+ printf("\tregion %p: name '%s' range 0x%lx - 0x%lx size 0x%zx flags 0x%x mmu_flags 0x%x\n",
+ r, r->name, r->base, r->base + r->size - 1, r->size, r->flags, r->arch_mmu_flags);
+}
+
+static void dump_aspace(const vmm_aspace_t *a)
+{
+ DEBUG_ASSERT(a);
+
+ printf("aspace %p: name '%s' range 0x%lx - 0x%lx size 0x%zx flags 0x%x\n",
+ a, a->name, a->base, a->base + a->size - 1, a->size, a->flags);
+
+ printf("regions:\n");
+ vmm_region_t *r;
+ list_for_every_entry(&a->region_list, r, vmm_region_t, node) {
+ dump_region(r);
+ }
+}
+
+static int cmd_vmm(int argc, const cmd_args *argv)
+{
+ if (argc < 2) {
+notenoughargs:
+ printf("not enough arguments\n");
+usage:
+ printf("usage:\n");
+ printf("%s aspaces\n", argv[0].str);
+ printf("%s alloc <size> <align_pow2>\n", argv[0].str);
+ printf("%s alloc_physical <paddr> <size> <align_pow2>\n", argv[0].str);
+ printf("%s alloc_contig <size> <align_pow2>\n", argv[0].str);
+ printf("%s create_aspace\n", argv[0].str);
+ return ERR_GENERIC;
+ }
+
+ if (!strcmp(argv[1].str, "aspaces")) {
+ vmm_aspace_t *a;
+ list_for_every_entry(&aspace_list, a, vmm_aspace_t, node) {
+ dump_aspace(a);
+ }
+ } else if (!strcmp(argv[1].str, "alloc")) {
+ if (argc < 4) goto notenoughargs;
+
+ void *ptr = (void *)0x99;
+ status_t err = vmm_alloc(vmm_get_kernel_aspace(), "alloc test", argv[2].u, &ptr, argv[3].u, 0, 0);
+ printf("vmm_alloc returns %d, ptr %p\n", err, ptr);
+ } else if (!strcmp(argv[1].str, "alloc_physical")) {
+ if (argc < 4) goto notenoughargs;
+
+ void *ptr = (void *)0x99;
+ status_t err = vmm_alloc_physical(vmm_get_kernel_aspace(), "physical test", argv[3].u, &ptr, argv[4].u, argv[2].u, 0, ARCH_MMU_FLAG_UNCACHED_DEVICE);
+ printf("vmm_alloc_physical returns %d, ptr %p\n", err, ptr);
+ } else if (!strcmp(argv[1].str, "alloc_contig")) {
+ if (argc < 4) goto notenoughargs;
+
+ void *ptr = (void *)0x99;
+ status_t err = vmm_alloc_contiguous(vmm_get_kernel_aspace(), "contig test", argv[2].u, &ptr, argv[3].u, 0, 0);
+ printf("vmm_alloc_contig returns %d, ptr %p\n", err, ptr);
+ } else if (!strcmp(argv[1].str, "create_aspace")) {
+ vmm_aspace_t *aspace;
+ status_t err = vmm_create_aspace(&aspace, "test", 0);
+ printf("vmm_create_aspace returns %d, aspace %p\n", err, aspace);
+ } else {
+ printf("unknown command\n");
+ goto usage;
+ }
+
+ return NO_ERROR;
+}
+
+STATIC_COMMAND_START
+#if LK_DEBUGLEVEL > 0
+STATIC_COMMAND("vmm", "virtual memory manager", &cmd_vmm)
+#endif
+STATIC_COMMAND_END(vmm);
+