path: root/core/arch/arm/mm/pgt_cache.c

// SPDX-License-Identifier: BSD-2-Clause
/*
 * Copyright (c) 2016, Linaro Limited
 */

#include <assert.h>
#include <kernel/mutex.h>
#include <kernel/tee_misc.h>
#include <mm/core_mmu.h>
#include <mm/pgt_cache.h>
#include <mm/tee_pager.h>
#include <stdlib.h>
#include <trace.h>
#include <util.h>

/*
 * With pager enabled we allocate page table from the pager.
 *
 * For LPAE each page table is a complete page which is allocated and freed
 * using the interface provided by the pager.
 *
 * For compat v7 page tables there's room for four page table in one page
 * so we need to keep track of how much of an allocated page is used. When
 * a page is completely unused it's returned to the pager.
 *
 * With pager disabled we have a static allocation of page tables instead.
 *
 * In all cases we limit the number of active page tables to
 * PGT_CACHE_SIZE.  This pool of page tables are shared between all
 * threads. In case a thread can't allocate the needed number of pager
 * tables it will release all its current tables and wait for some more to
 * be freed. A threads allocated tables are freed each time a TA is
 * unmapped so each thread should be able to allocate the needed tables in
 * turn if needed.
 */

#if defined(CFG_WITH_PAGER) && !defined(CFG_WITH_LPAE)
struct pgt_parent {
	size_t num_used;
	struct pgt_cache pgt_cache;
};

static struct pgt_parent pgt_parents[PGT_CACHE_SIZE / PGT_NUM_PGT_PER_PAGE];
#else

static struct pgt_cache pgt_free_list = SLIST_HEAD_INITIALIZER(pgt_free_list);
#endif

#ifdef CFG_PAGED_USER_TA
/*
 * When a user TA context is temporarily unmapped the used struct pgt's of
 * the context (page tables holding valid physical pages) are saved in this
 * cache in the hope that some of the valid physical pages may still be
 * valid when the context is mapped again.
 */
static struct pgt_cache pgt_cache_list = SLIST_HEAD_INITIALIZER(pgt_cache_list);
#endif

static struct pgt pgt_entries[PGT_CACHE_SIZE];

static struct mutex pgt_mu = MUTEX_INITIALIZER;
static struct condvar pgt_cv = CONDVAR_INITIALIZER;

#if defined(CFG_WITH_PAGER) && defined(CFG_WITH_LPAE)
void pgt_init(void)
{
	size_t n;

	for (n = 0; n < PGT_CACHE_SIZE; n++) {
		struct pgt *p = pgt_entries + n;

		p->tbl = tee_pager_alloc(PGT_SIZE, TEE_MATTR_LOCKED);
		SLIST_INSERT_HEAD(&pgt_free_list, p, link);
	}
}
#elif defined(CFG_WITH_PAGER) && !defined(CFG_WITH_LPAE)
void pgt_init(void)
{
	size_t n;
	size_t m;

	COMPILE_TIME_ASSERT(PGT_CACHE_SIZE % PGT_NUM_PGT_PER_PAGE == 0);
	COMPILE_TIME_ASSERT(PGT_SIZE * PGT_NUM_PGT_PER_PAGE == SMALL_PAGE_SIZE);

	for (n = 0; n < ARRAY_SIZE(pgt_parents); n++) {
		uint8_t *tbl = tee_pager_alloc(SMALL_PAGE_SIZE,
					       TEE_MATTR_LOCKED);

		SLIST_INIT(&pgt_parents[n].pgt_cache);
		for (m = 0; m < PGT_NUM_PGT_PER_PAGE; m++) {
			struct pgt *p = pgt_entries +
					n * PGT_NUM_PGT_PER_PAGE + m;

			p->tbl = tbl + m * PGT_SIZE;
			p->parent = &pgt_parents[n];
			SLIST_INSERT_HEAD(&pgt_parents[n].pgt_cache, p, link);
		}
	}
}
#else
void pgt_init(void)
{
	/*
	 * We're putting this in .nozi.* instead of .bss because .nozi.* already
	 * has a large alignment, while .bss has a small alignment. The current
	 * link script is optimized for small alignment in .bss
	 */
	static uint8_t pgt_tables[PGT_CACHE_SIZE][PGT_SIZE]
			__aligned(PGT_SIZE) __section(".nozi.pgt_cache");
	size_t n;

	for (n = 0; n < ARRAY_SIZE(pgt_tables); n++) {
		struct pgt *p = pgt_entries + n;

		p->tbl = pgt_tables[n];
		SLIST_INSERT_HEAD(&pgt_free_list, p, link);
	}
}
#endif

#if defined(CFG_WITH_LPAE) || !defined(CFG_WITH_PAGER)
static struct pgt *pop_from_free_list(void)
{
	struct pgt *p = SLIST_FIRST(&pgt_free_list);

	if (p) {
		SLIST_REMOVE_HEAD(&pgt_free_list, link);
		memset(p->tbl, 0, PGT_SIZE);
	}
	return p;
}

static void push_to_free_list(struct pgt *p)
{
	SLIST_INSERT_HEAD(&pgt_free_list, p, link);
#if defined(CFG_WITH_PAGER)
	tee_pager_release_phys(p->tbl, PGT_SIZE);
#endif
}
#else
static struct pgt *pop_from_free_list(void)
{
	size_t n;

	for (n = 0; n < ARRAY_SIZE(pgt_parents); n++) {
		struct pgt *p = SLIST_FIRST(&pgt_parents[n].pgt_cache);

		if (p) {
			SLIST_REMOVE_HEAD(&pgt_parents[n].pgt_cache, link);
			pgt_parents[n].num_used++;
			memset(p->tbl, 0, PGT_SIZE);
			return p;
		}
	}
	return NULL;
}

static void push_to_free_list(struct pgt *p)
{
	SLIST_INSERT_HEAD(&p->parent->pgt_cache, p, link);
	assert(p->parent->num_used > 0);
	p->parent->num_used--;
	if (!p->parent->num_used) {
		vaddr_t va = (vaddr_t)p->tbl & ~SMALL_PAGE_MASK;

		tee_pager_release_phys((void *)va, SMALL_PAGE_SIZE);
	}
}
#endif

#ifdef CFG_PAGED_USER_TA
static void push_to_cache_list(struct pgt *pgt)
{
	SLIST_INSERT_HEAD(&pgt_cache_list, pgt, link);
}

static bool match_pgt(struct pgt *pgt, vaddr_t vabase, void *ctx)
{
	return pgt->ctx == ctx && pgt->vabase == vabase;
}

static struct pgt *pop_from_cache_list(vaddr_t vabase, void *ctx)
{
	struct pgt *pgt;
	struct pgt *p;

	pgt = SLIST_FIRST(&pgt_cache_list);
	if (!pgt)
		return NULL;
	if (match_pgt(pgt, vabase, ctx)) {
		SLIST_REMOVE_HEAD(&pgt_cache_list, link);
		return pgt;
	}

	while (true) {
		p = SLIST_NEXT(pgt, link);
		if (!p)
			break;
		if (match_pgt(p, vabase, ctx)) {
			SLIST_REMOVE_AFTER(pgt, link);
			break;
		}
		pgt = p;
	}
	return p;
}

static struct pgt *pop_least_used_from_cache_list(void)
{
	struct pgt *pgt;
	struct pgt *p_prev = NULL;
	size_t least_used;

	pgt = SLIST_FIRST(&pgt_cache_list);
	if (!pgt)
		return NULL;
	if (!pgt->num_used_entries)
		goto out;
	least_used = pgt->num_used_entries;

	while (true) {
		if (!SLIST_NEXT(pgt, link))
			break;
		if (SLIST_NEXT(pgt, link)->num_used_entries <= least_used) {
			p_prev = pgt;
			least_used = SLIST_NEXT(pgt, link)->num_used_entries;
		}
		pgt = SLIST_NEXT(pgt, link);
	}

out:
	if (p_prev) {
		pgt = SLIST_NEXT(p_prev, link);
		SLIST_REMOVE_AFTER(p_prev, link);
	} else {
		pgt = SLIST_FIRST(&pgt_cache_list);
		SLIST_REMOVE_HEAD(&pgt_cache_list, link);
	}
	return pgt;
}

static void pgt_free_unlocked(struct pgt_cache *pgt_cache, bool save_ctx)
{
	while (!SLIST_EMPTY(pgt_cache)) {
		struct pgt *p = SLIST_FIRST(pgt_cache);

		SLIST_REMOVE_HEAD(pgt_cache, link);
		if (save_ctx && p->num_used_entries) {
			push_to_cache_list(p);
		} else {
			tee_pager_pgt_save_and_release_entries(p);
			assert(!p->num_used_entries);
			p->ctx = NULL;
			p->vabase = 0;

			push_to_free_list(p);
		}
	}
}

static struct pgt *pop_from_some_list(vaddr_t vabase, void *ctx)
{
	struct pgt *p = pop_from_cache_list(vabase, ctx);

	if (p)
		return p;
	p = pop_from_free_list();
	if (!p) {
		p = pop_least_used_from_cache_list();
		if (!p)
			return NULL;
		tee_pager_pgt_save_and_release_entries(p);
		memset(p->tbl, 0, PGT_SIZE);
	}
	assert(!p->num_used_entries);
	p->ctx = ctx;
	p->vabase = vabase;
	return p;
}

void pgt_flush_ctx(struct tee_ta_ctx *ctx)
{
	struct pgt *p;
	struct pgt *pp = NULL;

	mutex_lock(&pgt_mu);

	while (true) {
		p = SLIST_FIRST(&pgt_cache_list);
		if (!p)
			goto out;
		if (p->ctx != ctx)
			break;
		SLIST_REMOVE_HEAD(&pgt_cache_list, link);
		tee_pager_pgt_save_and_release_entries(p);
		assert(!p->num_used_entries);
		p->ctx = NULL;
		p->vabase = 0;
		push_to_free_list(p);
	}

	pp = p;
	while (true) {
		p = SLIST_NEXT(pp, link);
		if (!p)
			break;
		if (p->ctx == ctx) {
			SLIST_REMOVE_AFTER(pp, link);
			tee_pager_pgt_save_and_release_entries(p);
			assert(!p->num_used_entries);
			p->ctx = NULL;
			p->vabase = 0;
			push_to_free_list(p);
		} else {
			pp = p;
		}
	}

out:
	mutex_unlock(&pgt_mu);
}

static void flush_pgt_entry(struct pgt *p)
{
	tee_pager_pgt_save_and_release_entries(p);
	assert(!p->num_used_entries);
	p->ctx = NULL;
	p->vabase = 0;
}

static bool pgt_entry_matches(struct pgt *p, void *ctx, vaddr_t begin,
			      vaddr_t last)
{
	if (!p)
		return false;
	if (p->ctx != ctx)
		return false;
	if (last <= begin)
		return false;
	if (!core_is_buffer_inside(p->vabase, SMALL_PAGE_SIZE, begin,
				   last - begin))
		return false;

	return true;
}

static void flush_ctx_range_from_list(struct pgt_cache *pgt_cache, void *ctx,
				      vaddr_t begin, vaddr_t last)
{
	struct pgt *p;
	struct pgt *next_p;

	/*
	 * Do the special case where the first element in the list is
	 * removed first.
	 */
	p = SLIST_FIRST(pgt_cache);
	while (pgt_entry_matches(p, ctx, begin, last)) {
		flush_pgt_entry(p);
		SLIST_REMOVE_HEAD(pgt_cache, link);
		push_to_free_list(p);
		p = SLIST_FIRST(pgt_cache);
	}

	/*
	 * p either points to the first element in the list or it's NULL,
	 * if NULL the list is empty and we're done.
	 */
	if (!p)
		return;

	/*
	 * Do the common case where the next element in the list is
	 * removed.
	 */
	while (true) {
		next_p = SLIST_NEXT(p, link);
		if (!next_p)
			break;
		if (pgt_entry_matches(next_p, ctx, begin, last)) {
			flush_pgt_entry(next_p);
			SLIST_REMOVE_AFTER(p, link);
			push_to_free_list(next_p);
			continue;
		}

		p = SLIST_NEXT(p, link);
	}
}

void pgt_flush_ctx_range(struct pgt_cache *pgt_cache, void *ctx,
			 vaddr_t begin, vaddr_t last)
{
	mutex_lock(&pgt_mu);

	flush_ctx_range_from_list(pgt_cache, ctx, begin, last);
	flush_ctx_range_from_list(&pgt_cache_list, ctx, begin, last);

	condvar_broadcast(&pgt_cv);
	mutex_unlock(&pgt_mu);
}

static void transfer_tables(struct pgt_cache *pgt_cache, void *old_ctx,
			    vaddr_t old_va, void *new_ctx, vaddr_t new_va,
			    size_t size)
{
	const size_t pgtsize = CORE_MMU_PGDIR_SIZE;
	const vaddr_t new_base = ROUNDDOWN(new_va, pgtsize);
	const vaddr_t old_base = ROUNDDOWN(old_va, pgtsize);
	const size_t num_new_pgt = (size - 1 + new_va - new_base) / pgtsize + 1;
	const size_t num_old_pgt = (size - 1 + old_va - old_base) / pgtsize + 1;
	struct pgt *new_pgt[num_new_pgt];
	struct pgt *old_pgt[num_old_pgt];
	struct pgt *pgt;
	size_t n;

	/*
	 * Fill in new_pgt based on pgt_cache. Note that the pages should
	 * already have been allocated.
	 */
	SLIST_FOREACH(pgt, pgt_cache, link) {
		if (pgt->vabase < new_base)
			continue;
		n = (pgt->vabase - new_base) / pgtsize;
		if (n < num_new_pgt)
			new_pgt[n] = pgt;
	}
	for (n = 0; n < num_new_pgt; n++) {
		assert(new_pgt[n]);
		assert(new_pgt[n]->ctx == new_ctx);
	}

	mutex_lock(&pgt_mu);

	/* Extract the array of pgts that need their content transferred */
	for (n = 0; n < num_old_pgt; n++) {
		/*
		 * If the pgt isn't in the cache list there's nothing to
		 * transfer, so NULL here is OK.
		 */
		old_pgt[n] = pop_from_cache_list(old_base + n * pgtsize,
						 old_ctx);
	}

	tee_pager_transfer_uta_region(to_user_ta_ctx(old_ctx), old_va,
				      to_user_ta_ctx(new_ctx), new_va, new_pgt,
				      size);

	for (n = 0; n < num_old_pgt; n++) {
		if (!old_pgt[n])
			continue;

		if (old_pgt[n]->num_used_entries)
			push_to_cache_list(old_pgt[n]);
		else
			push_to_free_list(old_pgt[n]);
	}

	mutex_unlock(&pgt_mu);
}

void pgt_transfer(struct pgt_cache *pgt_cache, void *old_ctx, vaddr_t old_va,
		  void *new_ctx, vaddr_t new_va, size_t size)
{
	if (size)
		transfer_tables(pgt_cache, old_ctx, old_va, new_ctx,
				new_va, size);
}

#else /*!CFG_PAGED_USER_TA*/

static void pgt_free_unlocked(struct pgt_cache *pgt_cache,
			      bool save_ctx __unused)
{
	while (!SLIST_EMPTY(pgt_cache)) {
		struct pgt *p = SLIST_FIRST(pgt_cache);

		SLIST_REMOVE_HEAD(pgt_cache, link);
		push_to_free_list(p);
	}
}

static struct pgt *pop_from_some_list(vaddr_t vabase __unused,
				      void *ctx __unused)
{
	return pop_from_free_list();
}
#endif /*!CFG_PAGED_USER_TA*/

static bool pgt_alloc_unlocked(struct pgt_cache *pgt_cache, void *ctx,
			       vaddr_t begin, vaddr_t last)
{
	const vaddr_t base = ROUNDDOWN(begin, CORE_MMU_PGDIR_SIZE);
	const size_t num_tbls = ((last - base) >> CORE_MMU_PGDIR_SHIFT) + 1;
	size_t n = 0;
	struct pgt *p;
	struct pgt *pp = NULL;

	while (n < num_tbls) {
		p = pop_from_some_list(base + n * CORE_MMU_PGDIR_SIZE, ctx);
		if (!p) {
			pgt_free_unlocked(pgt_cache, ctx);
			return false;
		}

		if (pp)
			SLIST_INSERT_AFTER(pp, p, link);
		else
			SLIST_INSERT_HEAD(pgt_cache, p, link);
		pp = p;
		n++;
	}

	return true;
}

void pgt_alloc(struct pgt_cache *pgt_cache, void *ctx,
	       vaddr_t begin, vaddr_t last)
{
	if (last <= begin)
		return;

	mutex_lock(&pgt_mu);

	pgt_free_unlocked(pgt_cache, ctx);
	while (!pgt_alloc_unlocked(pgt_cache, ctx, begin, last)) {
		DMSG("Waiting for page tables");
		condvar_broadcast(&pgt_cv);
		condvar_wait(&pgt_cv, &pgt_mu);
	}

	mutex_unlock(&pgt_mu);
}

void pgt_free(struct pgt_cache *pgt_cache, bool save_ctx)
{
	if (SLIST_EMPTY(pgt_cache))
		return;

	mutex_lock(&pgt_mu);

	pgt_free_unlocked(pgt_cache, save_ctx);

	condvar_broadcast(&pgt_cv);
	mutex_unlock(&pgt_mu);
}