opensbi/lib/sbi/sbi_tlb.c

/*
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Copyright (c) 2019 Western Digital Corporation or its affiliates.
 *
 * Authors:
 *   Atish Patra <atish.patra@wdc.com>
 *   Anup Patel <anup.patel@wdc.com>
 */

#include <sbi/riscv_asm.h>
#include <sbi/riscv_atomic.h>
#include <sbi/riscv_barrier.h>
#include <sbi/sbi_error.h>
#include <sbi/sbi_fifo.h>
#include <sbi/sbi_hart.h>
#include <sbi/sbi_scratch.h>
#include <sbi/sbi_tlb.h>
#include <sbi/sbi_hfence.h>
#include <sbi/sbi_string.h>
#include <sbi/sbi_console.h>
#include <sbi/sbi_platform.h>

static unsigned long tlb_sync_off;
static unsigned long tlb_fifo_off;
static unsigned long tlb_fifo_mem_off;
static unsigned long tlb_range_flush_limit;

static void sbi_tlb_flush_all(void)
{
	__asm__ __volatile("sfence.vma");
}

static void sbi_tlb_fifo_hfence_vvma(struct sbi_tlb_info *tinfo)
{
	unsigned long start = tinfo->start;
	unsigned long size  = tinfo->size;
	unsigned long i;

	if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) {
		__sbi_hfence_vvma_all();
		return;
	}

	for (i = 0; i < size; i += PAGE_SIZE) {
		__sbi_hfence_vvma_va(start+i);
	}
}

static void sbi_tlb_fifo_hfence_gvma(struct sbi_tlb_info *tinfo)
{
	unsigned long start = tinfo->start;
	unsigned long size  = tinfo->size;
	unsigned long i;

	if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) {
		__sbi_hfence_gvma_all();
		return;
	}

	for (i = 0; i < size; i += PAGE_SIZE) {
		__sbi_hfence_gvma_gpa(start+i);
	}
}

static void sbi_tlb_fifo_sfence_vma(struct sbi_tlb_info *tinfo)
{
	unsigned long start = tinfo->start;
	unsigned long size  = tinfo->size;
	unsigned long i;

	if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) {
		sbi_tlb_flush_all();
		return;
	}

	for (i = 0; i < size; i += PAGE_SIZE) {
		__asm__ __volatile__("sfence.vma %0"
				     :
				     : "r"(start + i)
				     : "memory");
	}
}

static void sbi_tlb_fifo_hfence_vvma_asid(struct sbi_tlb_info *tinfo)
{
	unsigned long start = tinfo->start;
	unsigned long size  = tinfo->size;
	unsigned long asid  = tinfo->asid;
	unsigned long i;

	if (start == 0 && size == 0) {
		__sbi_hfence_vvma_all();
		return;
	}

	if (size == SBI_TLB_FLUSH_ALL) {
		__sbi_hfence_vvma_asid(asid);
		return;
	}

	for (i = 0; i < size; i += PAGE_SIZE) {
		__sbi_hfence_vvma_asid_va(asid, start + i);
	}
}

static void sbi_tlb_fifo_hfence_gvma_vmid(struct sbi_tlb_info *tinfo)
{
	unsigned long start = tinfo->start;
	unsigned long size  = tinfo->size;
	unsigned long vmid  = tinfo->asid;
	unsigned long i;

	if (start == 0 && size == 0) {
		__sbi_hfence_gvma_all();
		return;
	}

	if (size == SBI_TLB_FLUSH_ALL) {
		__sbi_hfence_gvma_vmid(vmid);
		return;
	}

	for (i = 0; i < size; i += PAGE_SIZE) {
		__sbi_hfence_gvma_vmid_gpa(vmid, start+i);
	}
}

static void sbi_tlb_fifo_sfence_vma_asid(struct sbi_tlb_info *tinfo)
{
	unsigned long start = tinfo->start;
	unsigned long size  = tinfo->size;
	unsigned long asid  = tinfo->asid;
	unsigned long i;

	if (start == 0 && size == 0) {
		sbi_tlb_flush_all();
		return;
	}

	/* Flush entire MM context for a given ASID */
	if (size == SBI_TLB_FLUSH_ALL) {
		__asm__ __volatile__("sfence.vma x0, %0"
				     :
				     : "r"(asid)
				     : "memory");
		return;
	}

	for (i = 0; i < size; i += PAGE_SIZE) {
		__asm__ __volatile__("sfence.vma %0, %1"
				     :
				     : "r"(start + i), "r"(asid)
				     : "memory");
	}
}

static void sbi_tlb_local_flush(struct sbi_tlb_info *tinfo)
{
	switch (tinfo->type) {
	case SBI_TLB_FLUSH_VMA:
		sbi_tlb_fifo_sfence_vma(tinfo);
		break;
	case SBI_TLB_FLUSH_VMA_ASID:
		sbi_tlb_fifo_sfence_vma_asid(tinfo);
		break;
	case SBI_TLB_FLUSH_GVMA:
		sbi_tlb_fifo_hfence_gvma(tinfo);
		break;
	case SBI_TLB_FLUSH_GVMA_VMID:
		sbi_tlb_fifo_hfence_gvma_vmid(tinfo);
		break;
	case SBI_TLB_FLUSH_VVMA:
		sbi_tlb_fifo_hfence_vvma(tinfo);
		break;
	case SBI_TLB_FLUSH_VVMA_ASID:
		sbi_tlb_fifo_hfence_vvma_asid(tinfo);
		break;
	case SBI_ITLB_FLUSH:
		__asm__ __volatile("fence.i");
		break;
	default:
		sbi_printf("Invalid tlb flush request type [%lu]\n",
			   tinfo->type);
	}
	return;
}

static void sbi_tlb_entry_process(struct sbi_scratch *scratch,
				  struct sbi_tlb_info *tinfo)
{
	u32 i;
	u64 m;
	struct sbi_scratch *rscratch = NULL;
	unsigned long *rtlb_sync = NULL;

	sbi_tlb_local_flush(tinfo);
	for (i = 0, m = tinfo->shart_mask; m; i++, m >>= 1) {
		if (!(m & 1UL))
			continue;

		rscratch = sbi_hart_id_to_scratch(scratch, i);
		rtlb_sync = sbi_scratch_offset_ptr(rscratch, tlb_sync_off);
		while (atomic_raw_xchg_ulong(rtlb_sync, 1)) ;
	}
}

static void sbi_tlb_fifo_process_count(struct sbi_scratch *scratch, int count)
{
	struct sbi_tlb_info tinfo;
	u32 deq_count = 0;
	struct sbi_fifo *tlb_fifo =
			sbi_scratch_offset_ptr(scratch, tlb_fifo_off);

	while (!sbi_fifo_dequeue(tlb_fifo, &tinfo)) {
		sbi_tlb_entry_process(scratch, &tinfo);
		deq_count++;
		if (deq_count > count)
			break;

	}
}

void sbi_tlb_fifo_process(struct sbi_scratch *scratch)
{
	struct sbi_tlb_info tinfo;
	struct sbi_fifo *tlb_fifo =
			sbi_scratch_offset_ptr(scratch, tlb_fifo_off);

	while (!sbi_fifo_dequeue(tlb_fifo, &tinfo))
		sbi_tlb_entry_process(scratch, &tinfo);
}

void sbi_tlb_fifo_sync(struct sbi_scratch *scratch)
{
	unsigned long *tlb_sync =
			sbi_scratch_offset_ptr(scratch, tlb_sync_off);

	while (!atomic_raw_xchg_ulong(tlb_sync, 0)) {
		/*
		 * While we are waiting for remote hart to set the sync,
		 * consume fifo requests to avoid deadlock.
		 */
		sbi_tlb_fifo_process_count(scratch, 1);
	}

	return;
}

static inline int __sbi_tlb_fifo_range_check(struct sbi_tlb_info *curr,
					     struct sbi_tlb_info *next)
{
	unsigned long curr_end;
	unsigned long next_end;
	int ret = SBI_FIFO_UNCHANGED;

	if (!curr || !next)
		return ret;

	next_end = next->start + next->size;
	curr_end = curr->start + curr->size;
	if (next->start <= curr->start && next_end > curr_end) {
		curr->start = next->start;
		curr->size  = next->size;
		curr->shart_mask = curr->shart_mask | next->shart_mask;
		ret	    = SBI_FIFO_UPDATED;
	} else if (next->start >= curr->start && next_end <= curr_end) {
		curr->shart_mask = curr->shart_mask | next->shart_mask;
		ret		 = SBI_FIFO_SKIP;
	}

	return ret;
}

/**
 * Call back to decide if an inplace fifo update is required or next entry can
 * can be skipped. Here are the different cases that are being handled.
 *
 * Case1:
 *	if next flush request range lies within one of the existing entry, skip
 *	the next entry.
 * Case2:
 *	if flush request range in current fifo entry lies within next flush
 *	request, update the current entry.
 *
 * Note:
 *	We can not issue a fifo reset anymore if a complete vma flush is requested.
 *	This is because we are queueing FENCE.I requests as well now.
 *	To ease up the pressure in enqueue/fifo sync path, try to dequeue 1 element
 *	before continuing the while loop. This method is preferred over wfi/ipi because
 *	of MMIO cost involved in later method.
 */
static int sbi_tlb_fifo_update_cb(void *in, void *data)
{
	struct sbi_tlb_info *curr;
	struct sbi_tlb_info *next;
	int ret = SBI_FIFO_UNCHANGED;

	if (!in || !data)
		return ret;

	curr = (struct sbi_tlb_info *)data;
	next = (struct sbi_tlb_info *)in;

	if (next->type == SBI_TLB_FLUSH_VMA_ASID &&
	    curr->type == SBI_TLB_FLUSH_VMA_ASID) {
		if (next->asid == curr->asid)
			ret = __sbi_tlb_fifo_range_check(curr, next);
	} else if (next->type == SBI_TLB_FLUSH_VMA &&
		   curr->type == SBI_TLB_FLUSH_VMA) {
			ret = __sbi_tlb_fifo_range_check(curr, next);
	}

	return ret;
}

int sbi_tlb_fifo_update(struct sbi_scratch *rscratch, u32 hartid, void *data)
{
	int ret;
	struct sbi_fifo *tlb_fifo_r;
	struct sbi_scratch *lscratch;
	struct sbi_tlb_info *tinfo = data;
	u32 curr_hartid = sbi_current_hartid();

	/*
	 * If address range to flush is too big then simply
	 * upgrade it to flush all because we can only flush
	 * 4KB at a time.
	 */
	if (tinfo->size > tlb_range_flush_limit) {
		tinfo->start = 0;
		tinfo->size = SBI_TLB_FLUSH_ALL;
	}

	/*
	 * If the request is to queue a tlb flush entry for itself
	 * then just do a local flush and return;
	 */
	if (hartid == curr_hartid) {
		sbi_tlb_local_flush(tinfo);
		return -1;
	}

	lscratch = sbi_hart_id_to_scratch(rscratch, curr_hartid);
	tlb_fifo_r = sbi_scratch_offset_ptr(rscratch, tlb_fifo_off);

	ret = sbi_fifo_inplace_update(tlb_fifo_r, data, sbi_tlb_fifo_update_cb);
	if (ret != SBI_FIFO_UNCHANGED) {
		return 1;
	}

	while (sbi_fifo_enqueue(tlb_fifo_r, data) < 0) {
		/**
		 * For now, Busy loop until there is space in the fifo.
		 * There may be case where target hart is also
		 * enqueue in source hart's fifo. Both hart may busy
		 * loop leading to a deadlock.
		 * TODO: Introduce a wait/wakeup event mechanism to handle
		 * this properly.
		 */
		sbi_tlb_fifo_process_count(lscratch, 1);
		sbi_dprintf(rscratch, "hart%d: hart%d tlb fifo full\n",
			    curr_hartid, hartid);
	}

	return 0;
}

int sbi_tlb_fifo_init(struct sbi_scratch *scratch, bool cold_boot)
{
	void *tlb_mem;
	unsigned long *tlb_sync;
	struct sbi_fifo *tlb_q;
	const struct sbi_platform *plat = sbi_platform_ptr(scratch);

	if (cold_boot) {
		tlb_sync_off = sbi_scratch_alloc_offset(sizeof(*tlb_sync),
							    "IPI_TLB_SYNC");
		if (!tlb_sync_off)
			return SBI_ENOMEM;
		tlb_fifo_off = sbi_scratch_alloc_offset(sizeof(*tlb_q),
							    "IPI_TLB_FIFO");
		if (!tlb_fifo_off) {
			sbi_scratch_free_offset(tlb_sync_off);
			return SBI_ENOMEM;
		}
		tlb_fifo_mem_off = sbi_scratch_alloc_offset(
				SBI_TLB_FIFO_NUM_ENTRIES * SBI_TLB_INFO_SIZE,
				"IPI_TLB_FIFO_MEM");
		if (!tlb_fifo_mem_off) {
			sbi_scratch_free_offset(tlb_fifo_off);
			sbi_scratch_free_offset(tlb_sync_off);
			return SBI_ENOMEM;
		}
		tlb_range_flush_limit = sbi_platform_tlbr_flush_limit(plat);
	} else {
		if (!tlb_sync_off ||
		    !tlb_fifo_off ||
		    !tlb_fifo_mem_off)
			return SBI_ENOMEM;
	}

	tlb_sync = sbi_scratch_offset_ptr(scratch, tlb_sync_off);
	tlb_q = sbi_scratch_offset_ptr(scratch, tlb_fifo_off);
	tlb_mem = sbi_scratch_offset_ptr(scratch, tlb_fifo_mem_off);

	*tlb_sync = 0;

	sbi_fifo_init(tlb_q, tlb_mem,
		      SBI_TLB_FIFO_NUM_ENTRIES, SBI_TLB_INFO_SIZE);

	return 0;
}