include: Bump-up version to 0.7

This patch updates OpenSBI version to 0.7 as part of
release preparation.

Signed-off-by: Anup Patel <anup.patel@wdc.com>

Makefile

@@ -12,29 +12,37 @@
 # o  Do not print "Entering directory ...";
 MAKEFLAGS += -r --no-print-directory
 
+# Readlink -f requires GNU readlink
+ifeq ($(shell uname -s),Darwin)
+READLINK ?= greadlink
+else
+READLINK ?= readlink
+endif
+
 # Find out source, build, and install directories
 src_dir=$(CURDIR)
 ifdef O
- build_dir=$(shell readlink -f $(O))
+ build_dir=$(shell $(READLINK) -f $(O))
 else
  build_dir=$(CURDIR)/build
 endif
 ifeq ($(build_dir),$(CURDIR))
 $(error Build directory is same as source directory.)
 endif
+install_root_dir_default=$(CURDIR)/install
 ifdef I
- install_dir=$(shell readlink -f $(I))
+ install_root_dir=$(shell $(READLINK) -f $(I))
 else
- install_dir=$(CURDIR)/install
+ install_root_dir=$(install_root_dir_default)/usr
 endif
-ifeq ($(install_dir),$(CURDIR))
-$(error Install directory is same as source directory.)
+ifeq ($(install_root_dir),$(CURDIR))
+$(error Install root directory is same as source directory.)
 endif
-ifeq ($(install_dir),$(build_dir))
-$(error Install directory is same as build directory.)
+ifeq ($(install_root_dir),$(build_dir))
+$(error Install root directory is same as build directory.)
 endif
 ifdef PLATFORM_DIR
-  platform_dir_path=$(shell readlink -f $(PLATFORM_DIR))
+  platform_dir_path=$(shell $(READLINK) -f $(PLATFORM_DIR))
   ifdef PLATFORM
     platform_parent_dir=$(platform_dir_path)
   else
@@ -149,6 +157,33 @@ ifndef PLATFORM_RISCV_CODE_MODEL
   PLATFORM_RISCV_CODE_MODEL = medany
 endif
 
+# Setup install directories
+ifdef INSTALL_INCLUDE_PATH
+	install_include_path=$(INSTALL_INCLUDE_PATH)
+else
+	install_include_path=include
+endif
+ifdef INSTALL_LIB_PATH
+	install_lib_path=$(INSTALL_LIB_PATH)
+else
+	ifneq ($(origin INSTALL_LIB_SUBDIR), undefined)
+		install_lib_subdir=$(INSTALL_LIB_SUBDIR)
+	else
+		install_lib_subdir=$(PLATFORM_RISCV_ABI)
+	endif
+	install_lib_path=lib$(subst 32,,$(PLATFORM_RISCV_XLEN))/$(install_lib_subdir)
+endif
+ifdef INSTALL_FIRMWARE_PATH
+	install_firmware_path=$(INSTALL_FIRMWARE_PATH)
+else
+	install_firmware_path=share/opensbi/$(PLATFORM_RISCV_ABI)
+endif
+ifdef INSTALL_DOCS_PATH
+	install_docs_path=$(INSTALL_DOCS_PATH)
+else
+	install_docs_path=share/opensbi/docs
+endif
+
 # Setup compilation commands flags
 GENFLAGS	=	-I$(platform_src_dir)/include
 GENFLAGS	+=	-I$(include_dir)
@@ -159,7 +194,7 @@ GENFLAGS	+=	$(libsbiutils-genflags-y)
 GENFLAGS	+=	$(platform-genflags-y)
 GENFLAGS	+=	$(firmware-genflags-y)
 
-CFLAGS		=	-g -Wall -Werror -nostdlib -fno-strict-aliasing -O2
+CFLAGS		=	-g -Wall -Werror -ffreestanding -nostdlib -fno-strict-aliasing -O2
 CFLAGS		+=	-fno-omit-frame-pointer -fno-optimize-sibling-calls
 CFLAGS		+=	-mno-save-restore -mstrict-align
 CFLAGS		+=	-mabi=$(PLATFORM_RISCV_ABI) -march=$(PLATFORM_RISCV_ISA)
@@ -208,12 +243,12 @@ copy_file =  $(CMD_PREFIX)mkdir -p `dirname $(1)`; \
 	     echo " COPY      $(subst $(build_dir)/,,$(1))"; \
 	     cp -f $(2) $(1)
 inst_file =  $(CMD_PREFIX)mkdir -p `dirname $(1)`; \
-	     echo " INSTALL   $(subst $(install_dir)/,,$(1))"; \
+	     echo " INSTALL   $(subst $(install_root_dir)/,,$(1))"; \
 	     cp -f $(2) $(1)
 inst_file_list = $(CMD_PREFIX)if [ ! -z "$(4)" ]; then \
 	     mkdir -p $(1)/$(3); \
 	     for file in $(4) ; do \
-	     rel_file=`echo $$file | sed -e 's@$(2)/$(3)/@@'`; \
+	     rel_file=`echo $$file | sed -e 's@$(2)/$(subst $(install_firmware_path),platform,$(3))@@'`; \
 	     dest_file=$(1)"/"$(3)"/"`echo $$rel_file`; \
 	     dest_dir=`dirname $$dest_file`; \
 	     echo " INSTALL   "$(3)"/"`echo $$rel_file`; \
@@ -222,7 +257,7 @@ inst_file_list = $(CMD_PREFIX)if [ ! -z "$(4)" ]; then \
 	     done \
 	     fi
 inst_header_dir =  $(CMD_PREFIX)mkdir -p $(1); \
-	     echo " INSTALL   $(subst $(install_dir)/,,$(1))"; \
+	     echo " INSTALL   $(subst $(install_root_dir)/,,$(1))"; \
 	     cp -rf $(2) $(1)
 compile_cpp = $(CMD_PREFIX)mkdir -p `dirname $(1)`; \
 	     echo " CPP       $(subst $(build_dir)/,,$(1))"; \
@@ -378,26 +413,26 @@ install: $(install_targets-y)
 
 .PHONY: install_libsbi
 install_libsbi: $(build_dir)/lib/libsbi.a
-	$(call inst_header_dir,$(install_dir)/include,$(include_dir)/sbi)
-	$(call inst_file,$(install_dir)/lib/libsbi.a,$(build_dir)/lib/libsbi.a)
+	$(call inst_header_dir,$(install_root_dir)/$(install_include_path),$(include_dir)/sbi)
+	$(call inst_file,$(install_root_dir)/$(install_lib_path)/libsbi.a,$(build_dir)/lib/libsbi.a)
 
 .PHONY: install_libsbiutils
 install_libsbiutils: $(build_dir)/lib/libsbiutils.a
-	$(call inst_header_dir,$(install_dir)/include,$(include_dir)/sbi_utils)
-	$(call inst_file,$(install_dir)/lib/libsbiutils.a,$(build_dir)/lib/libsbiutils.a)
+	$(call inst_header_dir,$(install_root_dir)/$(install_include_path),$(include_dir)/sbi_utils)
+	$(call inst_file,$(install_root_dir)/$(install_lib_path)/libsbiutils.a,$(build_dir)/lib/libsbiutils.a)
 
 .PHONY: install_libplatsbi
 install_libplatsbi: $(platform_build_dir)/lib/libplatsbi.a $(build_dir)/lib/libsbi.a $(build_dir)/lib/libsbiutils.a
-	$(call inst_file,$(install_dir)/platform/$(platform_subdir)/lib/libplatsbi.a,$(platform_build_dir)/lib/libplatsbi.a)
+	$(call inst_file,$(install_root_dir)/$(install_lib_path)/opensbi/$(platform_subdir)/lib/libplatsbi.a,$(platform_build_dir)/lib/libplatsbi.a)
 
 .PHONY: install_firmwares
 install_firmwares: $(platform_build_dir)/lib/libplatsbi.a $(build_dir)/lib/libsbi.a $(build_dir)/lib/libsbiutils.a $(firmware-bins-path-y)
-	$(call inst_file_list,$(install_dir),$(build_dir),platform/$(platform_subdir)/firmware,$(firmware-elfs-path-y))
-	$(call inst_file_list,$(install_dir),$(build_dir),platform/$(platform_subdir)/firmware,$(firmware-bins-path-y))
+	$(call inst_file_list,$(install_root_dir),$(build_dir),$(install_firmware_path)/$(platform_subdir)/firmware,$(firmware-elfs-path-y))
+	$(call inst_file_list,$(install_root_dir),$(build_dir),$(install_firmware_path)/$(platform_subdir)/firmware,$(firmware-bins-path-y))
 
 .PHONY: install_docs
 install_docs: $(build_dir)/docs/latex/refman.pdf
-	$(call inst_file,$(install_dir)/docs/refman.pdf,$(build_dir)/docs/latex/refman.pdf)
+	$(call inst_file,$(install_root_dir)/$(install_docs_path)/refman.pdf,$(build_dir)/docs/latex/refman.pdf)
 
 # Rule for "make clean"
 .PHONY: clean
@@ -422,7 +457,7 @@ ifeq ($(build_dir),$(CURDIR)/build)
 	$(if $(V), @echo " RM        $(build_dir)")
 	$(CMD_PREFIX)rm -rf $(build_dir)
 endif
-ifeq ($(install_dir),$(CURDIR)/install)
-	$(if $(V), @echo " RM        $(install_dir)")
-	$(CMD_PREFIX)rm -rf $(install_dir)
+ifeq ($(install_root_dir),$(install_root_dir_default)/usr)
+	$(if $(V), @echo " RM        $(install_root_dir_default)")
+	$(CMD_PREFIX)rm -rf $(install_root_dir_default)
 endif

README.md

@@ -39,6 +39,27 @@ OpenSBI also provides several runtime firmware examples built using the platform
 *riscv-pk* bootloader (aka BBL) and enable the use of well-known bootloaders
 such as [U-Boot] (https://git.denx.de/u-boot.git).
 
+Supported SBI version
+---------------------
+Currently, OpenSBI fully supports SBI specification *v0.2*. OpenSBI also
+supports Hart State Management (HSM) SBI extension starting from OpenSBI v0.7.
+HSM extension allows S-mode software to boot all the harts a defined order
+rather than legacy method of random booting of harts. As a result, many
+required features such as CPU hotplug, kexec/kdump can also be supported easily
+in S-mode. HSM extension in OpenSBI is implemented in a non-backward compatible
+manner to reduce the maintenance burden and avoid confusion. That's why, any
+S-mode software using OpenSBI will not be able to boot more than 1 hart if HSM
+extension is not supported in S-mode.
+
+Linux kernel already supports SBI v0.2 and HSM SBI extension starting from
+**v5.7-rc1**. If you are using an Linux kernel older than **5.7-rc1** or any
+other S-mode software without HSM SBI extension, you should stick to OpenSBI
+v0.6 to boot all the harts. For a UMP systems, it doesn't matter.
+
+N.B. Any S-mode boot loader (i.e. U-Boot) doesn't need to support HSM extension,
+as it doesn't need to boot all the harts. The operating system should be
+capable enough to bring up all other non-booting harts using HSM extension.
+
 Required Toolchain
 ------------------
 

ThirdPartyNotices.md

@@ -16,4 +16,3 @@ The libfdt source code is disjunctively dual licensed (GPL-2.0+ or
 BSD-2-Clause). Some of this project code is used in OpenSBI under the terms of
 the BSD 2-Clause license. The full text of this license can be found in the
 file [COPYING.BSD](COPYING.BSD).
-

docs/external/coreboot.md

@@ -1,7 +1,10 @@
 OpenSBI as coreboot payload
-==============================
+===========================
 
-[coreboot](https://www.coreboot.org/) is a free/libre and open source firmware platform support multiple hardware architectures( x86, ARMv7, arm64, PowerPC64, MIPS and RISC-V) and diverse hardware models. In RISC-V world, coreboot currently support HiFive Unleashed with OpenSBI as a payload to boot GNU/Linux:
+[coreboot] is a free/libre and open source firmware platform support multiple
+hardware architectures(x86, ARMv7, arm64, PowerPC64, MIPS and RISC-V) and
+diverse hardware models. In RISC-V world, coreboot currently support HiFive
+Unleashed with OpenSBI as a payload to boot GNU/Linux:
 
 ```
 SiFive HiFive unleashed's original firmware boot process:
@@ -21,4 +24,9 @@ coreboot boot process:
                       +---------------------------------------------+-------------+-------+-+
 ```
 
-The upstreaming work is still in progress. There's a [documentation](https://github.com/hardenedlinux/embedded-iot_profile/blob/master/docs/riscv/hifiveunleashed_coreboot_notes-en.md) about how to build [out-of-tree code](https://github.com/hardenedlinux/coreboot-HiFiveUnleashed) to load OpenSBI.
+The upstreaming work is still in progress. There's a [documentation] about how
+to build [out-of-tree code] to load OpenSBI.
+
+[coreboot]: https://www.coreboot.org/
+[documentation]: https://github.com/hardenedlinux/embedded-iot_profile/blob/master/docs/riscv/hifiveunleashed_coreboot_notes-en.md
+[out-of-tree code]: https://github.com/hardenedlinux/coreboot-HiFiveUnleashed

docs/firmware/fw_jump.md

@@ -38,15 +38,14 @@ follows:
 * **FW_JUMP_FDT_ADDR** - Address where the *flattened device tree (FDT file)*
   passed by the prior booting stage will be placed in memory before executing
   the booting stage following the OpenSBI firmware. If this option is not
-  provided, then the OpenSBI firmware will pass zero as the FDT address to the
-  following booting stage.
+  provided, then the OpenSBI firmware will pass the FDT address passed by the
+  previous booting stage to the next booting stage.
 
 *FW_JUMP* Example
 -----------------
 
-The *[qemu/virt]* and *[qemu/sifive_u]* platforms illustrate how to configure
-and use a *FW_JUMP* firmware. Detailed information regarding these platforms
-can be found in the platform documentation files.
+The *[qemu/virt]* platform illustrates how to configure and use a *FW_JUMP*
+firmware. Detailed information regarding these platforms can be found in the
+platform documentation files.
 
 [qemu/virt]: ../platform/qemu_virt.md
-[qemu/sifive_u]: ../platform/qemu_sifive_u.md

docs/firmware/fw_payload.md

@@ -73,17 +73,17 @@ file. The parameters currently defined are as follows:
   stage or specified by the *FW_PAYLOAD_FDT_PATH* parameter and embedded in
   the *.text* section will be placed before executing the next booting stage,
   that is, the payload firmware. If this option is not provided, then the
-  firmware will pass zero as the FDT address to the next booting stage.
+  firmware will pass the FDT address passed by the previous booting stage
+  to the next booting stage.
 
 *FW_PAYLOAD* Example
 --------------------
 
-The *[qemu/virt]* and *[qemu/sifive_u]* platforms illustrate how to configure
-and use a *FW_PAYLOAD* firmware. Detailed information regarding these platforms
-can be found in the platform documentation files.
+The *[qemu/virt]* platforms illustrate how to configure and use a *FW_PAYLOAD*
+firmware. Detailed information regarding these platforms can be found in the
+platform documentation files.
 
 The *kendryte/k210* platform also enables a build of a *FW_PAYLOAD* using an
 internally defined device tree file (*FW_PAYLOAD_FDT*).
 
 [qemu/virt]: ../platform/qemu_virt.md
-[qemu/sifive_u]: ../platform/qemu_sifive_u.md

docs/firmware/payload_linux.md

@@ -7,4 +7,3 @@ provided as a payload to OpenSBI.
 
 Detailed examples can be found in both the [QEMU](../platform/qemu_virt.md)
 and the [HiFive Unleashed](../platform/sifive_fu540.md) platform guides.
-

docs/firmware/payload_uboot.md

@@ -7,34 +7,9 @@ environment. In the context of OpenSBI, U-Boot can be specified as a payload to
 the OpenSBI firmware, becoming the boot stage following the OpenSBI firmware
 execution.
 
-The current stable upstream code of U-Boot does not yet include all patches
-necessary to fully support OpenSBI. To use U-Boot as an OpenSBI payload, the
-following out-of-tree patch series must be applied to the upstream U-Boot source
-code:
-
-HiFive Unleashed support for U-Boot
-
-https://lists.denx.de/pipermail/u-boot/2019-February/358058.html
-
-This patch series enables a single CPU to execute U-Boot. As a result, the next
-stage boot code such as a Linux kernel can also only execute on a single CPU.
-U-Boot SMP support for RISC-V can be enabled with the following additional
-patches:
-
-https://lists.denx.de/pipermail/u-boot/2019-February/358393.html
-
 Building and Generating U-Boot images
 =====================================
 Please refer to the U-Boot build documentation for detailed instructions on
-how to build U-Boot images.
+how to build U-Boot image and boot high level operating systems from U-Boot
+prompt.
 
-Once U-Boot images are built, the Linux kernel image needs to be converted
-into a format that U-Boot understands:
-
-```
-<uboot-dir>/tools/mkimage -A riscv -O linux -T kernel -C none -a 0x80200000 -e 0x80200000 -n Linux -d \
-		<linux_build_directory>arch/riscv/boot/Image \
-		<linux_build_directory>/arch/riscv/boot/uImage
-```
-
-Copy the uImage to your tftpboot server path if network boot is required.

docs/library_usage.md

@@ -66,3 +66,24 @@ bootloader to service the following interrupts and traps:
 
 **Note:** external firmwares or bootloaders can be more conservative by
 forwarding all traps and interrupts to *sbi_trap_handler()*.
+
+Definitions of OpenSBI Data Types for the External Firmware
+-----------------------------------------------------------
+
+OpenSBI can be built as library using external firmware build system such as EDK2
+code base (The open source of UEFI firmware implementation) and linked with external
+firmware drivers based on the external firmware architecture.
+
+**OPENSBI_EXTERNAL_SBI_TYPES** identifier is introduced to *sbi_types.h* for selecting
+external header file during the build preprocess in order to define OpensSBI data types
+based on external firmware data type binding.
+For example, *bool* is declared as *int* in sbi_types.h. However in EDK2 build system,
+*bool* is declared as *BOOLEAN* which is defined as *unsigned char* data type.
+
+External firmware can define **OPENSBI_EXTERNAL_SBI_TYPES** in CFLAGS and specify it to the
+header file maintained in its code tree. However, the external build system has to address
+the additional include directory for the external header file based on its own build system.
+For example,
+*-D***OPENSBI_EXTERNAL_SBI_TYPES***=OpensbiTypes.h*
+Above tells *sbi_types.h* to refer to *OpensbiTypes.h* instead of using original definitions of
+data types.

docs/platform/andes-ae350.md

@@ -1,13 +1,14 @@
 Andes AE350 SoC Platform
 ========================
-The AE350 AXI/AHB-based platform N25(F)/NX25(F)/D25F/A25/AX25 CPU with
-level-one memories,interrupt controller, debug module, AXI and AHB Bus
-Matrix Controller, AXI-to-AHB Bridge and a collection of fundamentalAHB/APB
-bus IP components pre-integrated together as a system design.The high-quality
-and configurable AHB/APB IPs suites a majority embedded systems,
-and the verified platform serves as a starting point to jump start SoC designs.
+The AE350 AXI/AHB-based platform N25(F)/NX25(F)/D25F/A25/AX25 CPU with level-one
+memories,interrupt controller, debug module, AXI and AHB Bus Matrix Controller,
+AXI-to-AHB Bridge and a collection of fundamentalAHB/APB bus IP components
+pre-integrated together as a system design.The high-quality and configurable
+AHB/APB IPs suites a majority embedded systems, and the verified platform serves
+as a starting point to jump start SoC designs.
 
-To build platform specific library and firmwares, provide the *PLATFORM=andes/ae350* parameter to the top level make command.
+To build platform specific library and firmwares, provide the
+*PLATFORM=andes/ae350* parameter to the top level make command.
 
 Platform Options
 ----------------
@@ -17,7 +18,8 @@ The Andes AE350 platform does not have any platform-specific options.
 Building Andes AE350 Platform
 -----------------------------
 
-To use Linux v5.2 should be used to build Andes AE350 OpenSBI binaries by using the compile time option FW_PAYLOAD_FDT_PATH.
+To use Linux v5.2 should be used to build Andes AE350 OpenSBI binaries by using
+the compile time option FW_PAYLOAD_FDT_PATH.
 
 AE350's dts is included in https://github.com/andestech/linux/tree/ast-v3_2_0-release-public
 

docs/platform/fpga-ariane.md

@@ -1,9 +1,11 @@
 Ariane FPGA SoC Platform
-==========================
-Ariane is a 6-stage, single issue, in-order CPU which implements the 64-bit RISC-V instruction set.
-The Ariane FPGA development platform is based on FPGA FPGA SoC(which currently supports only  Genesys 2 board) and is capable
-of running Linux.
- The FPGA SoC currently contains the following peripherals:
+========================
+Ariane is a 6-stage, single issue, in-order CPU which implements the 64-bit
+RISC-V instruction set. The Ariane FPGA development platform is based on FPGA
+SoC (which currently supports only Genesys 2 board) and is capable of running
+Linux.
+
+The FPGA SoC currently contains the following peripherals:
 - DDR3 memory controller
 - SPI controller to conncet to an SDCard
 - Ethernet controller
@@ -11,27 +13,26 @@ of running Linux.
 - Bootrom containing zero stage bootloader and device tree.
 
 To build platform specific library and firmwares, provide the
-*PLATFORM=ariane-fpga* parameter to the top level `make` command.
+*PLATFORM=fpga/ariane* parameter to the top level `make` command.
 
 Platform Options
 ----------------
 
-The *Ariane FPGA* platform does not have any platform-specific
-options.
+The *Ariane FPGA* platform does not have any platform-specific options.
 
 Building Ariane FPGA Platform
 -----------------------------
+
 **Linux Kernel Payload**
 
-
 ```
-make PLATFORM=ariane-fpga FW_PAYLOAD_PATH=<linux_build_directory>/arch/riscv/boot/Image
+make PLATFORM=fpga/ariane FW_PAYLOAD_PATH=<linux_build_directory>/arch/riscv/boot/Image
 ```
 
 Booting Ariane FPGA Platform
------------------------------
+----------------------------
 
 **Linux Kernel Payload**
 
-As Linux kernel image is embedded in the OpenSBI firmware binary, Ariane will directly
-boot into Linux directly after powered on.
+As Linux kernel image is embedded in the OpenSBI firmware binary, Ariane will
+directly boot into Linux directly after powered on.

docs/platform/fpga-openpiton.md

@@ -0,0 +1,33 @@
+OpenPiton FPGA SoC Platform
+========================
+OpenPiton is the world's first open source, general purpose, multithreaded
+manycore processor. It is a tiled manycore framework scalable from one to
+1/2 billion cores. Currently, OpenPiton supports the 64bit Ariane RISC-V
+processor from ETH Zurich. To this end, Ariane has been equipped with a
+different L1 cache subsystem that follows a write-through protocol and that has
+support for cache invalidations and atomics.
+
+To build platform specific library and firmwares, provide the
+*PLATFORM=fpga/openpiton* parameter to the top level `make` command.
+
+Platform Options
+----------------
+
+The *OpenPiton* platform does not have any platform-specific options.
+
+Building Ariane FPGA Platform
+-----------------------------
+
+**Linux Kernel Payload**
+
+```
+make PLATFORM=fpga/openpiton FW_PAYLOAD_PATH=<linux_build_directory>/arch/riscv/boot/Image
+```
+
+Booting Ariane FPGA Platform
+----------------------------
+
+**Linux Kernel Payload**
+
+As Linux kernel image is embedded in the OpenSBI firmware binary, Ariane will
+directly boot into Linux directly after powered on.

docs/platform/platform.md

@@ -8,14 +8,9 @@ OpenSBI currently supports the following virtual and hardware platforms:
   development and tests. More details on this platform can be found in the
   file *[qemu_virt.md]*.
 
-* **QEMU SiFive Unleashed Machine**: Platform support for the *sifive_u* QEMU
-  virtual RISC-V machine. This is an emulation machine of the HiFive Unleashed
-  board by SiFive. More details on this platform can be found in the file
-  *[qemu_sifive_u.md]*.
-
 * **SiFive FU540 SoC**: Platform support for SiFive FU540 SoC used on the
-  HiFive Unleashed board. This platform is very similar to the *QEMU sifive_u*
-  platform. More details on this platform can be found in the file
+  HiFive Unleashed board, as well as the *sifive_u* QEMU virtual RISC-V
+  machine. More details on this platform can be found in the file
   *[sifive_fu540.md]*.
 
 * **Kendryte K210 SoC**: Platform support for the Kendryte K210 SoC used on
@@ -26,6 +21,13 @@ OpenSBI currently supports the following virtual and hardware platforms:
 
 * **Andes AE350 SoC**: Platform support for the Andes's SoC (AE350).
 
+* **T-HEAD C910**: Platform support for the T-HEAD C910 Processor.
+
+* **Spike**: Platform support for the Spike emulator.
+
+* **OpenPiton FPGA SoC**: Platform support OpenPiton research platform based on
+ariane core.
+
 The code for these supported platforms can be used as example to implement
 support for other platforms. The *platform/template* directory also provides
 template files for implementing support for a new platform. The *object.mk*,
@@ -33,7 +35,9 @@ template files for implementing support for a new platform. The *object.mk*,
 facilitate the implementation.
 
 [qemu_virt.md]: qemu_virt.md
-[qemu_sifive_u.md]: qemu_sifive_u.md
 [sifive_fu540.md]: sifive_fu540.md
-[ariane-fpga.md]: ariane-fpga.md
+[fpga-ariane.md]: fpga-ariane.md
 [andes_ae350.md]: andes-ae350.md
+[thead-c910.md]: thead-c910.md
+[spike.md]: spike.md
+[fpga_openpiton.md]: fpga_openpiton.md

docs/platform/qemu_sifive_u.md

@@ -1,60 +0,0 @@
-QEMU SiFive Unleashed Machine Platform
-======================================
-
-The **QEMU SiFive Unleashed Machine** is an emulation of the SiFive Unleashed
-platform.
-
-To build this platform specific library and firmwares, provide the
-*PLATFORM=qemu/sifive_u* parameter to the top level `make` command line.
-
-Note with QEMU v4.2 release, the QEMU *sifive_u* machine has been updated to
-closely match the SiFive HiFive Unleashed hardware and can therefore run the
-same firmware as what gets loaded onto the board, and OpenSBI's *qemu/sifive_u*
-platform should only be used with QEMU v4.1 release or before.
-
-The special *qemu/sifive_u* platform support will be dropped in the future
-OpenSBI release.
-
-Platform Options
-----------------
-
-The *QEMU SiFive Unleashed Machine* platform does not have any platform specific
-options.
-
-Executing on QEMU RISC-V 64-bit
--------------------------------
-
-**No Payload Case**
-
-Build:
-```
-make PLATFORM=qemu/sifive_u
-```
-
-Run:
-```
-qemu-system-riscv64 -M sifive_u -m 256M -display none -serial stdio \
-	-kernel build/platform/qemu/sifive_u/firmware/fw_payload.elf
-```
-
-**U-Boot as a Payload**
-
-Note: the command line examples here assume that U-Boot was compiled using
-the `sifive_fu540_defconfig` configuration.
-
-Build:
-```
-make PLATFORM=qemu/sifive_u FW_PAYLOAD_PATH=<uboot_build_directory>/u-boot.bin
-```
-
-Run:
-```
-qemu-system-riscv64 -M sifive_u -m 256M -display none -serial stdio \
-	-kernel build/platform/qemu/sifive_u/firmware/fw_payload.elf
-```
-or
-```
-qemu-system-riscv64 -M sifive_u -m 256M -display none -serial stdio \
-	-kernel build/platform/qemu/sifive_u/firmware/fw_jump.elf \
-	-device loader,file=<uboot_build_directory>/u-boot.bin,addr=0x80200000
-```

docs/platform/qemu_virt.md

@@ -149,4 +149,3 @@ qemu-system-riscv32 -M virt -m 256M -nographic \
 	-device virtio-blk-device,drive=hd0 \
 	-append "root=/dev/vda rw console=ttyS0"
 ```
-

docs/platform/sifive_fu540.md

@@ -1,5 +1,5 @@
 SiFive FU540 SoC Platform
-==========================
+=========================
 The FU540-C000 is the world’s first 4+1 64-bit RISC-V SoC from SiFive.
 The HiFive Unleashed development platform is based on FU540-C000 and capable
 of running Linux.
@@ -13,22 +13,14 @@ To build platform specific library and firmwares, provide the
 Platform Options
 ----------------
 
-As hart0 in the FU540 doesn't have an MMU, only harts 1-4 boot by default.
-A hart mask i.e. *FU540_ENABLED_HART_MASK* compile time option is provided
-to select any other hart for booting. Please keep in mind that this is not
-a generic option and it can only be specified for FU540 platform in the
-following way:
-
-```
-make PLATFORM=sifive/fu540 FW_PAYLOAD_PATH=Image FU540_ENABLED_HART_MASK=0x02
-```
-This will let the board boot only hart1 instead of default 1-4.
+The *SiFive FU540 SoC* platform does not have any platform-specific
+options.
 
 Building SiFive Fu540 Platform
------------------------------
+------------------------------
 
 In order to boot SMP Linux in U-Boot, Linux v5.1 (or higher) and latest
-U-Boot v2019.04 (or higher) should be used.
+U-Boot v2020.01 (or higher) should be used.
 
 **Linux Kernel Payload**
 
@@ -49,24 +41,12 @@ make PLATFORM=sifive/fu540 FW_PAYLOAD_PATH=<linux_build_directory>/arch/riscv/bo
 **U-Boot Payload**
 
 The command-line example here assumes that U-Boot was compiled using the
-sifive_fu540_defconfig configuration and with U-Boot v2019.04 (or higher)
-having SMP support. From, Linux v5.2 (or higher) device tree is hosted in
-Linux kernel and compiled as a part of Linux kernel build process.
+sifive_fu540_defconfig configuration and with U-Boot v2020.01 (or higher).
 
-The detailed U-Boot booting guide is avaialble at [U-Boot](https://gitlab.denx.de/u-boot/u-boot/blob/master/doc/board/sifive/fu540.rst)
+The detailed U-Boot booting guide is avaialble at [U-Boot].
 
 ```
-make PLATFORM=sifive/fu540 FW_PAYLOAD_PATH=<u-boot_build_dir>/u-boot.bin
-or
-(For U-Boot which follows Linux v5.2 (or higher) DT bindings)
-make PLATFORM=sifive/fu540 FW_PAYLOAD_PATH=<u-boot_build_dir>/u-boot.bin FW_PAYLOAD_FDT_PATH=<hifive-unleashed-a00.dtb path from Linux kernel>
-```
-
-Generate the uImage from Linux Image.
-```
-mkimage -A riscv -O linux -T kernel -C none -a 0x80200000 -e 0x80200000 -n Linux -d \
-		<linux_build_directory>/arch/riscv/boot/Image \
-		<linux_build_directory>/arch/riscv/boot/uImage
+make PLATFORM=sifive/fu540 FW_PAYLOAD_PATH=<u-boot_build_dir>/u-boot-dtb.bin
 ```
 
 **U-Boot & Linux Kernel as a single payload**
@@ -74,37 +54,29 @@ mkimage -A riscv -O linux -T kernel -C none -a 0x80200000 -e 0x80200000 -n Linux
 A single monolithic image containing both U-Boot & Linux can also be used if
 network boot setup is not available.
 
-1. Generate the uImage from Linux Image.
-```
-mkimage -A riscv -O linux -T kernel -C none -a 0x80200000 -e 0x80200000 -n Linux -d \
-		<linux_build_directory>/arch/riscv/boot/Image \
-		<linux_build_directory>/arch/riscv/boot/uImage
-```
-
-2. Create a temporary image with u-boot.bin as the first payload. The
+1. Create a temporary image with u-boot-dtb.bin as the first payload. The
 command-line example here assumes that U-Boot was compiled using
 sifive_fu540_defconfig configuration.
 ```
-dd if=~/workspace/u-boot-riscv/u-boot.bin of=/tmp/temp.bin bs=1M
+dd if=~/workspace/u-boot-riscv/u-boot-dtb.bin of=/tmp/temp.bin bs=1M
 ```
-3. Append the Linux Kernel image generated in step 1.
+2. Append the Linux Kernel image.
 ```
-dd if=<linux_build_directory>/arch/riscv/boot/uImage of=/tmp/temp.bin bs=1M seek=4
+dd if=<linux_build_directory>/arch/riscv/boot/Image of=/tmp/temp.bin bs=1M seek=4
 ```
-4. Compile OpenSBI with temp.bin (generated in step 3) as payload.
+3. Compile OpenSBI with temp.bin (generated in step 2) as payload.
 ```
 make PLATFORM=sifive/fu540 FW_PAYLOAD_PATH=/tmp/temp.bin
 or
 (For U-Boot which follows Linux v5.2 (or higher) DT bindings)
-make PLATFORM=sifive/fu540 FW_PAYLOAD_PATH=/tmp/temp.bin FW_PAYLOAD_FDT_PATH=<hifive-unleashed-a00.dtb path from Linux kernel>
+make PLATFORM=sifive/fu540 FW_PAYLOAD_PATH=/tmp/temp.bin
 ```
 
 Flashing the OpenSBI firmware binary to storage media:
------------------------------------------------------
-The first stage boot loader([FSBL](https://github.com/sifive/freedom-u540-c000-bootloader))
-expects the storage media to have a GPT partition table. It tries to look for
-a partition with following GUID to load the next stage boot loader (OpenSBI
-in this case).
+------------------------------------------------------
+The first stage boot loader ([FSBL]) expects the storage media to have a GPT
+partition table. It tries to look for a partition with following GUID to load
+the next stage boot loader (OpenSBI in this case).
 
 ```
 2E54B353-1271-4842-806F-E436D6AF6985
@@ -144,43 +116,33 @@ As U-Boot image is used as payload, HiFive Unleashed will boot into a U-Boot
 prompt. U-Boot tftp boot method can be used to load kernel image in U-Boot
 prompt. Here are the steps do a tftpboot.
 
-1. Set the mac address of the board.
-```
-setenv ethaddr <mac address of the board>
-```
-(Note: This step is optional)
-
-2. Set the ip address of the board.
+1. Set the ip address of the board.
 ```
 setenv ipaddr <ipaddr of the board>
 ```
-
-3. Set the tftpboot server IP.
+2. Set the tftpboot server IP.
 ```
 setenv serverip <ipaddr of the tftp server>
 ```
-
-4. Set the network gateway address.
+3. Set the network gateway address.
 ```
 setenv gatewayip <ipaddress of the network gateway>
 ```
-
-5. Load the Linux kernel image from the tftp server.
+4. Load the Linux kernel image from the tftp server.
 ```
-tftpboot ${kernel_addr_r} <uImage path in tftpboot directory>
+tftpboot ${kernel_addr_r} <Image path in tftpboot directory>
 ```
-
-6. Load the ramdisk image from the tftp server. This is only required if
+5. Load the ramdisk image from the tftp server. This is only required if
 ramdisk is loaded from tftp server. This step is optional, if rootfs is
 already part of the kernel or loaded from an external storage by kernel.
 ```
 tftpboot ${ramdisk_addr_r} <ramdisk path in tftpboot directory>
 ```
-7. Load the pre-compiled device tree via tftpboot.
+6. Load the pre-compiled device tree via tftpboot.
 ```
-tftpboot ${fdt_addr_r} <linux source>/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dtb
+tftpboot ${fdt_addr_r} <hifive-unleashed-a00.dtb path in tftpboot directory>
 ```
-8. Set the boot command-line arguments.
+7. Set the boot command-line arguments.
 ```
 setenv bootargs "root=<root partition> rw console=ttySIF0 earlycon=sbi"
 ```
@@ -188,13 +150,12 @@ setenv bootargs "root=<root partition> rw console=ttySIF0 earlycon=sbi"
 ** /dev/ram ** - If a ramdisk is used
 ** root=/dev/mmcblk0pX ** - If a rootfs is already on some other partition
 of sdcard)
-
-9. Now boot into Linux.
+8. Now boot into Linux.
 ```
-bootm ${kernel_addr_r} ${ramdisk_addr_r} ${fdt_addr_r}
+booti ${kernel_addr_r} ${ramdisk_addr_r} ${fdt_addr_r}
 or
 (If ramdisk is not loaded from network)
-bootm ${kernel_addr_r} - ${fdt_addr_r}
+booti ${kernel_addr_r} - ${fdt_addr_r}
 ```
 
 **U-Boot & Linux Kernel as a single payload**
@@ -202,7 +163,7 @@ bootm ${kernel_addr_r} - ${fdt_addr_r}
 At U-Boot prompt execute the following boot command to boot Linux.
 
 ```
-bootm ${kernel_addr_r} - ${fdt_addr_r}
+booti ${kernel_addr_r} - ${fdt_addr_r}
 ```
 
 QEMU Specific Instructions
@@ -213,3 +174,20 @@ same instructions above, with the exception of not passing FW_PAYLOAD_FDT_PATH.
 This is because QEMU generates a device tree blob on the fly based on the
 command line parameters and it's compatible with the one used in the upstream
 Linux kernel.
+
+When U-Boot v2020.01 (or higher) is used as the payload, as the SiFive FU540
+DTB for the real hardware is embedded in U-Boot binary itself, due to the same
+reason above, we need to switch the U-Boot sifive_fu540_defconfig configuration
+from CONFIG_OF_SEPARATE to CONFIG_OF_PRIOR_STAGE so that U-Boot uses the DTB
+generated by QEMU, and u-boot.bin should be used as the payload image, like:
+
+```
+make PLATFORM=sifive/fu540 FW_PAYLOAD_PATH=<u-boot_build_dir>/u-boot.bin
+```
+
+While the real hardware operates at the 64-bit mode, it's possible for QEMU to
+test the 32-bit OpenSBI firmware. This can be helpful for testing 32-bit SiFive
+specific drivers.
+
+[U-Boot]: https://gitlab.denx.de/u-boot/u-boot/blob/master/doc/board/sifive/fu540.rst
+[FSBL]: https://github.com/sifive/freedom-u540-c000-bootloader

docs/platform/spike.md

@@ -0,0 +1,89 @@
+Spike Simulator Platform
+========================
+
+The **Spike** is a RISC-V ISA simulator which implements a functional model
+of one or more RISC-V harts. The **Spike** compatible virtual platform is
+also available on QEMU. In fact, we can use same OpenSBI firmware binaries
+on **Spike** simulator and QEMU Spike machine.
+
+For more details, refer [Spike on GitHub](https://github.com/riscv/riscv-isa-sim)
+
+To build the platform-specific library and firmware images, provide the
+*PLATFORM=spike* parameter to the top level `make` command.
+
+Platform Options
+----------------
+
+The *Spike* platform does not have any platform-specific options.
+
+Execution on Spike Simulator
+----------------------------
+
+**No Payload Case**
+
+Build:
+```
+make PLATFORM=spike
+```
+
+Run:
+```
+spike build/platform/spike/firmware/fw_payload.elf
+```
+
+**Linux Kernel Payload**
+
+Note: We assume that the Linux kernel is compiled using
+*arch/riscv/configs/defconfig*.
+
+Build:
+```
+make PLATFORM=spike FW_PAYLOAD_PATH=<linux_build_directory>/arch/riscv/boot/Image
+```
+
+Run:
+```
+spike --initrd <path_to_cpio_ramdisk> build/platform/spike/firmware/fw_payload.elf
+```
+
+Execution on QEMU RISC-V 64-bit
+-------------------------------
+
+**No Payload Case**
+
+Build:
+```
+make PLATFORM=spike
+```
+
+Run:
+```
+qemu-system-riscv64 -M spike -m 256M -nographic \
+	-kernel build/platform/spike/firmware/fw_payload.elf
+```
+
+**Linux Kernel Payload**
+
+Note: We assume that the Linux kernel is compiled using
+*arch/riscv/configs/defconfig*.
+
+Build:
+```
+make PLATFORM=spike FW_PAYLOAD_PATH=<linux_build_directory>/arch/riscv/boot/Image
+```
+
+Run:
+```
+qemu-system-riscv64 -M spike -m 256M -nographic \
+	-kernel build/platform/spike/firmware/fw_payload.elf \
+	-initrd <path_to_cpio_ramdisk> \
+	-append "root=/dev/ram rw console=hvc0 earlycon=sbi"
+```
+or
+```
+qemu-system-riscv64 -M spike -m 256M -nographic \
+	-bios build/platform/spike/firmware/fw_jump.elf \
+	-kernel <linux_build_directory>/arch/riscv/boot/Image \
+	-initrd <path_to_cpio_ramdisk> \
+	-append "root=/dev/ram rw console=hvc0 earlycon=sbi"
+```

docs/platform/thead-c910.md

@@ -0,0 +1,34 @@
+T-HEAD C910 Processor
+=====================
+C910 is a 12-stage, 3 issues, 8 executions, out-of-order 64-bit RISC-V CPU which
+supports 16 cores, runs with 2.5GHz, and is capable of running Linux.
+
+To build platform specific library and firmwares, provide the
+*PLATFORM=thead/c910* parameter to the top level make command.
+
+Platform Options
+----------------
+
+The *T-HEAD C910* platform does not have any platform-specific options.
+
+Building T-HEAD C910 Platform
+-----------------------------
+
+```
+make PLATFORM=thead/c910
+```
+
+Booting T-HEAD C910 Platform
+----------------------------
+
+**No Payload**
+
+As there's no payload, you may download vmlinux or u-boot to FW_JUMP_ADDR which
+specified in config.mk or compile commands with GDB. And the execution flow will
+turn to vmlinux or u-boot when opensbi ends.
+
+**Linux Kernel Payload**
+
+You can also choose to use Linux kernel as payload by enabling FW_PAYLOAD=y
+along with specifying FW_PAYLOAD_OFFSET. The kernel image will be embedded in
+the OPENSBI firmware binary, T-head will directly boot into Linux after OpenSBI.

firmware/fw_base.S

@@ -41,17 +41,26 @@
 999:
 .endm
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.globl _start
 	.globl _start_warm
 _start:
-	/*
-	 * Jump to warm-boot if this is not the first core booting,
-	 * that is, for mhartid != 0
-	 */
-	csrr	a6, CSR_MHARTID
-	blt	zero, a6, _wait_relocate_copy_done
+	/* Find preferred boot HART id */
+	MOV_3R	s0, a0, s1, a1, s2, a2
+	call	fw_boot_hart
+	add	a6, a0, zero
+	MOV_3R	a0, s0, a1, s1, a2, s2
+	li	a7, -1
+	beq	a6, a7, _try_lottery
+	/* Jump to relocation wait loop if we are not boot hart */
+	bne	a0, a6, _wait_relocate_copy_done
+_try_lottery:
+	/* Jump to relocation wait loop if we don't get relocation lottery */
+	la	a6, _relocate_lottery
+	li	a7, 1
+	amoadd.w a6, a7, (a6)
+	bnez	a6, _wait_relocate_copy_done
 
 	/* Save load address */
 	la	t0, _load_start
@@ -75,6 +84,8 @@ _relocate:
 	blt	t2, t0, _relocate_copy_to_upper
 _relocate_copy_to_lower:
 	ble	t1, t2, _relocate_copy_to_lower_loop
+	la	t3, _relocate_lottery
+	BRANGE	t2, t1, t3, _start_hang
 	la	t3, _boot_status
 	BRANGE	t2, t1, t3, _start_hang
 	la	t3, _relocate
@@ -91,6 +102,8 @@ _relocate_copy_to_lower_loop:
 	jr	t4
 _relocate_copy_to_upper:
 	ble	t3, t0, _relocate_copy_to_upper_loop
+	la	t2, _relocate_lottery
+	BRANGE	t0, t3, t2, _start_hang
 	la	t2, _boot_status
 	BRANGE	t0, t3, t2, _start_hang
 	la	t2, _relocate
@@ -338,6 +351,7 @@ _start_warm:
 	csrw	CSR_MIE, zero
 	csrw	CSR_MIP, zero
 
+	/* Find HART count and HART stack size */
 	la	a4, platform
 #if __riscv_xlen == 64
 	lwu	s7, SBI_PLATFORM_HART_COUNT_OFFSET(a4)
@@ -346,12 +360,29 @@ _start_warm:
 	lw	s7, SBI_PLATFORM_HART_COUNT_OFFSET(a4)
 	lw	s8, SBI_PLATFORM_HART_STACK_SIZE_OFFSET(a4)
 #endif
+	REG_L	s9, SBI_PLATFORM_HART_INDEX2ID_OFFSET(a4)
 
-	/* HART ID should be within expected limit */
+	/* Find HART id */
 	csrr	s6, CSR_MHARTID
-	bge	s6, s7, _start_hang
 
-	/* find the scratch space for this hart */
+	/* Find HART index */
+	beqz	s9, 3f
+	li	a4, 0
+1:
+#if __riscv_xlen == 64
+	lwu	a5, (s9)
+#else
+	lw	a5, (s9)
+#endif
+	beq	a5, s6, 2f
+	add	s9, s9, 4
+	add	a4, a4, 1
+	blt	a4, s7, 1b
+	li	a4, -1
+2:	add	s6, a4, zero
+3:	bge	s6, s7, _start_hang
+
+	/* Find the scratch space based on HART index */
 	la	tp, _fw_end
 	mul	a5, s7, s8
 	add	tp, tp, a5
@@ -381,6 +412,8 @@ _start_warm:
 	j	_start_hang
 
 	.align 3
+_relocate_lottery:
+	RISCV_PTR	0
 _boot_status:
 	RISCV_PTR	0
 _load_start:
@@ -390,50 +423,42 @@ _link_start:
 _link_end:
 	RISCV_PTR	_fw_reloc_end
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.globl _hartid_to_scratch
 _hartid_to_scratch:
-	add	sp, sp, -(3 * __SIZEOF_POINTER__)
-	REG_S	s0, (sp)
-	REG_S	s1, (__SIZEOF_POINTER__)(sp)
-	REG_S	s2, (__SIZEOF_POINTER__ * 2)(sp)
 	/*
 	 * a0 -> HART ID (passed by caller)
-	 * s0 -> HART Stack Size
-	 * s1 -> HART Stack End
-	 * s2 -> Temporary
+	 * a1 -> HART Index (passed by caller)
+	 * t0 -> HART Stack Size
+	 * t1 -> HART Stack End
+	 * t2 -> Temporary
 	 */
-	la	s2, platform
+	la	t2, platform
 #if __riscv_xlen == 64
-	lwu	s0, SBI_PLATFORM_HART_STACK_SIZE_OFFSET(s2)
-	lwu	s2, SBI_PLATFORM_HART_COUNT_OFFSET(s2)
+	lwu	t0, SBI_PLATFORM_HART_STACK_SIZE_OFFSET(t2)
+	lwu	t2, SBI_PLATFORM_HART_COUNT_OFFSET(t2)
 #else
-	lw	s0, SBI_PLATFORM_HART_STACK_SIZE_OFFSET(s2)
-	lw	s2, SBI_PLATFORM_HART_COUNT_OFFSET(s2)
+	lw	t0, SBI_PLATFORM_HART_STACK_SIZE_OFFSET(t2)
+	lw	t2, SBI_PLATFORM_HART_COUNT_OFFSET(t2)
 #endif
-	mul	s2, s2, s0
-	la	s1, _fw_end
-	add	s1, s1, s2
-	mul	s2, s0, a0
-	sub	s1, s1, s2
-	li	s2, SBI_SCRATCH_SIZE
-	sub	a0, s1, s2
-	REG_L	s0, (sp)
-	REG_L	s1, (__SIZEOF_POINTER__)(sp)
-	REG_L	s2, (__SIZEOF_POINTER__ * 2)(sp)
-	add	sp, sp, (3 * __SIZEOF_POINTER__)
+	sub	t2, t2, a1
+	mul	t2, t2, t0
+	la	t1, _fw_end
+	add	t1, t1, t2
+	li	t2, SBI_SCRATCH_SIZE
+	sub	a0, t1, t2
 	ret
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.globl _start_hang
 _start_hang:
 	wfi
 	j	_start_hang
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.globl _trap_handler
 _trap_handler:
 	/* Swap TP and MSCRATCH */
@@ -531,7 +556,6 @@ _skip_mstatush_save:
 
 	/* Call C routine */
 	add	a0, sp, zero
-	csrr	a1, CSR_MSCRATCH
 	call	sbi_trap_handler
 
 	/* Restore all general regisers except SP and T0 */
@@ -588,8 +612,8 @@ _skip_mstatush_restore:
 
 	mret
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.globl _reset_regs
 _reset_regs:
 

firmware/fw_dynamic.S

@@ -11,14 +11,40 @@
 
 #include "fw_base.S"
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 _bad_dynamic_info:
 	wfi
 	j	_bad_dynamic_info
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
+	.global fw_boot_hart
+	/*
+	 * This function is called very early even before
+	 * fw_save_info() is called.
+	 * We can only use a0, a1, and a2 registers here.
+	 * The boot HART id should be returned in 'a0'.
+	 */
+fw_boot_hart:
+	/* Sanity checks */
+	li	a1, FW_DYNAMIC_INFO_MAGIC_VALUE
+	REG_L	a0, FW_DYNAMIC_INFO_MAGIC_OFFSET(a2)
+	bne	a0, a1, _bad_dynamic_info
+	li	a1, FW_DYNAMIC_INFO_VERSION_MAX
+	REG_L	a0, FW_DYNAMIC_INFO_VERSION_OFFSET(a2)
+	bgt	a0, a1, _bad_dynamic_info
+
+	/* Read boot HART id */
+	li	a1, 0x2
+	blt	a0, a1, 2f
+	REG_L	a0, FW_DYNAMIC_INFO_BOOT_HART_OFFSET(a2)
+	ret
+2:	li	a0, -1
+	ret
+
+	.section .entry, "ax", %progbits
+	.align 3
 	.global fw_save_info
 	/*
 	 * We can only use a0, a1, a2, a3, and a4 registers here.
@@ -27,14 +53,19 @@ _bad_dynamic_info:
 	 * Nothing to be returned here.
 	 */
 fw_save_info:
+	/* Save next arg1 in 'a1' */
 	la	a4, _dynamic_next_arg1
 	REG_S	a1, (a4)
+
+	/* Sanity checks */
 	li	a4, FW_DYNAMIC_INFO_MAGIC_VALUE
 	REG_L	a3, FW_DYNAMIC_INFO_MAGIC_OFFSET(a2)
 	bne	a3, a4, _bad_dynamic_info
 	li	a4, FW_DYNAMIC_INFO_VERSION_MAX
 	REG_L	a3, FW_DYNAMIC_INFO_VERSION_OFFSET(a2)
 	bgt	a3, a4, _bad_dynamic_info
+
+	/* Save version == 0x1 fields */
 	la	a4, _dynamic_next_addr
 	REG_L	a3, FW_DYNAMIC_INFO_NEXT_ADDR_OFFSET(a2)
 	REG_S	a3, (a4)
@@ -44,24 +75,37 @@ fw_save_info:
 	la	a4, _dynamic_options
 	REG_L	a3, FW_DYNAMIC_INFO_OPTIONS_OFFSET(a2)
 	REG_S	a3, (a4)
+
+	/* Save version == 0x2 fields */
+	li	a4, 0x2
+	REG_L	a3, FW_DYNAMIC_INFO_VERSION_OFFSET(a2)
+	blt	a3, a4, 2f
+	la	a4, _dynamic_boot_hart
+	REG_L	a3, FW_DYNAMIC_INFO_BOOT_HART_OFFSET(a2)
+	REG_S	a3, (a4)
+2:
 	ret
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.global fw_prev_arg1
 	/*
 	 * We can only use a0, a1, and a2 registers here.
+	 * The a0, a1, and a2 registers will be same as passed by
+	 * previous booting stage.
 	 * The previous arg1 should be returned in 'a0'.
 	 */
 fw_prev_arg1:
 	add	a0, zero, zero
 	ret
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.global fw_next_arg1
 	/*
 	 * We can only use a0, a1, and a2 registers here.
+	 * The a0, a1, and a2 registers will be same as passed by
+	 * previous booting stage.
 	 * The next arg1 should be returned in 'a0'.
 	 */
 fw_next_arg1:
@@ -69,8 +113,8 @@ fw_next_arg1:
 	REG_L	a0, (a0)
 	ret
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.global fw_next_addr
 	/*
 	 * We can only use a0, a1, and a2 registers here.
@@ -81,8 +125,8 @@ fw_next_addr:
 	REG_L	a0, (a0)
 	ret
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.global fw_next_mode
 	/*
 	 * We can only use a0, a1, and a2 registers here.
@@ -93,8 +137,8 @@ fw_next_mode:
 	REG_L	a0, (a0)
 	ret
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.global fw_options
 	/*
 	 * We can only use a0, a1, and a2 registers here.
@@ -106,8 +150,8 @@ fw_options:
 	REG_L	a0, (a0)
 	ret
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 _dynamic_next_arg1:
 	RISCV_PTR 0x0
 _dynamic_next_addr:
@@ -116,3 +160,5 @@ _dynamic_next_mode:
 	RISCV_PTR PRV_S
 _dynamic_options:
 	RISCV_PTR 0x0
+_dynamic_boot_hart:
+	RISCV_PTR -1

firmware/fw_jump.S

@@ -9,8 +9,21 @@
 
 #include "fw_base.S"
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
+	.global fw_boot_hart
+	/*
+	 * This function is called very early even before
+	 * fw_save_info() is called.
+	 * We can only use a0, a1, and a2 registers here.
+	 * The boot HART id should be returned in 'a0'.
+	 */
+fw_boot_hart:
+	li	a0, -1
+	ret
+
+	.section .entry, "ax", %progbits
+	.align 3
 	.global fw_save_info
 	/*
 	 * We can only use a0, a1, a2, a3, and a4 registers here.
@@ -21,34 +34,38 @@
 fw_save_info:
 	ret
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.global fw_prev_arg1
 	/*
 	 * We can only use a0, a1, and a2 registers here.
+	 * The a0, a1, and a2 registers will be same as passed by
+	 * previous booting stage.
 	 * The previous arg1 should be returned in 'a0'.
 	 */
 fw_prev_arg1:
 	add	a0, zero, zero
 	ret
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.global fw_next_arg1
 	/*
 	 * We can only use a0, a1, and a2 registers here.
+	 * The a0, a1, and a2 registers will be same as passed by
+	 * previous booting stage.
 	 * The next arg1 should be returned in 'a0'.
 	 */
 fw_next_arg1:
 #ifdef FW_JUMP_FDT_ADDR
 	li	a0, FW_JUMP_FDT_ADDR
 #else
-	add	a0, zero, zero
+	add	a0, a1, zero
 #endif
 	ret
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.global fw_next_addr
 	/*
 	 * We can only use a0, a1, and a2 registers here.
@@ -59,8 +76,8 @@ fw_next_addr:
 	REG_L	a0, (a0)
 	ret
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.global fw_next_mode
 	/*
 	 * We can only use a0, a1, and a2 registers here.
@@ -70,8 +87,8 @@ fw_next_mode:
 	li	a0, PRV_S
 	ret
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.global fw_options
 	/*
 	 * We can only use a0, a1, and a2 registers here.
@@ -86,7 +103,7 @@ fw_options:
 #error "Must define FW_JUMP_ADDR"
 #endif
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 _jump_addr:
 	RISCV_PTR FW_JUMP_ADDR

firmware/fw_payload.S

@@ -9,8 +9,21 @@
 
 #include "fw_base.S"
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
+	.global fw_boot_hart
+	/*
+	 * This function is called very early even before
+	 * fw_save_info() is called.
+	 * We can only use a0, a1, and a2 registers here.
+	 * The boot HART id should be returned in 'a0'.
+	 */
+fw_boot_hart:
+	li	a0, -1
+	ret
+
+	.section .entry, "ax", %progbits
+	.align 3
 	.global fw_save_info
 	/*
 	 * We can only use a0, a1, a2, a3, and a4 registers here.
@@ -21,11 +34,13 @@
 fw_save_info:
 	ret
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.global fw_prev_arg1
 	/*
 	 * We can only use a0, a1, and a2 registers here.
+	 * The a0, a1, and a2 registers will be same as passed by
+	 * previous booting stage.
 	 * The previous arg1 should be returned in 'a0'.
 	 */
 fw_prev_arg1:
@@ -36,23 +51,25 @@ fw_prev_arg1:
 #endif
 	ret
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.global fw_next_arg1
 	/*
 	 * We can only use a0, a1, and a2 registers here.
+	 * The a0, a1, and a2 registers will be same as passed by
+	 * previous booting stage.
 	 * The next arg1 should be returned in 'a0'.
 	 */
 fw_next_arg1:
 #ifdef FW_PAYLOAD_FDT_ADDR
 	li	a0, FW_PAYLOAD_FDT_ADDR
 #else
-	add	a0, zero, zero
+	add	a0, a1, zero
 #endif
 	ret
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.global fw_next_addr
 	/*
 	 * We can only use a0, a1, and a2 registers here.
@@ -62,8 +79,8 @@ fw_next_addr:
 	la	a0, payload_bin
 	ret
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.global fw_next_mode
 	/*
 	 * We can only use a0, a1, and a2 registers here.
@@ -73,8 +90,8 @@ fw_next_mode:
 	li	a0, PRV_S
 	ret
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.global fw_options
 	/*
 	 * We can only use a0, a1, and a2 registers here.
@@ -86,15 +103,15 @@ fw_options:
 	ret
 
 #ifdef FW_PAYLOAD_FDT_PATH
-	.align 4
 	.section .text, "ax", %progbits
+	.align 4
 	.globl fdt_bin
 fdt_bin:
 	.incbin	FW_PAYLOAD_FDT_PATH
 #endif
 
-	.align 4
 	.section .payload, "ax", %progbits
+	.align 4
 	.globl payload_bin
 payload_bin:
 #ifndef FW_PAYLOAD_PATH

firmware/payloads/test_head.S

@@ -23,8 +23,8 @@
 #define REG_L		__REG_SEL(ld, lw)
 #define REG_S		__REG_SEL(sd, sw)
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.globl _start
 _start:
 	/* Pick one hart to run the main boot sequence */
@@ -71,15 +71,15 @@ _start_warm:
 	/* We don't expect to reach here hence just hang */
 	j	_start_hang
 
-	.align 3
 	.section .entry, "ax", %progbits
+	.align 3
 	.globl _start_hang
 _start_hang:
 	wfi
 	j	_start_hang
 
-	.align	3
 	.section .entry, "ax", %progbits
+	.align	3
 _hart_lottery:
 	RISCV_PTR	0
 _boot_a0:

include/sbi/fw_dynamic.h

@@ -18,18 +18,20 @@
 #define FW_DYNAMIC_INFO_MAGIC_OFFSET		(0 * __SIZEOF_POINTER__)
 /** Offset of version member in fw_dynamic_info */
 #define FW_DYNAMIC_INFO_VERSION_OFFSET		(1 * __SIZEOF_POINTER__)
-/** Offset of next_addr member in fw_dynamic_info */
+/** Offset of next_addr member in fw_dynamic_info (version >= 1) */
 #define FW_DYNAMIC_INFO_NEXT_ADDR_OFFSET	(2 * __SIZEOF_POINTER__)
-/** Offset of next_mode member in fw_dynamic_info */
+/** Offset of next_mode member in fw_dynamic_info  (version >= 1) */
 #define FW_DYNAMIC_INFO_NEXT_MODE_OFFSET	(3 * __SIZEOF_POINTER__)
-/** Offset of options member in fw_dynamic_info */
+/** Offset of options member in fw_dynamic_info  (version >= 1) */
 #define FW_DYNAMIC_INFO_OPTIONS_OFFSET		(4 * __SIZEOF_POINTER__)
+/** Offset of boot_hart member in fw_dynamic_info  (version >= 2) */
+#define FW_DYNAMIC_INFO_BOOT_HART_OFFSET	(5 * __SIZEOF_POINTER__)
 
 /** Expected value of info magic ('OSBI' ascii string in hex) */
 #define FW_DYNAMIC_INFO_MAGIC_VALUE		0x4942534f
 
 /** Maximum supported info version */
-#define FW_DYNAMIC_INFO_VERSION_MAX		0x1
+#define FW_DYNAMIC_INFO_VERSION_MAX		0x2
 
 /** Possible next mode values */
 #define FW_DYNAMIC_INFO_NEXT_MODE_U		0x0
@@ -54,6 +56,22 @@ struct fw_dynamic_info {
 	unsigned long next_mode;
 	/** Options for OpenSBI library */
 	unsigned long options;
+	/**
+	 * Preferred boot HART id
+	 *
+	 * It is possible that the previous booting stage uses same link
+	 * address as the FW_DYNAMIC firmware. In this case, the relocation
+	 * lottery mechanism can potentially overwrite the previous booting
+	 * stage while other HARTs are still running in the previous booting
+	 * stage leading to boot-time crash. To avoid this boot-time crash,
+	 * the previous booting stage can specify last HART that will jump
+	 * to the FW_DYNAMIC firmware as the preferred boot HART.
+	 *
+	 * To avoid specifying a preferred boot HART, the previous booting
+	 * stage can set it to -1UL which will force the FW_DYNAMIC firmware
+	 * to use the relocation lottery mechanism.
+	 */
+	unsigned long boot_hart;
 } __packed;
 
 #endif

include/sbi/riscv_asm.h

@@ -28,9 +28,9 @@
 #error "Unexpected __riscv_xlen"
 #endif
 
-#define PAGE_SHIFT      (12)
-#define PAGE_SIZE       (_AC(1, UL) << PAGE_SHIFT)
-#define PAGE_MASK       (~(PAGE_SIZE - 1))
+#define PAGE_SHIFT	(12)
+#define PAGE_SIZE	(_AC(1, UL) << PAGE_SHIFT)
+#define PAGE_MASK	(~(PAGE_SIZE - 1))
 
 #define REG_L		__REG_SEL(ld, lw)
 #define REG_S		__REG_SEL(sd, sw)
@@ -38,7 +38,6 @@
 #define LGREG		__REG_SEL(3, 2)
 
 #if __SIZEOF_POINTER__ == 8
-#define BITS_PER_LONG		64
 #ifdef __ASSEMBLY__
 #define RISCV_PTR		.dword
 #define RISCV_SZPTR		8
@@ -49,7 +48,6 @@
 #define RISCV_LGPTR		"3"
 #endif
 #elif __SIZEOF_POINTER__ == 4
-#define BITS_PER_LONG		32
 #ifdef __ASSEMBLY__
 #define RISCV_PTR		.word
 #define RISCV_SZPTR		4
@@ -159,22 +157,28 @@ void csr_write_num(int csr_num, unsigned long val);
 		__asm__ __volatile__("wfi" ::: "memory"); \
 	} while (0)
 
-static inline int misa_extension(char ext)
-{
-	return csr_read(CSR_MISA) & (1 << (ext - 'A'));
-}
+/* Get current HART id */
+#define current_hartid()	((unsigned int)csr_read(CSR_MHARTID))
 
-static inline int misa_xlen(void)
-{
-	return ((long)csr_read(CSR_MISA) < 0) ? 64 : 32;
-}
+/* determine CPU extension, return non-zero support */
+int misa_extension_imp(char ext);
+
+#define misa_extension(c)\
+({\
+	_Static_assert(((c >= 'A') && (c <= 'Z')),\
+		"The parameter of misa_extension must be [A-Z]");\
+	misa_extension_imp(c);\
+})
+
+/* Get MXL field of misa, return -1 on error */
+int misa_xlen(void);
 
 static inline void misa_string(char *out, unsigned int out_sz)
 {
-	unsigned long i, val = csr_read(CSR_MISA);
+	unsigned long i;
 
 	for (i = 0; i < 26; i++) {
-		if (val & (1 << i)) {
+		if (misa_extension_imp('A' + i)) {
 			*out = 'A' + i;
 			out++;
 		}
@@ -187,7 +191,7 @@ int pmp_set(unsigned int n, unsigned long prot, unsigned long addr,
 	    unsigned long log2len);
 
 int pmp_get(unsigned int n, unsigned long *prot_out, unsigned long *addr_out,
-	    unsigned long *log2len_out);
+	    unsigned long *size);
 
 #endif /* !__ASSEMBLY__ */
 

include/sbi/riscv_atomic.h

@@ -29,9 +29,9 @@ long atomic_add_return(atomic_t *atom, long value);
 
 long atomic_sub_return(atomic_t *atom, long value);
 
-long arch_atomic_cmpxchg(atomic_t *atom, long oldval, long newval);
+long atomic_cmpxchg(atomic_t *atom, long oldval, long newval);
 
-long arch_atomic_xchg(atomic_t *atom, long newval);
+long atomic_xchg(atomic_t *atom, long newval);
 
 unsigned int atomic_raw_xchg_uint(volatile unsigned int *ptr,
 				  unsigned int newval);

include/sbi/riscv_encoding.h

@@ -12,193 +12,127 @@
 
 #include <sbi/sbi_const.h>
 
-/* TODO: Make constants usable in assembly with _AC() macro */
-
 /* clang-format off */
-#define MSTATUS_UIE			0x00000001
-#define MSTATUS_SIE			0x00000002
-#define MSTATUS_HIE			0x00000004
-#define MSTATUS_MIE			0x00000008
-#define MSTATUS_UPIE			0x00000010
+#define MSTATUS_SIE			_UL(0x00000002)
+#define MSTATUS_MIE			_UL(0x00000008)
 #define MSTATUS_SPIE_SHIFT		5
-#define MSTATUS_SPIE			(1UL << MSTATUS_SPIE_SHIFT)
-#define MSTATUS_HPIE			0x00000040
-#define MSTATUS_MPIE			0x00000080
+#define MSTATUS_SPIE			(_UL(1) << MSTATUS_SPIE_SHIFT)
+#define MSTATUS_UBE			_UL(0x00000040)
+#define MSTATUS_MPIE			_UL(0x00000080)
 #define MSTATUS_SPP_SHIFT		8
-#define MSTATUS_SPP			(1UL << MSTATUS_SPP_SHIFT)
-#define MSTATUS_HPP			0x00000600
+#define MSTATUS_SPP			(_UL(1) << MSTATUS_SPP_SHIFT)
 #define MSTATUS_MPP_SHIFT		11
-#define MSTATUS_MPP			(3UL << MSTATUS_MPP_SHIFT)
-#define MSTATUS_FS			0x00006000
-#define MSTATUS_XS			0x00018000
-#define MSTATUS_MPRV			0x00020000
-#define MSTATUS_SUM			0x00040000
-#define MSTATUS_MXR			0x00080000
-#define MSTATUS_TVM			0x00100000
-#define MSTATUS_TW			0x00200000
-#define MSTATUS_TSR			0x00400000
-#define MSTATUS32_SD			0x80000000
+#define MSTATUS_MPP			(_UL(3) << MSTATUS_MPP_SHIFT)
+#define MSTATUS_FS			_UL(0x00006000)
+#define MSTATUS_XS			_UL(0x00018000)
+#define MSTATUS_VS			_UL(0x01800000)
+#define MSTATUS_MPRV			_UL(0x00020000)
+#define MSTATUS_SUM			_UL(0x00040000)
+#define MSTATUS_MXR			_UL(0x00080000)
+#define MSTATUS_TVM			_UL(0x00100000)
+#define MSTATUS_TW			_UL(0x00200000)
+#define MSTATUS_TSR			_UL(0x00400000)
+#define MSTATUS32_SD			_UL(0x80000000)
 #if __riscv_xlen == 64
-#define MSTATUS_UXL			0x0000000300000000
-#define MSTATUS_SXL			0x0000000C00000000
-#define MSTATUS_MTL			0x0000004000000000
-#define MSTATUS_MTL_SHIFT		38
-#define MSTATUS_MPV			0x0000008000000000
-#define MSTATUS_MPV_HIFT		39
+#define MSTATUS_UXL			_ULL(0x0000000300000000)
+#define MSTATUS_SXL			_ULL(0x0000000C00000000)
+#define MSTATUS_SBE			_ULL(0x0000001000000000)
+#define MSTATUS_MBE			_ULL(0x0000002000000000)
+#define MSTATUS_MPV			_ULL(0x0000008000000000)
 #else
-#define MSTATUSH_UXL			0x00000003
-#define MSTATUSH_SXL			0x0000000C
-#define MSTATUSH_MTL			0x00000040
-#define MSTATUSH_MTL_SHIFT		6
-#define MSTATUSH_MPV			0x00000080
-#define MSTATUSH_MPV_HIFT		7
+#define MSTATUSH_SBE			_UL(0x00000010)
+#define MSTATUSH_MBE			_UL(0x00000020)
+#define MSTATUSH_MPV			_UL(0x00000080)
 #endif
-#define MSTATUS64_SD			0x8000000000000000
+#define MSTATUS32_SD			_UL(0x80000000)
+#define MSTATUS64_SD			_ULL(0x8000000000000000)
 
-#define SSTATUS_UIE			0x00000001
-#define SSTATUS_SIE			0x00000002
-#define SSTATUS_UPIE			0x00000010
-#define SSTATUS_SPIE_SHIFT		5
-#define SSTATUS_SPIE			(1UL << MSTATUS_SPIE_SHIFT)
-#define SSTATUS_SPP_SHIFT		8
-#define SSTATUS_SPP			(1UL << MSTATUS_SPP_SHIFT)
-#define SSTATUS_FS			0x00006000
-#define SSTATUS_XS			0x00018000
-#define SSTATUS_SUM			0x00040000
-#define SSTATUS_MXR			0x00080000
-#define SSTATUS32_SD			0x80000000
-#define SSTATUS_UXL			0x0000000300000000
-#define SSTATUS64_SD			0x8000000000000000
+#define SSTATUS_SIE			MSTATUS_SIE
+#define SSTATUS_SPIE_SHIFT		MSTATUS_SPIE_SHIFT
+#define SSTATUS_SPIE			MSTATUS_SPIE
+#define SSTATUS_SPP_SHIFT		MSTATUS_SPP_SHIFT
+#define SSTATUS_SPP			MSTATUS_SPP
+#define SSTATUS_FS			MSTATUS_FS
+#define SSTATUS_XS			MSTATUS_XS
+#define SSTATUS_VS			MSTATUS_VS
+#define SSTATUS_SUM			MSTATUS_SUM
+#define SSTATUS_MXR			MSTATUS_MXR
+#define SSTATUS32_SD			MSTATUS32_SD
+#define SSTATUS64_UXL			MSTATUS_UXL
+#define SSTATUS64_SD			MSTATUS64_SD
 
-#define HSTATUS_VTSR			0x00400000
-#define HSTATUS_VTVM			0x00100000
-#define HSTATUS_SP2V			0x00000200
-#define HSTATUS_SP2P			0x00000100
-#define HSTATUS_SPV			0x00000080
-#define HSTATUS_STL			0x00000040
-#define HSTATUS_SPRV			0x00000001
-
-#define DCSR_XDEBUGVER			(3U<<30)
-#define DCSR_NDRESET			(1<<29)
-#define DCSR_FULLRESET			(1<<28)
-#define DCSR_EBREAKM			(1<<15)
-#define DCSR_EBREAKH			(1<<14)
-#define DCSR_EBREAKS			(1<<13)
-#define DCSR_EBREAKU			(1<<12)
-#define DCSR_STOPCYCLE			(1<<10)
-#define DCSR_STOPTIME			(1<<9)
-#define DCSR_CAUSE			(7<<6)
-#define DCSR_DEBUGINT			(1<<5)
-#define DCSR_HALT			(1<<3)
-#define DCSR_STEP			(1<<2)
-#define DCSR_PRV			(3<<0)
-
-#define DCSR_CAUSE_NONE			0
-#define DCSR_CAUSE_SWBP			1
-#define DCSR_CAUSE_HWBP			2
-#define DCSR_CAUSE_DEBUGINT		3
-#define DCSR_CAUSE_STEP			4
-#define DCSR_CAUSE_HALT			5
-
-#define MCONTROL_TYPE(xlen)		(0xfULL<<((xlen)-4))
-#define MCONTROL_DMODE(xlen)		(1ULL<<((xlen)-5))
-#define MCONTROL_MASKMAX(xlen)		(0x3fULL<<((xlen)-11))
-
-#define MCONTROL_SELECT			(1<<19)
-#define MCONTROL_TIMING			(1<<18)
-#define MCONTROL_ACTION			(0x3f<<12)
-#define MCONTROL_CHAIN			(1<<11)
-#define MCONTROL_MATCH			(0xf<<7)
-#define MCONTROL_M			(1<<6)
-#define MCONTROL_H			(1<<5)
-#define MCONTROL_S			(1<<4)
-#define MCONTROL_U			(1<<3)
-#define MCONTROL_EXECUTE		(1<<2)
-#define MCONTROL_STORE			(1<<1)
-#define MCONTROL_LOAD			(1<<0)
-
-#define MCONTROL_TYPE_NONE		0
-#define MCONTROL_TYPE_MATCH		2
-
-#define MCONTROL_ACTION_DEBUG_EXCEPTION	0
-#define MCONTROL_ACTION_DEBUG_MODE	1
-#define MCONTROL_ACTION_TRACE_START	2
-#define MCONTROL_ACTION_TRACE_STOP	3
-#define MCONTROL_ACTION_TRACE_EMIT	4
-
-#define MCONTROL_MATCH_EQUAL		0
-#define MCONTROL_MATCH_NAPOT		1
-#define MCONTROL_MATCH_GE		2
-#define MCONTROL_MATCH_LT		3
-#define MCONTROL_MATCH_MASK_LOW		4
-#define MCONTROL_MATCH_MASK_HIGH	5
+#define HSTATUS_VTSR			_UL(0x00400000)
+#define HSTATUS_VTVM			_UL(0x00100000)
+#define HSTATUS_SP2V			_UL(0x00000200)
+#define HSTATUS_SP2P			_UL(0x00000100)
+#define HSTATUS_SPV			_UL(0x00000080)
+#define HSTATUS_SPRV			_UL(0x00000001)
 
 #define IRQ_S_SOFT			1
-#define IRQ_H_SOFT			2
+#define IRQ_VS_SOFT			2
 #define IRQ_M_SOFT			3
 #define IRQ_S_TIMER			5
-#define IRQ_H_TIMER			6
+#define IRQ_VS_TIMER			6
 #define IRQ_M_TIMER			7
 #define IRQ_S_EXT			9
-#define IRQ_H_EXT			10
+#define IRQ_VS_EXT			10
 #define IRQ_M_EXT			11
-#define IRQ_COP				12
-#define IRQ_HOST			13
+#define IRQ_S_GEXT			12
 
-#define MIP_SSIP			(1 << IRQ_S_SOFT)
-#define MIP_HSIP			(1 << IRQ_H_SOFT)
-#define MIP_MSIP			(1 << IRQ_M_SOFT)
-#define MIP_STIP			(1 << IRQ_S_TIMER)
-#define MIP_HTIP			(1 << IRQ_H_TIMER)
-#define MIP_MTIP			(1 << IRQ_M_TIMER)
-#define MIP_SEIP			(1 << IRQ_S_EXT)
-#define MIP_HEIP			(1 << IRQ_H_EXT)
-#define MIP_MEIP			(1 << IRQ_M_EXT)
+#define MIP_SSIP			(_UL(1) << IRQ_S_SOFT)
+#define MIP_VSSIP			(_UL(1) << IRQ_VS_SOFT)
+#define MIP_MSIP			(_UL(1) << IRQ_M_SOFT)
+#define MIP_STIP			(_UL(1) << IRQ_S_TIMER)
+#define MIP_VSTIP			(_UL(1) << IRQ_VS_TIMER)
+#define MIP_MTIP			(_UL(1) << IRQ_M_TIMER)
+#define MIP_SEIP			(_UL(1) << IRQ_S_EXT)
+#define MIP_VSEIP			(_UL(1) << IRQ_VS_EXT)
+#define MIP_MEIP			(_UL(1) << IRQ_M_EXT)
+#define MIP_SGEIP			(_UL(1) << IRQ_S_GEXT)
 
 #define SIP_SSIP			MIP_SSIP
 #define SIP_STIP			MIP_STIP
 
-#define PRV_U				0
-#define PRV_S				1
-#define PRV_H				2
-#define PRV_M				3
+#define PRV_U				_UL(0)
+#define PRV_S				_UL(1)
+#define PRV_M				_UL(3)
 
-#define SATP32_MODE			0x80000000
-#define SATP32_ASID			0x7FC00000
-#define SATP32_PPN			0x003FFFFF
-#define SATP64_MODE			0xF000000000000000
-#define SATP64_ASID			0x0FFFF00000000000
-#define SATP64_PPN			0x00000FFFFFFFFFFF
+#define SATP32_MODE			_UL(0x80000000)
+#define SATP32_ASID			_UL(0x7FC00000)
+#define SATP32_PPN			_UL(0x003FFFFF)
+#define SATP64_MODE			_ULL(0xF000000000000000)
+#define SATP64_ASID			_ULL(0x0FFFF00000000000)
+#define SATP64_PPN			_ULL(0x00000FFFFFFFFFFF)
 
-#define SATP_MODE_OFF			0
-#define SATP_MODE_SV32			1
-#define SATP_MODE_SV39			8
-#define SATP_MODE_SV48			9
-#define SATP_MODE_SV57			10
-#define SATP_MODE_SV64			11
+#define SATP_MODE_OFF			_UL(0)
+#define SATP_MODE_SV32			_UL(1)
+#define SATP_MODE_SV39			_UL(8)
+#define SATP_MODE_SV48			_UL(9)
+#define SATP_MODE_SV57			_UL(10)
+#define SATP_MODE_SV64			_UL(11)
 
-#define PMP_R				0x01
-#define PMP_W				0x02
-#define PMP_X				0x04
-#define PMP_A				0x18
-#define PMP_A_TOR			0x08
-#define PMP_A_NA4			0x10
-#define PMP_A_NAPOT			0x18
-#define PMP_L				0x80
+#define PMP_R				_UL(0x01)
+#define PMP_W				_UL(0x02)
+#define PMP_X				_UL(0x04)
+#define PMP_A				_UL(0x18)
+#define PMP_A_TOR			_UL(0x08)
+#define PMP_A_NA4			_UL(0x10)
+#define PMP_A_NAPOT			_UL(0x18)
+#define PMP_L				_UL(0x80)
 
 #define PMP_SHIFT			2
 #define PMP_COUNT			16
 
 /* page table entry (PTE) fields */
-#define PTE_V				0x001 /* Valid */
-#define PTE_R				0x002 /* Read */
-#define PTE_W				0x004 /* Write */
-#define PTE_X				0x008 /* Execute */
-#define PTE_U				0x010 /* User */
-#define PTE_G				0x020 /* Global */
-#define PTE_A				0x040 /* Accessed */
-#define PTE_D				0x080 /* Dirty */
-#define PTE_SOFT			0x300 /* Reserved for Software */
+#define PTE_V				_UL(0x001) /* Valid */
+#define PTE_R				_UL(0x002) /* Read */
+#define PTE_W				_UL(0x004) /* Write */
+#define PTE_X				_UL(0x008) /* Execute */
+#define PTE_U				_UL(0x010) /* User */
+#define PTE_G				_UL(0x020) /* Global */
+#define PTE_A				_UL(0x040) /* Accessed */
+#define PTE_D				_UL(0x080) /* Dirty */
+#define PTE_SOFT			_UL(0x300) /* Reserved for Software */
 
 #define PTE_PPN_SHIFT			10
 
@@ -276,10 +210,16 @@
 #define CSR_HSTATUS			0x600
 #define CSR_HEDELEG			0x602
 #define CSR_HIDELEG			0x603
+#define CSR_HIE				0x604
 #define CSR_HTIMEDELTA			0x605
 #define CSR_HTIMEDELTAH			0x615
 #define CSR_HCOUNTERNEN			0x606
+#define CSR_HGEIE			0x607
+#define CSR_HTVAL			0x643
+#define CSR_HIP				0x644
+#define CSR_HTINST			0x64a
 #define CSR_HGATP			0x680
+#define CSR_HGEIP			0xe07
 
 #define CSR_VSSTATUS			0x200
 #define CSR_VSIE			0x204
@@ -304,6 +244,8 @@
 #define CSR_MCAUSE			0x342
 #define CSR_MTVAL			0x343
 #define CSR_MIP				0x344
+#define CSR_MTINST			0x34a
+#define CSR_MTVAL2			0x34b
 #define CSR_PMPCFG0			0x3a0
 #define CSR_PMPCFG1			0x3a1
 #define CSR_PMPCFG2			0x3a2
@@ -475,6 +417,9 @@
 #define CAUSE_FETCH_PAGE_FAULT		0xc
 #define CAUSE_LOAD_PAGE_FAULT		0xd
 #define CAUSE_STORE_PAGE_FAULT		0xf
+#define CAUSE_FETCH_GUEST_PAGE_FAULT	0x14
+#define CAUSE_LOAD_GUEST_PAGE_FAULT	0x15
+#define CAUSE_STORE_GUEST_PAGE_FAULT	0x17
 
 #define INSN_MATCH_LB			0x3
 #define INSN_MASK_LB			0x707f
@@ -549,7 +494,16 @@
 #define INSN_MASK_WFI			0xffffff00
 #define INSN_MATCH_WFI			0x10500000
 
-#define INSN_LEN(insn)			((((insn) & 0x3) < 0x3) ? 2 : 4)
+#define INSN_16BIT_MASK			0x3
+#define INSN_32BIT_MASK			0x1c
+
+#define INSN_IS_16BIT(insn)		\
+	(((insn) & INSN_16BIT_MASK) != INSN_16BIT_MASK)
+#define INSN_IS_32BIT(insn)		\
+	(((insn) & INSN_16BIT_MASK) == INSN_16BIT_MASK && \
+	 ((insn) & INSN_32BIT_MASK) != INSN_32BIT_MASK)
+
+#define INSN_LEN(insn)			(INSN_IS_16BIT(insn) ? 2 : 4)
 
 #if __riscv_xlen == 64
 #define LOG_REGBYTES			3

include/sbi/sbi_bitmap.h

@@ -0,0 +1,128 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2020 Western Digital Corporation or its affiliates.
+ *
+ * Authors:
+ *   Anup Patel <anup.patel@wdc.com>
+ */
+
+#ifndef __SBI_BITMAP_H__
+#define __SBI_BITMAP_H__
+
+#include <sbi/sbi_bitops.h>
+
+#define BITMAP_FIRST_WORD_MASK(start) (~0UL << ((start) % BITS_PER_LONG))
+#define BITMAP_LAST_WORD_MASK(nbits)					\
+(									\
+	((nbits) % BITS_PER_LONG) ?					\
+		((1UL << ((nbits) % BITS_PER_LONG)) - 1) : ~0UL		\
+)
+
+#define small_const_nbits(nbits) \
+	(__builtin_constant_p(nbits) && (nbits) <= BITS_PER_LONG)
+
+#define DECLARE_BITMAP(name, nbits)	unsigned long name[BITS_TO_LONGS(nbits)]
+#define DEFINE_BITMAP(name)		extern unsigned long name[]
+
+static inline unsigned long bitmap_estimate_size(int nbits)
+{
+	return (BITS_TO_LONGS(nbits) * sizeof(unsigned long));
+}
+
+void __bitmap_and(unsigned long *dst, const unsigned long *bitmap1,
+		  const unsigned long *bitmap2, int bits);
+void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1,
+		 const unsigned long *bitmap2, int bits);
+void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1,
+		  const unsigned long *bitmap2, int bits);
+
+static inline void bitmap_set(unsigned long *bmap, int start, int len)
+{
+	int bit;
+	for (bit = start; bit < (start + len); bit++)
+		bmap[BIT_WORD(bit)] |= (0x1UL << BIT_WORD_OFFSET(bit));
+}
+
+static inline void bitmap_clear(unsigned long *bmap, int start, int len)
+{
+	int bit;
+	for (bit = start; bit < (start + len); bit++)
+		bmap[BIT_WORD(bit)] &= ~(0x1UL << BIT_WORD_OFFSET(bit));
+}
+
+static inline void bitmap_zero(unsigned long *dst, int nbits)
+{
+	if (small_const_nbits(nbits))
+		*dst = 0UL;
+	else {
+		size_t i, len = BITS_TO_LONGS(nbits);
+		for (i = 0; i < len; i++)
+			dst[i] = 0;
+	}
+}
+
+static inline void bitmap_zero_except(unsigned long *dst,
+				      int exception, int nbits)
+{
+	if (small_const_nbits(nbits))
+		*dst = 0UL;
+	else {
+		size_t i, len = BITS_TO_LONGS(nbits);
+		for (i = 0; i < len; i++)
+			dst[i] = 0;
+	}
+	if (exception < nbits)
+		__set_bit(exception, dst);
+}
+
+static inline void bitmap_fill(unsigned long *dst, int nbits)
+{
+	size_t i, nlongs = BITS_TO_LONGS(nbits);
+	if (!small_const_nbits(nbits)) {
+		for (i = 0; i < (nlongs - 1); i++)
+			dst[i] = -1UL;
+	}
+	dst[nlongs - 1] = BITMAP_LAST_WORD_MASK(nbits);
+}
+
+static inline void bitmap_copy(unsigned long *dst,
+			       const unsigned long *src, int nbits)
+{
+	if (small_const_nbits(nbits))
+		*dst = *src;
+	else {
+		size_t i, len = BITS_TO_LONGS(nbits);
+		for (i = 0; i < len; i++)
+			dst[i] = src[i];
+	}
+}
+
+static inline void bitmap_and(unsigned long *dst, const unsigned long *src1,
+			      const unsigned long *src2, int nbits)
+{
+	if (small_const_nbits(nbits))
+		*dst = *src1 & *src2;
+	else
+		__bitmap_and(dst, src1, src2, nbits);
+}
+
+static inline void bitmap_or(unsigned long *dst, const unsigned long *src1,
+			     const unsigned long *src2, int nbits)
+{
+	if (small_const_nbits(nbits))
+		*dst = *src1 | *src2;
+	else
+		__bitmap_or(dst, src1, src2, nbits);
+}
+
+static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1,
+			      const unsigned long *src2, int nbits)
+{
+	if (small_const_nbits(nbits))
+		*dst = *src1 ^ *src2;
+	else
+		__bitmap_xor(dst, src1, src2, nbits);
+}
+
+#endif

include/sbi/sbi_bitops.h

@@ -4,14 +4,37 @@
  * Copyright (c) 2019 Western Digital Corporation or its affiliates.
  *
  * Authors:
- *   Atish Patra<atish.patra@wdc.com>
+ *   Atish Patra <atish.patra@wdc.com>
  */
 
 #ifndef __SBI_BITOPS_H__
 #define __SBI_BITOPS_H__
 
-#include <sbi/sbi_bits.h>
-#include <sbi/riscv_asm.h>
+#include <sbi/sbi_types.h>
+
+#if __SIZEOF_POINTER__ == 8
+#define BITS_PER_LONG		64
+#elif __SIZEOF_POINTER__ == 4
+#define BITS_PER_LONG		32
+#else
+#error "Unexpected __SIZEOF_POINTER__"
+#endif
+
+#define EXTRACT_FIELD(val, which) \
+	(((val) & (which)) / ((which) & ~((which)-1)))
+#define INSERT_FIELD(val, which, fieldval) \
+	(((val) & ~(which)) | ((fieldval) * ((which) & ~((which)-1))))
+
+#define BITS_TO_LONGS(nbits)	(((nbits) + BITS_PER_LONG - 1) / \
+				 BITS_PER_LONG)
+
+#define BIT(nr)			(1UL << (nr))
+#define BIT_MASK(nr)		(1UL << ((nr) % BITS_PER_LONG))
+#define BIT_WORD(bit)		((bit) / BITS_PER_LONG)
+#define BIT_WORD_OFFSET(bit)	((bit) & (BITS_PER_LONG - 1))
+
+#define GENMASK(h, l) \
+	(((~0UL) - (1UL << (l)) + 1) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
 
 /**
  * ffs - Find first bit set
@@ -95,4 +118,207 @@ static inline int __ffs(unsigned long word)
  */
 #define ffz(x) __ffs(~(x))
 
+/**
+ * fls - find last (most-significant) bit set
+ * @x: the word to search
+ *
+ * This is defined the same way as ffs.
+ * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
+ */
+
+static inline int fls(int x)
+{
+	int r = 32;
+
+	if (!x)
+		return 0;
+	if (!(x & 0xffff0000u)) {
+		x <<= 16;
+		r -= 16;
+	}
+	if (!(x & 0xff000000u)) {
+		x <<= 8;
+		r -= 8;
+	}
+	if (!(x & 0xf0000000u)) {
+		x <<= 4;
+		r -= 4;
+	}
+	if (!(x & 0xc0000000u)) {
+		x <<= 2;
+		r -= 2;
+	}
+	if (!(x & 0x80000000u)) {
+		x <<= 1;
+		r -= 1;
+	}
+	return r;
+}
+
+/**
+ * __fls - find last (most-significant) set bit in a long word
+ * @word: the word to search
+ *
+ * Undefined if no set bit exists, so code should check against 0 first.
+ */
+static inline unsigned long __fls(unsigned long word)
+{
+	int num = BITS_PER_LONG - 1;
+
+#if BITS_PER_LONG == 64
+	if (!(word & (~0ul << 32))) {
+		num -= 32;
+		word <<= 32;
+	}
+#endif
+	if (!(word & (~0ul << (BITS_PER_LONG-16)))) {
+		num -= 16;
+		word <<= 16;
+	}
+	if (!(word & (~0ul << (BITS_PER_LONG-8)))) {
+		num -= 8;
+		word <<= 8;
+	}
+	if (!(word & (~0ul << (BITS_PER_LONG-4)))) {
+		num -= 4;
+		word <<= 4;
+	}
+	if (!(word & (~0ul << (BITS_PER_LONG-2)))) {
+		num -= 2;
+		word <<= 2;
+	}
+	if (!(word & (~0ul << (BITS_PER_LONG-1))))
+		num -= 1;
+	return num;
+}
+
+#define for_each_set_bit(bit, addr, size) \
+	for ((bit) = find_first_bit((addr), (size));		\
+	     (bit) < (size);					\
+	     (bit) = find_next_bit((addr), (size), (bit) + 1))
+
+/* same as for_each_set_bit() but use bit as value to start with */
+#define for_each_set_bit_from(bit, addr, size) \
+	for ((bit) = find_next_bit((addr), (size), (bit));	\
+	     (bit) < (size);					\
+	     (bit) = find_next_bit((addr), (size), (bit) + 1))
+
+#define for_each_clear_bit(bit, addr, size) \
+	for ((bit) = find_first_zero_bit((addr), (size));	\
+	     (bit) < (size);					\
+	     (bit) = find_next_zero_bit((addr), (size), (bit) + 1))
+
+/* same as for_each_clear_bit() but use bit as value to start with */
+#define for_each_clear_bit_from(bit, addr, size) \
+	for ((bit) = find_next_zero_bit((addr), (size), (bit));	\
+	     (bit) < (size);					\
+	     (bit) = find_next_zero_bit((addr), (size), (bit) + 1))
+
+unsigned long find_first_bit(const unsigned long *addr,
+			     unsigned long size);
+
+unsigned long find_first_zero_bit(const unsigned long *addr,
+				  unsigned long size);
+
+unsigned long find_last_bit(const unsigned long *addr,
+			    unsigned long size);
+
+unsigned long find_next_bit(const unsigned long *addr,
+			    unsigned long size, unsigned long offset);
+
+unsigned long find_next_zero_bit(const unsigned long *addr,
+				 unsigned long size,
+				 unsigned long offset);
+
+/**
+ * __set_bit - Set a bit in memory
+ * @nr: the bit to set
+ * @addr: the address to start counting from
+ *
+ * This function is non-atomic and may be reordered.
+ */
+static inline void __set_bit(int nr, volatile unsigned long *addr)
+{
+	unsigned long mask = BIT_MASK(nr);
+	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
+
+	*p |= mask;
+}
+
+/**
+ * __clear_bit - Clear a bit in memory
+ * @nr: the bit to clear
+ * @addr: the address to start counting from
+ *
+ * This function is non-atomic and may be reordered.
+ */
+static inline void __clear_bit(int nr, volatile unsigned long *addr)
+{
+	unsigned long mask = BIT_MASK(nr);
+	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
+
+	*p &= ~mask;
+}
+
+/**
+ * __change_bit - Toggle a bit in memory
+ * @nr: the bit to change
+ * @addr: the address to start counting from
+ *
+ * This function is non-atomic and may be reordered.
+ */
+static inline void __change_bit(int nr, volatile unsigned long *addr)
+{
+	unsigned long mask = BIT_MASK(nr);
+	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
+
+	*p ^= mask;
+}
+
+/**
+ * __test_and_set_bit - Set a bit and return its old value
+ * @nr: Bit to set
+ * @addr: Address to count from
+ *
+ * This operation is non-atomic and can be reordered.
+ */
+static inline int __test_and_set_bit(int nr, volatile unsigned long *addr)
+{
+	unsigned long mask = BIT_MASK(nr);
+	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
+	unsigned long old = *p;
+
+	*p = old | mask;
+	return (old & mask) != 0;
+}
+
+/**
+ * __test_and_clear_bit - Clear a bit and return its old value
+ * @nr: Bit to clear
+ * @addr: Address to count from
+ *
+ * This operation is non-atomic and can be reordered.
+ */
+static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr)
+{
+	unsigned long mask = BIT_MASK(nr);
+	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
+	unsigned long old = *p;
+
+	*p = old & ~mask;
+	return (old & mask) != 0;
+}
+
+/**
+ * __test_bit - Determine whether a bit is set
+ * @nr: bit number to test
+ * @addr: Address to start counting from
+ *
+ * This operation is non-atomic and can be reordered.
+ */
+static inline int __test_bit(int nr, const volatile unsigned long *addr)
+{
+	return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
+}
+
 #endif

include/sbi/sbi_bits.h

@@ -1,32 +0,0 @@
-/*
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2019 Western Digital Corporation or its affiliates.
- *
- * Authors:
- *   Anup Patel <anup.patel@wdc.com>
- */
-
-#ifndef __SBI_BITS_H__
-#define __SBI_BITS_H__
-
-#define likely(x) __builtin_expect((x), 1)
-#define unlikely(x) __builtin_expect((x), 0)
-
-#define ROUNDUP(a, b) ((((a)-1) / (b) + 1) * (b))
-#define ROUNDDOWN(a, b) ((a) / (b) * (b))
-
-#define MAX(a, b) ((a) > (b) ? (a) : (b))
-#define MIN(a, b) ((a) < (b) ? (a) : (b))
-#define CLAMP(a, lo, hi) MIN(MAX(a, lo), hi)
-
-#define EXTRACT_FIELD(val, which) (((val) & (which)) / ((which) & ~((which)-1)))
-#define INSERT_FIELD(val, which, fieldval) \
-	(((val) & ~(which)) | ((fieldval) * ((which) & ~((which)-1))))
-
-#define STR(x) XSTR(x)
-#define XSTR(x) #x
-
-#define BIT_MASK(nr) (1UL << ((nr) % BITS_PER_LONG))
-#define BIT_WORD(nr) ((nr) / BITS_PER_LONG)
-#endif

include/sbi/sbi_console.h

@@ -30,10 +30,9 @@ int __printf(3, 4) sbi_snprintf(char *out, u32 out_sz, const char *format, ...);
 
 int __printf(1, 2) sbi_printf(const char *format, ...);
 
-struct sbi_scratch;
+int __printf(1, 2) sbi_dprintf(const char *format, ...);
 
-int __printf(2, 3) sbi_dprintf(struct sbi_scratch *scratch,
-			       const char *format, ...);
+struct sbi_scratch;
 
 int sbi_console_init(struct sbi_scratch *scratch);
 

include/sbi/sbi_const.h

@@ -10,7 +10,8 @@
 #ifndef __SBI_CONST_H__
 #define __SBI_CONST_H__
 
-/* Some constant macros are used in both assembler and
+/*
+ * Some constant macros are used in both assembler and
  * C code.  Therefore we cannot annotate them always with
  * 'UL' and other type specifiers unilaterally.  We
  * use the following macros to deal with this.
@@ -39,8 +40,8 @@
 #define UL(x)		(_UL(x))
 #define ULL(x)		(_ULL(x))
 
-#define	__STR(s)	#s
-#define	STRINGIFY(s)	__STR(s)
+#define __STR(s)	#s
+#define STRINGIFY(s)	__STR(s)
 
 /* clang-format on */
 

include/sbi/sbi_ecall.h

@@ -11,15 +11,49 @@
 #define __SBI_ECALL_H__
 
 #include <sbi/sbi_types.h>
+#include <sbi/sbi_list.h>
+
+#define SBI_ECALL_VERSION_MAJOR		0
+#define SBI_ECALL_VERSION_MINOR		2
+#define SBI_OPENSBI_IMPID		1
 
 struct sbi_trap_regs;
-struct sbi_scratch;
+struct sbi_trap_info;
+
+struct sbi_ecall_extension {
+	struct sbi_dlist head;
+	unsigned long extid_start;
+	unsigned long extid_end;
+	int (* probe)(unsigned long extid, unsigned long *out_val);
+	int (* handle)(unsigned long extid, unsigned long funcid,
+		       unsigned long *args, unsigned long *out_val,
+		       struct sbi_trap_info *out_trap);
+};
+
+extern struct sbi_ecall_extension ecall_base;
+extern struct sbi_ecall_extension ecall_legacy;
+extern struct sbi_ecall_extension ecall_time;
+extern struct sbi_ecall_extension ecall_rfence;
+extern struct sbi_ecall_extension ecall_ipi;
+extern struct sbi_ecall_extension ecall_vendor;
+extern struct sbi_ecall_extension ecall_hsm;
 
 u16 sbi_ecall_version_major(void);
 
 u16 sbi_ecall_version_minor(void);
 
-int sbi_ecall_handler(u32 hartid, ulong mcause, struct sbi_trap_regs *regs,
-		      struct sbi_scratch *scratch);
+unsigned long sbi_ecall_get_impid(void);
+
+void sbi_ecall_set_impid(unsigned long impid);
+
+struct sbi_ecall_extension *sbi_ecall_find_extension(unsigned long extid);
+
+int sbi_ecall_register_extension(struct sbi_ecall_extension *ext);
+
+void sbi_ecall_unregister_extension(struct sbi_ecall_extension *ext);
+
+int sbi_ecall_handler(struct sbi_trap_regs *regs);
+
+int sbi_ecall_init(void);
 
 #endif

include/sbi/sbi_ecall_interface.h

@@ -12,34 +12,61 @@
 
 /* clang-format off */
 
-enum sbi_ext_id {
-	SBI_EXT_0_1_SET_TIMER = 0x0,
-	SBI_EXT_0_1_CONSOLE_PUTCHAR = 0x1,
-	SBI_EXT_0_1_CONSOLE_GETCHAR = 0x2,
-	SBI_EXT_0_1_CLEAR_IPI = 0x3,
-	SBI_EXT_0_1_SEND_IPI = 0x4,
-	SBI_EXT_0_1_REMOTE_FENCE_I = 0x5,
-	SBI_EXT_0_1_REMOTE_SFENCE_VMA = 0x6,
-	SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID = 0x7,
-	SBI_EXT_0_1_SHUTDOWN = 0x8,
-	SBI_EXT_BASE = 0x10,
-};
+/* SBI Extension IDs */
+#define	SBI_EXT_0_1_SET_TIMER			0x0
+#define SBI_EXT_0_1_CONSOLE_PUTCHAR		0x1
+#define SBI_EXT_0_1_CONSOLE_GETCHAR		0x2
+#define SBI_EXT_0_1_CLEAR_IPI			0x3
+#define SBI_EXT_0_1_SEND_IPI			0x4
+#define SBI_EXT_0_1_REMOTE_FENCE_I		0x5
+#define SBI_EXT_0_1_REMOTE_SFENCE_VMA		0x6
+#define SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID	0x7
+#define SBI_EXT_0_1_SHUTDOWN			0x8
+#define SBI_EXT_BASE				0x10
+#define SBI_EXT_TIME				0x54494D45
+#define SBI_EXT_IPI				0x735049
+#define SBI_EXT_RFENCE				0x52464E43
+#define SBI_EXT_HSM				0x48534D
 
-enum sbi_ext_base_fid {
-	SBI_EXT_BASE_GET_SPEC_VERSION = 0,
-	SBI_EXT_BASE_GET_IMP_ID,
-	SBI_EXT_BASE_GET_IMP_VERSION,
-	SBI_EXT_BASE_PROBE_EXT,
-	SBI_EXT_BASE_GET_MVENDORID,
-	SBI_EXT_BASE_GET_MARCHID,
-	SBI_EXT_BASE_GET_MIMPID,
-};
+/* SBI function IDs for BASE extension*/
+#define SBI_EXT_BASE_GET_SPEC_VERSION		0x0
+#define SBI_EXT_BASE_GET_IMP_ID			0x1
+#define SBI_EXT_BASE_GET_IMP_VERSION		0x2
+#define SBI_EXT_BASE_PROBE_EXT			0x3
+#define SBI_EXT_BASE_GET_MVENDORID		0x4
+#define SBI_EXT_BASE_GET_MARCHID		0x5
+#define SBI_EXT_BASE_GET_MIMPID			0x6
 
-#define SBI_SPEC_VERSION_MAJOR_OFFSET	24
-#define SBI_SPEC_VERSION_MAJOR_MASK	0x7f
-#define SBI_SPEC_VERSION_MINOR_MASK	0xffffff
-#define SBI_EXT_VENDOR_START		0x09000000
-#define SBI_EXT_VENDOR_END		0x09FFFFFF
+/* SBI function IDs for TIME extension*/
+#define SBI_EXT_TIME_SET_TIMER			0x0
+
+/* SBI function IDs for IPI extension*/
+#define SBI_EXT_IPI_SEND_IPI			0x0
+
+/* SBI function IDs for RFENCE extension*/
+#define SBI_EXT_RFENCE_REMOTE_FENCE_I		0x0
+#define SBI_EXT_RFENCE_REMOTE_SFENCE_VMA	0x1
+#define SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID	0x2
+#define SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA	0x3
+#define SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID	0x4
+#define SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA	0x5
+#define SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID	0x6
+
+/* SBI function IDs for HSM extension */
+#define SBI_EXT_HSM_HART_START			0x0
+#define SBI_EXT_HSM_HART_STOP			0x1
+#define SBI_EXT_HSM_HART_GET_STATUS		0x2
+
+#define SBI_HSM_HART_STATUS_STARTED		0x0
+#define SBI_HSM_HART_STATUS_STOPPED		0x1
+#define SBI_HSM_HART_STATUS_START_PENDING	0x2
+#define SBI_HSM_HART_STATUS_STOP_PENDING	0x3
+
+#define SBI_SPEC_VERSION_MAJOR_OFFSET		24
+#define SBI_SPEC_VERSION_MAJOR_MASK		0x7f
+#define SBI_SPEC_VERSION_MINOR_MASK		0xffffff
+#define SBI_EXT_VENDOR_START			0x09000000
+#define SBI_EXT_VENDOR_END			0x09FFFFFF
 /* clang-format on */
 
 #endif

include/sbi/sbi_emulate_csr.h

@@ -13,12 +13,11 @@
 #include <sbi/sbi_types.h>
 
 struct sbi_trap_regs;
-struct sbi_scratch;
 
-int sbi_emulate_csr_read(int csr_num, u32 hartid, struct sbi_trap_regs *regs,
-			 struct sbi_scratch *scratch, ulong *csr_val);
+int sbi_emulate_csr_read(int csr_num, struct sbi_trap_regs *regs,
+			 ulong *csr_val);
 
-int sbi_emulate_csr_write(int csr_num, u32 hartid, struct sbi_trap_regs *regs,
-			  struct sbi_scratch *scratch, ulong csr_val);
+int sbi_emulate_csr_write(int csr_num, struct sbi_trap_regs *regs,
+			  ulong csr_val);
 
 #endif

include/sbi/sbi_error.h

@@ -12,22 +12,23 @@
 
 /* clang-format off */
 
-#define SBI_OK		0
-#define SBI_EFAIL	-1
-#define SBI_ENOTSUPP	-2
-#define SBI_EINVAL	-3
-#define SBI_DENIED	-4
-#define SBI_INVALID_ADDR -5
-#define SBI_ENODEV	-6
-#define SBI_ENOSYS	-7
-#define SBI_ETIMEDOUT	-8
-#define SBI_EIO		-9
-#define SBI_EILL	-10
-#define SBI_ENOSPC	-11
-#define SBI_ENOMEM	-12
-#define SBI_ETRAP	-13
-#define SBI_EUNKNOWN	-14
-#define SBI_ENOENT	-15
+#define SBI_OK			0
+#define SBI_EFAIL		-1
+#define SBI_ENOTSUPP		-2
+#define SBI_EINVAL		-3
+#define SBI_DENIED		-4
+#define SBI_INVALID_ADDR	-5
+#define SBI_ENODEV		-6
+#define SBI_ENOSYS		-7
+#define SBI_ETIMEDOUT		-8
+#define SBI_EIO			-9
+#define SBI_EILL		-10
+#define SBI_ENOSPC		-11
+#define SBI_ENOMEM		-12
+#define SBI_ETRAP		-13
+#define SBI_EUNKNOWN		-14
+#define SBI_ENOENT		-15
+#define SBI_EALREADY_STARTED	-16
 
 /* clang-format on */
 

include/sbi/sbi_hart.h

@@ -16,11 +16,16 @@ struct sbi_scratch;
 
 int sbi_hart_init(struct sbi_scratch *scratch, u32 hartid, bool cold_boot);
 
-void *sbi_hart_get_trap_info(struct sbi_scratch *scratch);
-
-void sbi_hart_set_trap_info(struct sbi_scratch *scratch, void *data);
+extern void (*sbi_hart_unpriv_trap)(void);
+static inline ulong sbi_hart_unpriv_trap_addr(void)
+{
+	return (ulong)sbi_hart_unpriv_trap;
+}
 
+void sbi_hart_delegation_dump(struct sbi_scratch *scratch);
 void sbi_hart_pmp_dump(struct sbi_scratch *scratch);
+int  sbi_hart_pmp_check_addr(struct sbi_scratch *scratch, unsigned long daddr,
+			     unsigned long attr);
 
 void __attribute__((noreturn)) sbi_hart_hang(void);
 
@@ -29,19 +34,4 @@ sbi_hart_switch_mode(unsigned long arg0, unsigned long arg1,
 		     unsigned long next_addr, unsigned long next_mode,
 		     bool next_virt);
 
-void sbi_hart_mark_available(u32 hartid);
-
-ulong sbi_hart_available_mask(void);
-
-void sbi_hart_unmark_available(u32 hartid);
-
-struct sbi_scratch *sbi_hart_id_to_scratch(struct sbi_scratch *scratch,
-					   u32 hartid);
-
-void sbi_hart_wait_for_coldboot(struct sbi_scratch *scratch, u32 hartid);
-
-void sbi_hart_wake_coldboot_harts(struct sbi_scratch *scratch, u32 hartid);
-
-u32 sbi_current_hartid(void);
-
 #endif

include/sbi/sbi_hartmask.h

@@ -0,0 +1,141 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2020 Western Digital Corporation or its affiliates.
+ *
+ * Authors:
+ *   Anup Patel <anup.patel@wdc.com>
+ */
+
+#ifndef __SBI_HARTMASK_H__
+#define __SBI_HARTMASK_H__
+
+#include <sbi/sbi_bitmap.h>
+
+/**
+ * Maximum number of bits in a hartmask
+ *
+ * The hartmask is indexed using physical HART id so this define
+ * also represents the maximum number of HART ids generic OpenSBI
+ * can handle.
+ */
+#define SBI_HARTMASK_MAX_BITS		128
+
+/** Representation of hartmask */
+struct sbi_hartmask {
+	DECLARE_BITMAP(bits, SBI_HARTMASK_MAX_BITS);
+};
+
+/** Initialize hartmask to zero */
+#define SBI_HARTMASK_INIT(__m)		\
+	bitmap_zero(((__m)->bits), SBI_HARTMASK_MAX_BITS)
+
+/** Initialize hartmask to zero except a particular HART id */
+#define SBI_HARTMASK_INIT_EXCEPT(__m, __h)	\
+	bitmap_zero_except(((__m)->bits), (__h), SBI_HARTMASK_MAX_BITS)
+
+/**
+ * Get underlying bitmap of hartmask
+ * @param m the hartmask pointer
+ */
+#define sbi_hartmask_bits(__m)		((__m)->bits)
+
+/**
+ * Set a HART in hartmask
+ * @param h HART id to set
+ * @param m the hartmask pointer
+ */
+static inline void sbi_hartmask_set_hart(u32 h, struct sbi_hartmask *m)
+{
+	if (h < SBI_HARTMASK_MAX_BITS)
+		__set_bit(h, m->bits);
+}
+
+/**
+ * Clear a HART in hartmask
+ * @param h HART id to clear
+ * @param m the hartmask pointer
+ */
+static inline void sbi_hartmask_clear_hart(u32 h, struct sbi_hartmask *m)
+{
+	if (h < SBI_HARTMASK_MAX_BITS)
+		__clear_bit(h, m->bits);
+}
+
+/**
+ * Test a HART in hartmask
+ * @param h HART id to test
+ * @param m the hartmask pointer
+ */
+static inline int sbi_hartmask_test_hart(u32 h, struct sbi_hartmask *m)
+{
+	if (h < SBI_HARTMASK_MAX_BITS)
+		return __test_bit(h, m->bits);
+	return 0;
+}
+
+/**
+ * Set all HARTs in a hartmask
+ * @param dstp the hartmask pointer
+ */
+static inline void sbi_hartmask_set_all(struct sbi_hartmask *dstp)
+{
+	bitmap_fill(sbi_hartmask_bits(dstp), SBI_HARTMASK_MAX_BITS);
+}
+
+/**
+ * Clear all HARTs in a hartmask
+ * @param dstp the hartmask pointer
+ */
+static inline void sbi_hartmask_clear_all(struct sbi_hartmask *dstp)
+{
+	bitmap_zero(sbi_hartmask_bits(dstp), SBI_HARTMASK_MAX_BITS);
+}
+
+/**
+ * *dstp = *src1p & *src2p
+ * @param dstp the hartmask result
+ * @param src1p the first input
+ * @param src2p the second input
+ */
+static inline void sbi_hartmask_and(struct sbi_hartmask *dstp,
+				    const struct sbi_hartmask *src1p,
+				    const struct sbi_hartmask *src2p)
+{
+	bitmap_and(sbi_hartmask_bits(dstp), sbi_hartmask_bits(src1p),
+		   sbi_hartmask_bits(src2p), SBI_HARTMASK_MAX_BITS);
+}
+
+/**
+ * *dstp = *src1p | *src2p
+ * @param dstp the hartmask result
+ * @param src1p the first input
+ * @param src2p the second input
+ */
+static inline void sbi_hartmask_or(struct sbi_hartmask *dstp,
+				   const struct sbi_hartmask *src1p,
+				   const struct sbi_hartmask *src2p)
+{
+	bitmap_or(sbi_hartmask_bits(dstp), sbi_hartmask_bits(src1p),
+		  sbi_hartmask_bits(src2p), SBI_HARTMASK_MAX_BITS);
+}
+
+/**
+ * *dstp = *src1p ^ *src2p
+ * @param dstp the hartmask result
+ * @param src1p the first input
+ * @param src2p the second input
+ */
+static inline void sbi_hartmask_xor(struct sbi_hartmask *dstp,
+				    const struct sbi_hartmask *src1p,
+				    const struct sbi_hartmask *src2p)
+{
+	bitmap_xor(sbi_hartmask_bits(dstp), sbi_hartmask_bits(src1p),
+		   sbi_hartmask_bits(src2p), SBI_HARTMASK_MAX_BITS);
+}
+
+/** Iterate over each HART in hartmask */
+#define sbi_hartmask_for_each_hart(__h, __m)	\
+	for_each_set_bit(__h, (__m)->bits, SBI_HARTMASK_MAX_BITS)
+
+#endif

include/sbi/sbi_hfence.h

@@ -0,0 +1,36 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2019 Western Digital Corporation or its affiliates.
+ *
+ * Authors:
+ *   Atish Patra <atish.patra@wdc.com>
+ *   Anup Patel <anup.patel@wdc.com>
+ */
+
+#ifndef __SBI_FENCE_H__
+#define __SBI_FENCE_H__
+/** Invalidate Stage2 TLBs for given VMID and guest physical address */
+void __sbi_hfence_gvma_vmid_gpa(unsigned long vmid, unsigned long gpa);
+
+/** Invalidate Stage2 TLBs for given VMID */
+void __sbi_hfence_gvma_vmid(unsigned long vmid);
+
+/** Invalidate Stage2 TLBs for given guest physical address */
+void __sbi_hfence_gvma_gpa(unsigned long gpa);
+
+/** Invalidate all possible Stage2 TLBs */
+void __sbi_hfence_gvma_all(void);
+
+/** Invalidate unified TLB entries for given asid and guest virtual address */
+void __sbi_hfence_vvma_asid_va(unsigned long asid, unsigned long va);
+
+/** Invalidate unified TLB entries for given ASID for a guest*/
+void __sbi_hfence_vvma_asid(unsigned long asid);
+
+/** Invalidate unified TLB entries for a given guest virtual address */
+void __sbi_hfence_vvma_va(unsigned long va);
+
+/** Invalidate all possible Stage2 TLBs */
+void __sbi_hfence_vvma_all(void);
+#endif

include/sbi/sbi_hsm.h

@@ -0,0 +1,36 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2020 Western Digital Corporation or its affiliates.
+ *
+ * Authors:
+ *   Atish Patra <atish.patra@wdc.com>
+ */
+
+#ifndef __SBI_HSM_H__
+#define __SBI_HSM_H__
+
+#include <sbi/sbi_types.h>
+
+/** Hart state values **/
+#define SBI_HART_STOPPED	0
+#define SBI_HART_STOPPING	1
+#define SBI_HART_STARTING	2
+#define SBI_HART_STARTED	3
+#define SBI_HART_UNKNOWN	4
+
+struct sbi_scratch;
+
+int sbi_hsm_init(struct sbi_scratch *scratch, u32 hartid, bool cold_boot);
+void __noreturn sbi_hsm_exit(struct sbi_scratch *scratch);
+
+int sbi_hsm_hart_start(struct sbi_scratch *scratch, u32 hartid,
+		       ulong saddr, ulong priv);
+int sbi_hsm_hart_stop(struct sbi_scratch *scratch, bool exitnow);
+int sbi_hsm_hart_get_state(u32 hartid);
+int sbi_hsm_hart_state_to_status(int state);
+bool sbi_hsm_hart_started(u32 hartid);
+int sbi_hsm_hart_started_mask(ulong hbase, ulong *out_hmask);
+void sbi_hsm_prepare_next_jump(struct sbi_scratch *scratch, u32 hartid);
+
+#endif

include/sbi/sbi_illegal_insn.h

@@ -13,10 +13,7 @@
 #include <sbi/sbi_types.h>
 
 struct sbi_trap_regs;
-struct sbi_scratch;
 
-int sbi_illegal_insn_handler(u32 hartid, ulong mcause,
-			     struct sbi_trap_regs *regs,
-			     struct sbi_scratch *scratch);
+int sbi_illegal_insn_handler(ulong insn, struct sbi_trap_regs *regs);
 
 #endif

include/sbi/sbi_init.h

@@ -16,4 +16,8 @@ struct sbi_scratch;
 
 void __noreturn sbi_init(struct sbi_scratch *scratch);
 
+unsigned long sbi_init_count(u32 hartid);
+
+void __noreturn sbi_exit(struct sbi_scratch *scratch);
+
 #endif

include/sbi/sbi_ipi.h

@@ -10,32 +10,61 @@
 #ifndef __SBI_IPI_H__
 #define __SBI_IPI_H__
 
-#include <sbi/riscv_unpriv.h>
 #include <sbi/sbi_types.h>
 
 /* clang-format off */
 
-#define SBI_IPI_EVENT_SOFT			0x1
-#define SBI_IPI_EVENT_FENCE_I			0x2
-#define SBI_IPI_EVENT_SFENCE_VMA		0x4
-#define SBI_IPI_EVENT_SFENCE_VMA_ASID		0x8
-#define SBI_IPI_EVENT_HALT			0x10
+#define SBI_IPI_EVENT_MAX			__riscv_xlen
 
 /* clang-format on */
 
 struct sbi_scratch;
 
-struct sbi_ipi_data {
-	unsigned long ipi_type;
+/** IPI event operations or callbacks */
+struct sbi_ipi_event_ops {
+	/** Name of the IPI event operations */
+	char name[32];
+
+	/**
+	 * Update callback to save/enqueue data for remote HART
+	 * Note: This is an optional callback and it is called just before
+	 * triggering IPI to remote HART.
+	 */
+	int (* update)(struct sbi_scratch *scratch,
+			struct sbi_scratch *remote_scratch,
+			u32 remote_hartid, void *data);
+
+	/**
+	 * Sync callback to wait for remote HART
+	 * Note: This is an optional callback and it is called just after
+	 * triggering IPI to remote HART.
+	 */
+	void (* sync)(struct sbi_scratch *scratch);
+
+	/**
+	 * Process callback to handle IPI event
+	 * Note: This is a mandatory callback and it is called on the
+	 * remote HART after IPI is triggered.
+	 */
+	void (* process)(struct sbi_scratch *scratch);
 };
 
-int sbi_ipi_send_many(struct sbi_scratch *scratch, struct unpriv_trap *uptrap,
-		      ulong *pmask, u32 event, void *data);
+int sbi_ipi_send_many(ulong hmask, ulong hbase, u32 event, void *data);
 
-void sbi_ipi_clear_smode(struct sbi_scratch *scratch);
+int sbi_ipi_event_create(const struct sbi_ipi_event_ops *ops);
 
-void sbi_ipi_process(struct sbi_scratch *scratch);
+void sbi_ipi_event_destroy(u32 event);
+
+int sbi_ipi_send_smode(ulong hmask, ulong hbase);
+
+void sbi_ipi_clear_smode(void);
+
+int sbi_ipi_send_halt(ulong hmask, ulong hbase);
+
+void sbi_ipi_process(void);
 
 int sbi_ipi_init(struct sbi_scratch *scratch, bool cold_boot);
 
+void sbi_ipi_exit(struct sbi_scratch *scratch);
+
 #endif

include/sbi/sbi_list.h

@@ -0,0 +1,152 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Simple doubly-linked list library.
+ *
+ * Adapted from Xvisor source file libs/include/libs/list.h
+ *
+ * Copyright (c) 2020 Western Digital Corporation or its affiliates.
+ *
+ * Authors:
+ *   Anup Patel <anup.patel@wdc.com>
+ */
+
+#ifndef __SBI_LIST_H__
+#define __SBI_LIST_H__
+
+#include <sbi/sbi_types.h>
+
+#define SBI_LIST_POISON_PREV	0xDEADBEEF
+#define SBI_LIST_POISON_NEXT	0xFADEBABE
+
+struct sbi_dlist {
+	struct sbi_dlist *next, *prev;
+};
+
+#define SBI_LIST_HEAD_INIT(__lname)	{ &(__lname), &(__lname) }
+
+#define SBI_LIST_HEAD(_lname)	\
+struct sbi_dlist _lname = SBI_LIST_HEAD_INIT(_lname)
+
+#define SBI_INIT_LIST_HEAD(ptr)	\
+do { \
+	(ptr)->next = ptr; (ptr)->prev = ptr; \
+} while (0);
+
+static inline void __sbi_list_add(struct sbi_dlist *new,
+				  struct sbi_dlist *prev,
+				  struct sbi_dlist *next)
+{
+	new->prev = prev;
+	new->next = next;
+	prev->next = new;
+	next->prev = new;
+}
+
+/**
+ * Adds the new node after the given head.
+ * @param new New node that needs to be added to list.
+ * @param head List head after which the "new" node should be added.
+ * Note: the new node is added after the head.
+ */
+static inline void sbi_list_add(struct sbi_dlist *new, struct sbi_dlist *head)
+{
+	__sbi_list_add(new, head, head->next);
+}
+
+/**
+ * Adds a node at the tail where tnode points to tail node.
+ * @param new The new node to be added before tail.
+ * @param tnode The current tail node.
+ * Note: the new node is added before tail node.
+ */
+static inline void sbi_list_add_tail(struct sbi_dlist *new,
+				     struct sbi_dlist *tnode)
+{
+	__sbi_list_add(new, tnode->prev, tnode);
+}
+
+static inline void __sbi_list_del(struct sbi_dlist *prev,
+				  struct sbi_dlist *next)
+{
+	prev->next = next;
+	next->prev = prev;
+}
+
+static inline void __sbi_list_del_entry(struct sbi_dlist *entry)
+{
+	__sbi_list_del(entry->prev, entry->next);
+}
+
+/**
+ * Deletes a given entry from list.
+ * @param node Node to be deleted.
+ */
+static inline void sbi_list_del(struct sbi_dlist *entry)
+{
+	__sbi_list_del(entry->prev, entry->next);
+	entry->next = (void *)SBI_LIST_POISON_NEXT;
+	entry->prev = (void *)SBI_LIST_POISON_PREV;
+}
+
+/**
+ * Deletes entry from list and reinitialize it.
+ * @param entry the element to delete from the list.
+ */
+static inline void sbi_list_del_init(struct sbi_dlist *entry)
+{
+	__sbi_list_del_entry(entry);
+	SBI_INIT_LIST_HEAD(entry);
+}
+
+/**
+ * Get the struct for this entry
+ * @param ptr the &struct list_head pointer.
+ * @param type the type of the struct this is embedded in.
+ * @param member the name of the list_struct within the struct.
+ */
+#define sbi_list_entry(ptr, type, member) \
+	container_of(ptr, type, member)
+
+/**
+ * Get the first element from a list
+ * @param ptr the list head to take the element from.
+ * @param type the type of the struct this is embedded in.
+ * @param member the name of the list_struct within the struct.
+ *
+ * Note: that list is expected to be not empty.
+ */
+#define sbi_list_first_entry(ptr, type, member) \
+	sbi_list_entry((ptr)->next, type, member)
+
+/**
+ * Get the last element from a list
+ * @param ptr the list head to take the element from.
+ * @param type the type of the struct this is embedded in.
+ * @param member the name of the list_head within the struct.
+ *
+ * Note: that list is expected to be not empty.
+ */
+#define sbi_list_last_entry(ptr, type, member) \
+	sbi_list_entry((ptr)->prev, type, member)
+
+/**
+ * Iterate over a list
+ * @param pos the &struct list_head to use as a loop cursor.
+ * @param head the head for your list.
+ */
+#define sbi_list_for_each(pos, head) \
+	for (pos = (head)->next; pos != (head); pos = pos->next)
+
+/**
+ * Iterate over list of given type
+ * @param pos the type * to use as a loop cursor.
+ * @param head the head for your list.
+ * @param member the name of the list_struct within the struct.
+ */
+#define sbi_list_for_each_entry(pos, head, member) \
+	for (pos = sbi_list_entry((head)->next, typeof(*pos), member);	\
+	     &pos->member != (head); 	\
+	     pos = sbi_list_entry(pos->member.next, typeof(*pos), member))
+
+#endif

include/sbi/sbi_misaligned_ldst.h

@@ -13,14 +13,11 @@
 #include <sbi/sbi_types.h>
 
 struct sbi_trap_regs;
-struct sbi_scratch;
 
-int sbi_misaligned_load_handler(u32 hartid, ulong mcause,
-				struct sbi_trap_regs *regs,
-				struct sbi_scratch *scratch);
+int sbi_misaligned_load_handler(ulong addr, ulong tval2, ulong tinst,
+				struct sbi_trap_regs *regs);
 
-int sbi_misaligned_store_handler(u32 hartid, ulong mcause,
-				 struct sbi_trap_regs *regs,
-				 struct sbi_scratch *scratch);
+int sbi_misaligned_store_handler(ulong addr, ulong tval2, ulong tinst,
+				 struct sbi_trap_regs *regs);
 
 #endif

include/sbi/sbi_platform.h

@@ -10,9 +10,10 @@
 #ifndef __SBI_PLATFORM_H__
 #define __SBI_PLATFORM_H__
 
-/** OpenSBI 32-bit platform version with:
- *  1. upper 16-bits as major number
- *  2. lower 16-bits as minor number
+/**
+ * OpenSBI 32-bit platform version with:
+ * 1. upper 16-bits as major number
+ * 2. lower 16-bits as minor number
  */
 #define SBI_PLATFORM_VERSION(Major, Minor) ((Major << 16) | Minor)
 
@@ -28,23 +29,22 @@
 #define SBI_PLATFORM_HART_COUNT_OFFSET (0x50)
 /** Offset of hart_stack_size in struct sbi_platform */
 #define SBI_PLATFORM_HART_STACK_SIZE_OFFSET (0x54)
-/** Offset of disabled_hart_mask in struct sbi_platform */
-#define SBI_PLATFORM_DISABLED_HART_OFFSET (0x58)
-/** Offset of tlb_range_flush_limit in struct sbi_platform */
-#define SBI_PLATFORM_TLB_RANGE_FLUSH_LIMIT_OFFSET (0x60)
 /** Offset of platform_ops_addr in struct sbi_platform */
-#define SBI_PLATFORM_OPS_OFFSET (0x68)
+#define SBI_PLATFORM_OPS_OFFSET (0x58)
 /** Offset of firmware_context in struct sbi_platform */
-#define SBI_PLATFORM_FIRMWARE_CONTEXT_OFFSET (0x68 + __SIZEOF_POINTER__)
+#define SBI_PLATFORM_FIRMWARE_CONTEXT_OFFSET (0x58 + __SIZEOF_POINTER__)
+/** Offset of hart_index2id in struct sbi_platform */
+#define SBI_PLATFORM_HART_INDEX2ID_OFFSET (0x58 + (__SIZEOF_POINTER__ * 2))
 
 #define SBI_PLATFORM_TLB_RANGE_FLUSH_LIMIT_DEFAULT		(1UL << 12)
 
 #ifndef __ASSEMBLY__
 
-#include <sbi/sbi_version.h>
-#include <sbi/sbi_scratch.h>
 #include <sbi/sbi_ecall.h>
 #include <sbi/sbi_error.h>
+#include <sbi/sbi_scratch.h>
+#include <sbi/sbi_trap.h>
+#include <sbi/sbi_version.h>
 
 /** Possible feature flags of a platform */
 enum sbi_platform_features {
@@ -60,6 +60,8 @@ enum sbi_platform_features {
 	SBI_PLATFORM_HAS_MCOUNTEREN = (1 << 4),
 	/** Platform has fault delegation support */
 	SBI_PLATFORM_HAS_MFAULTS_DELEGATION = (1 << 5),
+	/** Platform has custom secondary hart booting support */
+	SBI_PLATFORM_HAS_HART_SECONDARY_BOOT = (1 << 6),
 };
 
 /** Default feature set for a platform */
@@ -75,6 +77,23 @@ struct sbi_platform_operations {
 	/** Platform final initialization */
 	int (*final_init)(bool cold_boot);
 
+	/** Platform early exit */
+	void (*early_exit)(void);
+	/** Platform final exit */
+	void (*final_exit)(void);
+
+	/**
+	 * For platforms that do not implement misa, non-standard
+	 * methods are needed to determine cpu extension.
+	 */
+	int (*misa_check_extension)(char ext);
+
+	/**
+	 * For platforms that do not implement misa, non-standard
+	 * methods are needed to get MXL field of misa.
+	 */
+	int (*misa_get_xlen)(void);
+
 	/** Get number of PMP regions for given HART */
 	u32 (*pmp_region_count)(u32 hartid);
 	/**
@@ -93,6 +112,8 @@ struct sbi_platform_operations {
 
 	/** Initialize the platform interrupt controller for current HART */
 	int (*irqchip_init)(bool cold_boot);
+	/** Exit the platform interrupt controller for current HART */
+	void (*irqchip_exit)(void);
 
 	/** Send IPI to a target HART */
 	void (*ipi_send)(u32 target_hart);
@@ -100,6 +121,11 @@ struct sbi_platform_operations {
 	void (*ipi_clear)(u32 target_hart);
 	/** Initialize IPI for current HART */
 	int (*ipi_init)(bool cold_boot);
+	/** Exit IPI for current HART */
+	void (*ipi_exit)(void);
+
+	/** Get tlb flush limit value **/
+	u64 (*get_tlbr_flush_limit)(void);
 
 	/** Get platform timer value */
 	u64 (*timer_value)(void);
@@ -109,6 +135,16 @@ struct sbi_platform_operations {
 	void (*timer_event_stop)(void);
 	/** Initialize platform timer for current HART */
 	int (*timer_init)(bool cold_boot);
+	/** Exit platform timer for current HART */
+	void (*timer_exit)(void);
+
+	/** Bringup the given hart */
+	int (*hart_start)(u32 hartid, ulong saddr);
+	/**
+	 * Stop the current hart from running. This call doesn't expect to
+	 * return if success.
+	 */
+	int (*hart_stop)(void);
 
 	/** Reboot the platform */
 	int (*system_reboot)(u32 type);
@@ -119,11 +155,14 @@ struct sbi_platform_operations {
 	int (*vendor_ext_check)(long extid);
 	/** platform specific SBI extension implementation provider */
 	int (*vendor_ext_provider)(long extid, long funcid,
-			unsigned long *args, unsigned long *out_value,
-			unsigned long *out_trap_cause,
-			unsigned long *out_trap_val);
+				   unsigned long *args,
+				   unsigned long *out_value,
+				   struct sbi_trap_info *out_trap);
 } __packed;
 
+/** Platform default per-HART stack size for exception/interrupt handling */
+#define SBI_PLATFORM_DEFAULT_HART_STACK_SIZE	8192
+
 /** Representation of a platform */
 struct sbi_platform {
 	/**
@@ -146,14 +185,26 @@ struct sbi_platform {
 	u32 hart_count;
 	/** Per-HART stack size for exception/interrupt handling */
 	u32 hart_stack_size;
-	/** Mask representing the set of disabled HARTs */
-	u64 disabled_hart_mask;
-	/* Maximum value of tlb flush range request*/
-	u64 tlb_range_flush_limit;
 	/** Pointer to sbi platform operations */
 	unsigned long platform_ops_addr;
 	/** Pointer to system firmware specific context */
 	unsigned long firmware_context;
+	/**
+	 * HART index to HART id table
+	 *
+	 * For used HART index <abc>:
+	 *     hart_index2id[<abc>] = some HART id
+	 * For unused HART index <abc>:
+	 *     hart_index2id[<abc>] = -1U
+	 *
+	 * If hart_index2id == NULL then we assume identity mapping
+	 *     hart_index2id[<abc>] = <abc>
+	 *
+	 * We have only two restrictions:
+	 * 1. HART index < sbi_platform hart_count
+	 * 2. HART id < SBI_HARTMASK_MAX_BITS
+	 */
+	const u32 *hart_index2id;
 } __packed;
 
 /** Get pointer to sbi_platform for sbi_scratch pointer */
@@ -183,35 +234,22 @@ struct sbi_platform {
 /** Check whether the platform supports fault delegation */
 #define sbi_platform_has_mfaults_delegation(__p) \
 	((__p)->features & SBI_PLATFORM_HAS_MFAULTS_DELEGATION)
+/** Check whether the platform supports custom secondary hart booting support */
+#define sbi_platform_has_hart_secondary_boot(__p) \
+	((__p)->features & SBI_PLATFORM_HAS_HART_SECONDARY_BOOT)
 
 /**
  * Get name of the platform
  *
  * @param plat pointer to struct sbi_platform
  *
- * @return pointer to platform name on success and NULL on failure
+ * @return pointer to platform name on success and "Unknown" on failure
  */
 static inline const char *sbi_platform_name(const struct sbi_platform *plat)
 {
 	if (plat)
 		return plat->name;
-	return NULL;
-}
-
-/**
- * Check whether the given HART is disabled
- *
- * @param plat pointer to struct sbi_platform
- * @param hartid HART ID
- *
- * @return TRUE if HART is disabled and FALSE otherwise
- */
-static inline bool sbi_platform_hart_disabled(const struct sbi_platform *plat,
-					      u32 hartid)
-{
-	if (plat && (plat->disabled_hart_mask & (1 << hartid)))
-		return TRUE;
-	return FALSE;
+	return "Unknown";
 }
 
 /**
@@ -225,8 +263,8 @@ static inline bool sbi_platform_hart_disabled(const struct sbi_platform *plat,
  */
 static inline u64 sbi_platform_tlbr_flush_limit(const struct sbi_platform *plat)
 {
-	if (plat && plat->tlb_range_flush_limit)
-		return plat->tlb_range_flush_limit;
+	if (plat && sbi_platform_ops(plat)->get_tlbr_flush_limit)
+		return sbi_platform_ops(plat)->get_tlbr_flush_limit();
 	return SBI_PLATFORM_TLB_RANGE_FLUSH_LIMIT_DEFAULT;
 }
 
@@ -258,6 +296,83 @@ static inline u32 sbi_platform_hart_stack_size(const struct sbi_platform *plat)
 	return 0;
 }
 
+/**
+ * Get HART index for the given HART
+ *
+ * @param plat pointer to struct sbi_platform
+ * @param hartid HART ID
+ *
+ * @return 0 <= value < hart_count for valid HART otherwise -1U
+ */
+static inline u32 sbi_platform_hart_index(const struct sbi_platform *plat,
+					  u32 hartid)
+{
+	u32 i;
+
+	if (!plat)
+		return -1U;
+	if (plat->hart_index2id) {
+		for (i = 0; i < plat->hart_count; i++) {
+			if (plat->hart_index2id[i] == hartid)
+				return i;
+		}
+		return -1U;
+	}
+
+	return hartid;
+}
+
+/**
+ * Check whether given HART is invalid
+ *
+ * @param plat pointer to struct sbi_platform
+ * @param hartid HART ID
+ *
+ * @return TRUE if HART is invalid and FALSE otherwise
+ */
+static inline bool sbi_platform_hart_invalid(const struct sbi_platform *plat,
+					     u32 hartid)
+{
+	if (!plat)
+		return TRUE;
+	if (plat->hart_count <= sbi_platform_hart_index(plat, hartid))
+		return TRUE;
+	return FALSE;
+}
+
+/**
+ * Bringup a given hart from previous stage. Platform should implement this
+ * operation if they support a custom mechanism to start a hart. Otherwise,
+ * a generic WFI based approach will be used to start/stop a hart in OpenSBI.
+ *
+ * @param plat pointer to struct sbi_platform
+ * @param hartid HART id
+ * @param saddr M-mode start physical address for the HART
+ *
+ * @return 0 if sucessful and negative error code on failure
+ */
+static inline int sbi_platform_hart_start(const struct sbi_platform *plat,
+					  u32 hartid, ulong saddr)
+{
+	if (plat && sbi_platform_ops(plat)->hart_start)
+		return sbi_platform_ops(plat)->hart_start(hartid, saddr);
+	return SBI_ENOTSUPP;
+}
+
+/**
+ * Stop the current hart in OpenSBI.
+ *
+ * @param plat pointer to struct sbi_platform
+ *
+ * @return Negative error code on failure. It doesn't return on success.
+ */
+static inline int sbi_platform_hart_stop(const struct sbi_platform *plat)
+{
+	if (plat && sbi_platform_ops(plat)->hart_stop)
+		return sbi_platform_ops(plat)->hart_stop();
+	return SBI_ENOTSUPP;
+}
+
 /**
  * Early initialization for current HART
  *
@@ -290,6 +405,58 @@ static inline int sbi_platform_final_init(const struct sbi_platform *plat,
 	return 0;
 }
 
+/**
+ * Early exit for current HART
+ *
+ * @param plat pointer to struct sbi_platform
+ */
+static inline void sbi_platform_early_exit(const struct sbi_platform *plat)
+{
+	if (plat && sbi_platform_ops(plat)->early_exit)
+		sbi_platform_ops(plat)->early_exit();
+}
+
+/**
+ * Final exit for current HART
+ *
+ * @param plat pointer to struct sbi_platform
+ */
+static inline void sbi_platform_final_exit(const struct sbi_platform *plat)
+{
+	if (plat && sbi_platform_ops(plat)->final_exit)
+		sbi_platform_ops(plat)->final_exit();
+}
+
+/**
+ * Check CPU extension in MISA
+ *
+ * @param plat pointer to struct sbi_platform
+ * @param ext shorthand letter for CPU extensions
+ *
+ * @return zero for not-supported and non-zero for supported
+ */
+static inline int sbi_platform_misa_extension(const struct sbi_platform *plat,
+					      char ext)
+{
+	if (plat && sbi_platform_ops(plat)->misa_check_extension)
+		return sbi_platform_ops(plat)->misa_check_extension(ext);
+	return 0;
+}
+
+/**
+ * Get MXL field of MISA
+ *
+ * @param plat pointer to struct sbi_platform
+ *
+ * @return 1/2/3 on success and error code on failure
+ */
+static inline int sbi_platform_misa_xlen(const struct sbi_platform *plat)
+{
+	if (plat && sbi_platform_ops(plat)->misa_get_xlen)
+		return sbi_platform_ops(plat)->misa_get_xlen();
+	return -1;
+}
+
 /**
  * Get the number of PMP regions of a HART
  *
@@ -387,6 +554,17 @@ static inline int sbi_platform_irqchip_init(const struct sbi_platform *plat,
 	return 0;
 }
 
+/**
+ * Exit the platform interrupt controller for current HART
+ *
+ * @param plat pointer to struct sbi_platform
+ */
+static inline void sbi_platform_irqchip_exit(const struct sbi_platform *plat)
+{
+	if (plat && sbi_platform_ops(plat)->irqchip_exit)
+		sbi_platform_ops(plat)->irqchip_exit();
+}
+
 /**
  * Send IPI to a target HART
  *
@@ -429,6 +607,17 @@ static inline int sbi_platform_ipi_init(const struct sbi_platform *plat,
 	return 0;
 }
 
+/**
+ * Exit the platform IPI support for current HART
+ *
+ * @param plat pointer to struct sbi_platform
+ */
+static inline void sbi_platform_ipi_exit(const struct sbi_platform *plat)
+{
+	if (plat && sbi_platform_ops(plat)->ipi_exit)
+		sbi_platform_ops(plat)->ipi_exit();
+}
+
 /**
  * Get platform timer value
  *
@@ -484,6 +673,17 @@ static inline int sbi_platform_timer_init(const struct sbi_platform *plat,
 	return 0;
 }
 
+/**
+ * Exit the platform timer for current HART
+ *
+ * @param plat pointer to struct sbi_platform
+ */
+static inline void sbi_platform_timer_exit(const struct sbi_platform *plat)
+{
+	if (plat && sbi_platform_ops(plat)->timer_exit)
+		sbi_platform_ops(plat)->timer_exit();
+}
+
 /**
  * Reboot the platform
  *
@@ -540,25 +740,23 @@ static inline int sbi_platform_vendor_ext_check(const struct sbi_platform *plat,
  * @param extid	vendor SBI extension id
  * @param funcid SBI function id within the extension id
  * @param args pointer to arguments passed by the caller
- * @param out_value output value that can be filled the callee
- * @param out_tcause trap cause that can be filled the callee
- * @param out_tvalue possible trap value that can be filled the callee
+ * @param out_value output value that can be filled by the callee
+ * @param out_trap trap info that can be filled by the callee
  *
  * @return 0 on success and negative error code on failure
  */
-static inline int sbi_platform_vendor_ext_provider(const struct sbi_platform *plat,
-						   long extid, long funcid,
-						   unsigned long *args,
-						   unsigned long *out_value,
-						   unsigned long *out_tcause,
-						   unsigned long *out_tval)
+static inline int sbi_platform_vendor_ext_provider(
+					const struct sbi_platform *plat,
+					long extid, long funcid,
+					unsigned long *args,
+					unsigned long *out_value,
+					struct sbi_trap_info *out_trap)
 {
 	if (plat && sbi_platform_ops(plat)->vendor_ext_provider) {
 		return sbi_platform_ops(plat)->vendor_ext_provider(extid,
-								   funcid, args,
-								  out_value,
-								  out_tcause,
-								  out_tval);
+								funcid, args,
+								out_value,
+								out_trap);
 	}
 
 	return SBI_ENOTSUPP;

include/sbi/sbi_scratch.h

@@ -36,15 +36,14 @@
 #define SBI_SCRATCH_OPTIONS_OFFSET		(9 * __SIZEOF_POINTER__)
 /** Offset of extra space in sbi_scratch */
 #define SBI_SCRATCH_EXTRA_SPACE_OFFSET		(10 * __SIZEOF_POINTER__)
-/** Maximum size of sbi_scratch and sbi_ipi_data */
-#define SBI_SCRATCH_SIZE			(64 * __SIZEOF_POINTER__)
+/** Maximum size of sbi_scratch (4KB) */
+#define SBI_SCRATCH_SIZE			(0x1000)
 
 /* clang-format on */
 
 #ifndef __ASSEMBLY__
 
 #include <sbi/sbi_types.h>
-#include <sbi/sbi_ipi.h>
 
 /** Representation of per-HART scratch space */
 struct sbi_scratch {
@@ -86,7 +85,11 @@ enum sbi_scratch_options {
 #define sbi_scratch_thishart_arg1_ptr() \
 	((void *)(sbi_scratch_thishart_ptr()->next_arg1))
 
-/** Allocate from extra space in sbi_scratch
+/** Initialize scatch table and allocator */
+int sbi_scratch_init(struct sbi_scratch *scratch);
+
+/**
+ * Allocate from extra space in sbi_scratch
  *
  * @return zero on failure and non-zero (>= SBI_SCRATCH_EXTRA_SPACE_OFFSET)
  * on success
@@ -103,6 +106,19 @@ void sbi_scratch_free_offset(unsigned long offset);
 #define sbi_scratch_thishart_offset_ptr(offset)	\
 	((void *)sbi_scratch_thishart_ptr() + (offset))
 
+/** HART id to scratch table */
+extern struct sbi_scratch *hartid_to_scratch_table[];
+
+/** Get sbi_scratch from HART id */
+#define sbi_hartid_to_scratch(__hartid) \
+	hartid_to_scratch_table[__hartid]
+
+/** Last HART id having a sbi_scratch pointer */
+extern u32 last_hartid_having_scratch;
+
+/** Get last HART id having a sbi_scratch pointer */
+#define sbi_scratch_last_hartid()	last_hartid_having_scratch
+
 #endif
 
 #endif

include/sbi/sbi_system.h

@@ -12,16 +12,8 @@
 
 #include <sbi/sbi_types.h>
 
-struct sbi_scratch;
+void __noreturn sbi_system_reboot(u32 type);
 
-int sbi_system_early_init(struct sbi_scratch *scratch, bool cold_boot);
-
-int sbi_system_final_init(struct sbi_scratch *scratch, bool cold_boot);
-
-void __attribute__((noreturn))
-sbi_system_reboot(struct sbi_scratch *scratch, u32 type);
-
-void __attribute__((noreturn))
-sbi_system_shutdown(struct sbi_scratch *scratch, u32 type);
+void __noreturn sbi_system_shutdown(u32 type);
 
 #endif

include/sbi/sbi_timer.h

@@ -14,22 +14,31 @@
 
 struct sbi_scratch;
 
-u64 sbi_timer_value(struct sbi_scratch *scratch);
+/** Get timer value for current HART */
+u64 sbi_timer_value(void);
 
-u64 sbi_timer_virt_value(struct sbi_scratch *scratch);
+/** Get virtualized timer value for current HART */
+u64 sbi_timer_virt_value(void);
 
-u64 sbi_timer_get_delta(struct sbi_scratch *scratch);
+/** Get timer delta value for current HART */
+u64 sbi_timer_get_delta(void);
 
-void sbi_timer_set_delta(struct sbi_scratch *scratch, ulong delta);
+/** Set timer delta value for current HART */
+void sbi_timer_set_delta(ulong delta);
 
-void sbi_timer_set_delta_upper(struct sbi_scratch *scratch, ulong delta_upper);
+/** Set upper 32-bits of timer delta value for current HART */
+void sbi_timer_set_delta_upper(ulong delta_upper);
 
-void sbi_timer_event_stop(struct sbi_scratch *scratch);
+/** Start timer event for current HART */
+void sbi_timer_event_start(u64 next_event);
 
-void sbi_timer_event_start(struct sbi_scratch *scratch, u64 next_event);
-
-void sbi_timer_process(struct sbi_scratch *scratch);
+/** Process timer event for current HART */
+void sbi_timer_process(void);
 
+/* Initialize timer */
 int sbi_timer_init(struct sbi_scratch *scratch, bool cold_boot);
 
+/* Exit timer */
+void sbi_timer_exit(struct sbi_scratch *scratch);
+
 #endif

include/sbi/sbi_tlb.h

@@ -12,6 +12,7 @@
 #define __SBI_TLB_H__
 
 #include <sbi/sbi_types.h>
+#include <sbi/sbi_hartmask.h>
 
 /* clang-format off */
 
@@ -24,7 +25,10 @@
 enum sbi_tlb_info_types {
 	SBI_TLB_FLUSH_VMA,
 	SBI_TLB_FLUSH_VMA_ASID,
-	SBI_TLB_FLUSH_VMA_VMID,
+	SBI_TLB_FLUSH_GVMA,
+	SBI_TLB_FLUSH_GVMA_VMID,
+	SBI_TLB_FLUSH_VVMA,
+	SBI_TLB_FLUSH_VVMA_ASID,
 	SBI_ITLB_FLUSH
 };
 
@@ -35,17 +39,22 @@ struct sbi_tlb_info {
 	unsigned long size;
 	unsigned long asid;
 	unsigned long type;
-	unsigned long shart_mask;
+	struct sbi_hartmask smask;
 };
 
-#define SBI_TLB_INFO_SIZE			sizeof(struct sbi_tlb_info)
+#define SBI_TLB_INFO_INIT(__ptr, __start, __size, __asid, __type, __src_hart) \
+do { \
+	(__ptr)->start = (__start); \
+	(__ptr)->size = (__size); \
+	(__ptr)->asid = (__asid); \
+	(__ptr)->type = (__type); \
+	SBI_HARTMASK_INIT_EXCEPT(&(__ptr)->smask, (__src_hart)); \
+} while (0)
 
-int sbi_tlb_fifo_update(struct sbi_scratch *scratch, u32 hartid, void *data);
+#define SBI_TLB_INFO_SIZE		sizeof(struct sbi_tlb_info)
 
-void sbi_tlb_fifo_process(struct sbi_scratch *scratch);
+int sbi_tlb_request(ulong hmask, ulong hbase, struct sbi_tlb_info *tinfo);
 
-int sbi_tlb_fifo_init(struct sbi_scratch *scratch, bool cold_boot);
-
-void sbi_tlb_fifo_sync(struct sbi_scratch *scratch);
+int sbi_tlb_init(struct sbi_scratch *scratch, bool cold_boot);
 
 #endif

include/sbi/sbi_trap.h

@@ -85,6 +85,19 @@
 /** Last member index in sbi_trap_regs */
 #define SBI_TRAP_REGS_last			35
 
+/** Index of epc member in sbi_trap_info */
+#define SBI_TRAP_INFO_epc			0
+/** Index of cause member in sbi_trap_info */
+#define SBI_TRAP_INFO_cause			1
+/** Index of tval member in sbi_trap_info */
+#define SBI_TRAP_INFO_tval			2
+/** Index of tval2 member in sbi_trap_info */
+#define SBI_TRAP_INFO_tval2			3
+/** Index of tinst member in sbi_trap_info */
+#define SBI_TRAP_INFO_tinst			4
+/** Last member index in sbi_trap_info */
+#define SBI_TRAP_INFO_last			5
+
 /* clang-format on */
 
 /** Get offset of member with name 'x' in sbi_trap_regs */
@@ -92,6 +105,11 @@
 /** Size (in bytes) of sbi_trap_regs */
 #define SBI_TRAP_REGS_SIZE SBI_TRAP_REGS_OFFSET(last)
 
+/** Get offset of member with name 'x' in sbi_trap_info */
+#define SBI_TRAP_INFO_OFFSET(x) ((SBI_TRAP_INFO_##x) * __SIZEOF_POINTER__)
+/** Size (in bytes) of sbi_trap_info */
+#define SBI_TRAP_INFO_SIZE SBI_TRAP_INFO_OFFSET(last)
+
 #ifndef __ASSEMBLY__
 
 #include <sbi/sbi_types.h>
@@ -170,12 +188,24 @@ struct sbi_trap_regs {
 	unsigned long mstatusH;
 } __packed;
 
-struct sbi_scratch;
+/** Representation of trap details */
+struct sbi_trap_info {
+	/** epc Trap program counter */
+	unsigned long epc;
+	/** cause Trap exception cause */
+	unsigned long cause;
+	/** tval Trap value */
+	unsigned long tval;
+	/** tval2 Trap value 2 */
+	unsigned long tval2;
+	/** tinst Trap instruction */
+	unsigned long tinst;
+};
 
-int sbi_trap_redirect(struct sbi_trap_regs *regs, struct sbi_scratch *scratch,
-		      ulong epc, ulong cause, ulong tval);
+int sbi_trap_redirect(struct sbi_trap_regs *regs,
+		      struct sbi_trap_info *trap);
 
-void sbi_trap_handler(struct sbi_trap_regs *regs, struct sbi_scratch *scratch);
+void sbi_trap_handler(struct sbi_trap_regs *regs);
 
 #endif
 

include/sbi/sbi_types.h

@@ -10,6 +10,8 @@
 #ifndef __SBI_TYPES_H__
 #define __SBI_TYPES_H__
 
+#ifndef OPENSBI_EXTERNAL_SBI_TYPES
+
 /* clang-format off */
 
 typedef char			s8;
@@ -54,12 +56,52 @@ typedef unsigned long		physical_size_t;
 
 #define TRUE			1
 #define FALSE			0
+#define true			TRUE
+#define false			FALSE
 
 #define NULL			((void *)0)
 
 #define __packed		__attribute__((packed))
 #define __noreturn		__attribute__((noreturn))
 
+#define likely(x) __builtin_expect((x), 1)
+#define unlikely(x) __builtin_expect((x), 0)
+
+#undef offsetof
+#ifdef __compiler_offsetof
+#define offsetof(TYPE, MEMBER) __compiler_offsetof(TYPE,MEMBER)
+#else
+#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
+#endif
+
+#define container_of(ptr, type, member) ({			\
+	const typeof(((type *)0)->member) * __mptr = (ptr);	\
+	(type *)((char *)__mptr - offsetof(type, member)); })
+
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+#define CLAMP(a, lo, hi) MIN(MAX(a, lo), hi)
+
+#define STR(x) XSTR(x)
+#define XSTR(x) #x
+
+#define ROUNDUP(a, b) ((((a)-1) / (b) + 1) * (b))
+#define ROUNDDOWN(a, b) ((a) / (b) * (b))
+
 /* clang-format on */
 
+#else
+/*
+ * OPENSBI_EXTERNAL_SBI_TYPES could be defined in CFLAGS for using the
+ * external definitions of data types and common macros.
+ * OPENSBI_EXTERNAL_SBI_TYPES is the file name to external header file,
+ * the external build system should address the additional include
+ * directory ccordingly.
+ */
+
+#define XSTR(x) #x
+#define STR(x) XSTR(x)
+#include STR(OPENSBI_EXTERNAL_SBI_TYPES)
+#endif
+
 #endif

include/sbi/sbi_unpriv.h

@@ -7,28 +7,21 @@
  *   Anup Patel <anup.patel@wdc.com>
  */
 
-#ifndef __RISCV_UNPRIV_H__
-#define __RISCV_UNPRIV_H__
+#ifndef __SBI_UNPRIV_H__
+#define __SBI_UNPRIV_H__
 
 #include <sbi/sbi_types.h>
 
 struct sbi_scratch;
+struct sbi_trap_info;
 
-struct unpriv_trap {
-	unsigned long ilen;
-	unsigned long cause;
-	unsigned long tval;
-};
+#define DECLARE_UNPRIVILEGED_LOAD_FUNCTION(type)           \
+	type sbi_load_##type(const type *addr,             \
+			     struct sbi_trap_info *trap);
 
-#define DECLARE_UNPRIVILEGED_LOAD_FUNCTION(type)       \
-	type load_##type(const type *addr,             \
-			 struct sbi_scratch *scratch,  \
-			 struct unpriv_trap *trap);
-
-#define DECLARE_UNPRIVILEGED_STORE_FUNCTION(type)      \
-	void store_##type(type *addr, type val,        \
-			  struct sbi_scratch *scratch, \
-			  struct unpriv_trap *trap);
+#define DECLARE_UNPRIVILEGED_STORE_FUNCTION(type)          \
+	void sbi_store_##type(type *addr, type val,        \
+			      struct sbi_trap_info *trap);
 
 DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u8)
 DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u16)
@@ -43,7 +36,6 @@ DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u64)
 DECLARE_UNPRIVILEGED_STORE_FUNCTION(u64)
 DECLARE_UNPRIVILEGED_LOAD_FUNCTION(ulong)
 
-ulong get_insn(ulong mepc, bool virt, struct sbi_scratch *scratch,
-	       struct unpriv_trap *trap);
+ulong sbi_get_insn(ulong mepc, struct sbi_trap_info *trap);
 
 #endif

include/sbi/sbi_version.h

@@ -11,12 +11,12 @@
 #define __SBI_VERSION_H__
 
 #define OPENSBI_VERSION_MAJOR 0
-#define OPENSBI_VERSION_MINOR 5
+#define OPENSBI_VERSION_MINOR 7
 
 /**
- *  OpenSBI 32-bit version with:
- *  1. upper 16-bits as major number
- *  2. lower 16-bits as minor number
+ * OpenSBI 32-bit version with:
+ * 1. upper 16-bits as major number
+ * 2. lower 16-bits as minor number
  */
 #define OPENSBI_VERSION ((OPENSBI_VERSION_MAJOR << 16) | \
 			 (OPENSBI_VERSION_MINOR))

include/sbi_utils/fdt/fdt_fixup.h

@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: BSD-2-Clause
+/*
+ * fdt_fixup.h - Flat Device Tree manipulation helper routines
+ * Implement platform specific DT fixups on top of libfdt. 
+ *
+ * Copyright (C) 2020 Bin Meng <bmeng.cn@gmail.com>
+ */
+
+#ifndef __FDT_FIXUP_H__
+#define __FDT_FIXUP_H__
+
+/**
+ * Fix up the CPU node in the device tree
+ *
+ * This routine updates the "status" property of a CPU node in the device tree
+ * to "disabled" if that hart is in disabled state in OpenSBI.
+ *
+ * It is recommended that platform codes call this helper in their final_init()
+ *
+ * @param fdt: device tree blob
+ */
+void fdt_cpu_fixup(void *fdt);
+
+/**
+ * Fix up the PLIC node in the device tree
+ *
+ * This routine updates the "interrupt-extended" property of the PLIC node in
+ * the device tree to hide the M-mode external interrupt from CPUs.
+ *
+ * It is recommended that platform codes call this helper in their final_init()
+ *
+ * @param fdt: device tree blob
+ * @param compat: PLIC node compatible string
+ */
+void fdt_plic_fixup(void *fdt, const char *compat);
+
+/**
+ * Fix up the reserved memory node in the device tree
+ *
+ * This routine inserts a child node of the reserved memory node in the device
+ * tree that describes the protected memory region done by OpenSBI via PMP.
+ *
+ * It is recommended that platform codes call this helper in their final_init()
+ *
+ * @param fdt: device tree blob
+ * @return zero on success and -ve on failure
+ */
+int fdt_reserved_memory_fixup(void *fdt);
+
+/**
+ * General device tree fix-up
+ *
+ * This routine do all required device tree fix-ups for a typical platform.
+ * It fixes up the PLIC node and the reserved memory node in the device tree
+ * by calling the corresponding helper routines to accomplish the task.
+ *
+ * It is recommended that platform codes call this helper in their final_init()
+ *
+ * @param fdt: device tree blob
+ */
+void fdt_fixups(void *fdt);
+
+#endif /* __FDT_FIXUP_H__ */
+

include/sbi_utils/fdt/fdt_helper.h

@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: BSD-2-Clause
+/*
+ * fdt_helper.h - Flat Device Tree parsing helper routines
+ * Implement helper routines to parse FDT nodes on top of
+ * libfdt for OpenSBI usage
+ *
+ * Copyright (C) 2020 Bin Meng <bmeng.cn@gmail.com>
+ */
+
+#ifndef __FDT_HELPER_H__
+#define __FDT_HELPER_H__
+
+struct platform_uart_data {
+	unsigned long addr;
+	unsigned long freq;
+	unsigned long baud;
+};
+
+struct platform_plic_data {
+	unsigned long addr;
+	unsigned long num_src;
+};
+
+int fdt_parse_uart8250(void *fdt, struct platform_uart_data *uart,
+		       const char *compatible);
+
+int fdt_parse_plic(void *fdt, struct platform_plic_data *plic,
+		   const char *compatible);
+
+int fdt_parse_clint(void *fdt, unsigned long *clint_addr,
+		    const char *compatible);
+
+#endif /* __FDT_HELPER_H__ */

include/sbi_utils/irqchip/plic.h

@@ -12,8 +12,6 @@
 
 #include <sbi/sbi_types.h>
 
-void plic_fdt_fixup(void *fdt, const char *compat);
-
 int plic_warm_irqchip_init(u32 target_hart, int m_cntx_id, int s_cntx_id);
 
 int plic_cold_irqchip_init(unsigned long base, u32 num_sources, u32 hart_count);

include/sbi_utils/sys/clint.h

@@ -30,6 +30,7 @@ void clint_timer_event_start(u64 next_event);
 
 int clint_warm_timer_init(void);
 
-int clint_cold_timer_init(unsigned long base, u32 hart_count);
+int clint_cold_timer_init(unsigned long base, u32 hart_count,
+			  bool has_64bit_mmio);
 
 #endif

include/sbi_utils/sys/htif.h

@@ -0,0 +1,19 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright (c) 2010-2020, The Regents of the University of California
+ * (Regents).  All Rights Reserved.
+ */
+
+#ifndef __SYS_HTIF_H__
+#define __SYS_HTIF_H__
+
+#include <sbi/sbi_types.h>
+
+void htif_putc(char ch);
+
+int htif_getc(void);
+
+int htif_system_down(u32 type);
+
+#endif

lib/sbi/objects.mk

@@ -11,20 +11,30 @@ libsbi-objs-y += riscv_asm.o
 libsbi-objs-y += riscv_atomic.o
 libsbi-objs-y += riscv_hardfp.o
 libsbi-objs-y += riscv_locks.o
-libsbi-objs-y += riscv_unpriv.o
 
+libsbi-objs-y += sbi_bitmap.o
+libsbi-objs-y += sbi_bitops.o
 libsbi-objs-y += sbi_console.o
 libsbi-objs-y += sbi_ecall.o
+libsbi-objs-y += sbi_ecall_base.o
+libsbi-objs-y += sbi_ecall_hsm.o
+libsbi-objs-y += sbi_ecall_legacy.o
+libsbi-objs-y += sbi_ecall_replace.o
+libsbi-objs-y += sbi_ecall_vendor.o
 libsbi-objs-y += sbi_emulate_csr.o
 libsbi-objs-y += sbi_fifo.o
 libsbi-objs-y += sbi_hart.o
+libsbi-objs-y += sbi_hfence.o
+libsbi-objs-y += sbi_hsm.o
 libsbi-objs-y += sbi_illegal_insn.o
 libsbi-objs-y += sbi_init.o
 libsbi-objs-y += sbi_ipi.o
 libsbi-objs-y += sbi_misaligned_ldst.o
 libsbi-objs-y += sbi_scratch.o
+libsbi-objs-y += sbi_string.o
 libsbi-objs-y += sbi_system.o
 libsbi-objs-y += sbi_timer.o
 libsbi-objs-y += sbi_tlb.o
 libsbi-objs-y += sbi_trap.o
-libsbi-objs-y += sbi_string.o
+libsbi-objs-y += sbi_unpriv.o
+libsbi-objs-y += sbi_unpriv_trap.o

lib/sbi/riscv_asm.c

@@ -10,6 +10,39 @@
 #include <sbi/riscv_asm.h>
 #include <sbi/riscv_encoding.h>
 #include <sbi/sbi_error.h>
+#include <sbi/sbi_platform.h>
+
+/* determine CPU extension, return non-zero support */
+int misa_extension_imp(char ext)
+{
+	unsigned long misa = csr_read(CSR_MISA);
+
+	if (misa)
+		return misa & (1 << (ext - 'A'));
+
+	return sbi_platform_misa_extension(sbi_platform_thishart_ptr(), ext);
+}
+
+int misa_xlen(void)
+{
+	long r;
+
+	if (csr_read(CSR_MISA) == 0)
+		return sbi_platform_misa_xlen(sbi_platform_thishart_ptr());
+
+	__asm__ __volatile__(
+		"csrr   t0, misa\n\t"
+		"slti   t1, t0, 0\n\t"
+		"slli   t1, t1, 1\n\t"
+		"slli   t0, t0, 1\n\t"
+		"slti   t0, t0, 0\n\t"
+		"add    %0, t0, t1"
+		: "=r"(r)
+		:
+		: "t0", "t1");
+
+	return r ? r : -1;
+}
 
 unsigned long csr_read_num(int csr_num)
 {
@@ -174,7 +207,7 @@ int pmp_set(unsigned int n, unsigned long prot, unsigned long addr,
 	if (n >= PMP_COUNT || log2len > __riscv_xlen || log2len < PMP_SHIFT)
 		return SBI_EINVAL;
 
-		/* calculate PMP register and offset */
+	/* calculate PMP register and offset */
 #if __riscv_xlen == 32
 	pmpcfg_csr   = CSR_PMPCFG0 + (n >> 2);
 	pmpcfg_shift = (n & 3) << 3;
@@ -216,16 +249,16 @@ int pmp_set(unsigned int n, unsigned long prot, unsigned long addr,
 }
 
 int pmp_get(unsigned int n, unsigned long *prot_out, unsigned long *addr_out,
-	    unsigned long *log2len_out)
+	    unsigned long *size)
 {
 	int pmpcfg_csr, pmpcfg_shift, pmpaddr_csr;
 	unsigned long cfgmask, pmpcfg, prot;
 	unsigned long t1, addr, log2len;
 
 	/* check parameters */
-	if (n >= PMP_COUNT || !prot_out || !addr_out || !log2len_out)
+	if (n >= PMP_COUNT || !prot_out || !addr_out || !size)
 		return SBI_EINVAL;
-	*prot_out = *addr_out = *log2len_out = 0;
+	*prot_out = *addr_out = *size = 0;
 
 	/* calculate PMP register and offset */
 #if __riscv_xlen == 32
@@ -266,7 +299,9 @@ int pmp_get(unsigned int n, unsigned long *prot_out, unsigned long *addr_out,
 	/* return details */
 	*prot_out    = prot;
 	*addr_out    = addr;
-	*log2len_out = log2len;
+
+	if (log2len < __riscv_xlen)
+		*size = (1UL << log2len);
 
 	return 0;
 }

lib/sbi/riscv_atomic.c

@@ -7,11 +7,10 @@
  *   Anup Patel <anup.patel@wdc.com>
  */
 
-#include <sbi/sbi_types.h>
+#include <sbi/sbi_bitops.h>
 #include <sbi/riscv_asm.h>
 #include <sbi/riscv_atomic.h>
 #include <sbi/riscv_barrier.h>
-#include <sbi/sbi_bits.h>
 
 long atomic_read(atomic_t *atom)
 {
@@ -50,7 +49,7 @@ long atomic_sub_return(atomic_t *atom, long value)
 	return ret - value;
 }
 
-#define __axchg(ptr, new, size)						\
+#define __axchg(ptr, new, size)							\
 	({									\
 		__typeof__(ptr) __ptr = (ptr);					\
 		__typeof__(new) __new = (new);					\
@@ -71,15 +70,15 @@ long atomic_sub_return(atomic_t *atom, long value)
 				: "memory");					\
 			break;							\
 		default:							\
-			break;						\
+			break;							\
 		}								\
 		__ret;								\
 	})
 
-#define axchg(ptr, x)							\
+#define axchg(ptr, x)								\
 	({									\
 		__typeof__(*(ptr)) _x_ = (x);					\
-		(__typeof__(*(ptr))) __xchg((ptr), _x_, sizeof(*(ptr)));	\
+		(__typeof__(*(ptr))) __axchg((ptr), _x_, sizeof(*(ptr)));	\
 	})
 
 
@@ -91,20 +90,20 @@ long atomic_sub_return(atomic_t *atom, long value)
 		register unsigned int __rc;                               \
 		switch (size) {                                           \
 		case 4:                                                   \
-			__asm__ __volatile__("0:	lr.w %0, %2\n"           \
-					     "	sc.w.rl %1, %z3, %2\n"     \
-					     "	bnez %1, 0b\n"             \
-					     "	fence rw, rw\n"            \
+			__asm__ __volatile__("0:	lr.w %0, %2\n"    \
+					     "	sc.w.rl %1, %z3, %2\n"    \
+					     "	bnez %1, 0b\n"            \
+					     "	fence rw, rw\n"           \
 					     : "=&r"(__ret), "=&r"(__rc), \
 					       "+A"(*__ptr)               \
 					     : "rJ"(__new)                \
 					     : "memory");                 \
 			break;                                            \
 		case 8:                                                   \
-			__asm__ __volatile__("0:	lr.d %0, %2\n"           \
-					     "	sc.d.rl %1, %z3, %2\n"     \
-					     "	bnez %1, 0b\n"             \
-					     "	fence rw, rw\n"            \
+			__asm__ __volatile__("0:	lr.d %0, %2\n"    \
+					     "	sc.d.rl %1, %z3, %2\n"    \
+					     "	bnez %1, 0b\n"            \
+					     "	fence rw, rw\n"           \
 					     : "=&r"(__ret), "=&r"(__rc), \
 					       "+A"(*__ptr)               \
 					     : "rJ"(__new)                \
@@ -131,11 +130,11 @@ long atomic_sub_return(atomic_t *atom, long value)
 		register unsigned int __rc;                               \
 		switch (size) {                                           \
 		case 4:                                                   \
-			__asm__ __volatile__("0:	lr.w %0, %2\n"           \
-					     "	bne  %0, %z3, 1f\n"        \
-					     "	sc.w.rl %1, %z4, %2\n"     \
-					     "	bnez %1, 0b\n"             \
-					     "	fence rw, rw\n"            \
+			__asm__ __volatile__("0:	lr.w %0, %2\n"    \
+					     "	bne  %0, %z3, 1f\n"       \
+					     "	sc.w.rl %1, %z4, %2\n"    \
+					     "	bnez %1, 0b\n"            \
+					     "	fence rw, rw\n"           \
 					     "1:\n"                       \
 					     : "=&r"(__ret), "=&r"(__rc), \
 					       "+A"(*__ptr)               \
@@ -143,11 +142,11 @@ long atomic_sub_return(atomic_t *atom, long value)
 					     : "memory");                 \
 			break;                                            \
 		case 8:                                                   \
-			__asm__ __volatile__("0:	lr.d %0, %2\n"           \
-					     "	bne %0, %z3, 1f\n"         \
-					     "	sc.d.rl %1, %z4, %2\n"     \
-					     "	bnez %1, 0b\n"             \
-					     "	fence rw, rw\n"            \
+			__asm__ __volatile__("0:	lr.d %0, %2\n"    \
+					     "	bne %0, %z3, 1f\n"        \
+					     "	sc.d.rl %1, %z4, %2\n"    \
+					     "	bnez %1, 0b\n"            \
+					     "	fence rw, rw\n"           \
 					     "1:\n"                       \
 					     : "=&r"(__ret), "=&r"(__rc), \
 					       "+A"(*__ptr)               \
@@ -168,7 +167,7 @@ long atomic_sub_return(atomic_t *atom, long value)
 			__cmpxchg((ptr), _o_, _n_, sizeof(*(ptr))); \
 	})
 
-long arch_atomic_cmpxchg(atomic_t *atom, long oldval, long newval)
+long atomic_cmpxchg(atomic_t *atom, long oldval, long newval)
 {
 #ifdef __riscv_atomic
 	return __sync_val_compare_and_swap(&atom->counter, oldval, newval);
@@ -177,7 +176,7 @@ long arch_atomic_cmpxchg(atomic_t *atom, long oldval, long newval)
 #endif
 }
 
-long arch_atomic_xchg(atomic_t *atom, long newval)
+long atomic_xchg(atomic_t *atom, long newval)
 {
 	/* Atomically set new value and return old value. */
 #ifdef __riscv_atomic
@@ -209,12 +208,12 @@ unsigned long atomic_raw_xchg_ulong(volatile unsigned long *ptr,
 #endif
 }
 
-#if (BITS_PER_LONG == 64)
+#if (__SIZEOF_POINTER__ == 8)
 #define __AMO(op) "amo" #op ".d"
-#elif (BITS_PER_LONG == 32)
+#elif (__SIZEOF_POINTER__ == 4)
 #define __AMO(op) "amo" #op ".w"
 #else
-#error "Unexpected BITS_PER_LONG"
+#error "Unexpected __SIZEOF_POINTER__"
 #endif
 
 #define __atomic_op_bit_ord(op, mod, nr, addr, ord)                          \

lib/sbi/riscv_unpriv.c

@@ -1,158 +0,0 @@
-/*
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2019 Western Digital Corporation or its affiliates.
- *
- * Authors:
- *   Anup Patel <anup.patel@wdc.com>
- */
-
-#include <sbi/riscv_encoding.h>
-#include <sbi/riscv_unpriv.h>
-#include <sbi/sbi_bits.h>
-#include <sbi/sbi_hart.h>
-#include <sbi/sbi_scratch.h>
-
-#define DEFINE_UNPRIVILEGED_LOAD_FUNCTION(type, insn, insnlen)                \
-	type load_##type(const type *addr,                                    \
-			struct sbi_scratch *scratch,                          \
-			struct unpriv_trap *trap)                             \
-	{                                                                     \
-		register ulong __mstatus asm("a2");                           \
-		type val = 0;                                                 \
-		trap->ilen = insnlen;                                         \
-		trap->cause = 0;                                              \
-		trap->tval = 0;                                               \
-		sbi_hart_set_trap_info(scratch, trap);                        \
-		asm volatile(                                                 \
-			"csrrs %0, " STR(CSR_MSTATUS) ", %3\n"                \
-			#insn " %1, %2\n"                                     \
-			"csrw " STR(CSR_MSTATUS) ", %0"                       \
-		    : "+&r"(__mstatus), "=&r"(val)                            \
-		    : "m"(*addr), "r"(MSTATUS_MPRV));                         \
-		sbi_hart_set_trap_info(scratch, NULL);                        \
-		return val;                                                   \
-	}
-
-#define DEFINE_UNPRIVILEGED_STORE_FUNCTION(type, insn, insnlen)               \
-	void store_##type(type *addr, type val,                               \
-			struct sbi_scratch *scratch,                          \
-			struct unpriv_trap *trap)                             \
-	{                                                                     \
-		register ulong __mstatus asm("a3");                           \
-		trap->ilen = insnlen;                                         \
-		trap->cause = 0;                                              \
-		trap->tval = 0;                                               \
-		sbi_hart_set_trap_info(scratch, trap);                        \
-		asm volatile(                                                 \
-			"csrrs %0, " STR(CSR_MSTATUS) ", %3\n"                \
-			#insn " %1, %2\n"                                     \
-			"csrw " STR(CSR_MSTATUS) ", %0"                       \
-			: "+&r"(__mstatus)                                    \
-			: "r"(val), "m"(*addr), "r"(MSTATUS_MPRV));           \
-		sbi_hart_set_trap_info(scratch, NULL);                        \
-	}
-
-DEFINE_UNPRIVILEGED_LOAD_FUNCTION(u8, lbu, 4)
-DEFINE_UNPRIVILEGED_LOAD_FUNCTION(u16, lhu, 4)
-DEFINE_UNPRIVILEGED_LOAD_FUNCTION(s8, lb, 4)
-DEFINE_UNPRIVILEGED_LOAD_FUNCTION(s16, lh, 4)
-DEFINE_UNPRIVILEGED_LOAD_FUNCTION(s32, lw, 2)
-DEFINE_UNPRIVILEGED_STORE_FUNCTION(u8, sb, 4)
-DEFINE_UNPRIVILEGED_STORE_FUNCTION(u16, sh, 4)
-DEFINE_UNPRIVILEGED_STORE_FUNCTION(u32, sw, 2)
-#if __riscv_xlen == 64
-DEFINE_UNPRIVILEGED_LOAD_FUNCTION(u32, lwu, 4)
-DEFINE_UNPRIVILEGED_LOAD_FUNCTION(u64, ld, 2)
-DEFINE_UNPRIVILEGED_STORE_FUNCTION(u64, sd, 2)
-DEFINE_UNPRIVILEGED_LOAD_FUNCTION(ulong, ld, 2)
-#else
-DEFINE_UNPRIVILEGED_LOAD_FUNCTION(u32, lw, 2)
-DEFINE_UNPRIVILEGED_LOAD_FUNCTION(ulong, lw, 2)
-
-u64 load_u64(const u64 *addr,
-	     struct sbi_scratch *scratch, struct unpriv_trap *trap)
-{
-	u64 ret = load_u32((u32 *)addr, scratch, trap);
-
-	if (trap->cause)
-		return 0;
-	ret |= ((u64)load_u32((u32 *)addr + 1, scratch, trap) << 32);
-	if (trap->cause)
-		return 0;
-
-	return ret;
-}
-
-void store_u64(u64 *addr, u64 val,
-	       struct sbi_scratch *scratch, struct unpriv_trap *trap)
-{
-	store_u32((u32 *)addr, val, scratch, trap);
-	if (trap->cause)
-		return;
-
-	store_u32((u32 *)addr + 1, val >> 32, scratch, trap);
-	if (trap->cause)
-		return;
-}
-#endif
-
-ulong get_insn(ulong mepc, bool virt, struct sbi_scratch *scratch,
-	       struct unpriv_trap *trap)
-{
-	ulong __mstatus = 0, __vsstatus = 0, val = 0;
-#ifdef __riscv_compressed
-	ulong rvc_mask = 3, tmp;
-#endif
-
-	trap->ilen = 4;
-	trap->cause = 0;
-	trap->tval = 0;
-	sbi_hart_set_trap_info(scratch, trap);
-
-	if (virt)
-		__vsstatus = csr_read_set(CSR_VSSTATUS, SSTATUS_MXR);
-
-#ifndef __riscv_compressed
-	asm("csrrs %[mstatus], " STR(CSR_MSTATUS) ", %[mprv]\n"
-#if __riscv_xlen == 64
-	    STR(LWU) " %[insn], (%[addr])\n"
-#else
-	    STR(LW) " %[insn], (%[addr])\n"
-#endif
-	    "csrw " STR(CSR_MSTATUS) ", %[mstatus]"
-	    : [mstatus] "+&r"(__mstatus), [insn] "=&r"(val)
-	    : [mprv] "r"(MSTATUS_MPRV | MSTATUS_MXR), [addr] "r"(mepc));
-#else
-	asm("csrrs %[mstatus], " STR(CSR_MSTATUS) ", %[mprv]\n"
-	    "lhu %[insn], (%[addr])\n"
-	    "and %[tmp], %[insn], %[rvc_mask]\n"
-	    "bne %[tmp], %[rvc_mask], 2f\n"
-	    "lhu %[tmp], 2(%[addr])\n"
-	    "sll %[tmp], %[tmp], 16\n"
-	    "add %[insn], %[insn], %[tmp]\n"
-	    "2: csrw " STR(CSR_MSTATUS) ", %[mstatus]"
-	    : [mstatus] "+&r"(__mstatus), [insn] "=&r"(val), [tmp] "=&r"(tmp)
-	    : [mprv] "r"(MSTATUS_MPRV | MSTATUS_MXR), [addr] "r"(mepc),
-	      [rvc_mask] "r"(rvc_mask));
-#endif
-
-	if (virt)
-		csr_write(CSR_VSSTATUS, __vsstatus);
-
-	sbi_hart_set_trap_info(scratch, NULL);
-	switch (trap->cause) {
-	case CAUSE_LOAD_ACCESS:
-		trap->cause = CAUSE_FETCH_ACCESS;
-		trap->tval = mepc;
-		break;
-	case CAUSE_LOAD_PAGE_FAULT:
-		trap->cause = CAUSE_FETCH_PAGE_FAULT;
-		trap->tval = mepc;
-		break;
-	default:
-		break;
-	};
-
-	return val;
-}

lib/sbi/sbi_bitmap.c

@@ -0,0 +1,40 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2020 Western Digital Corporation or its affiliates.
+ *
+ * Authors:
+ *   Anup Patel <anup.patel@wdc.com>
+ */
+
+#include <sbi/sbi_bitmap.h>
+
+void __bitmap_and(unsigned long *dst, const unsigned long *bitmap1,
+		  const unsigned long *bitmap2, int bits)
+{
+	int k;
+	int nr = BITS_TO_LONGS(bits);
+
+	for (k = 0; k < nr; k++)
+		dst[k] = bitmap1[k] & bitmap2[k];
+}
+
+void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1,
+		 const unsigned long *bitmap2, int bits)
+{
+	int k;
+	int nr = BITS_TO_LONGS(bits);
+
+	for (k = 0; k < nr; k++)
+		dst[k] = bitmap1[k] | bitmap2[k];
+}
+
+void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1,
+		  const unsigned long *bitmap2, int bits)
+{
+	int k;
+	int nr = BITS_TO_LONGS(bits);
+
+	for (k = 0; k < nr; k++)
+		dst[k] = bitmap1[k] ^ bitmap2[k];
+}

lib/sbi/sbi_bitops.c

@@ -0,0 +1,200 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2020 Western Digital Corporation or its affiliates.
+ *
+ * Authors:
+ *   Atish Patra <atish.patra@wdc.com>
+ *   Anup Patel <anup.patel@wdc.com>
+ */
+
+#include <sbi/sbi_bitops.h>
+
+#define BITOP_WORD(nr)		((nr) / BITS_PER_LONG)
+
+/**
+ * find_first_bit - find the first set bit in a memory region
+ * @addr: The address to start the search at
+ * @size: The maximum size to search
+ *
+ * Returns the bit number of the first set bit.
+ */
+unsigned long find_first_bit(const unsigned long *addr,
+			     unsigned long size)
+{
+	const unsigned long *p = addr;
+	unsigned long result = 0;
+	unsigned long tmp;
+
+	while (size & ~(BITS_PER_LONG-1)) {
+		if ((tmp = *(p++)))
+			goto found;
+		result += BITS_PER_LONG;
+		size -= BITS_PER_LONG;
+	}
+	if (!size)
+		return result;
+
+	tmp = (*p) & (~0UL >> (BITS_PER_LONG - size));
+	if (tmp == 0UL)		/* Are any bits set? */
+		return result + size;	/* Nope. */
+found:
+	return result + __ffs(tmp);
+}
+
+/**
+ * find_first_zero_bit - find the first cleared bit in a memory region
+ * @addr: The address to start the search at
+ * @size: The maximum size to search
+ *
+ * Returns the bit number of the first cleared bit.
+ */
+unsigned long find_first_zero_bit(const unsigned long *addr,
+				  unsigned long size)
+{
+	const unsigned long *p = addr;
+	unsigned long result = 0;
+	unsigned long tmp;
+
+	while (size & ~(BITS_PER_LONG-1)) {
+		if (~(tmp = *(p++)))
+			goto found;
+		result += BITS_PER_LONG;
+		size -= BITS_PER_LONG;
+	}
+	if (!size)
+		return result;
+
+	tmp = (*p) | (~0UL << size);
+	if (tmp == ~0UL)	/* Are any bits zero? */
+		return result + size;	/* Nope. */
+found:
+	return result + ffz(tmp);
+}
+
+/**
+ * find_last_bit - find the last set bit in a memory region
+ * @addr: The address to start the search at
+ * @size: The maximum size to search
+ *
+ * Returns the bit number of the first set bit, or size.
+ */
+unsigned long find_last_bit(const unsigned long *addr,
+			    unsigned long size)
+{
+	unsigned long words;
+	unsigned long tmp;
+
+	/* Start at final word. */
+	words = size / BITS_PER_LONG;
+
+	/* Partial final word? */
+	if (size & (BITS_PER_LONG-1)) {
+		tmp = (addr[words] & (~0UL >> (BITS_PER_LONG
+					 - (size & (BITS_PER_LONG-1)))));
+		if (tmp)
+			goto found;
+	}
+
+	while (words) {
+		tmp = addr[--words];
+		if (tmp) {
+found:
+			return words * BITS_PER_LONG + __fls(tmp);
+		}
+	}
+
+	/* Not found */
+	return size;
+}
+
+/**
+ * find_next_bit - find the next set bit in a memory region
+ * @addr: The address to base the search on
+ * @offset: The bitnumber to start searching at
+ * @size: The bitmap size in bits
+ */
+unsigned long find_next_bit(const unsigned long *addr,
+			    unsigned long size, unsigned long offset)
+{
+	const unsigned long *p = addr + BITOP_WORD(offset);
+	unsigned long result = offset & ~(BITS_PER_LONG-1);
+	unsigned long tmp;
+
+	if (offset >= size)
+		return size;
+	size -= result;
+	offset %= BITS_PER_LONG;
+	if (offset) {
+		tmp = *(p++);
+		tmp &= (~0UL << offset);
+		if (size < BITS_PER_LONG)
+			goto found_first;
+		if (tmp)
+			goto found_middle;
+		size -= BITS_PER_LONG;
+		result += BITS_PER_LONG;
+	}
+	while (size & ~(BITS_PER_LONG-1)) {
+		if ((tmp = *(p++)))
+			goto found_middle;
+		result += BITS_PER_LONG;
+		size -= BITS_PER_LONG;
+	}
+	if (!size)
+		return result;
+	tmp = *p;
+
+found_first:
+	tmp &= (~0UL >> (BITS_PER_LONG - size));
+	if (tmp == 0UL)		/* Are any bits set? */
+		return result + size;	/* Nope. */
+found_middle:
+	return result + __ffs(tmp);
+}
+
+/**
+ * find_next_zero_bit - find the next cleared bit in a memory region
+ * @addr: The address to base the search on
+ * @offset: The bitnumber to start searching at
+ * @size: The bitmap size in bits
+ */
+unsigned long find_next_zero_bit(const unsigned long *addr,
+				 unsigned long size,
+				 unsigned long offset)
+{
+	const unsigned long *p = addr + BITOP_WORD(offset);
+	unsigned long result = offset & ~(BITS_PER_LONG-1);
+	unsigned long tmp;
+
+	if (offset >= size)
+		return size;
+	size -= result;
+	offset %= BITS_PER_LONG;
+	if (offset) {
+		tmp = *(p++);
+		tmp |= ~0UL >> (BITS_PER_LONG - offset);
+		if (size < BITS_PER_LONG)
+			goto found_first;
+		if (~tmp)
+			goto found_middle;
+		size -= BITS_PER_LONG;
+		result += BITS_PER_LONG;
+	}
+	while (size & ~(BITS_PER_LONG-1)) {
+		if (~(tmp = *(p++)))
+			goto found_middle;
+		result += BITS_PER_LONG;
+		size -= BITS_PER_LONG;
+	}
+	if (!size)
+		return result;
+	tmp = *p;
+
+found_first:
+	tmp |= ~0UL << size;
+	if (tmp == ~0UL)	/* Are any bits zero? */
+		return result + size;	/* Nope. */
+found_middle:
+	return result + ffz(tmp);
+}

lib/sbi/sbi_console.c

@@ -7,9 +7,10 @@
  *   Anup Patel <anup.patel@wdc.com>
  */
 
-#include <sbi/sbi_platform.h>
-#include <sbi/sbi_console.h>
 #include <sbi/riscv_locks.h>
+#include <sbi/sbi_console.h>
+#include <sbi/sbi_platform.h>
+#include <sbi/sbi_scratch.h>
 
 static const struct sbi_platform *console_plat = NULL;
 static spinlock_t console_out_lock	       = SPIN_LOCK_INITIALIZER;
@@ -375,10 +376,11 @@ int sbi_printf(const char *format, ...)
 	return retval;
 }
 
-int sbi_dprintf(struct sbi_scratch *scratch, const char *format, ...)
+int sbi_dprintf(const char *format, ...)
 {
 	va_list args;
 	int retval = 0;
+	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
 
 	va_start(args, format);
 	if (scratch->options & SBI_SCRATCH_DEBUG_PRINTS)

lib/sbi/sbi_ecall.c

@@ -7,23 +7,10 @@
  *   Anup Patel <anup.patel@wdc.com>
  */
 
-#include <sbi/sbi_console.h>
 #include <sbi/sbi_ecall.h>
 #include <sbi/sbi_ecall_interface.h>
 #include <sbi/sbi_error.h>
-#include <sbi/sbi_ipi.h>
-#include <sbi/sbi_platform.h>
-#include <sbi/sbi_system.h>
-#include <sbi/sbi_timer.h>
-#include <sbi/sbi_tlb.h>
 #include <sbi/sbi_trap.h>
-#include <sbi/sbi_hart.h>
-#include <sbi/sbi_version.h>
-#include <sbi/riscv_asm.h>
-
-#define SBI_ECALL_VERSION_MAJOR 0
-#define SBI_ECALL_VERSION_MINOR 2
-#define SBI_OPENSBI_IMPID 1
 
 u16 sbi_ecall_version_major(void)
 {
@@ -35,164 +22,83 @@ u16 sbi_ecall_version_minor(void)
 	return SBI_ECALL_VERSION_MINOR;
 }
 
-int sbi_check_extension(struct sbi_scratch *scratch, unsigned long extid,
-			unsigned long *out_val)
-{
-	/**
-	 * Each extension apart from base & 0.1, will be implemented as
-	 * platform specific feature. Thus, extension probing can be achieved
-	 * by checking the feature bits of the platform. We can create a map
-	 * between extension ID & feature and use a generic function to check
-	 * or just use a switch case for every new extension support added
-	 * TODO: Implement it.
-	 */
+static unsigned long ecall_impid = SBI_OPENSBI_IMPID;
 
-	if ((extid >= SBI_EXT_0_1_SET_TIMER &&
-	    extid <= SBI_EXT_0_1_SHUTDOWN) || (extid == SBI_EXT_BASE)) {
-		*out_val = 1;
-	} else if (extid >= SBI_EXT_VENDOR_START &&
-		   extid <= SBI_EXT_VENDOR_END) {
-		*out_val = sbi_platform_vendor_ext_check(
-						sbi_platform_ptr(scratch),
-						extid);
-	} else
-		*out_val = 0;
+unsigned long sbi_ecall_get_impid(void)
+{
+	return ecall_impid;
+}
+
+void sbi_ecall_set_impid(unsigned long impid)
+{
+	ecall_impid = impid;
+}
+
+static SBI_LIST_HEAD(ecall_exts_list);
+
+struct sbi_ecall_extension *sbi_ecall_find_extension(unsigned long extid)
+{
+	struct sbi_ecall_extension *t, *ret = NULL;
+
+	sbi_list_for_each_entry(t, &ecall_exts_list, head) {
+		if (t->extid_start <= extid && extid <= t->extid_end) {
+			ret = t;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+int sbi_ecall_register_extension(struct sbi_ecall_extension *ext)
+{
+	struct sbi_ecall_extension *t;
+
+	if (!ext || (ext->extid_end < ext->extid_start) || !ext->handle)
+		return SBI_EINVAL;
+
+	sbi_list_for_each_entry(t, &ecall_exts_list, head) {
+		unsigned long start = t->extid_start;
+		unsigned long end = t->extid_end;
+		if (end < ext->extid_start || ext->extid_end < start)
+			/* no overlap */;
+		else
+			return SBI_EINVAL;
+	}
+
+	SBI_INIT_LIST_HEAD(&ext->head);
+	sbi_list_add_tail(&ext->head, &ecall_exts_list);
 
 	return 0;
 }
 
-int sbi_ecall_vendor_ext_handler(struct sbi_scratch *scratch,
-				 unsigned long extid, unsigned long funcid,
-				 unsigned long *args, unsigned long *out_val,
-				 unsigned long *out_tcause,
-				 unsigned long *out_tval)
+void sbi_ecall_unregister_extension(struct sbi_ecall_extension *ext)
 {
-	return sbi_platform_vendor_ext_provider(sbi_platform_ptr(scratch),
-					       extid, funcid, args, out_val,
-					       out_tcause, out_tval);
-}
+	bool found = FALSE;
+	struct sbi_ecall_extension *t;
 
-int sbi_ecall_base_handler(struct sbi_scratch *scratch, unsigned long extid,
-			   unsigned long funcid, unsigned long *args,
-			   unsigned long *out_val, unsigned long *out_tcause,
-			   unsigned long *out_tval)
-{
-	int ret = 0;
+	if (!ext)
+		return;
 
-	switch (funcid) {
-	case SBI_EXT_BASE_GET_SPEC_VERSION:
-		*out_val = (SBI_ECALL_VERSION_MAJOR <<
-			   SBI_SPEC_VERSION_MAJOR_OFFSET) &
-			   (SBI_SPEC_VERSION_MAJOR_MASK <<
-			    SBI_SPEC_VERSION_MAJOR_OFFSET);
-		*out_val = *out_val | SBI_ECALL_VERSION_MINOR;
-		break;
-	case SBI_EXT_BASE_GET_IMP_ID:
-		*out_val = SBI_OPENSBI_IMPID;
-		break;
-	case SBI_EXT_BASE_GET_IMP_VERSION:
-		*out_val = OPENSBI_VERSION;
-		break;
-	case SBI_EXT_BASE_GET_MVENDORID:
-		*out_val = csr_read(CSR_MVENDORID);
-		break;
-	case SBI_EXT_BASE_GET_MARCHID:
-		*out_val = csr_read(CSR_MARCHID);
-		break;
-	case SBI_EXT_BASE_GET_MIMPID:
-		*out_val = csr_read(CSR_MIMPID);
-		break;
-	case SBI_EXT_BASE_PROBE_EXT:
-		ret = sbi_check_extension(scratch, args[0], out_val);
-	default:
-		ret = SBI_ENOTSUPP;
+	sbi_list_for_each_entry(t, &ecall_exts_list, head) {
+		if (t == ext) {
+			found = TRUE;
+			break;
+		}
 	}
 
-	return ret;
+	if (found)
+		sbi_list_del_init(&ext->head);
 }
 
-int sbi_ecall_0_1_handler(struct sbi_scratch *scratch, unsigned long extid,
-			   unsigned long *args, unsigned long *tval,
-			   unsigned long *tcause)
-{
-	int ret = 0;
-	struct sbi_tlb_info tlb_info;
-	u32 source_hart = sbi_current_hartid();
-	struct unpriv_trap uptrap = {0};
-
-	switch (extid) {
-	case SBI_EXT_0_1_SET_TIMER:
-#if __riscv_xlen == 32
-		sbi_timer_event_start(scratch,
-				      (((u64)args[1] << 32) | (u64)args[0]));
-#else
-		sbi_timer_event_start(scratch, (u64)args[0]);
-#endif
-		break;
-	case SBI_EXT_0_1_CONSOLE_PUTCHAR:
-		sbi_putc(args[0]);
-		break;
-	case SBI_EXT_0_1_CONSOLE_GETCHAR:
-		ret = sbi_getc();
-		break;
-	case SBI_EXT_0_1_CLEAR_IPI:
-		sbi_ipi_clear_smode(scratch);
-		break;
-	case SBI_EXT_0_1_SEND_IPI:
-		ret = sbi_ipi_send_many(scratch, &uptrap, (ulong *)args[0],
-					SBI_IPI_EVENT_SOFT, NULL);
-		break;
-	case SBI_EXT_0_1_REMOTE_FENCE_I:
-		tlb_info.start  = 0;
-		tlb_info.size  = 0;
-		tlb_info.type  = SBI_ITLB_FLUSH;
-		tlb_info.shart_mask = 1UL << source_hart;
-		ret = sbi_ipi_send_many(scratch, &uptrap, (ulong *)args[0],
-					SBI_IPI_EVENT_FENCE_I, &tlb_info);
-		break;
-	case SBI_EXT_0_1_REMOTE_SFENCE_VMA:
-		tlb_info.start = (unsigned long)args[1];
-		tlb_info.size  = (unsigned long)args[2];
-		tlb_info.type  = SBI_TLB_FLUSH_VMA;
-		tlb_info.shart_mask = 1UL << source_hart;
-
-		ret = sbi_ipi_send_many(scratch, &uptrap, (ulong *)args[0],
-					SBI_IPI_EVENT_SFENCE_VMA, &tlb_info);
-		break;
-	case SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID:
-		tlb_info.start = (unsigned long)args[1];
-		tlb_info.size  = (unsigned long)args[2];
-		tlb_info.asid  = (unsigned long)args[3];
-		tlb_info.type  = SBI_TLB_FLUSH_VMA_ASID;
-		tlb_info.shart_mask = 1UL << source_hart;
-
-		ret = sbi_ipi_send_many(scratch, &uptrap, (ulong *)args[0],
-					SBI_IPI_EVENT_SFENCE_VMA_ASID,
-					&tlb_info);
-		break;
-	case SBI_EXT_0_1_SHUTDOWN:
-		sbi_system_shutdown(scratch, 0);
-		break;
-	default:
-		ret = SBI_ENOTSUPP;
-	};
-
-	if (ret == SBI_ETRAP) {
-		*tcause = uptrap.cause;
-		*tval = uptrap.tval;
-	}
-	return ret;
-}
-
-int sbi_ecall_handler(u32 hartid, ulong mcause, struct sbi_trap_regs *regs,
-		      struct sbi_scratch *scratch)
+int sbi_ecall_handler(struct sbi_trap_regs *regs)
 {
 	int ret = 0;
+	struct sbi_ecall_extension *ext;
 	unsigned long extension_id = regs->a7;
 	unsigned long func_id = regs->a6;
-	unsigned long out_val;
-	unsigned long out_tval;
-	unsigned long out_tcause;
+	struct sbi_trap_info trap = {0};
+	unsigned long out_val = 0;
 	bool is_0_1_spec = 0;
 	unsigned long args[6];
 
@@ -203,29 +109,23 @@ int sbi_ecall_handler(u32 hartid, ulong mcause, struct sbi_trap_regs *regs,
 	args[4] = regs->a4;
 	args[5] = regs->a5;
 
-	if (extension_id >= SBI_EXT_0_1_SET_TIMER &&
-	    extension_id <= SBI_EXT_0_1_SHUTDOWN) {
-		ret = sbi_ecall_0_1_handler(scratch, extension_id, args,
-					    &out_tval, &out_tcause);
-		is_0_1_spec = 1;
-	} else if (extension_id == SBI_EXT_BASE)
-		ret = sbi_ecall_base_handler(scratch, extension_id, func_id,
-					     args, &out_val,
-					     &out_tval, &out_tcause);
-	else if (extension_id >= SBI_EXT_VENDOR_START &&
-		extension_id <= SBI_EXT_VENDOR_END) {
-		ret = sbi_ecall_vendor_ext_handler(scratch, extension_id,
-						   func_id, args, &out_val,
-						   &out_tval, &out_tcause);
+	ext = sbi_ecall_find_extension(extension_id);
+	if (ext && ext->handle) {
+		ret = ext->handle(extension_id, func_id,
+				  args, &out_val, &trap);
+		if (extension_id >= SBI_EXT_0_1_SET_TIMER &&
+		    extension_id <= SBI_EXT_0_1_SHUTDOWN)
+			is_0_1_spec = 1;
 	} else {
 		ret = SBI_ENOTSUPP;
 	}
 
 	if (ret == SBI_ETRAP) {
-		sbi_trap_redirect(regs, scratch, regs->mepc,
-				  out_tcause, out_tval);
+		trap.epc = regs->mepc;
+		sbi_trap_redirect(regs, &trap);
 	} else {
-		/* This function should return non-zero value only in case of
+		/*
+		 * This function should return non-zero value only in case of
 		 * fatal error. However, there is no good way to distinguish
 		 * between a fatal and non-fatal errors yet. That's why we treat
 		 * every return value except ETRAP as non-fatal and just return
@@ -240,3 +140,33 @@ int sbi_ecall_handler(u32 hartid, ulong mcause, struct sbi_trap_regs *regs,
 
 	return 0;
 }
+
+int sbi_ecall_init(void)
+{
+	int ret;
+
+	/* The order of below registrations is performance optimized */
+	ret = sbi_ecall_register_extension(&ecall_time);
+	if (ret)
+		return ret;
+	ret = sbi_ecall_register_extension(&ecall_rfence);
+	if (ret)
+		return ret;
+	ret = sbi_ecall_register_extension(&ecall_ipi);
+	if (ret)
+		return ret;
+	ret = sbi_ecall_register_extension(&ecall_base);
+	if (ret)
+		return ret;
+	ret = sbi_ecall_register_extension(&ecall_hsm);
+	if (ret)
+		return ret;
+	ret = sbi_ecall_register_extension(&ecall_legacy);
+	if (ret)
+		return ret;
+	ret = sbi_ecall_register_extension(&ecall_vendor);
+	if (ret)
+		return ret;
+
+	return 0;
+}

lib/sbi/sbi_ecall_base.c

@@ -0,0 +1,77 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2020 Western Digital Corporation or its affiliates.
+ *
+ * Authors:
+ *   Anup Patel <anup.patel@wdc.com>
+ *   Atish Patra <atish.patra@wdc.com>
+ */
+
+#include <sbi/sbi_ecall.h>
+#include <sbi/sbi_ecall_interface.h>
+#include <sbi/sbi_error.h>
+#include <sbi/sbi_version.h>
+#include <sbi/riscv_asm.h>
+
+static int sbi_ecall_base_probe(unsigned long extid, unsigned long *out_val)
+{
+	struct sbi_ecall_extension *ext;
+
+	ext = sbi_ecall_find_extension(extid);
+	if (!ext) {
+		*out_val = 0;
+		return 0;
+	}
+
+	if (ext->probe)
+		return ext->probe(extid, out_val);
+
+	*out_val = 1;
+	return 0;
+}
+
+static int sbi_ecall_base_handler(unsigned long extid, unsigned long funcid,
+				  unsigned long *args, unsigned long *out_val,
+				  struct sbi_trap_info *out_trap)
+{
+	int ret = 0;
+
+	switch (funcid) {
+	case SBI_EXT_BASE_GET_SPEC_VERSION:
+		*out_val = (SBI_ECALL_VERSION_MAJOR <<
+			   SBI_SPEC_VERSION_MAJOR_OFFSET) &
+			   (SBI_SPEC_VERSION_MAJOR_MASK <<
+			    SBI_SPEC_VERSION_MAJOR_OFFSET);
+		*out_val = *out_val | SBI_ECALL_VERSION_MINOR;
+		break;
+	case SBI_EXT_BASE_GET_IMP_ID:
+		*out_val = sbi_ecall_get_impid();
+		break;
+	case SBI_EXT_BASE_GET_IMP_VERSION:
+		*out_val = OPENSBI_VERSION;
+		break;
+	case SBI_EXT_BASE_GET_MVENDORID:
+		*out_val = csr_read(CSR_MVENDORID);
+		break;
+	case SBI_EXT_BASE_GET_MARCHID:
+		*out_val = csr_read(CSR_MARCHID);
+		break;
+	case SBI_EXT_BASE_GET_MIMPID:
+		*out_val = csr_read(CSR_MIMPID);
+		break;
+	case SBI_EXT_BASE_PROBE_EXT:
+		ret = sbi_ecall_base_probe(args[0], out_val);
+		break;
+	default:
+		ret = SBI_ENOTSUPP;
+	}
+
+	return ret;
+}
+
+struct sbi_ecall_extension ecall_base = {
+	.extid_start = SBI_EXT_BASE,
+	.extid_end = SBI_EXT_BASE,
+	.handle = sbi_ecall_base_handler,
+};

lib/sbi/sbi_ecall_hsm.c

@@ -0,0 +1,51 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2020 Western Digital Corporation or its affiliates.
+ *
+ * Authors:
+ *   Atish Patra <atish.patra@wdc.com>
+ */
+
+#include <sbi/sbi_ecall.h>
+#include <sbi/sbi_ecall_interface.h>
+#include <sbi/sbi_error.h>
+#include <sbi/sbi_version.h>
+#include <sbi/sbi_hsm.h>
+#include <sbi/sbi_scratch.h>
+#include <sbi/riscv_asm.h>
+
+static int sbi_ecall_hsm_handler(unsigned long extid, unsigned long funcid,
+				 unsigned long *args, unsigned long *out_val,
+				 struct sbi_trap_info *out_trap)
+{
+	int ret = 0, hstate;
+	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
+
+	switch (funcid) {
+	case SBI_EXT_HSM_HART_START:
+		ret = sbi_hsm_hart_start(scratch, args[0], args[1], args[2]);
+		break;
+	case SBI_EXT_HSM_HART_STOP:
+		ret = sbi_hsm_hart_stop(scratch, TRUE);
+		break;
+	case SBI_EXT_HSM_HART_GET_STATUS:
+		hstate = sbi_hsm_hart_get_state(args[0]);
+		ret = sbi_hsm_hart_state_to_status(hstate);
+		break;
+	default:
+		ret = SBI_ENOTSUPP;
+	};
+	if (ret >= 0) {
+		*out_val = ret;
+		ret = 0;
+	}
+
+	return ret;
+}
+
+struct sbi_ecall_extension ecall_hsm = {
+	.extid_start = SBI_EXT_HSM,
+	.extid_end = SBI_EXT_HSM,
+	.handle = sbi_ecall_hsm_handler,
+};

lib/sbi/sbi_ecall_legacy.c

@@ -0,0 +1,115 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2020 Western Digital Corporation or its affiliates.
+ *
+ * Authors:
+ *   Anup Patel <anup.patel@wdc.com>
+ *   Atish Patra <atish.patra@wdc.com>
+ */
+
+#include <sbi/riscv_asm.h>
+#include <sbi/sbi_console.h>
+#include <sbi/sbi_ecall.h>
+#include <sbi/sbi_ecall_interface.h>
+#include <sbi/sbi_error.h>
+#include <sbi/sbi_hsm.h>
+#include <sbi/sbi_ipi.h>
+#include <sbi/sbi_system.h>
+#include <sbi/sbi_timer.h>
+#include <sbi/sbi_tlb.h>
+#include <sbi/sbi_trap.h>
+#include <sbi/sbi_unpriv.h>
+#include <sbi/sbi_hart.h>
+
+static int sbi_load_hart_mask_unpriv(ulong *pmask, ulong *hmask,
+				     struct sbi_trap_info *uptrap)
+{
+	ulong mask = 0;
+
+	if (pmask) {
+		mask = sbi_load_ulong(pmask, uptrap);
+		if (uptrap->cause)
+			return SBI_ETRAP;
+	} else {
+		sbi_hsm_hart_started_mask(0, &mask);
+	}
+	*hmask = mask;
+
+	return 0;
+}
+
+static int sbi_ecall_legacy_handler(unsigned long extid, unsigned long funcid,
+				    unsigned long *args, unsigned long *out_val,
+				    struct sbi_trap_info *out_trap)
+{
+	int ret = 0;
+	struct sbi_tlb_info tlb_info;
+	u32 source_hart = current_hartid();
+	ulong hmask = 0;
+
+	switch (extid) {
+	case SBI_EXT_0_1_SET_TIMER:
+#if __riscv_xlen == 32
+		sbi_timer_event_start((((u64)args[1] << 32) | (u64)args[0]));
+#else
+		sbi_timer_event_start((u64)args[0]);
+#endif
+		break;
+	case SBI_EXT_0_1_CONSOLE_PUTCHAR:
+		sbi_putc(args[0]);
+		break;
+	case SBI_EXT_0_1_CONSOLE_GETCHAR:
+		ret = sbi_getc();
+		break;
+	case SBI_EXT_0_1_CLEAR_IPI:
+		sbi_ipi_clear_smode();
+		break;
+	case SBI_EXT_0_1_SEND_IPI:
+		ret = sbi_load_hart_mask_unpriv((ulong *)args[0],
+						&hmask, out_trap);
+		if (ret != SBI_ETRAP)
+			ret = sbi_ipi_send_smode(hmask, 0);
+		break;
+	case SBI_EXT_0_1_REMOTE_FENCE_I:
+		ret = sbi_load_hart_mask_unpriv((ulong *)args[0],
+						&hmask, out_trap);
+		if (ret != SBI_ETRAP) {
+			SBI_TLB_INFO_INIT(&tlb_info, 0, 0, 0,
+					  SBI_ITLB_FLUSH, source_hart);
+			ret = sbi_tlb_request(hmask, 0, &tlb_info);
+		}
+		break;
+	case SBI_EXT_0_1_REMOTE_SFENCE_VMA:
+		ret = sbi_load_hart_mask_unpriv((ulong *)args[0],
+						&hmask, out_trap);
+		if (ret != SBI_ETRAP) {
+			SBI_TLB_INFO_INIT(&tlb_info, args[1], args[2], 0,
+					  SBI_TLB_FLUSH_VMA, source_hart);
+			ret = sbi_tlb_request(hmask, 0, &tlb_info);
+		}
+		break;
+	case SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID:
+		ret = sbi_load_hart_mask_unpriv((ulong *)args[0],
+						&hmask, out_trap);
+		if (ret != SBI_ETRAP) {
+			SBI_TLB_INFO_INIT(&tlb_info, args[1], args[2], args[3],
+					  SBI_TLB_FLUSH_VMA_ASID, source_hart);
+			ret = sbi_tlb_request(hmask, 0, &tlb_info);
+		}
+		break;
+	case SBI_EXT_0_1_SHUTDOWN:
+		sbi_system_shutdown(0);
+		break;
+	default:
+		ret = SBI_ENOTSUPP;
+	};
+
+	return ret;
+}
+
+struct sbi_ecall_extension ecall_legacy = {
+	.extid_start = SBI_EXT_0_1_SET_TIMER,
+	.extid_end = SBI_EXT_0_1_SHUTDOWN,
+	.handle = sbi_ecall_legacy_handler,
+};

lib/sbi/sbi_ecall_replace.c

@@ -0,0 +1,124 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2020 Western Digital Corporation or its affiliates.
+ *
+ * Authors:
+ *   Anup Patel <anup.patel@wdc.com>
+ *   Atish Patra <atish.patra@wdc.com>
+ */
+
+#include <sbi/riscv_asm.h>
+#include <sbi/sbi_ecall.h>
+#include <sbi/sbi_ecall_interface.h>
+#include <sbi/sbi_error.h>
+#include <sbi/sbi_hart.h>
+#include <sbi/sbi_ipi.h>
+#include <sbi/sbi_timer.h>
+#include <sbi/sbi_tlb.h>
+
+static int sbi_ecall_time_handler(unsigned long extid, unsigned long funcid,
+				  unsigned long *args, unsigned long *out_val,
+				  struct sbi_trap_info *out_trap)
+{
+	int ret = 0;
+
+	if (funcid == SBI_EXT_TIME_SET_TIMER) {
+#if __riscv_xlen == 32
+		sbi_timer_event_start((((u64)args[1] << 32) | (u64)args[0]));
+#else
+		sbi_timer_event_start((u64)args[0]);
+#endif
+	} else
+		ret = SBI_ENOTSUPP;
+
+	return ret;
+}
+
+struct sbi_ecall_extension ecall_time = {
+	.extid_start = SBI_EXT_TIME,
+	.extid_end = SBI_EXT_TIME,
+	.handle = sbi_ecall_time_handler,
+};
+
+static int sbi_ecall_rfence_handler(unsigned long extid, unsigned long funcid,
+				    unsigned long *args, unsigned long *out_val,
+				    struct sbi_trap_info *out_trap)
+{
+	int ret = 0;
+	struct sbi_tlb_info tlb_info;
+	u32 source_hart = current_hartid();
+
+	if (funcid >= SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA &&
+	    funcid <= SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID)
+		if (!misa_extension('H'))
+			return SBI_ENOTSUPP;
+
+	switch (funcid) {
+	case SBI_EXT_RFENCE_REMOTE_FENCE_I:
+		SBI_TLB_INFO_INIT(&tlb_info, 0, 0, 0,
+				  SBI_ITLB_FLUSH, source_hart);
+		ret = sbi_tlb_request(args[0], args[1], &tlb_info);
+		break;
+	case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA:
+		SBI_TLB_INFO_INIT(&tlb_info, args[2], args[3], 0,
+				  SBI_TLB_FLUSH_GVMA, source_hart);
+		ret = sbi_tlb_request(args[0], args[1], &tlb_info);
+		break;
+	case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID:
+		SBI_TLB_INFO_INIT(&tlb_info, args[2], args[3], args[4],
+				  SBI_TLB_FLUSH_GVMA_VMID, source_hart);
+		ret = sbi_tlb_request(args[0], args[1], &tlb_info);
+		break;
+	case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA:
+		SBI_TLB_INFO_INIT(&tlb_info, args[2], args[3], 0,
+				  SBI_TLB_FLUSH_VVMA, source_hart);
+		ret = sbi_tlb_request(args[0], args[1], &tlb_info);
+		break;
+	case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID:
+		SBI_TLB_INFO_INIT(&tlb_info, args[2], args[3], args[4],
+				  SBI_TLB_FLUSH_VVMA_ASID, source_hart);
+		ret = sbi_tlb_request(args[0], args[1], &tlb_info);
+		break;
+	case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA:
+		SBI_TLB_INFO_INIT(&tlb_info, args[2], args[3], 0,
+				  SBI_TLB_FLUSH_VMA, source_hart);
+		ret = sbi_tlb_request(args[0], args[1], &tlb_info);
+		break;
+	case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID:
+		SBI_TLB_INFO_INIT(&tlb_info, args[2], args[3], args[4],
+				  SBI_TLB_FLUSH_VMA_ASID, source_hart);
+		ret = sbi_tlb_request(args[0], args[1], &tlb_info);
+		break;
+	default:
+		ret = SBI_ENOTSUPP;
+	};
+
+	return ret;
+}
+
+struct sbi_ecall_extension ecall_rfence = {
+	.extid_start = SBI_EXT_RFENCE,
+	.extid_end = SBI_EXT_RFENCE,
+	.handle = sbi_ecall_rfence_handler,
+};
+
+static int sbi_ecall_ipi_handler(unsigned long extid, unsigned long funcid,
+				 unsigned long *args, unsigned long *out_val,
+				 struct sbi_trap_info *out_trap)
+{
+	int ret = 0;
+
+	if (funcid == SBI_EXT_IPI_SEND_IPI)
+		ret = sbi_ipi_send_smode(args[0], args[1]);
+	else
+		ret = SBI_ENOTSUPP;
+
+	return ret;
+}
+
+struct sbi_ecall_extension ecall_ipi = {
+	.extid_start = SBI_EXT_IPI,
+	.extid_end = SBI_EXT_IPI,
+	.handle = sbi_ecall_ipi_handler,
+};

lib/sbi/sbi_ecall_vendor.c

@@ -0,0 +1,38 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2020 Western Digital Corporation or its affiliates.
+ *
+ * Authors:
+ *   Anup Patel <anup.patel@wdc.com>
+ *   Atish Patra <atish.patra@wdc.com>
+ */
+
+#include <sbi/sbi_ecall.h>
+#include <sbi/sbi_ecall_interface.h>
+#include <sbi/sbi_error.h>
+#include <sbi/sbi_platform.h>
+
+static int sbi_ecall_vendor_probe(unsigned long extid,
+				  unsigned long *out_val)
+{
+	*out_val = sbi_platform_vendor_ext_check(sbi_platform_thishart_ptr(),
+						 extid);
+	return 0;
+}
+
+static int sbi_ecall_vendor_handler(unsigned long extid, unsigned long funcid,
+				    unsigned long *args, unsigned long *out_val,
+				    struct sbi_trap_info *out_trap)
+{
+	return sbi_platform_vendor_ext_provider(sbi_platform_thishart_ptr(),
+						extid, funcid, args,
+						out_val, out_trap);
+}
+
+struct sbi_ecall_extension ecall_vendor = {
+	.extid_start = SBI_EXT_VENDOR_START,
+	.extid_end = SBI_EXT_VENDOR_END,
+	.probe = sbi_ecall_vendor_probe,
+	.handle = sbi_ecall_vendor_handler,
+};

lib/sbi/sbi_emulate_csr.c

@@ -9,15 +9,15 @@
 
 #include <sbi/riscv_asm.h>
 #include <sbi/riscv_encoding.h>
-#include <sbi/sbi_bits.h>
+#include <sbi/sbi_bitops.h>
 #include <sbi/sbi_console.h>
 #include <sbi/sbi_emulate_csr.h>
 #include <sbi/sbi_error.h>
 #include <sbi/sbi_timer.h>
 #include <sbi/sbi_trap.h>
 
-int sbi_emulate_csr_read(int csr_num, u32 hartid, struct sbi_trap_regs *regs,
-			 struct sbi_scratch *scratch, ulong *csr_val)
+int sbi_emulate_csr_read(int csr_num, struct sbi_trap_regs *regs,
+			 ulong *csr_val)
 {
 	int ret = 0;
 	ulong cen = -1UL;
@@ -34,7 +34,7 @@ int sbi_emulate_csr_read(int csr_num, u32 hartid, struct sbi_trap_regs *regs,
 	switch (csr_num) {
 	case CSR_HTIMEDELTA:
 		if (prev_mode == PRV_S && !virt)
-			*csr_val = sbi_timer_get_delta(scratch);
+			*csr_val = sbi_timer_get_delta();
 		else
 			ret = SBI_ENOTSUPP;
 		break;
@@ -46,8 +46,8 @@ int sbi_emulate_csr_read(int csr_num, u32 hartid, struct sbi_trap_regs *regs,
 	case CSR_TIME:
 		if (!((cen >> (CSR_TIME - CSR_CYCLE)) & 1))
 			return -1;
-		*csr_val = (virt) ? sbi_timer_virt_value(scratch):
-				    sbi_timer_value(scratch);
+		*csr_val = (virt) ? sbi_timer_virt_value():
+				    sbi_timer_value();
 		break;
 	case CSR_INSTRET:
 		if (!((cen >> (CSR_INSTRET - CSR_CYCLE)) & 1))
@@ -67,7 +67,7 @@ int sbi_emulate_csr_read(int csr_num, u32 hartid, struct sbi_trap_regs *regs,
 #if __riscv_xlen == 32
 	case CSR_HTIMEDELTAH:
 		if (prev_mode == PRV_S && !virt)
-			*csr_val = sbi_timer_get_delta(scratch) >> 32;
+			*csr_val = sbi_timer_get_delta() >> 32;
 		else
 			ret = SBI_ENOTSUPP;
 		break;
@@ -79,8 +79,8 @@ int sbi_emulate_csr_read(int csr_num, u32 hartid, struct sbi_trap_regs *regs,
 	case CSR_TIMEH:
 		if (!((cen >> (CSR_TIME - CSR_CYCLE)) & 1))
 			return -1;
-		*csr_val = (virt) ? sbi_timer_virt_value(scratch) >> 32:
-				    sbi_timer_value(scratch) >> 32;
+		*csr_val = (virt) ? sbi_timer_virt_value() >> 32:
+				    sbi_timer_value() >> 32;
 		break;
 	case CSR_INSTRETH:
 		if (!((cen >> (CSR_INSTRET - CSR_CYCLE)) & 1))
@@ -110,14 +110,14 @@ int sbi_emulate_csr_read(int csr_num, u32 hartid, struct sbi_trap_regs *regs,
 	};
 
 	if (ret)
-		sbi_dprintf(scratch, "%s: hartid%d: invalid csr_num=0x%x\n",
-			    __func__, hartid, csr_num);
+		sbi_dprintf("%s: hartid%d: invalid csr_num=0x%x\n",
+			    __func__, current_hartid(), csr_num);
 
 	return ret;
 }
 
-int sbi_emulate_csr_write(int csr_num, u32 hartid, struct sbi_trap_regs *regs,
-			  struct sbi_scratch *scratch, ulong csr_val)
+int sbi_emulate_csr_write(int csr_num, struct sbi_trap_regs *regs,
+			  ulong csr_val)
 {
 	int ret = 0;
 	ulong prev_mode = (regs->mstatus & MSTATUS_MPP) >> MSTATUS_MPP_SHIFT;
@@ -130,7 +130,7 @@ int sbi_emulate_csr_write(int csr_num, u32 hartid, struct sbi_trap_regs *regs,
 	switch (csr_num) {
 	case CSR_HTIMEDELTA:
 		if (prev_mode == PRV_S && !virt)
-			sbi_timer_set_delta(scratch, csr_val);
+			sbi_timer_set_delta(csr_val);
 		else
 			ret = SBI_ENOTSUPP;
 		break;
@@ -149,7 +149,7 @@ int sbi_emulate_csr_write(int csr_num, u32 hartid, struct sbi_trap_regs *regs,
 #if __riscv_xlen == 32
 	case CSR_HTIMEDELTAH:
 		if (prev_mode == PRV_S && !virt)
-			sbi_timer_set_delta_upper(scratch, csr_val);
+			sbi_timer_set_delta_upper(csr_val);
 		else
 			ret = SBI_ENOTSUPP;
 		break;
@@ -178,8 +178,8 @@ int sbi_emulate_csr_write(int csr_num, u32 hartid, struct sbi_trap_regs *regs,
 	};
 
 	if (ret)
-		sbi_dprintf(scratch, "%s: hartid%d: invalid csr_num=0x%x\n",
-			    __func__, hartid, csr_num);
+		sbi_dprintf("%s: hartid%d: invalid csr_num=0x%x\n",
+			    __func__, current_hartid(), csr_num);
 
 	return ret;
 }

lib/sbi/sbi_hart.c

@@ -11,28 +11,31 @@
 #include <sbi/riscv_barrier.h>
 #include <sbi/riscv_encoding.h>
 #include <sbi/riscv_fp.h>
-#include <sbi/riscv_locks.h>
-#include <sbi/sbi_bits.h>
+#include <sbi/sbi_bitops.h>
 #include <sbi/sbi_console.h>
 #include <sbi/sbi_error.h>
 #include <sbi/sbi_hart.h>
 #include <sbi/sbi_platform.h>
 
-/**
- * Return HART ID of the caller.
- */
-unsigned int sbi_current_hartid()
-{
-	return (u32)csr_read(CSR_MHARTID);
-}
+extern void __sbi_unpriv_trap(void);
+extern void __sbi_unpriv_trap_hext(void);
+
+void (*sbi_hart_unpriv_trap)(void) = &__sbi_unpriv_trap;
 
 static void mstatus_init(struct sbi_scratch *scratch, u32 hartid)
 {
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
+	unsigned long mstatus_val = 0;
 
 	/* Enable FPU */
 	if (misa_extension('D') || misa_extension('F'))
-		csr_write(CSR_MSTATUS, MSTATUS_FS);
+		mstatus_val |=  MSTATUS_FS;
+
+	/* Enable Vector context */
+	if (misa_extension('V'))
+		mstatus_val |=  MSTATUS_VS;
+
+	csr_write(CSR_MSTATUS, mstatus_val);
 
 	/* Enable user/supervisor use of perf counters */
 	if (misa_extension('S') && sbi_platform_has_scounteren(plat))
@@ -75,7 +78,7 @@ static int delegate_traps(struct sbi_scratch *scratch, u32 hartid)
 	unsigned long interrupts, exceptions;
 
 	if (!misa_extension('S'))
-		/* No delegation possible as mideleg does not exist*/
+		/* No delegation possible as mideleg does not exist */
 		return 0;
 
 	/* Send M-mode interrupts and most exceptions to S-mode */
@@ -88,24 +91,36 @@ static int delegate_traps(struct sbi_scratch *scratch, u32 hartid)
 			      (1U << CAUSE_STORE_PAGE_FAULT);
 
 	/*
-	 * If hypervisor extension available then we only handle
-	 * hypervisor calls (i.e. ecalls from HS-mode) and we let
-	 * HS-mode handle supervisor calls (i.e. ecalls from VS-mode)
+	 * If hypervisor extension available then we only handle hypervisor
+	 * calls (i.e. ecalls from HS-mode) in M-mode.
+	 *
+	 * The HS-mode will additionally handle supervisor calls (i.e. ecalls
+	 * from VS-mode), Guest page faults and Virtual interrupts.
 	 */
-	if (misa_extension('H'))
+	if (misa_extension('H')) {
 		exceptions |= (1U << CAUSE_SUPERVISOR_ECALL);
+		exceptions |= (1U << CAUSE_FETCH_GUEST_PAGE_FAULT);
+		exceptions |= (1U << CAUSE_LOAD_GUEST_PAGE_FAULT);
+		exceptions |= (1U << CAUSE_STORE_GUEST_PAGE_FAULT);
+	}
 
 	csr_write(CSR_MIDELEG, interrupts);
 	csr_write(CSR_MEDELEG, exceptions);
 
-	if (csr_read(CSR_MIDELEG) != interrupts)
-		return SBI_EFAIL;
-	if (csr_read(CSR_MEDELEG) != exceptions)
-		return SBI_EFAIL;
-
 	return 0;
 }
 
+void sbi_hart_delegation_dump(struct sbi_scratch *scratch)
+{
+#if __riscv_xlen == 32
+	sbi_printf("MIDELEG : 0x%08lx\n", csr_read(CSR_MIDELEG));
+	sbi_printf("MEDELEG : 0x%08lx\n", csr_read(CSR_MEDELEG));
+#else
+	sbi_printf("MIDELEG : 0x%016lx\n", csr_read(CSR_MIDELEG));
+	sbi_printf("MEDELEG : 0x%016lx\n", csr_read(CSR_MEDELEG));
+#endif
+}
+
 unsigned long log2roundup(unsigned long x)
 {
 	unsigned long ret = 0;
@@ -122,24 +137,20 @@ unsigned long log2roundup(unsigned long x)
 void sbi_hart_pmp_dump(struct sbi_scratch *scratch)
 {
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
-	unsigned long prot, addr, size, l2l;
+	unsigned long prot, addr, size;
 	unsigned int i;
 
 	if (!sbi_platform_has_pmp(plat))
 		return;
 
 	for (i = 0; i < PMP_COUNT; i++) {
-		pmp_get(i, &prot, &addr, &l2l);
+		pmp_get(i, &prot, &addr, &size);
 		if (!(prot & PMP_A))
 			continue;
-		if (l2l < __riscv_xlen)
-			size = (1UL << l2l);
-		else
-			size = 0;
 #if __riscv_xlen == 32
-		sbi_printf("PMP%d: 0x%08lx-0x%08lx (A",
+		sbi_printf("PMP%d    : 0x%08lx-0x%08lx (A",
 #else
-		sbi_printf("PMP%d: 0x%016lx-0x%016lx (A",
+		sbi_printf("PMP%d    : 0x%016lx-0x%016lx (A",
 #endif
 			   i, addr, addr + size - 1);
 		if (prot & PMP_L)
@@ -154,6 +165,27 @@ void sbi_hart_pmp_dump(struct sbi_scratch *scratch)
 	}
 }
 
+int sbi_hart_pmp_check_addr(struct sbi_scratch *scratch, unsigned long addr,
+			    unsigned long attr)
+{
+	unsigned long prot, size, i, tempaddr;
+	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
+
+	if (!sbi_platform_has_pmp(plat))
+		return SBI_OK;
+
+	for (i = 0; i < PMP_COUNT; i++) {
+		pmp_get(i, &prot, &tempaddr, &size);
+		if (!(prot & PMP_A))
+			continue;
+		if (tempaddr <= addr && addr <= tempaddr + size)
+			if (!(prot & attr))
+				return SBI_INVALID_ADDR;
+	}
+
+	return SBI_OK;
+}
+
 static int pmp_init(struct sbi_scratch *scratch, u32 hartid)
 {
 	u32 i, count;
@@ -183,17 +215,13 @@ static int pmp_init(struct sbi_scratch *scratch, u32 hartid)
 	return 0;
 }
 
-static unsigned long trap_info_offset;
-
 int sbi_hart_init(struct sbi_scratch *scratch, u32 hartid, bool cold_boot)
 {
 	int rc;
 
 	if (cold_boot) {
-		trap_info_offset = sbi_scratch_alloc_offset(__SIZEOF_POINTER__,
-							    "HART_TRAP_INFO");
-		if (!trap_info_offset)
-			return SBI_ENOMEM;
+		if (misa_extension('H'))
+			sbi_hart_unpriv_trap = &__sbi_unpriv_trap_hext;
 	}
 
 	mstatus_init(scratch, hartid);
@@ -209,29 +237,6 @@ int sbi_hart_init(struct sbi_scratch *scratch, u32 hartid, bool cold_boot)
 	return pmp_init(scratch, hartid);
 }
 
-void *sbi_hart_get_trap_info(struct sbi_scratch *scratch)
-{
-	unsigned long *trap_info;
-
-	if (!trap_info_offset)
-		return NULL;
-
-	trap_info = sbi_scratch_offset_ptr(scratch, trap_info_offset);
-
-	return (void *)(*trap_info);
-}
-
-void sbi_hart_set_trap_info(struct sbi_scratch *scratch, void *data)
-{
-	unsigned long *trap_info;
-
-	if (!trap_info_offset)
-		return;
-
-	trap_info = sbi_scratch_offset_ptr(scratch, trap_info_offset);
-	*trap_info = (unsigned long)data;
-}
-
 void __attribute__((noreturn)) sbi_hart_hang(void)
 {
 	while (1)
@@ -271,7 +276,6 @@ sbi_hart_switch_mode(unsigned long arg0, unsigned long arg1,
 #if __riscv_xlen == 32
 	if (misa_extension('H')) {
 		valH = csr_read(CSR_MSTATUSH);
-		valH = INSERT_FIELD(valH, MSTATUSH_MTL, 0);
 		if (next_virt)
 			valH = INSERT_FIELD(valH, MSTATUSH_MPV, 1);
 		else
@@ -280,7 +284,6 @@ sbi_hart_switch_mode(unsigned long arg0, unsigned long arg1,
 	}
 #else
 	if (misa_extension('H')) {
-		val = INSERT_FIELD(val, MSTATUS_MTL, 0);
 		if (next_virt)
 			val = INSERT_FIELD(val, MSTATUS_MPV, 1);
 		else
@@ -306,99 +309,3 @@ sbi_hart_switch_mode(unsigned long arg0, unsigned long arg1,
 	__asm__ __volatile__("mret" : : "r"(a0), "r"(a1));
 	__builtin_unreachable();
 }
-
-static spinlock_t avail_hart_mask_lock	      = SPIN_LOCK_INITIALIZER;
-static volatile unsigned long avail_hart_mask = 0;
-
-void sbi_hart_mark_available(u32 hartid)
-{
-	spin_lock(&avail_hart_mask_lock);
-	avail_hart_mask |= (1UL << hartid);
-	spin_unlock(&avail_hart_mask_lock);
-}
-
-void sbi_hart_unmark_available(u32 hartid)
-{
-	spin_lock(&avail_hart_mask_lock);
-	avail_hart_mask &= ~(1UL << hartid);
-	spin_unlock(&avail_hart_mask_lock);
-}
-
-ulong sbi_hart_available_mask(void)
-{
-	ulong ret;
-
-	spin_lock(&avail_hart_mask_lock);
-	ret = avail_hart_mask;
-	spin_unlock(&avail_hart_mask_lock);
-
-	return ret;
-}
-
-typedef struct sbi_scratch *(*h2s)(ulong hartid);
-
-struct sbi_scratch *sbi_hart_id_to_scratch(struct sbi_scratch *scratch,
-					   u32 hartid)
-{
-	return ((h2s)scratch->hartid_to_scratch)(hartid);
-}
-
-#define COLDBOOT_WAIT_BITMAP_SIZE __riscv_xlen
-static spinlock_t coldboot_lock = SPIN_LOCK_INITIALIZER;
-static unsigned long coldboot_done = 0;
-static unsigned long coldboot_wait_bitmap = 0;
-
-void sbi_hart_wait_for_coldboot(struct sbi_scratch *scratch, u32 hartid)
-{
-	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
-
-	if ((sbi_platform_hart_count(plat) <= hartid) ||
-	    (COLDBOOT_WAIT_BITMAP_SIZE <= hartid))
-		sbi_hart_hang();
-
-	/* Set MSIE bit to receive IPI */
-	csr_set(CSR_MIE, MIP_MSIP);
-
-	/* Acquire coldboot lock */
-	spin_lock(&coldboot_lock);
-
-	/* Mark current HART as waiting */
-	coldboot_wait_bitmap |= (1UL << hartid);
-
-	/* Wait for coldboot to finish using WFI */
-	while (!coldboot_done) {
-		spin_unlock(&coldboot_lock);
-		wfi();
-		spin_lock(&coldboot_lock);
-	};
-
-	/* Unmark current HART as waiting */
-	coldboot_wait_bitmap &= ~(1UL << hartid);
-
-	/* Release coldboot lock */
-	spin_unlock(&coldboot_lock);
-
-	/* Clear current HART IPI */
-	sbi_platform_ipi_clear(plat, hartid);
-}
-
-void sbi_hart_wake_coldboot_harts(struct sbi_scratch *scratch, u32 hartid)
-{
-	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
-	int max_hart			= sbi_platform_hart_count(plat);
-
-	/* Acquire coldboot lock */
-	spin_lock(&coldboot_lock);
-
-	/* Mark coldboot done */
-	coldboot_done = 1;
-
-	/* Send an IPI to all HARTs waiting for coldboot */
-	for (int i = 0; i < max_hart; i++) {
-		if ((i != hartid) && (coldboot_wait_bitmap & (1UL << i)))
-			sbi_platform_ipi_send(plat, i);
-	}
-
-	/* Release coldboot lock */
-	spin_unlock(&coldboot_lock);
-}

lib/sbi/sbi_hfence.S

@@ -0,0 +1,75 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2019 Western Digital Corporation or its affiliates.
+ *
+ * Authors:
+ *   Anup Patel <anup.patel@wdc.com>
+ *   Atish Patra <anup.patel@wdc.com>
+ */
+
+	/*
+	 * Instruction encoding of hfence.gvma is:
+	 * 0110001 rs2(5) rs1(5) 000 00000 1110011
+	 */
+
+	.align 3
+	.global __sbi_hfence_gvma_vmid_gpa
+__sbi_hfence_gvma_vmid_gpa:
+	/* hfence.gvma a1, a0 */
+	.word 0x62a60073
+	ret
+
+	.align 3
+	.global __sbi_hfence_gvma_vmid
+__sbi_hfence_gvma_vmid:
+	/* hfence.gvma zero, a0 */
+	.word 0x62a00073
+	ret
+
+	.align 3
+	.global __sbi_hfence_gvma_gpa
+__sbi_hfence_gvma_gpa:
+	/* hfence.gvma a0 */
+	.word 0x62050073
+	ret
+
+	.align 3
+	.global __sbi_hfence_gvma_all
+__sbi_hfence_gvma_all:
+	/* hfence.gvma */
+	.word 0x62000073
+	ret
+
+	/*
+	 * Instruction encoding of hfence.bvma is:
+	 * 0010001 rs2(5) rs1(5) 000 00000 1110011
+	 */
+
+	.align 3
+	.global __sbi_hfence_vvma_asid_va
+__sbi_hfence_vvma_asid_va:
+	/* hfence.bvma a1, a0 */
+	.word 0x22a60073
+	ret
+
+	.align 3
+	.global __sbi_hfence_vvma_asid
+__sbi_hfence_vvma_asid:
+	/* hfence.bvma zero, a0 */
+	.word 0x22a00073
+	ret
+
+	.align 3
+	.global __sbi_hfence_vvma_va
+__sbi_hfence_vvma_va:
+	/* hfence.bvma a0 */
+	.word 0x22050073
+	ret
+
+	.align 3
+	.global __sbi_hfence_vvma_all
+__sbi_hfence_vvma_all:
+	/* hfence.bvma */
+	.word 0x22000073
+	ret

lib/sbi/sbi_hsm.c

@@ -0,0 +1,273 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2020 Western Digital Corporation or its affiliates.
+ *
+ * Authors:
+ *   Atish Patra <atish.patra@wdc.com>
+ */
+
+#include <sbi/riscv_asm.h>
+#include <sbi/riscv_barrier.h>
+#include <sbi/riscv_encoding.h>
+#include <sbi/riscv_atomic.h>
+#include <sbi/sbi_bitops.h>
+#include <sbi/sbi_console.h>
+#include <sbi/sbi_error.h>
+#include <sbi/sbi_ecall_interface.h>
+#include <sbi/sbi_hart.h>
+#include <sbi/sbi_hartmask.h>
+#include <sbi/sbi_hsm.h>
+#include <sbi/sbi_init.h>
+#include <sbi/sbi_ipi.h>
+#include <sbi/sbi_platform.h>
+#include <sbi/sbi_system.h>
+#include <sbi/sbi_timer.h>
+#include <sbi/sbi_console.h>
+
+static unsigned long hart_data_offset;
+
+/** Per hart specific data to manage state transition **/
+struct sbi_hsm_data {
+	atomic_t state;
+};
+
+int sbi_hsm_hart_state_to_status(int state)
+{
+	int ret;
+
+	switch (state) {
+	case SBI_HART_STOPPED:
+		ret = SBI_HSM_HART_STATUS_STOPPED;
+		break;
+	case SBI_HART_STOPPING:
+		ret = SBI_HSM_HART_STATUS_STOP_PENDING;
+		break;
+	case SBI_HART_STARTING:
+		ret = SBI_HSM_HART_STATUS_START_PENDING;
+		break;
+	case SBI_HART_STARTED:
+		ret = SBI_HSM_HART_STATUS_STARTED;
+		break;
+	default:
+		ret = SBI_EINVAL;
+	}
+
+	return ret;
+}
+
+int sbi_hsm_hart_get_state(u32 hartid)
+{
+	struct sbi_hsm_data *hdata;
+	struct sbi_scratch *scratch;
+
+	scratch = sbi_hartid_to_scratch(hartid);
+	if (!scratch)
+		return SBI_HART_UNKNOWN;
+
+	hdata = sbi_scratch_offset_ptr(scratch, hart_data_offset);
+
+	return atomic_read(&hdata->state);
+}
+
+bool sbi_hsm_hart_started(u32 hartid)
+{
+	if (sbi_hsm_hart_get_state(hartid) == SBI_HART_STARTED)
+		return TRUE;
+	else
+		return FALSE;
+}
+
+/**
+ * Get ulong HART mask for given HART base ID
+ * @param hbase the HART base ID
+ * @param out_hmask the output ulong HART mask
+ * @return 0 on success and SBI_Exxx (< 0) on failure
+ * Note: the output HART mask will be set to zero on failure as well.
+ */
+int sbi_hsm_hart_started_mask(ulong hbase, ulong *out_hmask)
+{
+	ulong i;
+	ulong hcount = sbi_scratch_last_hartid() + 1;
+
+	*out_hmask = 0;
+	if (hcount <= hbase)
+		return SBI_EINVAL;
+	if (BITS_PER_LONG < (hcount - hbase))
+		hcount = BITS_PER_LONG;
+
+	for (i = hbase; i < hcount; i++) {
+		if (sbi_hsm_hart_get_state(i) == SBI_HART_STARTED)
+			*out_hmask |= 1UL << (i - hbase);
+	}
+
+	return 0;
+}
+
+void sbi_hsm_prepare_next_jump(struct sbi_scratch *scratch, u32 hartid)
+{
+	u32 oldstate;
+	struct sbi_hsm_data *hdata = sbi_scratch_offset_ptr(scratch,
+							    hart_data_offset);
+
+	oldstate = atomic_cmpxchg(&hdata->state, SBI_HART_STARTING,
+				  SBI_HART_STARTED);
+	if (oldstate != SBI_HART_STARTING)
+		sbi_hart_hang();
+}
+
+static void sbi_hsm_hart_wait(struct sbi_scratch *scratch, u32 hartid)
+{
+	unsigned long saved_mie;
+	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
+	struct sbi_hsm_data *hdata = sbi_scratch_offset_ptr(scratch,
+							    hart_data_offset);
+	/* Save MIE CSR */
+	saved_mie = csr_read(CSR_MIE);
+
+	/* Set MSIE bit to receive IPI */
+	csr_set(CSR_MIE, MIP_MSIP);
+
+	/* Wait for hart_add call*/
+	while (atomic_read(&hdata->state) != SBI_HART_STARTING) {
+		wfi();
+	};
+
+	/* Restore MIE CSR */
+	csr_write(CSR_MIE, saved_mie);
+
+	/* Clear current HART IPI */
+	sbi_platform_ipi_clear(plat, hartid);
+}
+
+int sbi_hsm_init(struct sbi_scratch *scratch, u32 hartid, bool cold_boot)
+{
+	u32 i;
+	struct sbi_scratch *rscratch;
+	struct sbi_hsm_data *hdata;
+
+	if (cold_boot) {
+		hart_data_offset = sbi_scratch_alloc_offset(sizeof(*hdata),
+							    "HART_DATA");
+		if (!hart_data_offset)
+			return SBI_ENOMEM;
+
+		/* Initialize hart state data for every hart */
+		for (i = 0; i <= sbi_scratch_last_hartid(); i++) {
+			rscratch = sbi_hartid_to_scratch(i);
+			if (!rscratch)
+				continue;
+
+			hdata = sbi_scratch_offset_ptr(rscratch,
+						       hart_data_offset);
+			ATOMIC_INIT(&hdata->state,
+			(i == hartid) ? SBI_HART_STARTING : SBI_HART_STOPPED);
+		}
+	} else {
+		sbi_hsm_hart_wait(scratch, hartid);
+	}
+
+	return 0;
+}
+
+void __noreturn sbi_hsm_exit(struct sbi_scratch *scratch)
+{
+	u32 hstate;
+	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
+	struct sbi_hsm_data *hdata = sbi_scratch_offset_ptr(scratch,
+							    hart_data_offset);
+	void (*jump_warmboot)(void) = (void (*)(void))scratch->warmboot_addr;
+
+	hstate = atomic_cmpxchg(&hdata->state, SBI_HART_STOPPING,
+				SBI_HART_STOPPED);
+	if (hstate != SBI_HART_STOPPING)
+		goto fail_exit;
+
+	if (sbi_platform_has_hart_hotplug(plat)) {
+		sbi_platform_hart_stop(plat);
+		/* It should never reach here */
+		goto fail_exit;
+	}
+
+	/**
+	 * As platform is lacking support for hotplug, directly jump to warmboot
+	 * and wait for interrupts in warmboot. We do it preemptively in order
+	 * preserve the hart states and reuse the code path for hotplug.
+	 */
+	jump_warmboot();
+
+fail_exit:
+	/* It should never reach here */
+	sbi_printf("ERR: Failed stop hart [%u]\n", current_hartid());
+	sbi_hart_hang();
+}
+
+int sbi_hsm_hart_start(struct sbi_scratch *scratch, u32 hartid,
+		       ulong saddr, ulong priv)
+{
+	int rc;
+	unsigned long init_count;
+	unsigned int hstate;
+	struct sbi_scratch *rscratch;
+	struct sbi_hsm_data *hdata;
+	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
+
+	rscratch = sbi_hartid_to_scratch(hartid);
+	if (!rscratch)
+		return SBI_EINVAL;
+	hdata = sbi_scratch_offset_ptr(rscratch, hart_data_offset);
+	hstate = atomic_cmpxchg(&hdata->state, SBI_HART_STOPPED,
+				SBI_HART_STARTING);
+	if (hstate == SBI_HART_STARTED)
+		return SBI_EALREADY_STARTED;
+
+	/**
+	 * if a hart is already transition to start or stop, another start call
+	 * is considered as invalid request.
+	 */
+	if (hstate != SBI_HART_STOPPED)
+		return SBI_EINVAL;
+
+	rc = sbi_hart_pmp_check_addr(scratch, saddr, PMP_X);
+	if (rc)
+		return rc;
+	//TODO: We also need to check saddr for valid physical address as well.
+
+	init_count = sbi_init_count(hartid);
+	rscratch->next_arg1 = priv;
+	rscratch->next_addr = saddr;
+
+	if (sbi_platform_has_hart_hotplug(plat) ||
+	   (sbi_platform_has_hart_secondary_boot(plat) && !init_count)) {
+		return sbi_platform_hart_start(plat, hartid,
+					       scratch->warmboot_addr);
+	} else {
+		sbi_platform_ipi_send(plat, hartid);
+	}
+
+	return 0;
+}
+
+int sbi_hsm_hart_stop(struct sbi_scratch *scratch, bool exitnow)
+{
+	int oldstate;
+	u32 hartid = current_hartid();
+	struct sbi_hsm_data *hdata = sbi_scratch_offset_ptr(scratch,
+							    hart_data_offset);
+
+	if (!sbi_hsm_hart_started(hartid))
+		return SBI_EINVAL;
+
+	oldstate = atomic_cmpxchg(&hdata->state, SBI_HART_STARTED,
+				  SBI_HART_STOPPING);
+	if (oldstate != SBI_HART_STARTED) {
+		sbi_printf("%s: ERR: The hart is in invalid state [%u]\n",
+			   __func__, oldstate);
+		return SBI_DENIED;
+	}
+
+	if (exitnow)
+		sbi_exit(scratch);
+
+	return 0;
+}

lib/sbi/sbi_illegal_insn.c

@@ -9,27 +9,29 @@
 
 #include <sbi/riscv_asm.h>
 #include <sbi/riscv_encoding.h>
-#include <sbi/riscv_unpriv.h>
-#include <sbi/sbi_bits.h>
+#include <sbi/sbi_bitops.h>
 #include <sbi/sbi_emulate_csr.h>
 #include <sbi/sbi_error.h>
 #include <sbi/sbi_illegal_insn.h>
 #include <sbi/sbi_trap.h>
+#include <sbi/sbi_unpriv.h>
 
-typedef int (*illegal_insn_func)(ulong insn, u32 hartid, ulong mcause,
-				 struct sbi_trap_regs *regs,
-				 struct sbi_scratch *scratch);
+typedef int (*illegal_insn_func)(ulong insn, struct sbi_trap_regs *regs);
 
-static int truly_illegal_insn(ulong insn, u32 hartid, ulong mcause,
-			      struct sbi_trap_regs *regs,
-			      struct sbi_scratch *scratch)
+static int truly_illegal_insn(ulong insn, struct sbi_trap_regs *regs)
 {
-	return sbi_trap_redirect(regs, scratch, regs->mepc, mcause, insn);
+	struct sbi_trap_info trap;
+
+	trap.epc = regs->mepc;
+	trap.cause = CAUSE_ILLEGAL_INSTRUCTION;
+	trap.tval = insn;
+	trap.tval2 = 0;
+	trap.tinst = 0;
+
+	return sbi_trap_redirect(regs, &trap);
 }
 
-static int system_opcode_insn(ulong insn, u32 hartid, ulong mcause,
-			      struct sbi_trap_regs *regs,
-			      struct sbi_scratch *scratch)
+static int system_opcode_insn(ulong insn, struct sbi_trap_regs *regs)
 {
 	int do_write, rs1_num = (insn >> 15) & 0x1f;
 	ulong rs1_val = GET_RS1(insn, regs);
@@ -46,12 +48,10 @@ static int system_opcode_insn(ulong insn, u32 hartid, ulong mcause,
 	if ((regs->mstatus & MSTATUS_MPV) &&
 #endif
 	    (insn & INSN_MASK_WFI) == INSN_MATCH_WFI)
-		return sbi_trap_redirect(regs, scratch,
-					 regs->mepc, mcause, insn);
+		return truly_illegal_insn(insn, regs);
 
-	if (sbi_emulate_csr_read(csr_num, hartid, regs, scratch, &csr_val))
-		return truly_illegal_insn(insn, hartid, mcause,
-					  regs, scratch);
+	if (sbi_emulate_csr_read(csr_num, regs, &csr_val))
+		return truly_illegal_insn(insn, regs);
 
 	do_write = rs1_num;
 	switch (GET_RM(insn)) {
@@ -76,12 +76,11 @@ static int system_opcode_insn(ulong insn, u32 hartid, ulong mcause,
 		new_csr_val = csr_val & ~rs1_num;
 		break;
 	default:
-		return truly_illegal_insn(insn, hartid, mcause, regs, scratch);
+		return truly_illegal_insn(insn, regs);
 	};
 
-	if (do_write && sbi_emulate_csr_write(csr_num, hartid, regs,
-					      scratch, new_csr_val))
-		return truly_illegal_insn(insn, hartid, mcause, regs, scratch);
+	if (do_write && sbi_emulate_csr_write(csr_num, regs, new_csr_val))
+		return truly_illegal_insn(insn, regs);
 
 	SET_RD(insn, regs, csr_val);
 
@@ -125,30 +124,21 @@ static illegal_insn_func illegal_insn_table[32] = {
 	truly_illegal_insn  /* 31 */
 };
 
-int sbi_illegal_insn_handler(u32 hartid, ulong mcause,
-			     struct sbi_trap_regs *regs,
-			     struct sbi_scratch *scratch)
+int sbi_illegal_insn_handler(ulong insn, struct sbi_trap_regs *regs)
 {
-	ulong insn = csr_read(CSR_MTVAL);
-#if __riscv_xlen == 32
-	bool virt = (regs->mstatusH & MSTATUSH_MPV) ? TRUE : FALSE;
-#else
-	bool virt = (regs->mstatus & MSTATUS_MPV) ? TRUE : FALSE;
-#endif
-	struct unpriv_trap uptrap;
+	struct sbi_trap_info uptrap;
 
 	if (unlikely((insn & 3) != 3)) {
 		if (insn == 0) {
-			insn = get_insn(regs->mepc, virt, scratch, &uptrap);
-			if (uptrap.cause)
-				return sbi_trap_redirect(regs, scratch,
-					regs->mepc, uptrap.cause, uptrap.tval);
+			insn = sbi_get_insn(regs->mepc, &uptrap);
+			if (uptrap.cause) {
+				uptrap.epc = regs->mepc;
+				return sbi_trap_redirect(regs, &uptrap);
+			}
 		}
 		if ((insn & 3) != 3)
-			return truly_illegal_insn(insn, hartid, mcause, regs,
-						  scratch);
+			return truly_illegal_insn(insn, regs);
 	}
 
-	return illegal_insn_table[(insn & 0x7c) >> 2](insn, hartid, mcause,
-						      regs, scratch);
+	return illegal_insn_table[(insn & 0x7c) >> 2](insn, regs);
 }

lib/sbi/sbi_init.c

@@ -9,13 +9,17 @@
 
 #include <sbi/riscv_asm.h>
 #include <sbi/riscv_atomic.h>
+#include <sbi/riscv_locks.h>
 #include <sbi/sbi_console.h>
 #include <sbi/sbi_ecall.h>
 #include <sbi/sbi_hart.h>
+#include <sbi/sbi_hartmask.h>
+#include <sbi/sbi_hsm.h>
 #include <sbi/sbi_ipi.h>
 #include <sbi/sbi_platform.h>
 #include <sbi/sbi_system.h>
 #include <sbi/sbi_timer.h>
+#include <sbi/sbi_tlb.h>
 #include <sbi/sbi_version.h>
 
 #define BANNER                                              \
@@ -30,25 +34,31 @@
 
 static void sbi_boot_prints(struct sbi_scratch *scratch, u32 hartid)
 {
+	int xlen;
 	char str[64];
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 
-	misa_string(str, sizeof(str));
 #ifdef OPENSBI_VERSION_GIT
-	sbi_printf("\nOpenSBI %s (%s %s)\n", OPENSBI_VERSION_GIT,
-		   __DATE__, __TIME__);
+	sbi_printf("\nOpenSBI %s\n", OPENSBI_VERSION_GIT);
 #else
-	sbi_printf("\nOpenSBI v%d.%d (%s %s)\n", OPENSBI_VERSION_MAJOR,
-		   OPENSBI_VERSION_MINOR, __DATE__, __TIME__);
+	sbi_printf("\nOpenSBI v%d.%d\n", OPENSBI_VERSION_MAJOR,
+		   OPENSBI_VERSION_MINOR);
 #endif
 
 	sbi_printf(BANNER);
 
+	/* Determine MISA XLEN and MISA string */
+	xlen = misa_xlen();
+	if (xlen < 1) {
+		sbi_printf("Error %d getting MISA XLEN\n", xlen);
+		sbi_hart_hang();
+	}
+	xlen = 16 * (1 << xlen);
+	misa_string(str, sizeof(str));
+
 	/* Platform details */
 	sbi_printf("Platform Name          : %s\n", sbi_platform_name(plat));
-	sbi_printf("Platform HART Features : RV%d%s\n", misa_xlen(), str);
-	sbi_printf("Platform Max HARTs     : %d\n",
-		   sbi_platform_hart_count(plat));
+	sbi_printf("Platform HART Features : RV%d%s\n", xlen, str);
 	sbi_printf("Current Hart           : %u\n", hartid);
 	/* Firmware details */
 	sbi_printf("Firmware Base          : 0x%lx\n", scratch->fw_start);
@@ -59,15 +69,98 @@ static void sbi_boot_prints(struct sbi_scratch *scratch, u32 hartid)
 		   sbi_ecall_version_major(), sbi_ecall_version_minor());
 	sbi_printf("\n");
 
+	sbi_hart_delegation_dump(scratch);
 	sbi_hart_pmp_dump(scratch);
 }
 
+static spinlock_t coldboot_lock = SPIN_LOCK_INITIALIZER;
+static unsigned long coldboot_done = 0;
+static struct sbi_hartmask coldboot_wait_hmask = { 0 };
+
+static void wait_for_coldboot(struct sbi_scratch *scratch, u32 hartid)
+{
+	unsigned long saved_mie, cmip;
+	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
+
+	/* Save MIE CSR */
+	saved_mie = csr_read(CSR_MIE);
+
+	/* Set MSIE bit to receive IPI */
+	csr_set(CSR_MIE, MIP_MSIP);
+
+	/* Acquire coldboot lock */
+	spin_lock(&coldboot_lock);
+
+	/* Mark current HART as waiting */
+	sbi_hartmask_set_hart(hartid, &coldboot_wait_hmask);
+
+	/* Wait for coldboot to finish using WFI */
+	while (!coldboot_done) {
+		spin_unlock(&coldboot_lock);
+		do {
+			wfi();
+			cmip = csr_read(CSR_MIP);
+		 } while (!(cmip & MIP_MSIP));
+		spin_lock(&coldboot_lock);
+	};
+
+	/* Unmark current HART as waiting */
+	sbi_hartmask_clear_hart(hartid, &coldboot_wait_hmask);
+
+	/* Release coldboot lock */
+	spin_unlock(&coldboot_lock);
+
+	/* Restore MIE CSR */
+	csr_write(CSR_MIE, saved_mie);
+
+	/* Clear current HART IPI */
+	sbi_platform_ipi_clear(plat, hartid);
+}
+
+static void wake_coldboot_harts(struct sbi_scratch *scratch, u32 hartid)
+{
+	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
+
+	/* Acquire coldboot lock */
+	spin_lock(&coldboot_lock);
+
+	/* Mark coldboot done */
+	coldboot_done = 1;
+
+	/* Send an IPI to all HARTs waiting for coldboot */
+	for (int i = 0; i <= sbi_scratch_last_hartid(); i++) {
+		if ((i != hartid) &&
+		    sbi_hartmask_test_hart(i, &coldboot_wait_hmask))
+			sbi_platform_ipi_send(plat, i);
+	}
+
+	/* Release coldboot lock */
+	spin_unlock(&coldboot_lock);
+}
+
+static unsigned long init_count_offset;
+
 static void __noreturn init_coldboot(struct sbi_scratch *scratch, u32 hartid)
 {
 	int rc;
+	unsigned long *init_count;
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 
-	rc = sbi_system_early_init(scratch, TRUE);
+	/* Note: This has to be first thing in coldboot init sequence */
+	rc = sbi_scratch_init(scratch);
+	if (rc)
+		sbi_hart_hang();
+
+	init_count_offset = sbi_scratch_alloc_offset(__SIZEOF_POINTER__,
+						     "INIT_COUNT");
+	if (!init_count_offset)
+		sbi_hart_hang();
+
+	rc = sbi_hsm_init(scratch, hartid, TRUE);
+	if (rc)
+		sbi_hart_hang();
+
+	rc = sbi_platform_early_init(plat, TRUE);
 	if (rc)
 		sbi_hart_hang();
 
@@ -87,20 +180,31 @@ static void __noreturn init_coldboot(struct sbi_scratch *scratch, u32 hartid)
 	if (rc)
 		sbi_hart_hang();
 
+	rc = sbi_tlb_init(scratch, TRUE);
+	if (rc)
+		sbi_hart_hang();
+
 	rc = sbi_timer_init(scratch, TRUE);
 	if (rc)
 		sbi_hart_hang();
 
-	rc = sbi_system_final_init(scratch, TRUE);
+	rc = sbi_ecall_init();
+	if (rc)
+		sbi_hart_hang();
+
+	rc = sbi_platform_final_init(plat, TRUE);
 	if (rc)
 		sbi_hart_hang();
 
 	if (!(scratch->options & SBI_SCRATCH_NO_BOOT_PRINTS))
 		sbi_boot_prints(scratch, hartid);
 
-	if (!sbi_platform_has_hart_hotplug(plat))
-		sbi_hart_wake_coldboot_harts(scratch, hartid);
-	sbi_hart_mark_available(hartid);
+	wake_coldboot_harts(scratch, hartid);
+
+	init_count = sbi_scratch_offset_ptr(scratch, init_count_offset);
+	(*init_count)++;
+
+	sbi_hsm_prepare_next_jump(scratch, hartid);
 	sbi_hart_switch_mode(hartid, scratch->next_arg1, scratch->next_addr,
 			     scratch->next_mode, FALSE);
 }
@@ -108,15 +212,19 @@ static void __noreturn init_coldboot(struct sbi_scratch *scratch, u32 hartid)
 static void __noreturn init_warmboot(struct sbi_scratch *scratch, u32 hartid)
 {
 	int rc;
+	unsigned long *init_count;
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 
-	if (!sbi_platform_has_hart_hotplug(plat))
-		sbi_hart_wait_for_coldboot(scratch, hartid);
+	wait_for_coldboot(scratch, hartid);
 
-	if (sbi_platform_hart_disabled(plat, hartid))
+	if (!init_count_offset)
 		sbi_hart_hang();
 
-	rc = sbi_system_early_init(scratch, FALSE);
+	rc = sbi_hsm_init(scratch, hartid, FALSE);
+	if (rc)
+		sbi_hart_hang();
+
+	rc = sbi_platform_early_init(plat, FALSE);
 	if (rc)
 		sbi_hart_hang();
 
@@ -132,23 +240,25 @@ static void __noreturn init_warmboot(struct sbi_scratch *scratch, u32 hartid)
 	if (rc)
 		sbi_hart_hang();
 
+	rc = sbi_tlb_init(scratch, FALSE);
+	if (rc)
+		sbi_hart_hang();
+
 	rc = sbi_timer_init(scratch, FALSE);
 	if (rc)
 		sbi_hart_hang();
 
-	rc = sbi_system_final_init(scratch, FALSE);
+	rc = sbi_platform_final_init(plat, FALSE);
 	if (rc)
 		sbi_hart_hang();
 
-	sbi_hart_mark_available(hartid);
+	init_count = sbi_scratch_offset_ptr(scratch, init_count_offset);
+	(*init_count)++;
 
-	if (sbi_platform_has_hart_hotplug(plat))
-		/* TODO: To be implemented in-future. */
-		sbi_hart_hang();
-	else
-		sbi_hart_switch_mode(hartid, scratch->next_arg1,
-				     scratch->next_addr,
-				     scratch->next_mode, FALSE);
+	sbi_hsm_prepare_next_jump(scratch, hartid);
+	sbi_hart_switch_mode(hartid, scratch->next_arg1,
+			     scratch->next_addr,
+			     scratch->next_mode, FALSE);
 }
 
 static atomic_t coldboot_lottery = ATOMIC_INITIALIZER(0);
@@ -168,13 +278,14 @@ static atomic_t coldboot_lottery = ATOMIC_INITIALIZER(0);
 void __noreturn sbi_init(struct sbi_scratch *scratch)
 {
 	bool coldboot			= FALSE;
-	u32 hartid			= sbi_current_hartid();
+	u32 hartid			= current_hartid();
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 
-	if (sbi_platform_hart_disabled(plat, hartid))
+	if ((SBI_HARTMASK_MAX_BITS <= hartid) ||
+	    sbi_platform_hart_invalid(plat, hartid))
 		sbi_hart_hang();
 
-	if (atomic_add_return(&coldboot_lottery, 1) == 1)
+	if (atomic_xchg(&coldboot_lottery, 1) == 0)
 		coldboot = TRUE;
 
 	if (coldboot)
@@ -182,3 +293,50 @@ void __noreturn sbi_init(struct sbi_scratch *scratch)
 	else
 		init_warmboot(scratch, hartid);
 }
+
+unsigned long sbi_init_count(u32 hartid)
+{
+	struct sbi_scratch *scratch;
+	unsigned long *init_count;
+
+	if (!init_count_offset)
+		return 0;
+
+	scratch = sbi_hartid_to_scratch(hartid);
+	if (!scratch)
+		return 0;
+
+	init_count = sbi_scratch_offset_ptr(scratch, init_count_offset);
+
+	return *init_count;
+}
+
+/**
+ * Exit OpenSBI library for current HART and stop HART
+ *
+ * The function expects following:
+ * 1. The 'mscratch' CSR is pointing to sbi_scratch of current HART
+ * 2. Stack pointer (SP) is setup for current HART
+ *
+ * @param scratch pointer to sbi_scratch of current HART
+ */
+void __noreturn sbi_exit(struct sbi_scratch *scratch)
+{
+	u32 hartid			= current_hartid();
+	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
+
+	if (sbi_platform_hart_invalid(plat, hartid))
+		sbi_hart_hang();
+
+	sbi_platform_early_exit(plat);
+
+	sbi_timer_exit(scratch);
+
+	sbi_ipi_exit(scratch);
+
+	sbi_platform_irqchip_exit(plat);
+
+	sbi_platform_final_exit(plat);
+
+	sbi_hsm_exit(scratch);
+}

lib/sbi/sbi_ipi.c

@@ -11,94 +11,176 @@
 #include <sbi/riscv_asm.h>
 #include <sbi/riscv_atomic.h>
 #include <sbi/riscv_barrier.h>
-#include <sbi/riscv_unpriv.h>
+#include <sbi/sbi_bitops.h>
 #include <sbi/sbi_error.h>
 #include <sbi/sbi_hart.h>
+#include <sbi/sbi_hsm.h>
+#include <sbi/sbi_init.h>
 #include <sbi/sbi_ipi.h>
 #include <sbi/sbi_platform.h>
-#include <sbi/sbi_tlb.h>
+
+struct sbi_ipi_data {
+	unsigned long ipi_type;
+};
 
 static unsigned long ipi_data_off;
 
-static int sbi_ipi_send(struct sbi_scratch *scratch, u32 hartid, u32 event,
-			void *data)
+static const struct sbi_ipi_event_ops *ipi_ops_array[SBI_IPI_EVENT_MAX];
+
+static int sbi_ipi_send(struct sbi_scratch *scratch, u32 remote_hartid,
+			u32 event, void *data)
 {
 	int ret;
 	struct sbi_scratch *remote_scratch = NULL;
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 	struct sbi_ipi_data *ipi_data;
+	const struct sbi_ipi_event_ops *ipi_ops;
 
-	if (sbi_platform_hart_disabled(plat, hartid))
-		return -1;
+	if ((SBI_IPI_EVENT_MAX <= event) ||
+	    !ipi_ops_array[event])
+		return SBI_EINVAL;
+	ipi_ops = ipi_ops_array[event];
+
+	remote_scratch = sbi_hartid_to_scratch(remote_hartid);
+	if (!remote_scratch)
+		return SBI_EINVAL;
+
+	ipi_data = sbi_scratch_offset_ptr(remote_scratch, ipi_data_off);
+
+	if (ipi_ops->update) {
+		ret = ipi_ops->update(scratch, remote_scratch,
+				      remote_hartid, data);
+		if (ret < 0)
+			return ret;
+	}
 
 	/*
 	 * Set IPI type on remote hart's scratch area and
 	 * trigger the interrupt
 	 */
-	remote_scratch = sbi_hart_id_to_scratch(scratch, hartid);
-	ipi_data = sbi_scratch_offset_ptr(remote_scratch, ipi_data_off);
-	if (event == SBI_IPI_EVENT_SFENCE_VMA ||
-	    event == SBI_IPI_EVENT_SFENCE_VMA_ASID ||
-	    event == SBI_IPI_EVENT_FENCE_I ) {
-		ret = sbi_tlb_fifo_update(remote_scratch, hartid, data);
-		if (ret < 0)
-			return ret;
-	}
 	atomic_raw_set_bit(event, &ipi_data->ipi_type);
 	smp_wmb();
-	sbi_platform_ipi_send(plat, hartid);
+	sbi_platform_ipi_send(plat, remote_hartid);
 
-	if (event == SBI_IPI_EVENT_SFENCE_VMA ||
-	    event == SBI_IPI_EVENT_SFENCE_VMA_ASID ||
-	    event == SBI_IPI_EVENT_FENCE_I ) {
-		sbi_tlb_fifo_sync(scratch);
-	}
+	if (ipi_ops->sync)
+		ipi_ops->sync(scratch);
 
 	return 0;
 }
 
-int sbi_ipi_send_many(struct sbi_scratch *scratch, struct unpriv_trap *uptrap,
-		      ulong *pmask, u32 event, void *data)
+/**
+ * As this this function only handlers scalar values of hart mask, it must be
+ * set to all online harts if the intention is to send IPIs to all the harts.
+ * If hmask is zero, no IPIs will be sent.
+ */
+int sbi_ipi_send_many(ulong hmask, ulong hbase, u32 event, void *data)
 {
+	int rc;
 	ulong i, m;
-	ulong mask = sbi_hart_available_mask();
-	u32 hartid = sbi_current_hartid();
+	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
 
-	if (pmask) {
-		mask &= load_ulong(pmask, scratch, uptrap);
-		if (uptrap->cause)
-			return SBI_ETRAP;
+	if (hbase != -1UL) {
+		rc = sbi_hsm_hart_started_mask(hbase, &m);
+		if (rc)
+			return rc;
+		m &= hmask;
+
+		/* Send IPIs */
+		for (i = hbase; m; i++, m >>= 1) {
+			if (m & 1UL)
+				sbi_ipi_send(scratch, i, event, data);
+		}
+	} else {
+		hbase = 0;
+		while (!sbi_hsm_hart_started_mask(hbase, &m)) {
+			/* Send IPIs */
+			for (i = hbase; m; i++, m >>= 1) {
+				if (m & 1UL)
+					sbi_ipi_send(scratch, i, event, data);
+			}
+			hbase += BITS_PER_LONG;
+		}
 	}
 
-	/* Send IPIs to every other hart on the set */
-	for (i = 0, m = mask; m; i++, m >>= 1)
-		if ((m & 1UL) && (i != hartid))
-			sbi_ipi_send(scratch, i, event, data);
-
-	/*
-	 * If the current hart is on the set, send an IPI
-	 * to it as well
-	 */
-	if (mask & (1UL << hartid))
-		sbi_ipi_send(scratch, hartid, event, data);
-
 	return 0;
 }
 
-void sbi_ipi_clear_smode(struct sbi_scratch *scratch)
+int sbi_ipi_event_create(const struct sbi_ipi_event_ops *ops)
+{
+	int i, ret = SBI_ENOSPC;
+
+	if (!ops || !ops->process)
+		return SBI_EINVAL;
+
+	for (i = 0; i < SBI_IPI_EVENT_MAX; i++) {
+		if (!ipi_ops_array[i]) {
+			ret = i;
+			ipi_ops_array[i] = ops;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+void sbi_ipi_event_destroy(u32 event)
+{
+	if (SBI_IPI_EVENT_MAX <= event)
+		return;
+
+	ipi_ops_array[event] = NULL;
+}
+
+static void sbi_ipi_process_smode(struct sbi_scratch *scratch)
+{
+	csr_set(CSR_MIP, MIP_SSIP);
+}
+
+static struct sbi_ipi_event_ops ipi_smode_ops = {
+	.name = "IPI_SMODE",
+	.process = sbi_ipi_process_smode,
+};
+
+static u32 ipi_smode_event = SBI_IPI_EVENT_MAX;
+
+int sbi_ipi_send_smode(ulong hmask, ulong hbase)
+{
+	return sbi_ipi_send_many(hmask, hbase, ipi_smode_event, NULL);
+}
+
+void sbi_ipi_clear_smode(void)
 {
 	csr_clear(CSR_MIP, MIP_SSIP);
 }
 
-void sbi_ipi_process(struct sbi_scratch *scratch)
+static void sbi_ipi_process_halt(struct sbi_scratch *scratch)
+{
+	sbi_hsm_hart_stop(scratch, TRUE);
+}
+
+static struct sbi_ipi_event_ops ipi_halt_ops = {
+	.name = "IPI_HALT",
+	.process = sbi_ipi_process_halt,
+};
+
+static u32 ipi_halt_event = SBI_IPI_EVENT_MAX;
+
+int sbi_ipi_send_halt(ulong hmask, ulong hbase)
+{
+	return sbi_ipi_send_many(hmask, hbase, ipi_halt_event, NULL);
+}
+
+void sbi_ipi_process(void)
 {
 	unsigned long ipi_type;
 	unsigned int ipi_event;
+	const struct sbi_ipi_event_ops *ipi_ops;
+	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 	struct sbi_ipi_data *ipi_data =
 			sbi_scratch_offset_ptr(scratch, ipi_data_off);
 
-	u32 hartid = sbi_current_hartid();
+	u32 hartid = current_hartid();
 	sbi_platform_ipi_clear(plat, hartid);
 
 	ipi_type = atomic_raw_xchg_ulong(&ipi_data->ipi_type, 0);
@@ -107,21 +189,9 @@ void sbi_ipi_process(struct sbi_scratch *scratch)
 		if (!(ipi_type & 1UL))
 			goto skip;
 
-		switch (ipi_event) {
-		case SBI_IPI_EVENT_SOFT:
-			csr_set(CSR_MIP, MIP_SSIP);
-			break;
-		case SBI_IPI_EVENT_FENCE_I:
-		case SBI_IPI_EVENT_SFENCE_VMA:
-		case SBI_IPI_EVENT_SFENCE_VMA_ASID:
-			sbi_tlb_fifo_process(scratch);
-			break;
-		case SBI_IPI_EVENT_HALT:
-			sbi_hart_hang();
-			break;
-		default:
-			break;
-		};
+		ipi_ops = ipi_ops_array[ipi_event];
+		if (ipi_ops && ipi_ops->process)
+			ipi_ops->process(scratch);
 
 skip:
 		ipi_type = ipi_type >> 1;
@@ -139,20 +209,44 @@ int sbi_ipi_init(struct sbi_scratch *scratch, bool cold_boot)
 							"IPI_DATA");
 		if (!ipi_data_off)
 			return SBI_ENOMEM;
+		ret = sbi_ipi_event_create(&ipi_smode_ops);
+		if (ret < 0)
+			return ret;
+		ipi_smode_event = ret;
+		ret = sbi_ipi_event_create(&ipi_halt_ops);
+		if (ret < 0)
+			return ret;
+		ipi_halt_event = ret;
 	} else {
 		if (!ipi_data_off)
 			return SBI_ENOMEM;
+		if (SBI_IPI_EVENT_MAX <= ipi_smode_event ||
+		    SBI_IPI_EVENT_MAX <= ipi_halt_event)
+			return SBI_ENOSPC;
 	}
 
 	ipi_data = sbi_scratch_offset_ptr(scratch, ipi_data_off);
 	ipi_data->ipi_type = 0x00;
 
-	ret = sbi_tlb_fifo_init(scratch, cold_boot);
+	/* Platform init */
+	ret = sbi_platform_ipi_init(sbi_platform_ptr(scratch), cold_boot);
 	if (ret)
 		return ret;
 
 	/* Enable software interrupts */
 	csr_set(CSR_MIE, MIP_MSIP);
 
-	return sbi_platform_ipi_init(sbi_platform_ptr(scratch), cold_boot);
+	return 0;
+}
+
+void sbi_ipi_exit(struct sbi_scratch *scratch)
+{
+	/* Disable software interrupts */
+	csr_clear(CSR_MIE, MIP_MSIP);
+
+	/* Process pending IPIs */
+	sbi_ipi_process();
+
+	/* Platform exit */
+	sbi_platform_ipi_exit(sbi_platform_ptr(scratch));
 }

lib/sbi/sbi_misaligned_ldst.c

@@ -9,11 +9,11 @@
 
 #include <sbi/riscv_asm.h>
 #include <sbi/riscv_encoding.h>
-#include <sbi/riscv_unpriv.h>
 #include <sbi/riscv_fp.h>
 #include <sbi/sbi_error.h>
 #include <sbi/sbi_misaligned_ldst.h>
 #include <sbi/sbi_trap.h>
+#include <sbi/sbi_unpriv.h>
 
 union reg_data {
 	u8 data_bytes[8];
@@ -21,24 +21,31 @@ union reg_data {
 	u64 data_u64;
 };
 
-int sbi_misaligned_load_handler(u32 hartid, ulong mcause,
-				struct sbi_trap_regs *regs,
-				struct sbi_scratch *scratch)
+int sbi_misaligned_load_handler(ulong addr, ulong tval2, ulong tinst,
+				struct sbi_trap_regs *regs)
 {
+	ulong insn;
 	union reg_data val;
-	struct unpriv_trap uptrap;
-	ulong addr = csr_read(CSR_MTVAL);
+	struct sbi_trap_info uptrap;
 	int i, fp = 0, shift = 0, len = 0;
-#if __riscv_xlen == 32
-	bool virt = (regs->mstatusH & MSTATUSH_MPV) ? TRUE : FALSE;
-#else
-	bool virt = (regs->mstatus & MSTATUS_MPV) ? TRUE : FALSE;
-#endif
-	ulong insn = get_insn(regs->mepc, virt, scratch, &uptrap);
 
-	if (uptrap.cause)
-		return sbi_trap_redirect(regs, scratch, regs->mepc,
-					 uptrap.cause, uptrap.tval);
+	if (tinst & 0x1) {
+		/*
+		 * Bit[0] == 1 implies trapped instruction value is
+		 * transformed instruction or custom instruction.
+		 */
+		insn = tinst | INSN_16BIT_MASK;
+	} else {
+		/*
+		 * Bit[0] == 0 implies trapped instruction value is
+		 * zero or special value.
+		 */
+		insn = sbi_get_insn(regs->mepc, &uptrap);
+		if (uptrap.cause) {
+			uptrap.epc = regs->mepc;
+			return sbi_trap_redirect(regs, &uptrap);
+		}
+	}
 
 	if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) {
 		len   = 4;
@@ -63,7 +70,6 @@ int sbi_misaligned_load_handler(u32 hartid, ulong mcause,
 		shift = 8 * (sizeof(ulong) - len);
 	} else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) {
 		len = 2;
-#ifdef __riscv_compressed
 #if __riscv_xlen >= 64
 	} else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) {
 		len   = 8;
@@ -100,18 +106,23 @@ int sbi_misaligned_load_handler(u32 hartid, ulong mcause,
 		len = 4;
 #endif
 #endif
-#endif
-	} else
-		return sbi_trap_redirect(regs, scratch, regs->mepc,
-					 mcause, addr);
+	} else {
+		uptrap.epc = regs->mepc;
+		uptrap.cause = CAUSE_MISALIGNED_LOAD;
+		uptrap.tval = addr;
+		uptrap.tval2 = tval2;
+		uptrap.tinst = tinst;
+		return sbi_trap_redirect(regs, &uptrap);
+	}
 
 	val.data_u64 = 0;
 	for (i = 0; i < len; i++) {
-		val.data_bytes[i] = load_u8((void *)(addr + i),
-					    scratch, &uptrap);
-		if (uptrap.cause)
-			return sbi_trap_redirect(regs, scratch, regs->mepc,
-						 uptrap.cause, uptrap.tval);
+		val.data_bytes[i] = sbi_load_u8((void *)(addr + i),
+						&uptrap);
+		if (uptrap.cause) {
+			uptrap.epc = regs->mepc;
+			return sbi_trap_redirect(regs, &uptrap);
+		}
 	}
 
 	if (!fp)
@@ -128,24 +139,31 @@ int sbi_misaligned_load_handler(u32 hartid, ulong mcause,
 	return 0;
 }
 
-int sbi_misaligned_store_handler(u32 hartid, ulong mcause,
-				 struct sbi_trap_regs *regs,
-				 struct sbi_scratch *scratch)
+int sbi_misaligned_store_handler(ulong addr, ulong tval2, ulong tinst,
+				 struct sbi_trap_regs *regs)
 {
+	ulong insn;
 	union reg_data val;
-	struct unpriv_trap uptrap;
-	ulong addr = csr_read(CSR_MTVAL);
+	struct sbi_trap_info uptrap;
 	int i, len = 0;
-#if __riscv_xlen == 32
-	bool virt = (regs->mstatusH & MSTATUSH_MPV) ? TRUE : FALSE;
-#else
-	bool virt = (regs->mstatus & MSTATUS_MPV) ? TRUE : FALSE;
-#endif
-	ulong insn = get_insn(regs->mepc, virt, scratch, &uptrap);
 
-	if (uptrap.cause)
-		return sbi_trap_redirect(regs, scratch, regs->mepc,
-					 uptrap.cause, uptrap.tval);
+	if (tinst & 0x1) {
+		/*
+		 * Bit[0] == 1 implies trapped instruction value is
+		 * transformed instruction or custom instruction.
+		 */
+		insn = tinst | INSN_16BIT_MASK;
+	} else {
+		/*
+		 * Bit[0] == 0 implies trapped instruction value is
+		 * zero or special value.
+		 */
+		insn = sbi_get_insn(regs->mepc, &uptrap);
+		if (uptrap.cause) {
+			uptrap.epc = regs->mepc;
+			return sbi_trap_redirect(regs, &uptrap);
+		}
+	}
 
 	val.data_ulong = GET_RS2(insn, regs);
 
@@ -165,7 +183,6 @@ int sbi_misaligned_store_handler(u32 hartid, ulong mcause,
 #endif
 	} else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) {
 		len = 2;
-#ifdef __riscv_compressed
 #if __riscv_xlen >= 64
 	} else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) {
 		len	       = 8;
@@ -198,17 +215,22 @@ int sbi_misaligned_store_handler(u32 hartid, ulong mcause,
 		val.data_ulong = GET_F32_RS2C(insn, regs);
 #endif
 #endif
-#endif
-	} else
-		return sbi_trap_redirect(regs, scratch, regs->mepc,
-					 mcause, addr);
+	} else {
+		uptrap.epc = regs->mepc;
+		uptrap.cause = CAUSE_MISALIGNED_STORE;
+		uptrap.tval = addr;
+		uptrap.tval2 = tval2;
+		uptrap.tinst = tinst;
+		return sbi_trap_redirect(regs, &uptrap);
+	}
 
 	for (i = 0; i < len; i++) {
-		store_u8((void *)(addr + i), val.data_bytes[i],
-			 scratch, &uptrap);
-		if (uptrap.cause)
-			return sbi_trap_redirect(regs, scratch, regs->mepc,
-						 uptrap.cause, uptrap.tval);
+		sbi_store_u8((void *)(addr + i), val.data_bytes[i],
+			     &uptrap);
+		if (uptrap.cause) {
+			uptrap.epc = regs->mepc;
+			return sbi_trap_redirect(regs, &uptrap);
+		}
 	}
 
 	regs->mepc += INSN_LEN(insn);

lib/sbi/sbi_scratch.c

@@ -8,14 +8,44 @@
  */
 
 #include <sbi/riscv_locks.h>
+#include <sbi/sbi_hart.h>
+#include <sbi/sbi_hartmask.h>
+#include <sbi/sbi_platform.h>
 #include <sbi/sbi_scratch.h>
+#include <sbi/sbi_string.h>
+
+u32 last_hartid_having_scratch = SBI_HARTMASK_MAX_BITS;
+struct sbi_scratch *hartid_to_scratch_table[SBI_HARTMASK_MAX_BITS] = { 0 };
 
 static spinlock_t extra_lock = SPIN_LOCK_INITIALIZER;
 static unsigned long extra_offset = SBI_SCRATCH_EXTRA_SPACE_OFFSET;
 
+typedef struct sbi_scratch *(*hartid2scratch)(ulong hartid, ulong hartindex);
+
+int sbi_scratch_init(struct sbi_scratch *scratch)
+{
+	u32 i;
+	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
+
+	for (i = 0; i < SBI_HARTMASK_MAX_BITS; i++) {
+		if (sbi_platform_hart_invalid(plat, i))
+			continue;
+		hartid_to_scratch_table[i] =
+			((hartid2scratch)scratch->hartid_to_scratch)(i,
+					sbi_platform_hart_index(plat, i));
+		if (hartid_to_scratch_table[i])
+			last_hartid_having_scratch = i;
+	}
+
+	return 0;
+}
+
 unsigned long sbi_scratch_alloc_offset(unsigned long size, const char *owner)
 {
+	u32 i;
+	void *ptr;
 	unsigned long ret = 0;
+	struct sbi_scratch *rscratch;
 
 	/*
 	 * We have a simple brain-dead allocator which never expects
@@ -29,8 +59,8 @@ unsigned long sbi_scratch_alloc_offset(unsigned long size, const char *owner)
 	if (!size)
 		return 0;
 
-	while (size & (__SIZEOF_POINTER__ - 1))
-		size++;
+	if (size & (__SIZEOF_POINTER__ - 1))
+		size = (size & ~(__SIZEOF_POINTER__ - 1)) + __SIZEOF_POINTER__;
 
 	spin_lock(&extra_lock);
 
@@ -43,6 +73,16 @@ unsigned long sbi_scratch_alloc_offset(unsigned long size, const char *owner)
 done:
 	spin_unlock(&extra_lock);
 
+	if (ret) {
+		for (i = 0; i < sbi_scratch_last_hartid(); i++) {
+			rscratch = sbi_hartid_to_scratch(i);
+			if (!rscratch)
+				continue;
+			ptr = sbi_scratch_offset_ptr(rscratch, ret);
+			sbi_memset(ptr, 0, size);
+		}
+	}
+
 	return ret;
 }
 

lib/sbi/sbi_system.c

@@ -8,38 +8,61 @@
  *   Nick Kossifidis <mick@ics.forth.gr>
  */
 
+#include <sbi/riscv_asm.h>
+#include <sbi/sbi_bitops.h>
 #include <sbi/sbi_hart.h>
+#include <sbi/sbi_hsm.h>
 #include <sbi/sbi_platform.h>
 #include <sbi/sbi_system.h>
 #include <sbi/sbi_ipi.h>
+#include <sbi/sbi_init.h>
 
-int sbi_system_early_init(struct sbi_scratch *scratch, bool cold_boot)
+void __noreturn sbi_system_reboot(u32 type)
 {
-	return sbi_platform_early_init(sbi_platform_ptr(scratch), cold_boot);
-}
+	ulong hbase = 0, hmask;
+	u32 cur_hartid = current_hartid();
+	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
 
-int sbi_system_final_init(struct sbi_scratch *scratch, bool cold_boot)
-{
-	return sbi_platform_final_init(sbi_platform_ptr(scratch), cold_boot);
-}
+	/* Send HALT IPI to every hart other than the current hart */
+	while (!sbi_hsm_hart_started_mask(hbase, &hmask)) {
+		if (hbase <= cur_hartid)
+			hmask &= ~(1UL << (cur_hartid - hbase));
+		if (hmask)
+			sbi_ipi_send_halt(hmask, hbase);
+		hbase += BITS_PER_LONG;
+	}
 
-void __attribute__((noreturn))
-sbi_system_reboot(struct sbi_scratch *scratch, u32 type)
+	/* Stop current HART */
+	sbi_hsm_hart_stop(scratch, FALSE);
 
-{
+	/* Platform specific reooot */
 	sbi_platform_system_reboot(sbi_platform_ptr(scratch), type);
-	sbi_hart_hang();
+
+	/* If platform specific reboot did not work then do sbi_exit() */
+	sbi_exit(scratch);
 }
 
-void __attribute__((noreturn))
-sbi_system_shutdown(struct sbi_scratch *scratch, u32 type)
+void __noreturn sbi_system_shutdown(u32 type)
 {
-	/* First try the platform-specific method */
+	ulong hbase = 0, hmask;
+	u32 cur_hartid = current_hartid();
+	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
+
+	/* Send HALT IPI to every hart other than the current hart */
+	while (!sbi_hsm_hart_started_mask(hbase, &hmask)) {
+		if (hbase <= cur_hartid)
+			hmask &= ~(1UL << (cur_hartid - hbase));
+		if (hmask)
+			sbi_ipi_send_halt(hmask, hbase);
+		hbase += BITS_PER_LONG;
+	}
+
+	/* Stop current HART */
+	sbi_hsm_hart_stop(scratch, FALSE);
+
+	/* Platform specific shutdown */
 	sbi_platform_system_shutdown(sbi_platform_ptr(scratch), type);
 
-	/* If that fails (or is not implemented) send an IPI on every
-	 * hart to hang and then hang the current hart */
-	sbi_ipi_send_many(scratch, NULL, NULL, SBI_IPI_EVENT_HALT, NULL);
-
-	sbi_hart_hang();
+	/* If platform specific shutdown did not work then do sbi_exit() */
+	sbi_exit(scratch);
 }

lib/sbi/sbi_timer.c

@@ -11,6 +11,7 @@
 #include <sbi/riscv_encoding.h>
 #include <sbi/sbi_error.h>
 #include <sbi/sbi_platform.h>
+#include <sbi/sbi_scratch.h>
 #include <sbi/sbi_timer.h>
 
 static unsigned long time_delta_off;
@@ -37,9 +38,9 @@ u64 get_ticks(void)
 }
 #endif
 
-u64 sbi_timer_value(struct sbi_scratch *scratch)
+u64 sbi_timer_value(void)
 {
-	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
+	const struct sbi_platform *plat = sbi_platform_thishart_ptr();
 
 	if (sbi_platform_has_timer_value(plat))
 		return sbi_platform_timer_value(plat);
@@ -47,48 +48,47 @@ u64 sbi_timer_value(struct sbi_scratch *scratch)
 		return get_ticks();
 }
 
-u64 sbi_timer_virt_value(struct sbi_scratch *scratch)
+u64 sbi_timer_virt_value(void)
 {
-	u64 *time_delta = sbi_scratch_offset_ptr(scratch, time_delta_off);
+	u64 *time_delta = sbi_scratch_offset_ptr(sbi_scratch_thishart_ptr(),
+						 time_delta_off);
 
-	return sbi_timer_value(scratch) + *time_delta;
+	return sbi_timer_value() + *time_delta;
 }
 
-u64 sbi_timer_get_delta(struct sbi_scratch *scratch)
+u64 sbi_timer_get_delta(void)
 {
-	u64 *time_delta = sbi_scratch_offset_ptr(scratch, time_delta_off);
+	u64 *time_delta = sbi_scratch_offset_ptr(sbi_scratch_thishart_ptr(),
+						 time_delta_off);
 
 	return *time_delta;
 }
 
-void sbi_timer_set_delta(struct sbi_scratch *scratch, ulong delta)
+void sbi_timer_set_delta(ulong delta)
 {
-	u64 *time_delta = sbi_scratch_offset_ptr(scratch, time_delta_off);
+	u64 *time_delta = sbi_scratch_offset_ptr(sbi_scratch_thishart_ptr(),
+						 time_delta_off);
 
 	*time_delta = (u64)delta;
 }
 
-void sbi_timer_set_delta_upper(struct sbi_scratch *scratch, ulong delta_upper)
+void sbi_timer_set_delta_upper(ulong delta_upper)
 {
-	u64 *time_delta = sbi_scratch_offset_ptr(scratch, time_delta_off);
+	u64 *time_delta = sbi_scratch_offset_ptr(sbi_scratch_thishart_ptr(),
+						 time_delta_off);
 
 	*time_delta &= 0xffffffffULL;
 	*time_delta |= ((u64)delta_upper << 32);
 }
 
-void sbi_timer_event_stop(struct sbi_scratch *scratch)
+void sbi_timer_event_start(u64 next_event)
 {
-	sbi_platform_timer_event_stop(sbi_platform_ptr(scratch));
-}
-
-void sbi_timer_event_start(struct sbi_scratch *scratch, u64 next_event)
-{
-	sbi_platform_timer_event_start(sbi_platform_ptr(scratch), next_event);
+	sbi_platform_timer_event_start(sbi_platform_thishart_ptr(), next_event);
 	csr_clear(CSR_MIP, MIP_STIP);
 	csr_set(CSR_MIE, MIP_MTIP);
 }
 
-void sbi_timer_process(struct sbi_scratch *scratch)
+void sbi_timer_process(void)
 {
 	csr_clear(CSR_MIE, MIP_MTIP);
 	csr_set(CSR_MIP, MIP_STIP);
@@ -113,3 +113,13 @@ int sbi_timer_init(struct sbi_scratch *scratch, bool cold_boot)
 
 	return sbi_platform_timer_init(sbi_platform_ptr(scratch), cold_boot);
 }
+
+void sbi_timer_exit(struct sbi_scratch *scratch)
+{
+	sbi_platform_timer_event_stop(sbi_platform_ptr(scratch));
+
+	csr_clear(CSR_MIP, MIP_STIP);
+	csr_clear(CSR_MIE, MIP_MTIP);
+
+	sbi_platform_timer_exit(sbi_platform_ptr(scratch));
+}

lib/sbi/sbi_tlb.c

@@ -14,8 +14,10 @@
 #include <sbi/sbi_error.h>
 #include <sbi/sbi_fifo.h>
 #include <sbi/sbi_hart.h>
+#include <sbi/sbi_ipi.h>
 #include <sbi/sbi_scratch.h>
 #include <sbi/sbi_tlb.h>
+#include <sbi/sbi_hfence.h>
 #include <sbi/sbi_string.h>
 #include <sbi/sbi_console.h>
 #include <sbi/sbi_platform.h>
@@ -30,7 +32,39 @@ static void sbi_tlb_flush_all(void)
 	__asm__ __volatile("sfence.vma");
 }
 
-static void sbi_tlb_fifo_sfence_vma(struct sbi_tlb_info *tinfo)
+static void sbi_tlb_hfence_vvma(struct sbi_tlb_info *tinfo)
+{
+	unsigned long start = tinfo->start;
+	unsigned long size  = tinfo->size;
+	unsigned long i;
+
+	if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) {
+		__sbi_hfence_vvma_all();
+		return;
+	}
+
+	for (i = 0; i < size; i += PAGE_SIZE) {
+		__sbi_hfence_vvma_va(start+i);
+	}
+}
+
+static void sbi_tlb_hfence_gvma(struct sbi_tlb_info *tinfo)
+{
+	unsigned long start = tinfo->start;
+	unsigned long size  = tinfo->size;
+	unsigned long i;
+
+	if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) {
+		__sbi_hfence_gvma_all();
+		return;
+	}
+
+	for (i = 0; i < size; i += PAGE_SIZE) {
+		__sbi_hfence_gvma_gpa(start+i);
+	}
+}
+
+static void sbi_tlb_sfence_vma(struct sbi_tlb_info *tinfo)
 {
 	unsigned long start = tinfo->start;
 	unsigned long size  = tinfo->size;
@@ -49,7 +83,51 @@ static void sbi_tlb_fifo_sfence_vma(struct sbi_tlb_info *tinfo)
 	}
 }
 
-static void sbi_tlb_fifo_sfence_vma_asid(struct sbi_tlb_info *tinfo)
+static void sbi_tlb_hfence_vvma_asid(struct sbi_tlb_info *tinfo)
+{
+	unsigned long start = tinfo->start;
+	unsigned long size  = tinfo->size;
+	unsigned long asid  = tinfo->asid;
+	unsigned long i;
+
+	if (start == 0 && size == 0) {
+		__sbi_hfence_vvma_all();
+		return;
+	}
+
+	if (size == SBI_TLB_FLUSH_ALL) {
+		__sbi_hfence_vvma_asid(asid);
+		return;
+	}
+
+	for (i = 0; i < size; i += PAGE_SIZE) {
+		__sbi_hfence_vvma_asid_va(asid, start + i);
+	}
+}
+
+static void sbi_tlb_hfence_gvma_vmid(struct sbi_tlb_info *tinfo)
+{
+	unsigned long start = tinfo->start;
+	unsigned long size  = tinfo->size;
+	unsigned long vmid  = tinfo->asid;
+	unsigned long i;
+
+	if (start == 0 && size == 0) {
+		__sbi_hfence_gvma_all();
+		return;
+	}
+
+	if (size == SBI_TLB_FLUSH_ALL) {
+		__sbi_hfence_gvma_vmid(vmid);
+		return;
+	}
+
+	for (i = 0; i < size; i += PAGE_SIZE) {
+		__sbi_hfence_gvma_vmid_gpa(vmid, start+i);
+	}
+}
+
+static void sbi_tlb_sfence_vma_asid(struct sbi_tlb_info *tinfo)
 {
 	unsigned long start = tinfo->start;
 	unsigned long size  = tinfo->size;
@@ -80,38 +158,54 @@ static void sbi_tlb_fifo_sfence_vma_asid(struct sbi_tlb_info *tinfo)
 
 static void sbi_tlb_local_flush(struct sbi_tlb_info *tinfo)
 {
-	if (tinfo->type == SBI_TLB_FLUSH_VMA) {
-		sbi_tlb_fifo_sfence_vma(tinfo);
-	} else if (tinfo->type == SBI_TLB_FLUSH_VMA_ASID) {
-		sbi_tlb_fifo_sfence_vma_asid(tinfo);
-	} else if (tinfo->type == SBI_ITLB_FLUSH)
+	switch (tinfo->type) {
+	case SBI_TLB_FLUSH_VMA:
+		sbi_tlb_sfence_vma(tinfo);
+		break;
+	case SBI_TLB_FLUSH_VMA_ASID:
+		sbi_tlb_sfence_vma_asid(tinfo);
+		break;
+	case SBI_TLB_FLUSH_GVMA:
+		sbi_tlb_hfence_gvma(tinfo);
+		break;
+	case SBI_TLB_FLUSH_GVMA_VMID:
+		sbi_tlb_hfence_gvma_vmid(tinfo);
+		break;
+	case SBI_TLB_FLUSH_VVMA:
+		sbi_tlb_hfence_vvma(tinfo);
+		break;
+	case SBI_TLB_FLUSH_VVMA_ASID:
+		sbi_tlb_hfence_vvma_asid(tinfo);
+		break;
+	case SBI_ITLB_FLUSH:
 		__asm__ __volatile("fence.i");
-	else
+		break;
+	default:
 		sbi_printf("Invalid tlb flush request type [%lu]\n",
 			   tinfo->type);
+	}
 	return;
 }
 
-static void sbi_tlb_entry_process(struct sbi_scratch *scratch,
-				  struct sbi_tlb_info *tinfo)
+static void sbi_tlb_entry_process(struct sbi_tlb_info *tinfo)
 {
-	u32 i;
-	u64 m;
+	u32 rhartid;
 	struct sbi_scratch *rscratch = NULL;
 	unsigned long *rtlb_sync = NULL;
 
 	sbi_tlb_local_flush(tinfo);
-	for (i = 0, m = tinfo->shart_mask; m; i++, m >>= 1) {
-		if (!(m & 1UL))
+
+	sbi_hartmask_for_each_hart(rhartid, &tinfo->smask) {
+		rscratch = sbi_hartid_to_scratch(rhartid);
+		if (!rscratch)
 			continue;
 
-		rscratch = sbi_hart_id_to_scratch(scratch, i);
 		rtlb_sync = sbi_scratch_offset_ptr(rscratch, tlb_sync_off);
 		while (atomic_raw_xchg_ulong(rtlb_sync, 1)) ;
 	}
 }
 
-static void sbi_tlb_fifo_process_count(struct sbi_scratch *scratch, int count)
+static void sbi_tlb_process_count(struct sbi_scratch *scratch, int count)
 {
 	struct sbi_tlb_info tinfo;
 	u32 deq_count = 0;
@@ -119,7 +213,7 @@ static void sbi_tlb_fifo_process_count(struct sbi_scratch *scratch, int count)
 			sbi_scratch_offset_ptr(scratch, tlb_fifo_off);
 
 	while (!sbi_fifo_dequeue(tlb_fifo, &tinfo)) {
-		sbi_tlb_entry_process(scratch, &tinfo);
+		sbi_tlb_entry_process(&tinfo);
 		deq_count++;
 		if (deq_count > count)
 			break;
@@ -127,17 +221,17 @@ static void sbi_tlb_fifo_process_count(struct sbi_scratch *scratch, int count)
 	}
 }
 
-void sbi_tlb_fifo_process(struct sbi_scratch *scratch)
+static void sbi_tlb_process(struct sbi_scratch *scratch)
 {
 	struct sbi_tlb_info tinfo;
 	struct sbi_fifo *tlb_fifo =
 			sbi_scratch_offset_ptr(scratch, tlb_fifo_off);
 
 	while (!sbi_fifo_dequeue(tlb_fifo, &tinfo))
-		sbi_tlb_entry_process(scratch, &tinfo);
+		sbi_tlb_entry_process(&tinfo);
 }
 
-void sbi_tlb_fifo_sync(struct sbi_scratch *scratch)
+static void sbi_tlb_sync(struct sbi_scratch *scratch)
 {
 	unsigned long *tlb_sync =
 			sbi_scratch_offset_ptr(scratch, tlb_sync_off);
@@ -147,14 +241,14 @@ void sbi_tlb_fifo_sync(struct sbi_scratch *scratch)
 		 * While we are waiting for remote hart to set the sync,
 		 * consume fifo requests to avoid deadlock.
 		 */
-		sbi_tlb_fifo_process_count(scratch, 1);
+		sbi_tlb_process_count(scratch, 1);
 	}
 
 	return;
 }
 
-static inline int __sbi_tlb_fifo_range_check(struct sbi_tlb_info *curr,
-					     struct sbi_tlb_info *next)
+static inline int __sbi_tlb_range_check(struct sbi_tlb_info *curr,
+					struct sbi_tlb_info *next)
 {
 	unsigned long curr_end;
 	unsigned long next_end;
@@ -168,11 +262,11 @@ static inline int __sbi_tlb_fifo_range_check(struct sbi_tlb_info *curr,
 	if (next->start <= curr->start && next_end > curr_end) {
 		curr->start = next->start;
 		curr->size  = next->size;
-		curr->shart_mask = curr->shart_mask | next->shart_mask;
-		ret	    = SBI_FIFO_UPDATED;
+		sbi_hartmask_or(&curr->smask, &curr->smask, &next->smask);
+		ret = SBI_FIFO_UPDATED;
 	} else if (next->start >= curr->start && next_end <= curr_end) {
-		curr->shart_mask = curr->shart_mask | next->shart_mask;
-		ret		 = SBI_FIFO_SKIP;
+		sbi_hartmask_or(&curr->smask, &curr->smask, &next->smask);
+		ret = SBI_FIFO_SKIP;
 	}
 
 	return ret;
@@ -196,7 +290,7 @@ static inline int __sbi_tlb_fifo_range_check(struct sbi_tlb_info *curr,
  *	before continuing the while loop. This method is preferred over wfi/ipi because
  *	of MMIO cost involved in later method.
  */
-static int sbi_tlb_fifo_update_cb(void *in, void *data)
+static int sbi_tlb_update_cb(void *in, void *data)
 {
 	struct sbi_tlb_info *curr;
 	struct sbi_tlb_info *next;
@@ -211,22 +305,23 @@ static int sbi_tlb_fifo_update_cb(void *in, void *data)
 	if (next->type == SBI_TLB_FLUSH_VMA_ASID &&
 	    curr->type == SBI_TLB_FLUSH_VMA_ASID) {
 		if (next->asid == curr->asid)
-			ret = __sbi_tlb_fifo_range_check(curr, next);
+			ret = __sbi_tlb_range_check(curr, next);
 	} else if (next->type == SBI_TLB_FLUSH_VMA &&
 		   curr->type == SBI_TLB_FLUSH_VMA) {
-			ret = __sbi_tlb_fifo_range_check(curr, next);
+			ret = __sbi_tlb_range_check(curr, next);
 	}
 
 	return ret;
 }
 
-int sbi_tlb_fifo_update(struct sbi_scratch *rscratch, u32 hartid, void *data)
+static int sbi_tlb_update(struct sbi_scratch *scratch,
+			  struct sbi_scratch *remote_scratch,
+			  u32 remote_hartid, void *data)
 {
 	int ret;
 	struct sbi_fifo *tlb_fifo_r;
-	struct sbi_scratch *lscratch;
 	struct sbi_tlb_info *tinfo = data;
-	u32 curr_hartid = sbi_current_hartid();
+	u32 curr_hartid = current_hartid();
 
 	/*
 	 * If address range to flush is too big then simply
@@ -242,15 +337,14 @@ int sbi_tlb_fifo_update(struct sbi_scratch *rscratch, u32 hartid, void *data)
 	 * If the request is to queue a tlb flush entry for itself
 	 * then just do a local flush and return;
 	 */
-	if (hartid == curr_hartid) {
+	if (remote_hartid == curr_hartid) {
 		sbi_tlb_local_flush(tinfo);
 		return -1;
 	}
 
-	lscratch = sbi_hart_id_to_scratch(rscratch, curr_hartid);
-	tlb_fifo_r = sbi_scratch_offset_ptr(rscratch, tlb_fifo_off);
+	tlb_fifo_r = sbi_scratch_offset_ptr(remote_scratch, tlb_fifo_off);
 
-	ret = sbi_fifo_inplace_update(tlb_fifo_r, data, sbi_tlb_fifo_update_cb);
+	ret = sbi_fifo_inplace_update(tlb_fifo_r, data, sbi_tlb_update_cb);
 	if (ret != SBI_FIFO_UNCHANGED) {
 		return 1;
 	}
@@ -264,16 +358,31 @@ int sbi_tlb_fifo_update(struct sbi_scratch *rscratch, u32 hartid, void *data)
 		 * TODO: Introduce a wait/wakeup event mechanism to handle
 		 * this properly.
 		 */
-		sbi_tlb_fifo_process_count(lscratch, 1);
-		sbi_dprintf(rscratch, "hart%d: hart%d tlb fifo full\n",
-			    curr_hartid, hartid);
+		sbi_tlb_process_count(scratch, 1);
+		sbi_dprintf("hart%d: hart%d tlb fifo full\n",
+			    curr_hartid, remote_hartid);
 	}
 
 	return 0;
 }
 
-int sbi_tlb_fifo_init(struct sbi_scratch *scratch, bool cold_boot)
+static struct sbi_ipi_event_ops tlb_ops = {
+	.name = "IPI_TLB",
+	.update = sbi_tlb_update,
+	.sync = sbi_tlb_sync,
+	.process = sbi_tlb_process,
+};
+
+static u32 tlb_event = SBI_IPI_EVENT_MAX;
+
+int sbi_tlb_request(ulong hmask, ulong hbase, struct sbi_tlb_info *tinfo)
 {
+	return sbi_ipi_send_many(hmask, hbase, tlb_event, tinfo);
+}
+
+int sbi_tlb_init(struct sbi_scratch *scratch, bool cold_boot)
+{
+	int ret;
 	void *tlb_mem;
 	unsigned long *tlb_sync;
 	struct sbi_fifo *tlb_q;
@@ -281,11 +390,11 @@ int sbi_tlb_fifo_init(struct sbi_scratch *scratch, bool cold_boot)
 
 	if (cold_boot) {
 		tlb_sync_off = sbi_scratch_alloc_offset(sizeof(*tlb_sync),
-							    "IPI_TLB_SYNC");
+							"IPI_TLB_SYNC");
 		if (!tlb_sync_off)
 			return SBI_ENOMEM;
 		tlb_fifo_off = sbi_scratch_alloc_offset(sizeof(*tlb_q),
-							    "IPI_TLB_FIFO");
+							"IPI_TLB_FIFO");
 		if (!tlb_fifo_off) {
 			sbi_scratch_free_offset(tlb_sync_off);
 			return SBI_ENOMEM;
@@ -298,12 +407,22 @@ int sbi_tlb_fifo_init(struct sbi_scratch *scratch, bool cold_boot)
 			sbi_scratch_free_offset(tlb_sync_off);
 			return SBI_ENOMEM;
 		}
+		ret = sbi_ipi_event_create(&tlb_ops);
+		if (ret < 0) {
+			sbi_scratch_free_offset(tlb_fifo_mem_off);
+			sbi_scratch_free_offset(tlb_fifo_off);
+			sbi_scratch_free_offset(tlb_sync_off);
+			return ret;
+		}
+		tlb_event = ret;
 		tlb_range_flush_limit = sbi_platform_tlbr_flush_limit(plat);
 	} else {
 		if (!tlb_sync_off ||
 		    !tlb_fifo_off ||
 		    !tlb_fifo_mem_off)
 			return SBI_ENOMEM;
+		if (SBI_IPI_EVENT_MAX <= tlb_event)
+			return SBI_ENOSPC;
 	}
 
 	tlb_sync = sbi_scratch_offset_ptr(scratch, tlb_sync_off);

lib/sbi/sbi_trap.c

@@ -9,7 +9,6 @@
 
 #include <sbi/riscv_asm.h>
 #include <sbi/riscv_encoding.h>
-#include <sbi/riscv_unpriv.h>
 #include <sbi/sbi_console.h>
 #include <sbi/sbi_ecall.h>
 #include <sbi/sbi_error.h>
@@ -20,13 +19,20 @@
 #include <sbi/sbi_timer.h>
 #include <sbi/sbi_trap.h>
 
-static void __noreturn sbi_trap_error(const char *msg, int rc, u32 hartid,
-				      ulong mcause, ulong mtval,
-				      struct sbi_trap_regs *regs)
+static void __noreturn sbi_trap_error(const char *msg, int rc,
+				      ulong mcause, ulong mtval, ulong mtval2,
+				      ulong mtinst, struct sbi_trap_regs *regs)
 {
+	u32 hartid = current_hartid();
+
 	sbi_printf("%s: hart%d: %s (error %d)\n", __func__, hartid, msg, rc);
 	sbi_printf("%s: hart%d: mcause=0x%" PRILX " mtval=0x%" PRILX "\n",
 		   __func__, hartid, mcause, mtval);
+	if (misa_extension('H')) {
+		sbi_printf("%s: hart%d: mtval2=0x%" PRILX
+			   " mtinst=0x%" PRILX "\n",
+			   __func__, hartid, mtval2, mtinst);
+	}
 	sbi_printf("%s: hart%d: mepc=0x%" PRILX " mstatus=0x%" PRILX "\n",
 		   __func__, hartid, regs->mepc, regs->mstatus);
 	sbi_printf("%s: hart%d: %s=0x%" PRILX " %s=0x%" PRILX "\n", __func__,
@@ -69,23 +75,18 @@ static void __noreturn sbi_trap_error(const char *msg, int rc, u32 hartid,
  * Redirect trap to lower privledge mode (S-mode or U-mode)
  *
  * @param regs pointer to register state
- * @param scratch pointer to sbi_scratch of current HART
- * @param epc error PC for lower privledge mode
- * @param cause exception cause for lower privledge mode
- * @param tval trap value for lower privledge mode
+ * @param trap pointer to trap details
  *
  * @return 0 on success and negative error code on failure
  */
-int sbi_trap_redirect(struct sbi_trap_regs *regs, struct sbi_scratch *scratch,
-		      ulong epc, ulong cause, ulong tval)
+int sbi_trap_redirect(struct sbi_trap_regs *regs,
+		      struct sbi_trap_info *trap)
 {
 	ulong hstatus, vsstatus, prev_mode;
 #if __riscv_xlen == 32
 	bool prev_virt = (regs->mstatusH & MSTATUSH_MPV) ? TRUE : FALSE;
-	bool prev_stage2 = (regs->mstatusH & MSTATUSH_MTL) ? TRUE : FALSE;
 #else
 	bool prev_virt = (regs->mstatus & MSTATUS_MPV) ? TRUE : FALSE;
-	bool prev_stage2 = (regs->mstatus & MSTATUS_MTL) ? TRUE : FALSE;
 #endif
 	/* By default, we redirect to HS-mode */
 	bool next_virt = FALSE;
@@ -96,8 +97,8 @@ int sbi_trap_redirect(struct sbi_trap_regs *regs, struct sbi_scratch *scratch,
 		return SBI_ENOTSUPP;
 
 	/* For certain exceptions from VS/VU-mode we redirect to VS-mode */
-	if (misa_extension('H') && prev_virt && !prev_stage2) {
-		switch (cause) {
+	if (misa_extension('H') && prev_virt) {
+		switch (trap->cause) {
 		case CAUSE_FETCH_PAGE_FAULT:
 		case CAUSE_LOAD_PAGE_FAULT:
 		case CAUSE_STORE_PAGE_FAULT:
@@ -108,20 +109,18 @@ int sbi_trap_redirect(struct sbi_trap_regs *regs, struct sbi_scratch *scratch,
 		};
 	}
 
-	/* Update MSTATUS MPV and MTL bits */
+	/* Update MSTATUS MPV bits */
 #if __riscv_xlen == 32
 	regs->mstatusH &= ~MSTATUSH_MPV;
 	regs->mstatusH |= (next_virt) ? MSTATUSH_MPV : 0UL;
-	regs->mstatusH &= ~MSTATUSH_MTL;
 #else
 	regs->mstatus &= ~MSTATUS_MPV;
 	regs->mstatus |= (next_virt) ? MSTATUS_MPV : 0UL;
-	regs->mstatus &= ~MSTATUS_MTL;
 #endif
 
 	/* Update HSTATUS for VS/VU-mode to HS-mode transition */
 	if (misa_extension('H') && prev_virt && !next_virt) {
-		/* Update HSTATUS SP2P, SP2V, SPV, and STL bits */
+		/* Update HSTATUS SP2P, SP2V, and SPV bits */
 		hstatus = csr_read(CSR_HSTATUS);
 		hstatus &= ~HSTATUS_SP2P;
 		hstatus |= (regs->mstatus & MSTATUS_SPP) ? HSTATUS_SP2P : 0;
@@ -129,17 +128,17 @@ int sbi_trap_redirect(struct sbi_trap_regs *regs, struct sbi_scratch *scratch,
 		hstatus |= (hstatus & HSTATUS_SPV) ? HSTATUS_SP2V : 0;
 		hstatus &= ~HSTATUS_SPV;
 		hstatus |= (prev_virt) ? HSTATUS_SPV : 0;
-		hstatus &= ~HSTATUS_STL;
-		hstatus |= (prev_stage2) ? HSTATUS_STL : 0;
 		csr_write(CSR_HSTATUS, hstatus);
+		csr_write(CSR_HTVAL, trap->tval2);
+		csr_write(CSR_HTINST, trap->tinst);
 	}
 
 	/* Update exception related CSRs */
 	if (next_virt) {
 		/* Update VS-mode exception info */
-		csr_write(CSR_VSTVAL, tval);
-		csr_write(CSR_VSEPC, epc);
-		csr_write(CSR_VSCAUSE, cause);
+		csr_write(CSR_VSTVAL, trap->tval);
+		csr_write(CSR_VSEPC, trap->epc);
+		csr_write(CSR_VSCAUSE, trap->cause);
 
 		/* Set MEPC to VS-mode exception vector base */
 		regs->mepc = csr_read(CSR_VSTVEC);
@@ -168,9 +167,9 @@ int sbi_trap_redirect(struct sbi_trap_regs *regs, struct sbi_scratch *scratch,
 		csr_write(CSR_VSSTATUS, vsstatus);
 	} else {
 		/* Update S-mode exception info */
-		csr_write(CSR_STVAL, tval);
-		csr_write(CSR_SEPC, epc);
-		csr_write(CSR_SCAUSE, cause);
+		csr_write(CSR_STVAL, trap->tval);
+		csr_write(CSR_SEPC, trap->epc);
+		csr_write(CSR_SCAUSE, trap->cause);
 
 		/* Set MEPC to S-mode exception vector base */
 		regs->mepc = csr_read(CSR_STVEC);
@@ -179,7 +178,7 @@ int sbi_trap_redirect(struct sbi_trap_regs *regs, struct sbi_scratch *scratch,
 		regs->mstatus &= ~MSTATUS_MPP;
 		regs->mstatus |= (PRV_S << MSTATUS_MPP_SHIFT);
 
-		/* Set SPP for S-mode*/
+		/* Set SPP for S-mode */
 		regs->mstatus &= ~MSTATUS_SPP;
 		if (prev_mode == PRV_S)
 			regs->mstatus |= (1UL << MSTATUS_SPP_SHIFT);
@@ -205,29 +204,34 @@ int sbi_trap_redirect(struct sbi_trap_regs *regs, struct sbi_scratch *scratch,
  * 1. The 'mscratch' CSR is pointing to sbi_scratch of current HART
  * 2. The 'mcause' CSR is having exception/interrupt cause
  * 3. The 'mtval' CSR is having additional trap information
- * 4. Stack pointer (SP) is setup for current HART
- * 5. Interrupts are disabled in MSTATUS CSR
+ * 4. The 'mtval2' CSR is having additional trap information
+ * 5. The 'mtinst' CSR is having decoded trap instruction
+ * 6. Stack pointer (SP) is setup for current HART
+ * 7. Interrupts are disabled in MSTATUS CSR
  *
  * @param regs pointer to register state
- * @param scratch pointer to sbi_scratch of current HART
  */
-void sbi_trap_handler(struct sbi_trap_regs *regs, struct sbi_scratch *scratch)
+void sbi_trap_handler(struct sbi_trap_regs *regs)
 {
 	int rc = SBI_ENOTSUPP;
 	const char *msg = "trap handler failed";
-	u32 hartid = sbi_current_hartid();
 	ulong mcause = csr_read(CSR_MCAUSE);
-	ulong mtval = csr_read(CSR_MTVAL);
-	struct unpriv_trap *uptrap;
+	ulong mtval = csr_read(CSR_MTVAL), mtval2 = 0, mtinst = 0;
+	struct sbi_trap_info trap;
+
+	if (misa_extension('H')) {
+		mtval2 = csr_read(CSR_MTVAL2);
+		mtinst = csr_read(CSR_MTINST);
+	}
 
 	if (mcause & (1UL << (__riscv_xlen - 1))) {
 		mcause &= ~(1UL << (__riscv_xlen - 1));
 		switch (mcause) {
 		case IRQ_M_TIMER:
-			sbi_timer_process(scratch);
+			sbi_timer_process();
 			break;
 		case IRQ_M_SOFT:
-			sbi_ipi_process(scratch);
+			sbi_ipi_process();
 			break;
 		default:
 			msg = "unhandled external interrupt";
@@ -238,49 +242,34 @@ void sbi_trap_handler(struct sbi_trap_regs *regs, struct sbi_scratch *scratch)
 
 	switch (mcause) {
 	case CAUSE_ILLEGAL_INSTRUCTION:
-		rc  = sbi_illegal_insn_handler(hartid, mcause, regs, scratch);
+		rc  = sbi_illegal_insn_handler(mtval, regs);
 		msg = "illegal instruction handler failed";
 		break;
 	case CAUSE_MISALIGNED_LOAD:
-		rc = sbi_misaligned_load_handler(hartid, mcause, regs, scratch);
+		rc = sbi_misaligned_load_handler(mtval, mtval2, mtinst, regs);
 		msg = "misaligned load handler failed";
 		break;
 	case CAUSE_MISALIGNED_STORE:
-		rc  = sbi_misaligned_store_handler(hartid, mcause, regs,
-						   scratch);
+		rc  = sbi_misaligned_store_handler(mtval, mtval2, mtinst, regs);
 		msg = "misaligned store handler failed";
 		break;
 	case CAUSE_SUPERVISOR_ECALL:
 	case CAUSE_HYPERVISOR_ECALL:
-		rc  = sbi_ecall_handler(hartid, mcause, regs, scratch);
+		rc  = sbi_ecall_handler(regs);
 		msg = "ecall handler failed";
 		break;
-	case CAUSE_LOAD_ACCESS:
-	case CAUSE_STORE_ACCESS:
-	case CAUSE_LOAD_PAGE_FAULT:
-	case CAUSE_STORE_PAGE_FAULT:
-		uptrap = sbi_hart_get_trap_info(scratch);
-		if ((regs->mstatus & MSTATUS_MPRV) && uptrap) {
-			rc = 0;
-			regs->mepc += uptrap->ilen;
-			uptrap->cause = mcause;
-			uptrap->tval = mtval;
-		} else {
-			rc = sbi_trap_redirect(regs, scratch, regs->mepc,
-					       mcause, mtval);
-		}
-		msg = "page/access fault handler failed";
-		break;
 	default:
 		/* If the trap came from S or U mode, redirect it there */
-		rc = sbi_trap_redirect(regs, scratch, regs->mepc,
-				       mcause, mtval);
+		trap.epc = regs->mepc;
+		trap.cause = mcause;
+		trap.tval = mtval;
+		trap.tval2 = mtval2;
+		trap.tinst = mtinst;
+		rc = sbi_trap_redirect(regs, &trap);
 		break;
 	};
 
 trap_error:
-	if (rc) {
-		sbi_trap_error(msg, rc, hartid, mcause, csr_read(CSR_MTVAL),
-			       regs);
-	}
+	if (rc)
+		sbi_trap_error(msg, rc, mcause, mtval, mtval2, mtinst, regs);
 }

lib/sbi/sbi_unpriv.c

@@ -0,0 +1,166 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2019 Western Digital Corporation or its affiliates.
+ *
+ * Authors:
+ *   Anup Patel <anup.patel@wdc.com>
+ */
+
+#include <sbi/riscv_encoding.h>
+#include <sbi/sbi_bitops.h>
+#include <sbi/sbi_hart.h>
+#include <sbi/sbi_scratch.h>
+#include <sbi/sbi_trap.h>
+#include <sbi/sbi_unpriv.h>
+
+#define DEFINE_UNPRIVILEGED_LOAD_FUNCTION(type, insn)                         \
+	type sbi_load_##type(const type *addr,                                \
+			     struct sbi_trap_info *trap)                      \
+	{                                                                     \
+		register ulong tinfo asm("a3");                               \
+		register ulong ttmp asm("a4");                                \
+		register ulong mstatus asm("a5");                             \
+		register ulong mtvec asm("a6") = sbi_hart_unpriv_trap_addr(); \
+		type ret = 0;                                                 \
+		trap->cause = 0;                                              \
+		asm volatile(                                                 \
+			"add %[tinfo], %[taddr], zero\n"                      \
+			"add %[ttmp], %[taddr], zero\n"                       \
+			"csrrw %[mtvec], " STR(CSR_MTVEC) ", %[mtvec]\n"      \
+			"csrrs %[mstatus], " STR(CSR_MSTATUS) ", %[mprv]\n"   \
+			".option push\n"                                      \
+			".option norvc\n"                                     \
+			#insn " %[ret], %[addr]\n"                            \
+			".option pop\n"                                       \
+			"csrw " STR(CSR_MSTATUS) ", %[mstatus]\n"             \
+			"csrw " STR(CSR_MTVEC) ", %[mtvec]"                   \
+		    : [mstatus] "+&r"(mstatus), [mtvec] "+&r"(mtvec),         \
+		      [tinfo] "+&r"(tinfo), [ttmp] "+&r"(ttmp),               \
+		      [ret] "=&r"(ret)                                        \
+		    : [addr] "m"(*addr), [mprv] "r"(MSTATUS_MPRV),            \
+		      [taddr] "r"((ulong)trap)                                \
+		    : "memory");                                              \
+		return ret;                                                   \
+	}
+
+#define DEFINE_UNPRIVILEGED_STORE_FUNCTION(type, insn)                        \
+	void sbi_store_##type(type *addr, type val,                           \
+			      struct sbi_trap_info *trap)                     \
+	{                                                                     \
+		register ulong tinfo asm("a3");                               \
+		register ulong ttmp asm("a4");                                \
+		register ulong mstatus asm("a5");                             \
+		register ulong mtvec asm("a6") = sbi_hart_unpriv_trap_addr(); \
+		trap->cause = 0;                                              \
+		asm volatile(                                                 \
+			"add %[tinfo], %[taddr], zero\n"                      \
+			"add %[ttmp], %[taddr], zero\n"                       \
+			"csrrw %[mtvec], " STR(CSR_MTVEC) ", %[mtvec]\n"      \
+			"csrrs %[mstatus], " STR(CSR_MSTATUS) ", %[mprv]\n"   \
+			".option push\n"                                      \
+			".option norvc\n"                                     \
+			#insn " %[val], %[addr]\n"                            \
+			".option pop\n"                                       \
+			"csrw " STR(CSR_MSTATUS) ", %[mstatus]\n"             \
+			"csrw " STR(CSR_MTVEC) ", %[mtvec]"                   \
+		    : [mstatus] "+&r"(mstatus), [mtvec] "+&r"(mtvec),         \
+		      [tinfo] "+&r"(tinfo), [ttmp] "+&r"(ttmp)                \
+		    : [addr] "m"(*addr), [mprv] "r"(MSTATUS_MPRV),            \
+		      [taddr] "r"((ulong)trap), [val] "r"(val)                \
+		    : "memory");              \
+	}
+
+DEFINE_UNPRIVILEGED_LOAD_FUNCTION(u8, lbu)
+DEFINE_UNPRIVILEGED_LOAD_FUNCTION(u16, lhu)
+DEFINE_UNPRIVILEGED_LOAD_FUNCTION(s8, lb)
+DEFINE_UNPRIVILEGED_LOAD_FUNCTION(s16, lh)
+DEFINE_UNPRIVILEGED_LOAD_FUNCTION(s32, lw)
+DEFINE_UNPRIVILEGED_STORE_FUNCTION(u8, sb)
+DEFINE_UNPRIVILEGED_STORE_FUNCTION(u16, sh)
+DEFINE_UNPRIVILEGED_STORE_FUNCTION(u32, sw)
+#if __riscv_xlen == 64
+DEFINE_UNPRIVILEGED_LOAD_FUNCTION(u32, lwu)
+DEFINE_UNPRIVILEGED_LOAD_FUNCTION(u64, ld)
+DEFINE_UNPRIVILEGED_STORE_FUNCTION(u64, sd)
+DEFINE_UNPRIVILEGED_LOAD_FUNCTION(ulong, ld)
+#else
+DEFINE_UNPRIVILEGED_LOAD_FUNCTION(u32, lw)
+DEFINE_UNPRIVILEGED_LOAD_FUNCTION(ulong, lw)
+
+u64 sbi_load_u64(const u64 *addr,
+		 struct sbi_trap_info *trap)
+{
+	u64 ret = sbi_load_u32((u32 *)addr, trap);
+
+	if (trap->cause)
+		return 0;
+	ret |= ((u64)sbi_load_u32((u32 *)addr + 1, trap) << 32);
+	if (trap->cause)
+		return 0;
+
+	return ret;
+}
+
+void sbi_store_u64(u64 *addr, u64 val,
+		   struct sbi_trap_info *trap)
+{
+	sbi_store_u32((u32 *)addr, val, trap);
+	if (trap->cause)
+		return;
+
+	sbi_store_u32((u32 *)addr + 1, val >> 32, trap);
+	if (trap->cause)
+		return;
+}
+#endif
+
+ulong sbi_get_insn(ulong mepc, struct sbi_trap_info *trap)
+{
+	register ulong tinfo asm("a3");
+	register ulong ttmp asm("a4");
+	register ulong mstatus asm("a5");
+	register ulong mtvec asm("a6") = sbi_hart_unpriv_trap_addr();
+	ulong insn = 0;
+
+	trap->cause = 0;
+
+	asm volatile(
+	    "add %[tinfo], %[taddr], zero\n"
+	    "csrrw %[mtvec], " STR(CSR_MTVEC) ", %[mtvec]\n"
+	    "csrrs %[mstatus], " STR(CSR_MSTATUS) ", %[mprv]\n"
+	    "lhu %[insn], (%[addr])\n"
+	    "andi %[ttmp], %[insn], 3\n"
+	    "addi %[ttmp], %[ttmp], -3\n"
+	    "bne %[ttmp], zero, 2f\n"
+	    "lhu %[ttmp], 2(%[addr])\n"
+	    "sll %[ttmp], %[ttmp], 16\n"
+	    "add %[insn], %[insn], %[ttmp]\n"
+	    "2: csrw " STR(CSR_MSTATUS) ", %[mstatus]\n"
+	    "csrw " STR(CSR_MTVEC) ", %[mtvec]"
+	    : [mstatus] "+&r"(mstatus), [mtvec] "+&r"(mtvec),
+	      [tinfo] "+&r"(tinfo), [ttmp] "+&r"(ttmp),
+	      [insn] "=&r"(insn)
+	    : [mprv] "r"(MSTATUS_MPRV | MSTATUS_MXR),
+	      [taddr] "r"((ulong)trap), [addr] "r"(mepc)
+	    : "memory");
+
+	switch (trap->cause) {
+	case CAUSE_LOAD_ACCESS:
+		trap->cause = CAUSE_FETCH_ACCESS;
+		trap->tval = mepc;
+		break;
+	case CAUSE_LOAD_PAGE_FAULT:
+		trap->cause = CAUSE_FETCH_PAGE_FAULT;
+		trap->tval = mepc;
+		break;
+	case CAUSE_LOAD_GUEST_PAGE_FAULT:
+		trap->cause = CAUSE_FETCH_GUEST_PAGE_FAULT;
+		trap->tval = mepc;
+		break;
+	default:
+		break;
+	};
+
+	return insn;
+}

lib/sbi/sbi_unpriv_trap.S

@@ -0,0 +1,56 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2020 Western Digital Corporation or its affiliates.
+ *
+ * Authors:
+ *   Anup Patel <anup.patel@wdc.com>
+ */
+
+#include <sbi/riscv_asm.h>
+#include <sbi/sbi_trap.h>
+
+	/*
+	 * We assume that faulting unpriv load/store instruction is
+	 * is 4-byte long and blindly increment SEPC by 4.
+	 *
+	 * The trap info will be saved as follows:
+	 * A3 <- pointer struct sbi_trap_info
+	 * A4 <- temporary
+	 */
+
+	.align 3
+	.global __sbi_unpriv_trap
+__sbi_unpriv_trap:
+	/* Without H-extension so, MTVAL2 and MTINST CSRs not available */
+	csrr	a4, CSR_MEPC
+	REG_S	a4, SBI_TRAP_INFO_OFFSET(epc)(a3)
+	csrr	a4, CSR_MCAUSE
+	REG_S	a4, SBI_TRAP_INFO_OFFSET(cause)(a3)
+	csrr	a4, CSR_MTVAL
+	REG_S	a4, SBI_TRAP_INFO_OFFSET(tval)(a3)
+	REG_S	zero, SBI_TRAP_INFO_OFFSET(tval2)(a3)
+	REG_S	zero, SBI_TRAP_INFO_OFFSET(tinst)(a3)
+	csrr	a4, CSR_MEPC
+	addi	a4, a4, 4
+	csrw	CSR_MEPC, a4
+	mret
+
+	.align 3
+	.global __sbi_unpriv_trap_hext
+__sbi_unpriv_trap_hext:
+	/* With H-extension so, MTVAL2 and MTINST CSRs available */
+	csrr	a4, CSR_MEPC
+	REG_S	a4, SBI_TRAP_INFO_OFFSET(epc)(a3)
+	csrr	a4, CSR_MCAUSE
+	REG_S	a4, SBI_TRAP_INFO_OFFSET(cause)(a3)
+	csrr	a4, CSR_MTVAL
+	REG_S	a4, SBI_TRAP_INFO_OFFSET(tval)(a3)
+	csrr	a4, CSR_MTVAL2
+	REG_S	a4, SBI_TRAP_INFO_OFFSET(tval2)(a3)
+	csrr	a4, CSR_MTINST
+	REG_S	a4, SBI_TRAP_INFO_OFFSET(tinst)(a3)
+	csrr	a4, CSR_MEPC
+	addi	a4, a4, 4
+	csrw	CSR_MEPC, a4
+	mret

lib/utils/fdt/fdt_fixup.c

@@ -0,0 +1,216 @@
+// SPDX-License-Identifier: BSD-2-Clause
+/*
+ * fdt_fixup.c - Flat Device Tree parsing helper routines
+ * Implement helper routines to parse FDT nodes on top of
+ * libfdt for OpenSBI usage
+ *
+ * Copyright (C) 2020 Bin Meng <bmeng.cn@gmail.com>
+ */
+
+#include <libfdt.h>
+#include <sbi/riscv_asm.h>
+#include <sbi/sbi_console.h>
+#include <sbi/sbi_platform.h>
+#include <sbi/sbi_scratch.h>
+#include <sbi/sbi_string.h>
+
+void fdt_cpu_fixup(void *fdt)
+{
+	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
+	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
+	int err, len, cpu_offset, cpus_offset;
+	const fdt32_t *val;
+	const void *prop;
+	u32 hartid;
+
+	err = fdt_open_into(fdt, fdt, fdt_totalsize(fdt) + 32);
+	if (err < 0)
+		return;
+
+	cpus_offset = fdt_path_offset(fdt, "/cpus");
+	if (cpus_offset < 0)
+		return;
+
+	fdt_for_each_subnode(cpu_offset, fdt, cpus_offset) {
+		prop = fdt_getprop(fdt, cpu_offset, "device_type", &len);
+		if (!prop || !len)
+			continue;
+		if (sbi_strcmp(prop, "cpu"))
+			continue;
+
+		val = fdt_getprop(fdt, cpu_offset, "reg", &len);
+		if (!val || len < sizeof(fdt32_t))
+			continue;
+
+		if (len > sizeof(fdt32_t))
+			val++;
+		hartid = fdt32_to_cpu(*val);
+
+		if (sbi_platform_hart_invalid(plat, hartid))
+			fdt_setprop_string(fdt, cpu_offset, "status",
+					   "disabled");
+	}
+}
+
+void fdt_plic_fixup(void *fdt, const char *compat)
+{
+	u32 *cells;
+	int i, cells_count;
+	int plic_off;
+
+	plic_off = fdt_node_offset_by_compatible(fdt, 0, compat);
+	if (plic_off < 0)
+		return;
+
+	cells = (u32 *)fdt_getprop(fdt, plic_off,
+				   "interrupts-extended", &cells_count);
+	if (!cells)
+		return;
+
+	cells_count = cells_count / sizeof(u32);
+	if (!cells_count)
+		return;
+
+	for (i = 0; i < (cells_count / 2); i++) {
+		if (fdt32_to_cpu(cells[2 * i + 1]) == IRQ_M_EXT)
+			cells[2 * i + 1] = cpu_to_fdt32(0xffffffff);
+	}
+}
+
+/**
+ * We use PMP to protect OpenSBI firmware to safe-guard it from buggy S-mode
+ * software, see pmp_init() in lib/sbi/sbi_hart.c. The protected memory region
+ * information needs to be conveyed to S-mode software (e.g.: operating system)
+ * via some well-known method.
+ *
+ * With device tree, this can be done by inserting a child node of the reserved
+ * memory node which is used to specify one or more regions of reserved memory.
+ *
+ * For the reserved memory node bindings, see Linux kernel documentation at
+ * Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt
+ *
+ * Some additional memory spaces may be protected by platform codes via PMP as
+ * well, and corresponding child nodes will be inserted.
+ */
+int fdt_reserved_memory_fixup(void *fdt)
+{
+	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
+	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
+	unsigned long prot, addr, size;
+	int na = fdt_address_cells(fdt, 0);
+	int ns = fdt_size_cells(fdt, 0);
+	fdt32_t addr_high, addr_low;
+	fdt32_t size_high, size_low;
+	fdt32_t reg[4];
+	fdt32_t *val;
+	char name[32];
+	int parent, subnode;
+	int i, j;
+	int err;
+
+	if (!sbi_platform_has_pmp(plat))
+		return 0;
+
+	/* expand the device tree to accommodate new node */
+	err  = fdt_open_into(fdt, fdt, fdt_totalsize(fdt) + 256);
+	if (err < 0)
+		return err;
+
+	/* try to locate the reserved memory node */
+	parent = fdt_path_offset(fdt, "/reserved-memory");
+	if (parent < 0) {
+		/* if such node does not exist, create one */
+		parent = fdt_add_subnode(fdt, 0, "reserved-memory");
+		if (parent < 0)
+			return parent;
+
+		/*
+		 * reserved-memory node has 3 required properties:
+		 * - #address-cells: the same value as the root node
+		 * - #size-cells: the same value as the root node
+		 * - ranges: should be empty
+		 */
+
+		err = fdt_setprop_empty(fdt, parent, "ranges");
+		if (err < 0)
+			return err;
+
+		err = fdt_setprop_u32(fdt, parent, "#size-cells", ns);
+		if (err < 0)
+			return err;
+
+		err = fdt_setprop_u32(fdt, parent, "#address-cells", na);
+		if (err < 0)
+			return err;
+	}
+
+	/*
+	 * We assume the given device tree does not contain any memory region
+	 * child node protected by PMP. Normally PMP programming happens at
+	 * M-mode firmware. The memory space used by OpenSBI is protected.
+	 * Some additional memory spaces may be protected by platform codes.
+	 *
+	 * With above assumption, we create child nodes directly.
+	 */
+
+	for (i = 0, j = 0; i < PMP_COUNT; i++) {
+		pmp_get(i, &prot, &addr, &size);
+		if (!(prot & PMP_A))
+			continue;
+		if (!(prot & (PMP_R | PMP_W | PMP_X))) {
+			addr_high = (u64)addr >> 32;
+			addr_low = addr;
+			size_high = (u64)size >> 32;
+			size_low = size;
+
+			if (na > 1 && addr_high)
+				sbi_snprintf(name, sizeof(name),
+					     "mmode_pmp%d@%x,%x", j,
+					     addr_high, addr_low);
+			else
+				sbi_snprintf(name, sizeof(name),
+					     "mmode_pmp%d@%x", j,
+					     addr_low);
+
+			subnode = fdt_add_subnode(fdt, parent, name);
+			if (subnode < 0)
+				return subnode;
+
+			/*
+			 * Tell operating system not to create a virtual
+			 * mapping of the region as part of its standard
+			 * mapping of system memory.
+			 */
+			err = fdt_setprop_empty(fdt, subnode, "no-map");
+			if (err < 0)
+				return err;
+
+			/* encode the <reg> property value */
+			val = reg;
+			if (na > 1)
+				*val++ = cpu_to_fdt32(addr_high);
+			*val++ = cpu_to_fdt32(addr_low);
+			if (ns > 1)
+				*val++ = cpu_to_fdt32(size_high);
+			*val++ = cpu_to_fdt32(size_low);
+
+			err = fdt_setprop(fdt, subnode, "reg", reg,
+					  (na + ns) * sizeof(fdt32_t));
+			if (err < 0)
+				return err;
+
+			j++;
+		}
+	}
+
+	return 0;
+}
+
+void fdt_fixups(void *fdt)
+{
+	fdt_plic_fixup(fdt, "riscv,plic0");
+
+	fdt_reserved_memory_fixup(fdt);
+}
+
+

lib/utils/fdt/fdt_helper.c

@@ -0,0 +1,129 @@
+// SPDX-License-Identifier: BSD-2-Clause
+/*
+ * fdt_helper.c - Flat Device Tree manipulation helper routines
+ * Implement helper routines on top of libfdt for OpenSBI usage
+ *
+ * Copyright (C) 2020 Bin Meng <bmeng.cn@gmail.com>
+ */
+
+#include <libfdt.h>
+#include <sbi/riscv_asm.h>
+#include <sbi/sbi_console.h>
+#include <sbi/sbi_platform.h>
+#include <sbi/sbi_scratch.h>
+#include <sbi_utils/fdt/fdt_helper.h>
+
+static int fdt_get_node_addr_size(void *fdt, int node, unsigned long *addr,
+				  unsigned long *size)
+{
+	int parent, len, i;
+	int cell_addr, cell_size;
+	const fdt32_t *prop_addr, *prop_size;
+	uint64_t temp = 0;
+
+	parent = fdt_parent_offset(fdt, node);
+	if (parent < 0)
+		return parent;
+	cell_addr = fdt_address_cells(fdt, parent);
+	if (cell_addr < 1)
+		return SBI_ENODEV;
+
+	cell_size = fdt_size_cells(fdt, parent);
+	if (cell_size < 0)
+		return SBI_ENODEV;
+
+	prop_addr = fdt_getprop(fdt, node, "reg", &len);
+	if (!prop_addr)
+		return SBI_ENODEV;
+	prop_size = prop_addr + cell_addr;
+
+	if (addr) {
+		for (i = 0; i < cell_addr; i++)
+			temp = (temp << 32) | fdt32_to_cpu(*prop_addr++);
+		*addr = temp;
+	}
+	temp = 0;
+
+	if (size) {
+		for (i = 0; i < cell_size; i++)
+			temp = (temp << 32) | fdt32_to_cpu(*prop_size++);
+		*size = temp;
+	}
+
+	return 0;
+}
+
+int fdt_parse_uart8250(void *fdt, struct platform_uart_data *uart,
+		   const char *compatible)
+{
+	int nodeoffset, len, rc;
+	fdt32_t *val;
+	unsigned long reg_addr, reg_size;
+
+	/**
+	 * TODO: We don't know how to handle multiple nodes with the same
+	 * compatible sring. Just return the first node for now.
+	 */
+
+	nodeoffset = fdt_node_offset_by_compatible(fdt, -1, compatible);
+	if (nodeoffset < 0)
+		return nodeoffset;
+
+	rc = fdt_get_node_addr_size(fdt, nodeoffset, &reg_addr, &reg_size);
+	if (rc < 0 || !reg_addr || !reg_size)
+		return SBI_ENODEV;
+	uart->addr = reg_addr;
+
+	/**
+	 * UART address is mandaotry. clock-frequency and current-speed may not
+	 * be present. Don't return error.
+	 */
+	val = (fdt32_t *)fdt_getprop(fdt, nodeoffset, "clock-frequency", &len);
+	if (len > 0 && val)
+		uart->freq = fdt32_to_cpu(*val);
+
+	val = (fdt32_t *)fdt_getprop(fdt, nodeoffset, "current-speed", &len);
+	if (len > 0 && val)
+		uart->baud = fdt32_to_cpu(*val);
+
+	return 0;
+}
+
+int fdt_parse_plic(void *fdt, struct platform_plic_data *plic,
+		   const char *compatible)
+{
+	int nodeoffset, len, rc;
+	const fdt32_t *val;
+	unsigned long reg_addr, reg_size;
+
+	nodeoffset = fdt_node_offset_by_compatible(fdt, -1, compatible);
+	if (nodeoffset < 0)
+		return nodeoffset;
+
+	rc = fdt_get_node_addr_size(fdt, nodeoffset, &reg_addr, &reg_size);
+	if (rc < 0 || !reg_addr || !reg_size)
+		return SBI_ENODEV;
+	plic->addr = reg_addr;
+
+	val = fdt_getprop(fdt, nodeoffset, "riscv,ndev", &len);
+	if (len > 0)
+		plic->num_src = fdt32_to_cpu(*val);
+
+	return 0;
+}
+
+int fdt_parse_clint(void *fdt, unsigned long *clint_addr,
+		    const char *compatible)
+{
+	int nodeoffset, rc;
+
+	nodeoffset = fdt_node_offset_by_compatible(fdt, -1, compatible);
+	if (nodeoffset < 0)
+		return nodeoffset;
+
+	rc = fdt_get_node_addr_size(fdt, nodeoffset, clint_addr, NULL);
+	if (rc < 0 || !clint_addr)
+		return SBI_ENODEV;
+
+	return 0;
+}

lib/utils/fdt/objects.mk

@@ -0,0 +1,8 @@
+#
+# SPDX-License-Identifier: BSD-2-Clause
+#
+# Copyright (C) 2020 Bin Meng <bmeng.cn@gmail.com>
+#
+
+libsbiutils-objs-y += fdt/fdt_helper.o
+libsbiutils-objs-y += fdt/fdt_fixup.o

lib/utils/irqchip/plic.c

@@ -12,8 +12,6 @@
 #include <sbi/sbi_console.h>
 #include <sbi/sbi_string.h>
 #include <sbi_utils/irqchip/plic.h>
-#include <libfdt.h>
-#include <fdt.h>
 
 #define PLIC_PRIORITY_BASE 0x0
 #define PLIC_PENDING_BASE 0x1000
@@ -47,31 +45,6 @@ void plic_set_ie(u32 cntxid, u32 word_index, u32 val)
 	writel(val, plic_ie + word_index * 4);
 }
 
-void plic_fdt_fixup(void *fdt, const char *compat)
-{
-	u32 *cells;
-	int i, cells_count;
-	int plic_off;
-
-	plic_off = fdt_node_offset_by_compatible(fdt, 0, compat);
-	if (plic_off < 0)
-		return;
-
-	cells = (u32 *)fdt_getprop(fdt, plic_off,
-				   "interrupts-extended", &cells_count);
-	if (!cells)
-		return;
-
-	cells_count = cells_count / sizeof(u32);
-	if (!cells_count)
-		return;
-
-	for (i = 0; i < (cells_count / 2); i++) {
-		if (fdt32_to_cpu(cells[2 * i + 1]) == IRQ_M_EXT)
-			cells[2 * i + 1] = cpu_to_fdt32(0xffffffff);
-	}
-}
-
 int plic_warm_irqchip_init(u32 target_hart, int m_cntx_id, int s_cntx_id)
 {
 	size_t i, ie_words = plic_num_sources / 32 + 1;
@@ -91,13 +64,13 @@ int plic_warm_irqchip_init(u32 target_hart, int m_cntx_id, int s_cntx_id)
 			plic_set_ie(s_cntx_id, i, 0);
 	}
 
-	/* By default, enable M-mode threshold */
+	/* By default, disable M-mode threshold */
 	if (m_cntx_id > -1)
-		plic_set_thresh(m_cntx_id, 1);
+		plic_set_thresh(m_cntx_id, 0x7);
 
 	/* By default, disable S-mode threshold */
 	if (s_cntx_id > -1)
-		plic_set_thresh(s_cntx_id, 0);
+		plic_set_thresh(s_cntx_id, 0x7);
 
 	return 0;
 }
@@ -112,7 +85,7 @@ int plic_cold_irqchip_init(unsigned long base, u32 num_sources, u32 hart_count)
 
 	/* Configure default priorities of all IRQs */
 	for (i = 1; i <= plic_num_sources; i++)
-		plic_set_priority(i, 1);
+		plic_set_priority(i, 0);
 
 	return 0;
 }

lib/utils/libfdt/Makefile.libfdt

@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause)
 # Makefile.libfdt
 #
 # This is not a complete Makefile of itself.  Instead, it is designed to
@@ -9,7 +10,9 @@ LIBFDT_VERSION = version.lds
 LIBFDT_SRCS = fdt.c fdt_ro.c fdt_wip.c fdt_sw.c fdt_rw.c fdt_strerror.c fdt_empty_tree.c \
 	fdt_addresses.c fdt_overlay.c
 LIBFDT_OBJS = $(LIBFDT_SRCS:%.c=%.o)
+LIBFDT_LIB = libfdt-$(DTC_VERSION).$(SHAREDLIB_EXT)
 
 libfdt_clean:
 	@$(VECHO) CLEAN "(libfdt)"
 	rm -f $(STD_CLEANFILES:%=$(LIBFDT_dir)/%)
+	rm -f $(LIBFDT_dir)/$(LIBFDT_soname)

lib/utils/libfdt/fdt.c

@@ -1,52 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause)
 /*
  * libfdt - Flat Device Tree manipulation
  * Copyright (C) 2006 David Gibson, IBM Corporation.
- *
- * libfdt is dual licensed: you can use it either under the terms of
- * the GPL, or the BSD license, at your option.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- *     You should have received a copy of the GNU General Public
- *     License along with this library; if not, write to the Free
- *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
- *     MA 02110-1301 USA
- *
- * Alternatively,
- *
- *  b) Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *     1. Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *     2. Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "libfdt_env.h"
 
@@ -60,8 +15,10 @@
  * that the given buffer contains what appears to be a flattened
  * device tree with sane information in its header.
  */
-int fdt_ro_probe_(const void *fdt)
+int32_t fdt_ro_probe_(const void *fdt)
 {
+	uint32_t totalsize = fdt_totalsize(fdt);
+
 	if (fdt_magic(fdt) == FDT_MAGIC) {
 		/* Complete tree */
 		if (fdt_version(fdt) < FDT_FIRST_SUPPORTED_VERSION)
@@ -76,7 +33,10 @@ int fdt_ro_probe_(const void *fdt)
 		return -FDT_ERR_BADMAGIC;
 	}
 
-	return 0;
+	if (totalsize < INT32_MAX)
+		return totalsize;
+	else
+		return -FDT_ERR_TRUNCATED;
 }
 
 static int check_off_(uint32_t hdrsize, uint32_t totalsize, uint32_t off)

lib/utils/libfdt/fdt.h

@@ -1,55 +1,10 @@
+/* SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) */
 #ifndef FDT_H
 #define FDT_H
 /*
  * libfdt - Flat Device Tree manipulation
  * Copyright (C) 2006 David Gibson, IBM Corporation.
  * Copyright 2012 Kim Phillips, Freescale Semiconductor.
- *
- * libfdt is dual licensed: you can use it either under the terms of
- * the GPL, or the BSD license, at your option.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- *     You should have received a copy of the GNU General Public
- *     License along with this library; if not, write to the Free
- *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
- *     MA 02110-1301 USA
- *
- * Alternatively,
- *
- *  b) Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *     1. Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *     2. Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #ifndef __ASSEMBLY__

lib/utils/libfdt/fdt_addresses.c

@@ -1,53 +1,8 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause)
 /*
  * libfdt - Flat Device Tree manipulation
  * Copyright (C) 2014 David Gibson <david@gibson.dropbear.id.au>
  * Copyright (C) 2018 embedded brains GmbH
- *
- * libfdt is dual licensed: you can use it either under the terms of
- * the GPL, or the BSD license, at your option.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- *     You should have received a copy of the GNU General Public
- *     License along with this library; if not, write to the Free
- *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
- *     MA 02110-1301 USA
- *
- * Alternatively,
- *
- *  b) Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *     1. Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *     2. Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "libfdt_env.h"
 
@@ -59,7 +14,7 @@
 static int fdt_cells(const void *fdt, int nodeoffset, const char *name)
 {
 	const fdt32_t *c;
-	int val;
+	uint32_t val;
 	int len;
 
 	c = fdt_getprop(fdt, nodeoffset, name, &len);
@@ -70,10 +25,10 @@ static int fdt_cells(const void *fdt, int nodeoffset, const char *name)
 		return -FDT_ERR_BADNCELLS;
 
 	val = fdt32_to_cpu(*c);
-	if ((val <= 0) || (val > FDT_MAX_NCELLS))
+	if (val > FDT_MAX_NCELLS)
 		return -FDT_ERR_BADNCELLS;
 
-	return val;
+	return (int)val;
 }
 
 int fdt_address_cells(const void *fdt, int nodeoffset)
@@ -81,6 +36,8 @@ int fdt_address_cells(const void *fdt, int nodeoffset)
 	int val;
 
 	val = fdt_cells(fdt, nodeoffset, "#address-cells");
+	if (val == 0)
+		return -FDT_ERR_BADNCELLS;
 	if (val == -FDT_ERR_NOTFOUND)
 		return 2;
 	return val;
@@ -95,3 +52,50 @@ int fdt_size_cells(const void *fdt, int nodeoffset)
 		return 1;
 	return val;
 }
+
+/* This function assumes that [address|size]_cells is 1 or 2 */
+int fdt_appendprop_addrrange(void *fdt, int parent, int nodeoffset,
+			     const char *name, uint64_t addr, uint64_t size)
+{
+	int addr_cells, size_cells, ret;
+	uint8_t data[sizeof(fdt64_t) * 2], *prop;
+
+	ret = fdt_address_cells(fdt, parent);
+	if (ret < 0)
+		return ret;
+	addr_cells = ret;
+
+	ret = fdt_size_cells(fdt, parent);
+	if (ret < 0)
+		return ret;
+	size_cells = ret;
+
+	/* check validity of address */
+	prop = data;
+	if (addr_cells == 1) {
+		if ((addr > UINT32_MAX) || ((UINT32_MAX + 1 - addr) < size))
+			return -FDT_ERR_BADVALUE;
+
+		fdt32_st(prop, (uint32_t)addr);
+	} else if (addr_cells == 2) {
+		fdt64_st(prop, addr);
+	} else {
+		return -FDT_ERR_BADNCELLS;
+	}
+
+	/* check validity of size */
+	prop += addr_cells * sizeof(fdt32_t);
+	if (size_cells == 1) {
+		if (size > UINT32_MAX)
+			return -FDT_ERR_BADVALUE;
+
+		fdt32_st(prop, (uint32_t)size);
+	} else if (size_cells == 2) {
+		fdt64_st(prop, size);
+	} else {
+		return -FDT_ERR_BADNCELLS;
+	}
+
+	return fdt_appendprop(fdt, nodeoffset, name, data,
+			      (addr_cells + size_cells) * sizeof(fdt32_t));
+}

lib/utils/libfdt/fdt_empty_tree.c

@@ -1,52 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause)
 /*
  * libfdt - Flat Device Tree manipulation
  * Copyright (C) 2012 David Gibson, IBM Corporation.
- *
- * libfdt is dual licensed: you can use it either under the terms of
- * the GPL, or the BSD license, at your option.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- *     You should have received a copy of the GNU General Public
- *     License along with this library; if not, write to the Free
- *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
- *     MA 02110-1301 USA
- *
- * Alternatively,
- *
- *  b) Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *     1. Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *     2. Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "libfdt_env.h"
 

lib/utils/libfdt/fdt_overlay.c

@@ -1,53 +1,8 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause)
 /*
  * libfdt - Flat Device Tree manipulation
  * Copyright (C) 2016 Free Electrons
  * Copyright (C) 2016 NextThing Co.
- *
- * libfdt is dual licensed: you can use it either under the terms of
- * the GPL, or the BSD license, at your option.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- *     You should have received a copy of the GNU General Public
- *     License along with this library; if not, write to the Free
- *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
- *     MA 02110-1301 USA
- *
- * Alternatively,
- *
- *  b) Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *     1. Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *     2. Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "libfdt_env.h"
 
@@ -93,11 +48,11 @@ static uint32_t overlay_get_target_phandle(const void *fdto, int fragment)
  * @pathp: pointer which receives the path of the target (or NULL)
  *
  * overlay_get_target() retrieves the target offset in the base
- * device tree of a fragment, no matter how the actual targetting is
+ * device tree of a fragment, no matter how the actual targeting is
  * done (through a phandle or a path)
  *
  * returns:
- *      the targetted node offset in the base device tree
+ *      the targeted node offset in the base device tree
  *      Negative error code on error
  */
 static int overlay_get_target(const void *fdt, const void *fdto,
@@ -778,26 +733,36 @@ static int overlay_symbol_update(void *fdt, void *fdto)
 		/* keep end marker to avoid strlen() */
 		e = path + path_len;
 
-		/* format: /<fragment-name>/__overlay__/<relative-subnode-path> */
-
 		if (*path != '/')
 			return -FDT_ERR_BADVALUE;
 
 		/* get fragment name first */
 		s = strchr(path + 1, '/');
-		if (!s)
-			return -FDT_ERR_BADOVERLAY;
+		if (!s) {
+			/* Symbol refers to something that won't end
+			 * up in the target tree */
+			continue;
+		}
 
 		frag_name = path + 1;
 		frag_name_len = s - path - 1;
 
 		/* verify format; safe since "s" lies in \0 terminated prop */
 		len = sizeof("/__overlay__/") - 1;
-		if ((e - s) < len || memcmp(s, "/__overlay__/", len))
-			return -FDT_ERR_BADOVERLAY;
-
-		rel_path = s + len;
-		rel_path_len = e - rel_path;
+		if ((e - s) > len && (memcmp(s, "/__overlay__/", len) == 0)) {
+			/* /<fragment-name>/__overlay__/<relative-subnode-path> */
+			rel_path = s + len;
+			rel_path_len = e - rel_path;
+		} else if ((e - s) == len
+			   && (memcmp(s, "/__overlay__", len - 1) == 0)) {
+			/* /<fragment-name>/__overlay__ */
+			rel_path = "";
+			rel_path_len = 0;
+		} else {
+			/* Symbol refers to something that won't end
+			 * up in the target tree */
+			continue;
+		}
 
 		/* find the fragment index in which the symbol lies */
 		ret = fdt_subnode_offset_namelen(fdto, 0, frag_name,
@@ -863,12 +828,16 @@ static int overlay_symbol_update(void *fdt, void *fdto)
 
 int fdt_overlay_apply(void *fdt, void *fdto)
 {
-	uint32_t delta = fdt_get_max_phandle(fdt);
+	uint32_t delta;
 	int ret;
 
 	FDT_RO_PROBE(fdt);
 	FDT_RO_PROBE(fdto);
 
+	ret = fdt_find_max_phandle(fdt, &delta);
+	if (ret)
+		goto err;
+
 	ret = overlay_adjust_local_phandles(fdto, delta);
 	if (ret)
 		goto err;

lib/utils/libfdt/fdt_ro.c

@@ -1,52 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause)
 /*
  * libfdt - Flat Device Tree manipulation
  * Copyright (C) 2006 David Gibson, IBM Corporation.
- *
- * libfdt is dual licensed: you can use it either under the terms of
- * the GPL, or the BSD license, at your option.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- *     You should have received a copy of the GNU General Public
- *     License along with this library; if not, write to the Free
- *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
- *     MA 02110-1301 USA
- *
- * Alternatively,
- *
- *  b) Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *     1. Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *     2. Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "libfdt_env.h"
 
@@ -78,19 +33,20 @@ static int fdt_nodename_eq_(const void *fdt, int offset,
 
 const char *fdt_get_string(const void *fdt, int stroffset, int *lenp)
 {
+	int32_t totalsize = fdt_ro_probe_(fdt);
 	uint32_t absoffset = stroffset + fdt_off_dt_strings(fdt);
 	size_t len;
 	int err;
 	const char *s, *n;
 
-	err = fdt_ro_probe_(fdt);
-	if (err != 0)
+	err = totalsize;
+	if (totalsize < 0)
 		goto fail;
 
 	err = -FDT_ERR_BADOFFSET;
-	if (absoffset >= fdt_totalsize(fdt))
+	if (absoffset >= totalsize)
 		goto fail;
-	len = fdt_totalsize(fdt) - absoffset;
+	len = totalsize - absoffset;
 
 	if (fdt_magic(fdt) == FDT_MAGIC) {
 		if (stroffset < 0)
@@ -144,29 +100,49 @@ static int fdt_string_eq_(const void *fdt, int stroffset,
 	return p && (slen == len) && (memcmp(p, s, len) == 0);
 }
 
-uint32_t fdt_get_max_phandle(const void *fdt)
+int fdt_find_max_phandle(const void *fdt, uint32_t *phandle)
 {
-	uint32_t max_phandle = 0;
-	int offset;
+	uint32_t max = 0;
+	int offset = -1;
 
-	for (offset = fdt_next_node(fdt, -1, NULL);;
-	     offset = fdt_next_node(fdt, offset, NULL)) {
-		uint32_t phandle;
+	while (true) {
+		uint32_t value;
 
-		if (offset == -FDT_ERR_NOTFOUND)
-			return max_phandle;
+		offset = fdt_next_node(fdt, offset, NULL);
+		if (offset < 0) {
+			if (offset == -FDT_ERR_NOTFOUND)
+				break;
 
-		if (offset < 0)
-			return (uint32_t)-1;
+			return offset;
+		}
 
-		phandle = fdt_get_phandle(fdt, offset);
-		if (phandle == (uint32_t)-1)
-			continue;
+		value = fdt_get_phandle(fdt, offset);
 
-		if (phandle > max_phandle)
-			max_phandle = phandle;
+		if (value > max)
+			max = value;
 	}
 
+	if (phandle)
+		*phandle = max;
+
+	return 0;
+}
+
+int fdt_generate_phandle(const void *fdt, uint32_t *phandle)
+{
+	uint32_t max;
+	int err;
+
+	err = fdt_find_max_phandle(fdt, &max);
+	if (err < 0)
+		return err;
+
+	if (max == FDT_MAX_PHANDLE)
+		return -FDT_ERR_NOPHANDLES;
+
+	if (phandle)
+		*phandle = max + 1;
+
 	return 0;
 }
 
@@ -313,7 +289,7 @@ const char *fdt_get_name(const void *fdt, int nodeoffset, int *len)
 	const char *nameptr;
 	int err;
 
-	if (((err = fdt_ro_probe_(fdt)) != 0)
+	if (((err = fdt_ro_probe_(fdt)) < 0)
 	    || ((err = fdt_check_node_offset_(fdt, nodeoffset)) < 0))
 			goto fail;
 

lib/utils/libfdt/fdt_rw.c

@@ -1,52 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause)
 /*
  * libfdt - Flat Device Tree manipulation
  * Copyright (C) 2006 David Gibson, IBM Corporation.
- *
- * libfdt is dual licensed: you can use it either under the terms of
- * the GPL, or the BSD license, at your option.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- *     You should have received a copy of the GNU General Public
- *     License along with this library; if not, write to the Free
- *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
- *     MA 02110-1301 USA
- *
- * Alternatively,
- *
- *  b) Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *     1. Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *     2. Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "libfdt_env.h"
 
@@ -136,6 +91,14 @@ static int fdt_splice_struct_(void *fdt, void *p,
 	return 0;
 }
 
+/* Must only be used to roll back in case of error */
+static void fdt_del_last_string_(void *fdt, const char *s)
+{
+	int newlen = strlen(s) + 1;
+
+	fdt_set_size_dt_strings(fdt, fdt_size_dt_strings(fdt) - newlen);
+}
+
 static int fdt_splice_string_(void *fdt, int newlen)
 {
 	void *p = (char *)fdt
@@ -149,7 +112,7 @@ static int fdt_splice_string_(void *fdt, int newlen)
 	return 0;
 }
 
-static int fdt_find_add_string_(void *fdt, const char *s)
+static int fdt_find_add_string_(void *fdt, const char *s, int *allocated)
 {
 	char *strtab = (char *)fdt + fdt_off_dt_strings(fdt);
 	const char *p;
@@ -157,6 +120,8 @@ static int fdt_find_add_string_(void *fdt, const char *s)
 	int len = strlen(s) + 1;
 	int err;
 
+	*allocated = 0;
+
 	p = fdt_find_string_(strtab, fdt_size_dt_strings(fdt), s);
 	if (p)
 		/* found it */
@@ -167,6 +132,8 @@ static int fdt_find_add_string_(void *fdt, const char *s)
 	if (err)
 		return err;
 
+	*allocated = 1;
+
 	memcpy(new, s, len);
 	return (new - strtab);
 }
@@ -225,11 +192,12 @@ static int fdt_add_property_(void *fdt, int nodeoffset, const char *name,
 	int nextoffset;
 	int namestroff;
 	int err;
+	int allocated;
 
 	if ((nextoffset = fdt_check_node_offset_(fdt, nodeoffset)) < 0)
 		return nextoffset;
 
-	namestroff = fdt_find_add_string_(fdt, name);
+	namestroff = fdt_find_add_string_(fdt, name, &allocated);
 	if (namestroff < 0)
 		return namestroff;
 
@@ -237,8 +205,11 @@ static int fdt_add_property_(void *fdt, int nodeoffset, const char *name,
 	proplen = sizeof(**prop) + FDT_TAGALIGN(len);
 
 	err = fdt_splice_struct_(fdt, *prop, 0, proplen);
-	if (err)
+	if (err) {
+		if (allocated)
+			fdt_del_last_string_(fdt, name);
 		return err;
+	}
 
 	(*prop)->tag = cpu_to_fdt32(FDT_PROP);
 	(*prop)->nameoff = cpu_to_fdt32(namestroff);

lib/utils/libfdt/fdt_strerror.c

@@ -1,51 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause)
 /*
  * libfdt - Flat Device Tree manipulation
  * Copyright (C) 2006 David Gibson, IBM Corporation.
- *
- * libfdt is dual licensed: you can use it either under the terms of
- * the GPL, or the BSD license, at your option.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- *     You should have received a copy of the GNU General Public
- *     License along with this library; if not, write to the Free
- *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
- *     MA 02110-1301 USA
- *
- * Alternatively,
- *
- *  b) Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *     1. Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *     2. Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "libfdt_env.h"
@@ -82,6 +38,7 @@ static struct fdt_errtabent fdt_errtable[] = {
 	FDT_ERRTABENT(FDT_ERR_BADVALUE),
 	FDT_ERRTABENT(FDT_ERR_BADOVERLAY),
 	FDT_ERRTABENT(FDT_ERR_NOPHANDLES),
+	FDT_ERRTABENT(FDT_ERR_BADFLAGS),
 };
 #define FDT_ERRTABSIZE	(sizeof(fdt_errtable) / sizeof(fdt_errtable[0]))