include: Bump-up version to 1.5

This patch updates OpenSBI version to 1.5 as part of
release preparation.

Signed-off-by: Anup Patel <apatel@ventanamicro.com>
Signed-off-by: Anup Patel <anup@brainfault.org>

.gitignore

@@ -1,3 +1,10 @@
+# ignore anything begin with dot
+.*
+
+# exceptions we need even begin with dot
+!.clang-format
+!.gitignore
+
 # Object files
 *.o
 *.a
@@ -10,4 +17,3 @@ install/
 # Development friendly files
 tags
 cscope*
-*.swp

Makefile

@@ -79,6 +79,7 @@ export PYTHONDONTWRITEBYTECODE=1
 export KCONFIG_DIR=$(platform_build_dir)/kconfig
 export KCONFIG_AUTOLIST=$(KCONFIG_DIR)/auto.list
 export KCONFIG_AUTOHEADER=$(KCONFIG_DIR)/autoconf.h
+export KCONFIG_AUTOCONFIG=$(KCONFIG_DIR)/auto.conf
 export KCONFIG_AUTOCMD=$(KCONFIG_DIR)/auto.conf.cmd
 export KCONFIG_CONFIG=$(KCONFIG_DIR)/.config
 # Additional exports for include paths in Kconfig files
@@ -167,12 +168,22 @@ endif
 # Check whether the linker supports creating PIEs
 OPENSBI_LD_PIE := $(shell $(CC) $(CLANG_TARGET) $(RELAX_FLAG) $(USE_LD_FLAG) -fPIE -nostdlib -Wl,-pie -x c /dev/null -o /dev/null >/dev/null 2>&1 && echo y || echo n)
 
+# Check whether the linker supports --exclude-libs
+OPENSBI_LD_EXCLUDE_LIBS := $(shell $(CC) $(CLANG_TARGET) $(RELAX_FLAG) $(USE_LD_FLAG) "-Wl,--exclude-libs,ALL" -x c /dev/null -o /dev/null >/dev/null 2>&1 && echo y || echo n)
+
 # Check whether the compiler supports -m(no-)save-restore
-CC_SUPPORT_SAVE_RESTORE := $(shell $(CC) $(CLANG_TARGET) $(RELAX_FLAG) -nostdlib -mno-save-restore -x c /dev/null -o /dev/null 2>&1 | grep "\-save\-restore" >/dev/null && echo n || echo y)
+CC_SUPPORT_SAVE_RESTORE := $(shell $(CC) $(CLANG_TARGET) $(RELAX_FLAG) -nostdlib -mno-save-restore -x c /dev/null -o /dev/null 2>&1 | grep -e "-save-restore" >/dev/null && echo n || echo y)
+
+# Check whether the compiler supports -m(no-)strict-align
+CC_SUPPORT_STRICT_ALIGN := $(shell $(CC) $(CLANG_TARGET) $(RELAX_FLAG) -nostdlib -mstrict-align -x c /dev/null -o /dev/null 2>&1 | grep -e "-mstrict-align\|-mno-unaligned-access" >/dev/null && echo n || echo y)
 
 # Check whether the assembler and the compiler support the Zicsr and Zifencei extensions
 CC_SUPPORT_ZICSR_ZIFENCEI := $(shell $(CC) $(CLANG_TARGET) $(RELAX_FLAG) -nostdlib -march=rv$(OPENSBI_CC_XLEN)imafd_zicsr_zifencei -x c /dev/null -o /dev/null 2>&1 | grep "zicsr\|zifencei" > /dev/null && echo n || echo y)
 
+ifneq ($(OPENSBI_LD_PIE),y)
+$(error Your linker does not support creating PIEs, opensbi requires this.)
+endif
+
 # Build Info:
 # OPENSBI_BUILD_TIME_STAMP -- the compilation time stamp
 # OPENSBI_BUILD_COMPILER_VERSION -- the compiler version info
@@ -210,24 +221,28 @@ ifdef PLATFORM
 menuconfig: $(platform_src_dir)/Kconfig $(src_dir)/Kconfig
 	$(CMD_PREFIX)mkdir -p $(KCONFIG_DIR)
 	$(CMD_PREFIX)$(src_dir)/scripts/Kconfiglib/menuconfig.py $(src_dir)/Kconfig
-	$(CMD_PREFIX)$(src_dir)/scripts/Kconfiglib/genconfig.py --header-path $(KCONFIG_AUTOHEADER) --sync-deps $(KCONFIG_DIR) --file-list $(KCONFIG_AUTOLIST) $(src_dir)/Kconfig
 
 .PHONY: savedefconfig
 savedefconfig: $(platform_src_dir)/Kconfig $(src_dir)/Kconfig
 	$(CMD_PREFIX)mkdir -p $(KCONFIG_DIR)
 	$(CMD_PREFIX)$(src_dir)/scripts/Kconfiglib/savedefconfig.py --kconfig $(src_dir)/Kconfig --out $(KCONFIG_DIR)/defconfig
 
-$(KCONFIG_CONFIG): $(platform_src_dir)/configs/$(PLATFORM_DEFCONFIG) $(platform_src_dir)/Kconfig $(src_dir)/Kconfig
+$(KCONFIG_CONFIG): $(platform_src_dir)/configs/$(PLATFORM_DEFCONFIG)
 	$(CMD_PREFIX)mkdir -p $(KCONFIG_DIR)
 	$(CMD_PREFIX)$(src_dir)/scripts/Kconfiglib/defconfig.py --kconfig $(src_dir)/Kconfig $(platform_src_dir)/configs/$(PLATFORM_DEFCONFIG)
+
+$(KCONFIG_AUTOCONFIG): $(KCONFIG_CONFIG)
 	$(CMD_PREFIX)$(src_dir)/scripts/Kconfiglib/genconfig.py --header-path $(KCONFIG_AUTOHEADER) --sync-deps $(KCONFIG_DIR) --file-list $(KCONFIG_AUTOLIST) $(src_dir)/Kconfig
 
-$(KCONFIG_AUTOCMD): $(KCONFIG_CONFIG)
-	$(CMD_PREFIX)mkdir -p $(KCONFIG_DIR)
+$(KCONFIG_AUTOHEADER): $(KCONFIG_AUTOCONFIG);
+
+$(KCONFIG_AUTOLIST): $(KCONFIG_AUTOCONFIG);
+
+$(KCONFIG_AUTOCMD): $(KCONFIG_AUTOLIST)
 	$(CMD_PREFIX)printf "%s: " $(KCONFIG_CONFIG) > $(KCONFIG_AUTOCMD)
 	$(CMD_PREFIX)cat $(KCONFIG_AUTOLIST) | tr '\n' ' ' >> $(KCONFIG_AUTOCMD)
 
-include $(KCONFIG_CONFIG)
+include $(KCONFIG_AUTOCONFIG)
 include $(KCONFIG_AUTOCMD)
 endif
 
@@ -337,17 +352,20 @@ CFLAGS		+=	-O0
 else
 CFLAGS		+=	-O2
 endif
-CFLAGS		+=	-fno-omit-frame-pointer -fno-optimize-sibling-calls -mstrict-align
-# enable -m(no-)save-restore option by CC_SUPPORT_SAVE_RESTORE
+CFLAGS		+=	-fno-omit-frame-pointer -fno-optimize-sibling-calls
+# Optionally supported flags
 ifeq ($(CC_SUPPORT_SAVE_RESTORE),y)
 CFLAGS		+=	-mno-save-restore
 endif
+ifeq ($(CC_SUPPORT_STRICT_ALIGN),y)
+CFLAGS		+=	-mstrict-align
+endif
 CFLAGS		+=	-mabi=$(PLATFORM_RISCV_ABI) -march=$(PLATFORM_RISCV_ISA)
 CFLAGS		+=	-mcmodel=$(PLATFORM_RISCV_CODE_MODEL)
 CFLAGS		+=	$(RELAX_FLAG)
 CFLAGS		+=	$(GENFLAGS)
 CFLAGS		+=	$(platform-cflags-y)
-CFLAGS		+=	-fno-pie -no-pie
+CFLAGS		+=	-fPIE -pie
 CFLAGS		+=	$(firmware-cflags-y)
 
 CPPFLAGS	+=	$(GENFLAGS)
@@ -355,11 +373,15 @@ CPPFLAGS	+=	$(platform-cppflags-y)
 CPPFLAGS	+=	$(firmware-cppflags-y)
 
 ASFLAGS		=	-g -Wall -nostdlib
-ASFLAGS		+=	-fno-omit-frame-pointer -fno-optimize-sibling-calls -mstrict-align
-# enable -m(no-)save-restore option by CC_SUPPORT_SAVE_RESTORE
+ASFLAGS		+=	-fno-omit-frame-pointer -fno-optimize-sibling-calls
+ASFLAGS		+=	-fPIE
+# Optionally supported flags
 ifeq ($(CC_SUPPORT_SAVE_RESTORE),y)
 ASFLAGS		+=	-mno-save-restore
 endif
+ifeq ($(CC_SUPPORT_STRICT_ALIGN),y)
+ASFLAGS		+=	-mstrict-align
+endif
 ASFLAGS		+=	-mabi=$(PLATFORM_RISCV_ABI) -march=$(PLATFORM_RISCV_ISA)
 ASFLAGS		+=	-mcmodel=$(PLATFORM_RISCV_CODE_MODEL)
 ASFLAGS		+=	$(RELAX_FLAG)
@@ -375,7 +397,11 @@ ASFLAGS		+=	$(firmware-asflags-y)
 ARFLAGS		=	rcs
 
 ELFFLAGS	+=	$(USE_LD_FLAG)
+ifeq ($(OPENSBI_LD_EXCLUDE_LIBS),y)
+ELFFLAGS	+=	-Wl,--exclude-libs,ALL
+endif
 ELFFLAGS	+=	-Wl,--build-id=none
+ELFFLAGS	+=	-Wl,--no-dynamic-linker -Wl,-pie
 ELFFLAGS	+=	$(platform-ldflags-y)
 ELFFLAGS	+=	$(firmware-ldflags-y)
 
@@ -489,14 +515,14 @@ $(build_dir)/lib/libsbi.a: $(libsbi-objs-path-y)
 $(platform_build_dir)/lib/libplatsbi.a: $(libsbi-objs-path-y) $(libsbiutils-objs-path-y) $(platform-objs-path-y)
 	$(call compile_ar,$@,$^)
 
-$(build_dir)/%.dep: $(src_dir)/%.carray $(KCONFIG_CONFIG)
-	$(call compile_gen_dep,$@,.c,$< $(KCONFIG_CONFIG))
+$(build_dir)/%.dep: $(src_dir)/%.carray $(KCONFIG_AUTOHEADER)
+	$(call compile_gen_dep,$@,.c,$< $(KCONFIG_AUTOHEADER))
 	$(call compile_gen_dep,$@,.o,$(@:.dep=.c))
 
 $(build_dir)/%.c: $(src_dir)/%.carray
 	$(call compile_carray,$@,$<)
 
-$(build_dir)/%.dep: $(src_dir)/%.c $(KCONFIG_CONFIG)
+$(build_dir)/%.dep: $(src_dir)/%.c $(KCONFIG_AUTOHEADER)
 	$(call compile_cc_dep,$@,$<)
 
 $(build_dir)/%.o: $(src_dir)/%.c
@@ -510,24 +536,24 @@ $(build_dir)/lib/sbi/sbi_init.o: $(libsbi_dir)/sbi_init.c FORCE
 	$(call compile_cc,$@,$<)
 endif
 
-$(build_dir)/%.dep: $(src_dir)/%.S $(KCONFIG_CONFIG)
+$(build_dir)/%.dep: $(src_dir)/%.S $(KCONFIG_AUTOHEADER)
 	$(call compile_as_dep,$@,$<)
 
 $(build_dir)/%.o: $(src_dir)/%.S
 	$(call compile_as,$@,$<)
 
 # Rules for platform sources
-$(platform_build_dir)/%.dep: $(platform_src_dir)/%.carray $(KCONFIG_CONFIG)
-	$(call compile_gen_dep,$@,.c,$< $(KCONFIG_CONFIG))
+$(platform_build_dir)/%.dep: $(platform_src_dir)/%.carray $(KCONFIG_AUTOHEADER)
+	$(call compile_gen_dep,$@,.c,$< $(KCONFIG_AUTOHEADER))
 	$(call compile_gen_dep,$@,.o,$(@:.dep=.c))
 
 $(platform_build_dir)/%.c: $(platform_src_dir)/%.carray
 	$(call compile_carray,$@,$<)
 
-$(platform_build_dir)/%.dep: $(platform_src_dir)/%.c $(KCONFIG_CONFIG)
+$(platform_build_dir)/%.dep: $(platform_src_dir)/%.c $(KCONFIG_AUTOHEADER)
 	$(call compile_cc_dep,$@,$<)
 
-$(platform_build_dir)/%.o: $(platform_src_dir)/%.c $(KCONFIG_CONFIG)
+$(platform_build_dir)/%.o: $(platform_src_dir)/%.c $(KCONFIG_AUTOHEADER)
 	$(call compile_cc,$@,$<)
 
 $(platform_build_dir)/%.dep: $(platform_src_dir)/%.S
@@ -536,8 +562,8 @@ $(platform_build_dir)/%.dep: $(platform_src_dir)/%.S
 $(platform_build_dir)/%.o: $(platform_src_dir)/%.S
 	$(call compile_as,$@,$<)
 
-$(platform_build_dir)/%.dep: $(platform_src_dir)/%.dts $(KCONFIG_CONFIG)
-	$(call compile_gen_dep,$@,.dtb,$< $(KCONFIG_CONFIG))
+$(platform_build_dir)/%.dep: $(platform_src_dir)/%.dts $(KCONFIG_AUTOHEADER)
+	$(call compile_gen_dep,$@,.dtb,$< $(KCONFIG_AUTOHEADER))
 	$(call compile_gen_dep,$@,.c,$(@:.dep=.dtb))
 	$(call compile_gen_dep,$@,.o,$(@:.dep=.c))
 
@@ -554,26 +580,26 @@ $(platform_build_dir)/%.bin: $(platform_build_dir)/%.elf
 $(platform_build_dir)/%.elf: $(platform_build_dir)/%.o $(platform_build_dir)/%.elf.ld $(platform_build_dir)/lib/libplatsbi.a
 	$(call compile_elf,$@,$@.ld,$< $(platform_build_dir)/lib/libplatsbi.a)
 
-$(platform_build_dir)/%.dep: $(src_dir)/%.ldS $(KCONFIG_CONFIG)
+$(platform_build_dir)/%.dep: $(src_dir)/%.ldS $(KCONFIG_AUTOHEADER)
 	$(call compile_cpp_dep,$@,.ld,$<)
 
 $(platform_build_dir)/%.ld: $(src_dir)/%.ldS
 	$(call compile_cpp,$@,$<)
 
-$(platform_build_dir)/%.dep: $(src_dir)/%.carray $(KCONFIG_CONFIG)
-	$(call compile_gen_dep,$@,.c,$< $(KCONFIG_CONFIG))
+$(platform_build_dir)/%.dep: $(src_dir)/%.carray $(KCONFIG_AUTOHEADER)
+	$(call compile_gen_dep,$@,.c,$< $(KCONFIG_AUTOHEADER))
 	$(call compile_gen_dep,$@,.o,$(@:.dep=.c))
 
 $(platform_build_dir)/%.c: $(src_dir)/%.carray
 	$(call compile_carray,$@,$<)
 
-$(platform_build_dir)/%.dep: $(src_dir)/%.c $(KCONFIG_CONFIG)
+$(platform_build_dir)/%.dep: $(src_dir)/%.c $(KCONFIG_AUTOHEADER)
 	$(call compile_cc_dep,$@,$<)
 
 $(platform_build_dir)/%.o: $(src_dir)/%.c
 	$(call compile_cc,$@,$<)
 
-$(platform_build_dir)/%.dep: $(src_dir)/%.S $(KCONFIG_CONFIG)
+$(platform_build_dir)/%.dep: $(src_dir)/%.S $(KCONFIG_AUTOHEADER)
 	$(call compile_as_dep,$@,$<)
 
 $(platform_build_dir)/%.o: $(src_dir)/%.S

README.md

@@ -276,8 +276,7 @@ document.
 
 NOTE: Using Clang with a `riscv*-linux-gnu` GNU binutils linker has been seen
 to produce broken binaries with missing relocations; it is therefore currently
-recommended that this combination be avoided or *FW_PIC=n* be used to disable
-building OpenSBI as a position-independent binary.
+recommended that this combination be avoided.
 
 Building with timestamp and compiler info
 -----------------------------------------

docs/domain_support.md

@@ -41,6 +41,7 @@ has following details:
 * **name** - Name of this domain
 * **assigned_harts** - HARTs assigned to this domain
 * **possible_harts** - HARTs possible in this domain
+* **hartindex_to_context_table** - Contexts corresponding to possible HARTs
 * **regions** - Array of memory regions terminated by a memory region
   with order zero
 * **boot_hartid** - HART id of the HART booting this domain. The domain
@@ -80,6 +81,7 @@ following manner:
   platform support
 * **possible_harts** - All valid HARTs of a RISC-V platform are possible
   HARTs of the ROOT domain
+* **hartindex_to_context_table** - Contexts corresponding to ROOT domain's possible HARTs
 * **regions** - Two memory regions available to the ROOT domain:
   **A)** A memory region to protect OpenSBI firmware from S-mode and U-mode
   **B)** A memory region of **order=__riscv_xlen** allowing S-mode and
@@ -126,9 +128,6 @@ The DT properties of a domain configuration DT node are as follows:
 * **compatible** (Mandatory) - The compatible string of the domain
   configuration. This DT property should have value *"opensbi,domain,config"*
 
-* **system-suspend-test** (Optional) - When present, enable a system
-  suspend test implementation which simply waits five seconds and issues a WFI.
-
 ### Domain Memory Region Node
 
 The domain memory region DT node describes details of a memory region and
@@ -237,7 +236,6 @@ be done:
     chosen {
         opensbi-domains {
             compatible = "opensbi,domain,config";
-            system-suspend-test;
 
             tmem: tmem {
                 compatible = "opensbi,domain,memregion";

docs/doxygen.cfg

@@ -796,6 +796,8 @@ INPUT                  = @@SRC_DIR@@/README.md \
                          @@SRC_DIR@@/docs/platform_requirements.md \
                          @@SRC_DIR@@/docs/library_usage.md \
                          @@SRC_DIR@@/docs/domain_support.md \
+                         @@SRC_DIR@@/docs/opensbi_config.md \
+                         @@SRC_DIR@@/docs/writing_tests.md \
                          @@SRC_DIR@@/docs/firmware \
                          @@SRC_DIR@@/docs/platform \
                          @@SRC_DIR@@/include \

docs/firmware/fw.md

@@ -61,20 +61,15 @@ Firmware Configuration and Compilation
 All firmware types support the following common compile time configuration
 parameters:
 
-* **FW_TEXT_START** - Defines the execution address of the OpenSBI firmware.
-  This configuration parameter is mandatory.
+* **FW_TEXT_START** - Defines the compile time address of the OpenSBI
+  firmware. This configuration parameter is optional and assumed to be
+  `0` if not specified.
 * **FW_FDT_PATH** - Path to an external flattened device tree binary file to
   be embedded in the *.rodata* section of the final firmware. If this option
   is not provided then the firmware will expect the FDT to be passed as an
   argument by the prior booting stage.
 * **FW_FDT_PADDING** - Optional zero bytes padding to the embedded flattened
   device tree binary file specified by **FW_FDT_PATH** option.
-* **FW_PIC** - "FW_PIC=y" generates position independent executable firmware
-  images. OpenSBI can run at arbitrary address with appropriate alignment.
-  Therefore, the original relocation mechanism ("FW_PIC=n") will be skipped.
-  In other words, OpenSBI will directly run at the load address without any
-  code movement. This option requires a toolchain with PIE support, and it
-  is on by default.
 
 Additionally, each firmware type as a set of type specific configuration
 parameters. Detailed information for each firmware type can be found in the

docs/firmware/fw_jump.md

@@ -31,9 +31,14 @@ follows:
 
 * **FW_JUMP_ADDR** - Address of the entry point of the booting stage to be
   executed following OpenSBI firmware. This address generally corresponds
-  exactly to the address where this next booting stage was loaded. This is a
-  mandatory parameter. Compilation errors will result from not defining this
-  address.
+  exactly to the address where this next booting stage was loaded.
+  At least one of *FW_JUMP_ADDR* and *FW_JUMP_OFFSET* (see below) should be
+  defined. Compilation errors will result from not defining one of them.
+
+* **FW_JUMP_OFFSET** - Address offset from the opensbi load address where the
+  entry point of the next booting stage is located. This offset is used as
+  relocatable address of the next booting stage entry point. If *FW_JUMP_ADDR*
+  is also defined, the firmware will prefer *FW_JUMP_ADDR*.
 
 * **FW_JUMP_FDT_ADDR** - Address where the *flattened device tree (FDT file)*
   passed by the prior booting stage will be placed in memory before executing
@@ -57,6 +62,12 @@ follows:
   echo fdt overlaps kernel, increase FW_JUMP_FDT_ADDR
   ```
 
+* **FW_JUMP_FDT_OFFSET** - Address offset from the opensbi load address where
+  the FDT will be passed to the next booting stage. This offset is used
+  as relocatable address of the FDT passed to the next booting stage. If
+  *FW_JUMP_FDT_ADDR* is also defined, the firmware will prefer
+  *FW_JUMP_FDT_ADDR*.
+
 *FW_JUMP* Example
 -----------------
 

docs/firmware/fw_payload.md

@@ -23,7 +23,7 @@ The *FW_PAYLOAD* firmware can be enabled by any of the following methods:
 2. Specifying `FW_PAYLOAD=y` in the target platform *objects.mk* configuration
    file.
 
-The compiled *FW_PAYLOAD* firmware ELF file is named *fw_jump.elf*. Its
+The compiled *FW_PAYLOAD* firmware ELF file is named *fw_payload.elf*. Its
 expanded image file is *fw_payload.bin*. Both files are created in the
 platform-specific build directory under the
 *build/platform/<platform_subdir>/firmware* directory.
@@ -36,8 +36,8 @@ options. These configuration parameters can be defined using either the top
 level `make` command line or the target platform *objects.mk* configuration
 file. The parameters currently defined are as follows:
 
-* **FW_PAYLOAD_OFFSET** - Offset from *FW_TEXT_BASE* where the payload binary
-  will be linked in the final *FW_PAYLOAD* firmware binary image. This
+* **FW_PAYLOAD_OFFSET** - Offset from the opensbi load address where the payload
+  binary will be linked in the final *FW_PAYLOAD* firmware binary image. This
   configuration parameter is mandatory if *FW_PAYLOAD_ALIGN* is not defined.
   Compilation errors will result from an incorrect definition of
   *FW_PAYLOAD_OFFSET* or of *FW_PAYLOAD_ALIGN*, or if neither of these
@@ -62,6 +62,11 @@ file. The parameters currently defined are as follows:
   firmware will pass the FDT address passed by the previous booting stage
   to the next booting stage.
 
+* **FW_PAYLOAD_FDT_OFFSET** - Address offset from the opensbi load address where
+  the FDT will be passed to the next booting stage. This offset is used as
+  relocatable address of the FDT passed to the next booting stage. If
+  *FW_PAYLOAD_FDT_ADDR* is also defined, the firmware will prefer *FW_PAYLOAD_FDT_ADDR*.
+
 *FW_PAYLOAD* Example
 --------------------
 

docs/opensbi_config.md

@@ -0,0 +1,87 @@
+OpenSBI Device Tree Configuration Guideline
+==================================
+
+Some configurations of OpenSBI's Generic Platform can be described
+in the **device tree (DT) blob** (or flattened device tree) passed
+to the OpenSBI firmwares by the previous booting stage. OpenSBI will
+parse and use these configurations during the boot phase, but delete
+them from the device tree at the end of cold boot.
+
+### OpenSBI Configuration Node
+
+All nodes related to OpenSBI configuration should be under the OpenSBI
+configuration DT node. The **/chosen** DT node is the preferred parent
+of the OpenSBI configuration DT node.
+
+The DT properties of a domain configuration DT node are as follows:
+
+* **compatible** (Mandatory) - The compatible string of the OpenSBI
+  configuration. This DT property should have value *"opensbi,config"*
+
+* **cold-boot-harts** (Optional) - If a platform lacks an override
+  cold_boot_allowed() mechanism, this DT property specifies that a
+  set of harts is permitted to perform a cold boot. Otherwise, all
+  harts are allowed to cold boot.
+
+* **system-suspend-test** (Optional) - When present, enable a system
+  suspend test implementation which simply waits five seconds and issues a WFI.
+
+The OpenSBI Configuration Node will be deleted at the end of cold boot
+(replace the node (subtree) with nop tags).
+
+### Example
+
+```text
+    chosen {
+        opensbi-config {
+            compatible = "opensbi,config";
+            cold-boot-harts = <&cpu1 &cpu2 &cpu3 &cpu4>;
+            system-suspend-test;
+        };
+    };
+
+    cpus {
+        #address-cells = <1>;
+        #size-cells = <0>;
+        timebase-frequency = <10000000>;
+
+        cpu0: cpu@0 {
+            device_type = "cpu";
+            reg = <0x00>;
+            compatible = "riscv";
+            ...
+        };
+
+        cpu1: cpu@1 {
+            device_type = "cpu";
+            reg = <0x01>;
+            compatible = "riscv";
+            ...
+        };
+
+        cpu2: cpu@2 {
+            device_type = "cpu";
+            reg = <0x02>;
+            compatible = "riscv";
+            ...
+        };
+
+        cpu3: cpu@3 {
+            device_type = "cpu";
+            reg = <0x03>;
+            compatible = "riscv";
+            ...
+        };
+
+        cpu4: cpu@4 {
+            device_type = "cpu";
+            reg = <0x04>;
+            compatible = "riscv";
+            ...
+        };
+    };
+
+    uart1: serial@10011000 {
+        ...
+    };
+```

docs/platform/generic.md

@@ -9,10 +9,9 @@ boards.
 
 By default, the generic FDT platform makes following assumptions:
 
-1. platform FW_TEXT_START is 0x80000000
-2. platform features are default
-3. platform stack size is default
-4. platform has no quirks or work-arounds
+1. platform features are default
+2. platform stack size is default
+3. platform has no quirks or work-arounds
 
 The above assumptions (except 1) can be overridden by adding special platform
 callbacks which will be called based on FDT root node compatible string.
@@ -33,10 +32,6 @@ Users of the generic FDT platform will have to ensure that:
 To build the platform-specific library and firmware images, provide the
 *PLATFORM=generic* parameter to the top level `make` command.
 
-For custom FW_TEXT_START, we can build the platform-specific library and
-firmware images by passing *PLATFORM=generic FW_TEXT_START=<custom_text_start>*
-parameter to the top level `make` command.
-
 Platform Options
 ----------------
 

docs/platform/thead-c9xx.md

@@ -1,7 +1,7 @@
 T-HEAD C9xx Series Processors
 =============================
 
-The **C9xx** series processors are high-performance RISC-V architecture
+The C9xx series processors are high-performance RISC-V architecture
 multi-core processors with AI vector acceleration engine.
 
 For more details, refer [T-HEAD.CN](https://www.t-head.cn/)
@@ -12,185 +12,16 @@ To build the platform-specific library and firmware images, provide the
 Platform Options
 ----------------
 
-The *T-HEAD C9xx* does not have any platform-specific compile options
+The T-HEAD C9xx does not have any platform-specific compile options
 because it uses generic platform.
 
 ```
-CROSS_COMPILE=riscv64-linux-gnu- PLATFORM=generic /usr/bin/make
+CROSS_COMPILE=riscv64-linux-gnu- PLATFORM=generic make
 ```
 
-The *T-HEAD C9xx* DTB provided to OpenSBI generic firmwares will usually have
-"riscv,clint0", "riscv,plic0", "thead,reset-sample" compatible strings.
+Here is the simplest boot flow for a fpga prototype:
 
-DTS Example1: (Single core, eg: Allwinner D1 - c906)
-----------------------------------------------------
+ (Jtag gdbinit) -> (zsb) -> (opensbi) -> (linux)
 
-```
-	cpus {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		timebase-frequency = <3000000>;
-		cpu@0 {
-			device_type = "cpu";
-			reg = <0>;
-			status = "okay";
-			compatible = "riscv";
-			riscv,isa = "rv64imafdcv";
-			mmu-type = "riscv,sv39";
-			cpu0_intc: interrupt-controller {
-				#interrupt-cells = <1>;
-				compatible = "riscv,cpu-intc";
-				interrupt-controller;
-			};
-		};
-	};
-
-	soc {
-		#address-cells = <2>;
-		#size-cells = <2>;
-		compatible = "simple-bus";
-		ranges;
-
-		clint0: clint@14000000 {
-			compatible = "allwinner,sun20i-d1-clint";
-			interrupts-extended = <
-				&cpu0_intc  3 &cpu0_intc  7
-				>;
-			reg = <0x0 0x14000000 0x0 0x04000000>;
-		};
-
-		intc: interrupt-controller@10000000 {
-			#interrupt-cells = <1>;
-			compatible = "allwinner,sun20i-d1-plic",
-				     "thead,c900-plic";
-			interrupt-controller;
-			interrupts-extended = <
-				&cpu0_intc  0xffffffff &cpu0_intc  9
-				>;
-			reg = <0x0 0x10000000 0x0 0x04000000>;
-			reg-names = "control";
-			riscv,max-priority = <7>;
-			riscv,ndev = <200>;
-		};
-	}
-```
-
-DTS Example2: (Multi cores with soc reset-regs)
------------------------------------------------
-
-```
-	cpus {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		timebase-frequency = <3000000>;
-		cpu@0 {
-			device_type = "cpu";
-			reg = <0>;
-			status = "okay";
-			compatible = "riscv";
-			riscv,isa = "rv64imafdc";
-			mmu-type = "riscv,sv39";
-			cpu0_intc: interrupt-controller {
-				#interrupt-cells = <1>;
-				compatible = "riscv,cpu-intc";
-				interrupt-controller;
-			};
-		};
-		cpu@1 {
-			device_type = "cpu";
-			reg = <1>;
-			status = "fail";
-			compatible = "riscv";
-			riscv,isa = "rv64imafdc";
-			mmu-type = "riscv,sv39";
-			cpu1_intc: interrupt-controller {
-				#interrupt-cells = <1>;
-				compatible = "riscv,cpu-intc";
-				interrupt-controller;
-			};
-		};
-		cpu@2 {
-			device_type = "cpu";
-			reg = <2>;
-			status = "fail";
-			compatible = "riscv";
-			riscv,isa = "rv64imafdc";
-			mmu-type = "riscv,sv39";
-			cpu2_intc: interrupt-controller {
-				#interrupt-cells = <1>;
-				compatible = "riscv,cpu-intc";
-				interrupt-controller;
-			};
-		};
-		cpu@3 {
-			device_type = "cpu";
-			reg = <3>;
-			status = "fail";
-			compatible = "riscv";
-			riscv,isa = "rv64imafdc";
-			mmu-type = "riscv,sv39";
-			cpu3_intc: interrupt-controller {
-				#interrupt-cells = <1>;
-				compatible = "riscv,cpu-intc";
-				interrupt-controller;
-			};
-		};
-	};
-
-	soc {
-		#address-cells = <2>;
-		#size-cells = <2>;
-		compatible = "simple-bus";
-		ranges;
-
-		reset: reset-sample {
-			compatible = "thead,reset-sample";
-			entry-reg = <0xff 0xff019050>;
-			entry-cnt = <4>;
-			control-reg = <0xff 0xff015004>;
-			control-val = <0x1c>;
-			csr-copy = <0x7f3 0x7c0 0x7c1 0x7c2 0x7c3 0x7c5 0x7cc>;
-		};
-
-		clint0: clint@ffdc000000 {
-			compatible = "riscv,clint0";
-			interrupts-extended = <
-				&cpu0_intc  3 &cpu0_intc  7
-				&cpu1_intc  3 &cpu1_intc  7
-				&cpu2_intc  3 &cpu2_intc  7
-				&cpu3_intc  3 &cpu3_intc  7
-				&cpu4_intc  3 &cpu4_intc  7
-				>;
-			reg = <0xff 0xdc000000 0x0 0x04000000>;
-		};
-
-		intc: interrupt-controller@ffd8000000 {
-			#interrupt-cells = <1>;
-			compatible = "thead,c900-plic";
-			interrupt-controller;
-			interrupts-extended = <
-				&cpu0_intc  0xffffffff &cpu0_intc  9
-				&cpu1_intc  0xffffffff &cpu1_intc  9
-				&cpu2_intc  0xffffffff &cpu2_intc  9
-				&cpu3_intc  0xffffffff &cpu3_intc  9
-				>;
-			reg = <0xff 0xd8000000 0x0 0x04000000>;
-			reg-names = "control";
-			riscv,max-priority = <7>;
-			riscv,ndev = <80>;
-		};
-	}
-```
-
-DTS Example2: (Multi cores with old reset csrs)
------------------------------------------------
-```
-	reset: reset-sample {
-		compatible = "thead,reset-sample";
-		using-csr-reset;
-		csr-copy = <0x7c0 0x7c1 0x7c2 0x7c3 0x7c5 0x7cc
-			    0x3b0 0x3b1 0x3b2 0x3b3
-			    0x3b4 0x3b5 0x3b6 0x3b7
-			    0x3a0>;
-	};
-```
+For more details, refer:
+ [zero stage boot](https://github.com/c-sky/zero_stage_boot)

docs/platform_requirements.md

@@ -18,7 +18,7 @@ Base Platform Requirements
 
 The base RISC-V platform requirements for OpenSBI are as follows:
 
-1. At least rv32ima or rv64ima required on all HARTs
+1. At least rv32ima_zicsr or rv64ima_zicsr required on all HARTs
 2. At least one HART should have S-mode support because:
 
      * SBI calls are meant for RISC-V S-mode (Supervisor mode)
@@ -33,7 +33,7 @@ The base RISC-V platform requirements for OpenSBI are as follows:
 6. Hardware support for injecting M-mode software interrupts on
    a multi-HART platform
 
-The RISC-V extensions not covered by rv32ima or rv64ima are optional
+The RISC-V extensions not covered by rv32ima_zicsr or rv64ima_zicsr are optional
 for OpenSBI. Although, OpenSBI will detect and handle some of these
 optional RISC-V extensions at runtime.
 

docs/pmu_support.md

@@ -125,3 +125,85 @@ pmu {
 					  <0x0 0x2 0xffffffff 0xffffe0ff 0x18>;
 };
 ```
+
+### Example 3
+
+```
+/*
+ * For Andes 45-series platforms. The encodings can be found in the
+ * "Machine Performance Monitoring Event Selector" section
+ * http://www.andestech.com/wp-content/uploads/AX45MP-1C-Rev.-5.0.0-Datasheet.pdf
+ */
+pmu {
+	compatible 			= "riscv,pmu";
+	riscv,event-to-mhpmevent =
+					 <0x1 0x0000 0x10>, /* CPU_CYCLES          -> Cycle count */
+					 <0x2 0x0000 0x20>, /* INSTRUCTIONS        -> Retired instruction count */
+					 <0x3 0x0000 0x41>, /* CACHE_REFERENCES    -> D-Cache access */
+					 <0x4 0x0000 0x51>, /* CACHE_MISSES        -> D-Cache miss */
+					 <0x5 0x0000 0x80>, /* BRANCH_INSTRUCTIONS -> Conditional branch instruction count */
+					 <0x6 0x0000 0x02>, /* BRANCH_MISSES       -> Misprediction of conditional branches */
+					 <0x10000 0x0000 0x61>,  /* L1D_READ_ACCESS  -> D-Cache load access */
+					 <0x10001 0x0000 0x71>,  /* L1D_READ_MISS    -> D-Cache load miss */
+					 <0x10002 0x0000 0x81>,  /* L1D_WRITE_ACCESS -> D-Cache store access */
+					 <0x10003 0x0000 0x91>,  /* L1D_WRITE_MISS   -> D-Cache store miss */
+					 <0x10008 0x0000 0x21>,  /* L1I_READ_ACCESS  -> I-Cache access */
+					 <0x10009 0x0000 0x31>;  /* L1I_READ_MISS    -> I-Cache miss */
+	riscv,event-to-mhpmcounters = <0x1 0x6 0x78>,
+							<0x10000 0x10003 0x78>,
+							<0x10008 0x10009 0x78>;
+	riscv,raw-event-to-mhpmcounters =
+						<0x0 0x10 0xffffffff 0xffffffff 0x78>, /* Cycle count */
+						<0x0 0x20 0xffffffff 0xffffffff 0x78>, /* Retired instruction count */
+						<0x0 0x30 0xffffffff 0xffffffff 0x78>, /* Integer load instruction count */
+						<0x0 0x40 0xffffffff 0xffffffff 0x78>, /* Integer store instruction count */
+						<0x0 0x50 0xffffffff 0xffffffff 0x78>, /* Atomic instruction count */
+						<0x0 0x60 0xffffffff 0xffffffff 0x78>, /* System instruction count */
+						<0x0 0x70 0xffffffff 0xffffffff 0x78>, /* Integer computational instruction count */
+						<0x0 0x80 0xffffffff 0xffffffff 0x78>, /* Conditional branch instruction count */
+						<0x0 0x90 0xffffffff 0xffffffff 0x78>, /* Taken conditional branch instruction count */
+						<0x0 0xA0 0xffffffff 0xffffffff 0x78>, /* JAL instruction count */
+						<0x0 0xB0 0xffffffff 0xffffffff 0x78>, /* JALR instruction count */
+						<0x0 0xC0 0xffffffff 0xffffffff 0x78>, /* Return instruction count */
+						<0x0 0xD0 0xffffffff 0xffffffff 0x78>, /* Control transfer instruction count */
+						<0x0 0xE0 0xffffffff 0xffffffff 0x78>, /* EXEC.IT instruction count */
+						<0x0 0xF0 0xffffffff 0xffffffff 0x78>, /* Integer multiplication instruction count */
+						<0x0 0x100 0xffffffff 0xffffffff 0x78>, /* Integer division instruction count */
+						<0x0 0x110 0xffffffff 0xffffffff 0x78>, /* Floating-point load instruction count */
+						<0x0 0x120 0xffffffff 0xffffffff 0x78>, /* Floating-point store instruction count */
+						<0x0 0x130 0xffffffff 0xffffffff 0x78>, /* Floating-point addition/subtraction instruction count */
+						<0x0 0x140 0xffffffff 0xffffffff 0x78>, /* Floating-point multiplication instruction count */
+						<0x0 0x150 0xffffffff 0xffffffff 0x78>, /* Floating-point fused multiply-add instruction count */
+						<0x0 0x160 0xffffffff 0xffffffff 0x78>, /* Floating-point division or square-root instruction count */
+						<0x0 0x170 0xffffffff 0xffffffff 0x78>, /* Other floating-point instruction count */
+						<0x0 0x180 0xffffffff 0xffffffff 0x78>, /* Integer multiplication and add/sub instruction count */
+						<0x0 0x190 0xffffffff 0xffffffff 0x78>, /* Retired operation count */
+						<0x0 0x01 0xffffffff 0xffffffff 0x78>, /* ILM access */
+						<0x0 0x11 0xffffffff 0xffffffff 0x78>, /* DLM access */
+						<0x0 0x21 0xffffffff 0xffffffff 0x78>, /* I-Cache access */
+						<0x0 0x31 0xffffffff 0xffffffff 0x78>, /* I-Cache miss */
+						<0x0 0x41 0xffffffff 0xffffffff 0x78>, /* D-Cache access */
+						<0x0 0x51 0xffffffff 0xffffffff 0x78>, /* D-Cache miss */
+						<0x0 0x61 0xffffffff 0xffffffff 0x78>, /* D-Cache load access */
+						<0x0 0x71 0xffffffff 0xffffffff 0x78>, /* D-Cache load miss */
+						<0x0 0x81 0xffffffff 0xffffffff 0x78>, /* D-Cache store access */
+						<0x0 0x91 0xffffffff 0xffffffff 0x78>, /* D-Cache store miss */
+						<0x0 0xA1 0xffffffff 0xffffffff 0x78>, /* D-Cache writeback */
+						<0x0 0xB1 0xffffffff 0xffffffff 0x78>, /* Cycles waiting for I-Cache fill data */
+						<0x0 0xC1 0xffffffff 0xffffffff 0x78>, /* Cycles waiting for D-Cache fill data */
+						<0x0 0xD1 0xffffffff 0xffffffff 0x78>, /* Uncached fetch data access from bus */
+						<0x0 0xE1 0xffffffff 0xffffffff 0x78>, /* Uncached load data access from bus */
+						<0x0 0xF1 0xffffffff 0xffffffff 0x78>, /* Cycles waiting for uncached fetch data from bus */
+						<0x0 0x101 0xffffffff 0xffffffff 0x78>, /* Cycles waiting for uncached load data from bus */
+						<0x0 0x111 0xffffffff 0xffffffff 0x78>, /* Main ITLB access */
+						<0x0 0x121 0xffffffff 0xffffffff 0x78>, /* Main ITLB miss */
+						<0x0 0x131 0xffffffff 0xffffffff 0x78>, /* Main DTLB access */
+						<0x0 0x141 0xffffffff 0xffffffff 0x78>, /* Main DTLB miss */
+						<0x0 0x151 0xffffffff 0xffffffff 0x78>, /* Cycles waiting for Main ITLB fill data */
+						<0x0 0x161 0xffffffff 0xffffffff 0x78>, /* Pipeline stall cycles caused by Main DTLB miss */
+						<0x0 0x171 0xffffffff 0xffffffff 0x78>, /* Hardware prefetch bus access */
+						<0x0 0x02 0xffffffff 0xffffffff 0x78>, /* Misprediction of conditional branches */
+						<0x0 0x12 0xffffffff 0xffffffff 0x78>, /* Misprediction of taken conditional branches */
+						<0x0 0x22 0xffffffff 0xffffffff 0x78>; /* Misprediction of targets of Return instructions */
+};
+```

docs/writing_tests.md

@@ -0,0 +1,129 @@
+Writing tests for OpenSBI
+=========================
+
+SBIUnit
+-------
+SBIUnit is a set of macros and functions which simplify the test development and
+automate the test execution and evaluation. All of the SBIUnit definitions are
+in the `include/sbi/sbi_unit_test.h` header file, and implementations are
+available in `lib/sbi/tests/sbi_unit_test.c`.
+
+Simple SBIUnit test
+-------------------
+
+For instance, we would like to test the following function from
+`lib/sbi/sbi_string.c`:
+
+```c
+size_t sbi_strlen(const char *str)
+{
+	unsigned long ret = 0;
+
+	while (*str != '\0') {
+		ret++;
+		str++;
+	}
+
+	return ret;
+}
+```
+
+which calculates the string length.
+
+Create the file `lib/sbi/tests/sbi_string_test.c` with the following content:
+
+```c
+#include <sbi/sbi_unit_test.h>
+#include <sbi/sbi_string.h>
+
+static void strlen_test(struct sbiunit_test_case *test)
+{
+	SBIUNIT_EXPECT_EQ(test, sbi_strlen("Hello"), 5);
+	SBIUNIT_EXPECT_EQ(test, sbi_strlen("Hell\0o"), 4);
+}
+
+static struct sbiunit_test_case string_test_cases[] = {
+	SBIUNIT_TEST_CASE(strlen_test),
+	SBIUNIT_END_CASE,
+};
+
+SBIUNIT_TEST_SUITE(string_test_suite, string_test_cases);
+```
+
+Then, add the corresponding Makefile entries to `lib/sbi/tests/objects.mk`:
+```lang-makefile
+...
+carray-sbi_unit_tests-$(CONFIG_SBIUNIT) += string_test_suite
+libsbi-objs-$(CONFIG_SBIUNIT) += tests/sbi_string_test.o
+```
+
+If you compiled OpenSBI with CONFIG_SBIUNIT enabled before, you may need to
+manually remove the build folder in order to regenerate the carray files:
+`rm -rf build/`.
+
+Recompile OpenSBI with the CONFIG_SBIUNIT option enabled and run it in QEMU.
+You will see something like this:
+```
+# make PLATFORM=generic run
+...
+# Running SBIUNIT tests #
+...
+## Running test suite: string_test_suite
+[PASSED] strlen_test
+1 PASSED / 0 FAILED / 1 TOTAL
+```
+
+Now let's try to change this test in the way that it will fail:
+
+```c
+- SBIUNIT_EXPECT_EQ(test, sbi_strlen("Hello"), 5);
++ SBIUNIT_EXPECT_EQ(test, sbi_strlen("Hello"), 100);
+```
+
+`make all` and `make run` it again:
+```
+...
+# Running SBIUNIT tests #
+...
+## Running test suite: string_test_suite
+[SBIUnit] [.../opensbi/lib/sbi/tests/sbi_string_test.c:6]: strlen_test: Condition "(sbi_strlen("Hello")) == (100)" expected to be true!
+[FAILED] strlen_test
+0 PASSED / 1 FAILED / 1 TOTAL
+```
+Covering the static functions / using the static definitions
+------------------------------------------------------------
+
+SBIUnit also allows you to test static functions. In order to do so, simply
+include your test source in the file you would like to test. Complementing the
+example above, just add this to the `lib/sbi/sbi_string.c` file:
+
+```c
+#ifdef CONFIG_SBIUNIT
+#include "tests/sbi_string_test.c"
+#endif
+```
+
+In this case you should only add a new carray entry pointing to the test suite
+to `lib/sbi/tests/objects.mk`:
+```lang-makefile
+...
+carray-sbi_unit_tests-$(CONFIG_SBIUNIT) += string_test_suite
+```
+
+You don't have to compile the `sbi_string_test.o` separately, because the
+test code will be included into the `sbi_string` object file.
+
+"Mocking" the structures
+------------------------
+See the example of structure "mocking" in `lib/sbi/tests/sbi_console_test.c`,
+where the sbi_console_device structure was mocked to be used in various
+console-related functions in order to test them.
+
+API Reference
+-------------
+All of the `SBIUNIT_EXPECT_*` macros will cause a test case to fail if the
+corresponding conditions are not met, however, the execution of a particular
+test case will not be stopped.
+
+All of the `SBIUNIT_ASSERT_*` macros will cause a test case to fail and stop
+immediately, triggering a panic.

firmware/fw_base.S

@@ -14,7 +14,7 @@
 #include <sbi/sbi_scratch.h>
 #include <sbi/sbi_trap.h>
 
-#define BOOT_STATUS_RELOCATE_DONE	1
+#define BOOT_STATUS_LOTTERY_DONE	1
 #define BOOT_STATUS_BOOT_HART_DONE	2
 
 .macro	MOV_3R __d0, __s0, __d1, __s1, __d2, __s2
@@ -31,17 +31,6 @@
 	add	\__d4, \__s4, zero
 .endm
 
-/*
- * If __start_reg <= __check_reg and __check_reg < __end_reg then
- *   jump to __pass
- */
-.macro BRANGE __start_reg, __end_reg, __check_reg, __jump_lable
-	blt	\__check_reg, \__start_reg, 999f
-	bge	\__check_reg, \__end_reg, 999f
-	j	\__jump_lable
-999:
-.endm
-
 	.section .entry, "ax", %progbits
 	.align 3
 	.globl _start
@@ -55,27 +44,18 @@ _start:
 	li	a7, -1
 	beq	a6, a7, _try_lottery
 	/* Jump to relocation wait loop if we are not boot hart */
-	bne	a0, a6, _wait_relocate_copy_done
+	bne	a0, a6, _wait_for_boot_hart
 _try_lottery:
 	/* Jump to relocation wait loop if we don't get relocation lottery */
-	lla	a6, _relocate_lottery
-	li	a7, 1
-	amoadd.w a6, a7, (a6)
-	bnez	a6, _wait_relocate_copy_done
+	lla	a6, _boot_status
+	li	a7, BOOT_STATUS_LOTTERY_DONE
+	amoswap.w a6, a7, (a6)
+	bnez	a6, _wait_for_boot_hart
 
-	/* Save load address */
-	lla	t0, _load_start
-	lla	t1, _fw_start
-	REG_S	t1, 0(t0)
-
-#ifdef FW_PIC
 	/* relocate the global table content */
-	lla	t0, _link_start
-	REG_L	t0, 0(t0)
-	/* t1 shall has the address of _fw_start */
-	sub	t2, t1, t0
-	lla	t3, _runtime_offset
-	REG_S	t2, (t3)
+	li	t0, FW_TEXT_START	/* link start */
+	lla	t1, _fw_start		/* load start */
+	sub	t2, t1, t0		/* load offset */
 	lla	t0, __rel_dyn_start
 	lla	t1, __rel_dyn_end
 	beq	t0, t1, _relocate_done
@@ -88,129 +68,14 @@ _try_lottery:
 	add	t5, t5, t2
 	add	t3, t3, t2
 	REG_S	t5, 0(t3)		/* store runtime address to the GOT entry */
-	j	5f
 
 3:
-	lla	t4, __dyn_sym_start
-
-4:
-	srli	t6, t5, SYM_INDEX	/* t6 <--- sym table index */
-	andi	t5, t5, 0xFF		/* t5 <--- relocation type */
-	li	t3, RELOC_TYPE
-	bne	t5, t3, 5f
-
-	/* address R_RISCV_64 or R_RISCV_32 cases*/
-	REG_L	t3, 0(t0)
-	li	t5, SYM_SIZE
-	mul	t6, t6, t5
-	add	s5, t4, t6
-	REG_L	t6, (REGBYTES * 2)(t0)	/* t0 <-- addend */
-	REG_L	t5, REGBYTES(s5)
-	add	t5, t5, t6
-	add	t5, t5, t2		/* t5 <-- location to fix up in RAM */
-	add	t3, t3, t2		/* t3 <-- location to fix up in RAM */
-	REG_S	t5, 0(t3)		/* store runtime address to the variable */
-
-5:
 	addi	t0, t0, (REGBYTES * 3)
 	blt	t0, t1, 2b
-	j	_relocate_done
-_wait_relocate_copy_done:
-	j	_wait_for_boot_hart
-#else
-	/* Relocate if load address != link address */
-_relocate:
-	lla	t0, _link_start
-	REG_L	t0, 0(t0)
-	lla	t1, _link_end
-	REG_L	t1, 0(t1)
-	lla	t2, _load_start
-	REG_L	t2, 0(t2)
-	beq	t0, t2, _relocate_done
-	sub	t3, t1, t0
-	add	t3, t3, t2
-	lla	t4, _relocate_done
-	sub	t4, t4, t2
-	add	t4, t4, t0
-	blt	t2, t0, _relocate_copy_to_upper
-_relocate_copy_to_lower:
-	ble	t1, t2, _relocate_copy_to_lower_loop
-	lla	t3, _relocate_lottery
-	BRANGE	t2, t1, t3, _start_hang
-	lla	t3, _boot_status
-	BRANGE	t2, t1, t3, _start_hang
-	lla	t3, _relocate
-	lla	t5, _relocate_done
-	BRANGE	t2, t1, t3, _start_hang
-	BRANGE	t2, t1, t5, _start_hang
-	BRANGE  t3, t5, t2, _start_hang
-_relocate_copy_to_lower_loop:
-	REG_L	t3, 0(t2)
-	REG_S	t3, 0(t0)
-	add	t0, t0, __SIZEOF_POINTER__
-	add	t2, t2, __SIZEOF_POINTER__
-	blt	t0, t1, _relocate_copy_to_lower_loop
-	jr	t4
-_relocate_copy_to_upper:
-	ble	t3, t0, _relocate_copy_to_upper_loop
-	lla	t2, _relocate_lottery
-	BRANGE	t0, t3, t2, _start_hang
-	lla	t2, _boot_status
-	BRANGE	t0, t3, t2, _start_hang
-	lla	t2, _relocate
-	lla	t5, _relocate_done
-	BRANGE	t0, t3, t2, _start_hang
-	BRANGE	t0, t3, t5, _start_hang
-	BRANGE	t2, t5, t0, _start_hang
-_relocate_copy_to_upper_loop:
-	add	t3, t3, -__SIZEOF_POINTER__
-	add	t1, t1, -__SIZEOF_POINTER__
-	REG_L	t2, 0(t3)
-	REG_S	t2, 0(t1)
-	blt	t0, t1, _relocate_copy_to_upper_loop
-	jr	t4
-_wait_relocate_copy_done:
-	lla	t0, _fw_start
-	lla	t1, _link_start
-	REG_L	t1, 0(t1)
-	beq	t0, t1, _wait_for_boot_hart
-	lla	t2, _boot_status
-	lla	t3, _wait_for_boot_hart
-	sub	t3, t3, t0
-	add	t3, t3, t1
-1:
-	/* waitting for relocate copy done (_boot_status == 1) */
-	li	t4, BOOT_STATUS_RELOCATE_DONE
-	REG_L	t5, 0(t2)
-	/* Reduce the bus traffic so that boot hart may proceed faster */
-	nop
-	nop
-	nop
-	bgt     t4, t5, 1b
-	jr	t3
-#endif
 _relocate_done:
-
-	/*
-	 * Mark relocate copy done
-	 * Use _boot_status copy relative to the load address
-	 */
-	lla	t0, _boot_status
-#ifndef FW_PIC
-	lla	t1, _link_start
-	REG_L	t1, 0(t1)
-	lla	t2, _load_start
-	REG_L	t2, 0(t2)
-	sub	t0, t0, t1
-	add	t0, t0, t2
-#endif
-	li	t1, BOOT_STATUS_RELOCATE_DONE
-	REG_S	t1, 0(t0)
-	fence	rw, rw
-
 	/* At this point we are running from link address */
 
-	/* Reset all registers for boot HART */
+	/* Reset all registers except ra, a0, a1, a2, a3 and a4 for boot HART */
 	li	ra, 0
 	call	_reset_regs
 
@@ -309,8 +174,8 @@ _scratch_init:
 	REG_S	a5, SBI_SCRATCH_FW_SIZE_OFFSET(tp)
 
 	/* Store R/W section's offset in scratch space */
-	lla	a4, __fw_rw_offset
-	REG_L	a5, 0(a4)
+	lla	a5, _fw_rw_start
+	sub	a5, a5, a4
 	REG_S	a5, SBI_SCRATCH_FW_RW_OFFSET(tp)
 
 	/* Store fw_heap_offset and fw_heap_size in scratch space */
@@ -341,10 +206,8 @@ _scratch_init:
 	/* Store hartid-to-scratch function address in scratch space */
 	lla	a4, _hartid_to_scratch
 	REG_S	a4, SBI_SCRATCH_HARTID_TO_SCRATCH_OFFSET(tp)
-	/* Store trap-exit function address in scratch space */
-	lla	a4, _trap_exit
-	REG_S	a4, SBI_SCRATCH_TRAP_EXIT_OFFSET(tp)
-	/* Clear tmp0 in scratch space */
+	/* Clear trap_context and tmp0 in scratch space */
+	REG_S	zero, SBI_SCRATCH_TRAP_CONTEXT_OFFSET(tp)
 	REG_S	zero, SBI_SCRATCH_TMP0_OFFSET(tp)
 	/* Store firmware options in scratch space */
 	MOV_3R	s0, a0, s1, a1, s2, a2
@@ -380,7 +243,7 @@ _scratch_init:
 	/* t0 = source FDT start address */
 	add	t0, a1, zero
 	/* t2 = source FDT size in big-endian */
-#if __riscv_xlen == 64
+#if __riscv_xlen > 32
 	lwu	t2, 4(t0)
 #else
 	lw	t2, 4(t0)
@@ -421,8 +284,8 @@ _fdt_reloc_done:
 	/* mark boot hart done */
 	li	t0, BOOT_STATUS_BOOT_HART_DONE
 	lla	t1, _boot_status
-	REG_S	t0, 0(t1)
 	fence	rw, rw
+	REG_S	t0, 0(t1)
 	j	_start_warm
 
 	/* waiting for boot hart to be done (_boot_status == 2) */
@@ -431,13 +294,13 @@ _wait_for_boot_hart:
 	lla	t1, _boot_status
 	REG_L	t1, 0(t1)
 	/* Reduce the bus traffic so that boot hart may proceed faster */
-	nop
-	nop
-	nop
+	div	t2, t2, zero
+	div	t2, t2, zero
+	div	t2, t2, zero
 	bne	t0, t1, _wait_for_boot_hart
 
 _start_warm:
-	/* Reset all registers for non-boot HARTs */
+	/* Reset all registers except ra, a0, a1, a2, a3 and a4 for non-boot HART */
 	li	ra, 0
 	call	_reset_regs
 
@@ -446,7 +309,7 @@ _start_warm:
 
 	/* Find HART count and HART stack size */
 	lla	a4, platform
-#if __riscv_xlen == 64
+#if __riscv_xlen > 32
 	lwu	s7, SBI_PLATFORM_HART_COUNT_OFFSET(a4)
 	lwu	s8, SBI_PLATFORM_HART_STACK_SIZE_OFFSET(a4)
 #else
@@ -462,7 +325,7 @@ _start_warm:
 	beqz	s9, 3f
 	li	a4, 0
 1:
-#if __riscv_xlen == 64
+#if __riscv_xlen > 32
 	lwu	a5, (s9)
 #else
 	lw	a5, (s9)
@@ -491,28 +354,14 @@ _start_warm:
 
 	/* Setup trap handler */
 	lla	a4, _trap_handler
-#if __riscv_xlen == 32
 	csrr	a5, CSR_MISA
 	srli	a5, a5, ('H' - 'A')
 	andi	a5, a5, 0x1
-	beq	a5, zero, _skip_trap_handler_rv32_hyp
-	lla	a4, _trap_handler_rv32_hyp
-_skip_trap_handler_rv32_hyp:
-#endif
+	beq	a5, zero, _skip_trap_handler_hyp
+	lla	a4, _trap_handler_hyp
+_skip_trap_handler_hyp:
 	csrw	CSR_MTVEC, a4
 
-#if __riscv_xlen == 32
-	/* Override trap exit for H-extension */
-	csrr	a5, CSR_MISA
-	srli	a5, a5, ('H' - 'A')
-	andi	a5, a5, 0x1
-	beq	a5, zero, _skip_trap_exit_rv32_hyp
-	lla	a4, _trap_exit_rv32_hyp
-	csrr	a5, CSR_MSCRATCH
-	REG_S	a4, SBI_SCRATCH_TRAP_EXIT_OFFSET(a5)
-_skip_trap_exit_rv32_hyp:
-#endif
-
 	/* Initialize SBI runtime */
 	csrr	a0, CSR_MSCRATCH
 	call	sbi_init
@@ -522,22 +371,8 @@ _skip_trap_exit_rv32_hyp:
 
 	.data
 	.align 3
-#ifdef FW_PIC
-_runtime_offset:
-	RISCV_PTR	0
-#endif
-_relocate_lottery:
-	RISCV_PTR	0
 _boot_status:
 	RISCV_PTR	0
-_load_start:
-	RISCV_PTR	_fw_start
-_link_start:
-	RISCV_PTR	FW_TEXT_START
-_link_end:
-	RISCV_PTR	_fw_reloc_end
-__fw_rw_offset:
-	RISCV_PTR	_fw_rw_start - _fw_start
 
 	.section .entry, "ax", %progbits
 	.align 3
@@ -551,7 +386,7 @@ _hartid_to_scratch:
 	 * t2 -> Temporary
 	 */
 	lla	t2, platform
-#if __riscv_xlen == 64
+#if __riscv_xlen > 32
 	lwu	t0, SBI_PLATFORM_HART_STACK_SIZE_OFFSET(t2)
 	lwu	t2, SBI_PLATFORM_HART_COUNT_OFFSET(t2)
 #else
@@ -635,10 +470,10 @@ memcmp:
 	xor	t0, tp, t0
 
 	/* Save original SP on exception stack */
-	REG_S	sp, (SBI_TRAP_REGS_OFFSET(sp) - SBI_TRAP_REGS_SIZE)(t0)
+	REG_S	sp, (SBI_TRAP_REGS_OFFSET(sp) - SBI_TRAP_CONTEXT_SIZE)(t0)
 
-	/* Set SP to exception stack and make room for trap registers */
-	add	sp, t0, -(SBI_TRAP_REGS_SIZE)
+	/* Set SP to exception stack and make room for trap context */
+	add	sp, t0, -(SBI_TRAP_CONTEXT_SIZE)
 
 	/* Restore T0 from scratch space */
 	REG_L	t0, SBI_SCRATCH_TMP0_OFFSET(tp)
@@ -698,6 +533,32 @@ memcmp:
 	REG_S	t6, SBI_TRAP_REGS_OFFSET(t6)(sp)
 .endm
 
+.macro	TRAP_SAVE_INFO have_mstatush have_h_extension
+	csrr	t0, CSR_MCAUSE
+	REG_S	t0, (SBI_TRAP_REGS_SIZE + SBI_TRAP_INFO_OFFSET(cause))(sp)
+	csrr	t0, CSR_MTVAL
+	REG_S	t0, (SBI_TRAP_REGS_SIZE + SBI_TRAP_INFO_OFFSET(tval))(sp)
+.if \have_h_extension
+	csrr	t0, CSR_MTVAL2
+	REG_S	t0, (SBI_TRAP_REGS_SIZE + SBI_TRAP_INFO_OFFSET(tval2))(sp)
+	csrr	t0, CSR_MTINST
+	REG_S	t0, (SBI_TRAP_REGS_SIZE + SBI_TRAP_INFO_OFFSET(tinst))(sp)
+	.if \have_mstatush
+	csrr	t0, CSR_MSTATUSH
+	srli	t0, t0, MSTATUSH_GVA_SHIFT
+	.else
+	csrr	t0, CSR_MSTATUS
+	srli	t0, t0, MSTATUS_GVA_SHIFT
+	.endif
+	and	t0, t0, 0x1
+.else
+	REG_S	zero, (SBI_TRAP_REGS_SIZE + SBI_TRAP_INFO_OFFSET(tval2))(sp)
+	REG_S	zero, (SBI_TRAP_REGS_SIZE + SBI_TRAP_INFO_OFFSET(tinst))(sp)
+	li	t0, 0
+.endif
+	REG_S	t0, (SBI_TRAP_REGS_SIZE + SBI_TRAP_INFO_OFFSET(gva))(sp)
+.endm
+
 .macro	TRAP_CALL_C_ROUTINE
 	/* Call C routine */
 	add	a0, sp, zero
@@ -760,7 +621,6 @@ memcmp:
 	.section .entry, "ax", %progbits
 	.align 3
 	.globl _trap_handler
-	.globl _trap_exit
 _trap_handler:
 	TRAP_SAVE_AND_SETUP_SP_T0
 
@@ -768,9 +628,10 @@ _trap_handler:
 
 	TRAP_SAVE_GENERAL_REGS_EXCEPT_SP_T0
 
+	TRAP_SAVE_INFO 0 0
+
 	TRAP_CALL_C_ROUTINE
 
-_trap_exit:
 	TRAP_RESTORE_GENERAL_REGS_EXCEPT_A0_T0
 
 	TRAP_RESTORE_MEPC_MSTATUS 0
@@ -779,29 +640,39 @@ _trap_exit:
 
 	mret
 
-#if __riscv_xlen == 32
 	.section .entry, "ax", %progbits
 	.align 3
-	.globl _trap_handler_rv32_hyp
-	.globl _trap_exit_rv32_hyp
-_trap_handler_rv32_hyp:
+	.globl _trap_handler_hyp
+_trap_handler_hyp:
 	TRAP_SAVE_AND_SETUP_SP_T0
 
+#if __riscv_xlen == 32
 	TRAP_SAVE_MEPC_MSTATUS 1
+#else
+	TRAP_SAVE_MEPC_MSTATUS 0
+#endif
 
 	TRAP_SAVE_GENERAL_REGS_EXCEPT_SP_T0
 
+#if __riscv_xlen == 32
+	TRAP_SAVE_INFO 1 1
+#else
+	TRAP_SAVE_INFO 0 1
+#endif
+
 	TRAP_CALL_C_ROUTINE
 
-_trap_exit_rv32_hyp:
 	TRAP_RESTORE_GENERAL_REGS_EXCEPT_A0_T0
 
+#if __riscv_xlen == 32
 	TRAP_RESTORE_MEPC_MSTATUS 1
+#else
+	TRAP_RESTORE_MEPC_MSTATUS 0
+#endif
 
 	TRAP_RESTORE_A0_T0
 
 	mret
-#endif
 
 	.section .entry, "ax", %progbits
 	.align 3
@@ -810,7 +681,7 @@ _reset_regs:
 
 	/* flush the instruction cache */
 	fence.i
-	/* Reset all registers except ra, a0, a1 and a2 */
+	/* Reset all registers except ra, a0, a1, a2, a3 and a4 */
 	li sp, 0
 	li gp, 0
 	li tp, 0
@@ -819,8 +690,6 @@ _reset_regs:
 	li t2, 0
 	li s0, 0
 	li s1, 0
-	li a3, 0
-	li a4, 0
 	li a5, 0
 	li a6, 0
 	li a7, 0

firmware/fw_base.ldS

@@ -38,18 +38,16 @@
 		. = ALIGN(8);
 	}
 
-	. = ALIGN(0x1000); /* Ensure next section is page aligned */
-
-	.dynsym : {
-		PROVIDE(__dyn_sym_start = .);
+	.dynsym :
+	{
 		*(.dynsym)
-		PROVIDE(__dyn_sym_end = .);
 	}
 
+	. = ALIGN(0x1000); /* Ensure next section is page aligned */
+
 	.rela.dyn : {
 		PROVIDE(__rel_dyn_start = .);
 		*(.rela*)
-		. = ALIGN(8);
 		PROVIDE(__rel_dyn_end = .);
 	}
 

firmware/fw_dynamic.S

@@ -11,12 +11,6 @@
 
 #include "fw_base.S"
 
-	.section .entry, "ax", %progbits
-	.align 3
-_bad_dynamic_info:
-	wfi
-	j	_bad_dynamic_info
-
 	.section .entry, "ax", %progbits
 	.align 3
 	.global fw_boot_hart
@@ -30,10 +24,10 @@ fw_boot_hart:
 	/* Sanity checks */
 	li	a1, FW_DYNAMIC_INFO_MAGIC_VALUE
 	REG_L	a0, FW_DYNAMIC_INFO_MAGIC_OFFSET(a2)
-	bne	a0, a1, _bad_dynamic_info
+	bne	a0, a1, _start_hang
 	li	a1, FW_DYNAMIC_INFO_VERSION_MAX
 	REG_L	a0, FW_DYNAMIC_INFO_VERSION_OFFSET(a2)
-	bgt	a0, a1, _bad_dynamic_info
+	bgt	a0, a1, _start_hang
 
 	/* Read boot HART id */
 	li	a1, FW_DYNAMIC_INFO_VERSION_2
@@ -129,7 +123,7 @@ fw_options:
 	REG_L	a0, (a0)
 	ret
 
-	.section .entry, "ax", %progbits
+	.section .data
 	.align 3
 _dynamic_next_arg1:
 	RISCV_PTR 0x0

firmware/fw_jump.S

@@ -46,6 +46,10 @@ fw_save_info:
 fw_next_arg1:
 #ifdef FW_JUMP_FDT_ADDR
 	li	a0, FW_JUMP_FDT_ADDR
+#elif defined(FW_JUMP_FDT_OFFSET)
+	lla	a0, _fw_start
+	li	a1, FW_JUMP_FDT_OFFSET
+	add	a0, a0, a1
 #else
 	add	a0, a1, zero
 #endif
@@ -59,8 +63,16 @@ fw_next_arg1:
 	 * The next address should be returned in 'a0'.
 	 */
 fw_next_addr:
+#ifdef FW_JUMP_ADDR
 	lla	a0, _jump_addr
 	REG_L	a0, (a0)
+#elif defined(FW_JUMP_OFFSET)
+	lla	a0, _fw_start
+	li	a1, FW_JUMP_OFFSET
+	add	a0, a0, a1
+#else
+#error "Must define at least FW_JUMP_ADDR or FW_JUMP_OFFSET"
+#endif
 	ret
 
 	.section .entry, "ax", %progbits
@@ -86,11 +98,9 @@ fw_options:
 	add	a0, zero, zero
 	ret
 
-#ifndef FW_JUMP_ADDR
-#error "Must define FW_JUMP_ADDR"
-#endif
-
-	.section .entry, "ax", %progbits
+#ifdef FW_JUMP_ADDR
+	.section .rodata
 	.align 3
 _jump_addr:
 	RISCV_PTR FW_JUMP_ADDR
+#endif

firmware/fw_payload.S

@@ -46,6 +46,10 @@ fw_save_info:
 fw_next_arg1:
 #ifdef FW_PAYLOAD_FDT_ADDR
 	li	a0, FW_PAYLOAD_FDT_ADDR
+#elif defined(FW_PAYLOAD_FDT_OFFSET)
+	lla	a0, _fw_start
+	li	a1, FW_PAYLOAD_FDT_OFFSET
+	add	a0, a0, a1
 #else
 	add	a0, a1, zero
 #endif

firmware/objects.mk

@@ -13,19 +13,10 @@ firmware-cflags-y +=
 firmware-asflags-y +=
 firmware-ldflags-y +=
 
-ifndef FW_PIC
-FW_PIC := $(OPENSBI_LD_PIE)
-endif
-
-ifeq ($(FW_PIC),y)
-firmware-genflags-y +=	-DFW_PIC
-firmware-asflags-y  +=	-fpic
-firmware-cflags-y   +=	-fPIE -pie
-firmware-ldflags-y  +=	-Wl,--no-dynamic-linker -Wl,-pie
-endif
-
 ifdef FW_TEXT_START
 firmware-genflags-y += -DFW_TEXT_START=$(FW_TEXT_START)
+else
+firmware-genflags-y += -DFW_TEXT_START=0x0
 endif
 
 ifdef FW_FDT_PATH
@@ -38,9 +29,15 @@ endif
 firmware-bins-$(FW_DYNAMIC) += fw_dynamic.bin
 
 firmware-bins-$(FW_JUMP) += fw_jump.bin
+ifdef FW_JUMP_OFFSET
+firmware-genflags-$(FW_JUMP) += -DFW_JUMP_OFFSET=$(FW_JUMP_OFFSET)
+endif
 ifdef FW_JUMP_ADDR
 firmware-genflags-$(FW_JUMP) += -DFW_JUMP_ADDR=$(FW_JUMP_ADDR)
 endif
+ifdef FW_JUMP_FDT_OFFSET
+firmware-genflags-$(FW_JUMP) += -DFW_JUMP_FDT_OFFSET=$(FW_JUMP_FDT_OFFSET)
+endif
 ifdef FW_JUMP_FDT_ADDR
 firmware-genflags-$(FW_JUMP) += -DFW_JUMP_FDT_ADDR=$(FW_JUMP_FDT_ADDR)
 endif
@@ -59,6 +56,9 @@ ifdef FW_PAYLOAD_ALIGN
 firmware-genflags-$(FW_PAYLOAD) += -DFW_PAYLOAD_ALIGN=$(FW_PAYLOAD_ALIGN)
 endif
 
+ifdef FW_PAYLOAD_FDT_OFFSET
+firmware-genflags-$(FW_PAYLOAD) += -DFW_PAYLOAD_FDT_OFFSET=$(FW_PAYLOAD_FDT_OFFSET)
+endif
 ifdef FW_PAYLOAD_FDT_ADDR
 firmware-genflags-$(FW_PAYLOAD) += -DFW_PAYLOAD_FDT_ADDR=$(FW_PAYLOAD_FDT_ADDR)
 endif

firmware/payloads/test_head.S

@@ -78,7 +78,7 @@ _start_hang:
 	wfi
 	j	_start_hang
 
-	.section .entry, "ax", %progbits
+	.section .data
 	.align	3
 _hart_lottery:
 	RISCV_PTR	0

firmware/payloads/test_main.c

@@ -8,31 +8,42 @@
  */
 
 #include <sbi/sbi_ecall_interface.h>
+#include <sbi/sbi_string.h>
 
-#define SBI_ECALL(__eid, __fid, __a0, __a1, __a2)                             \
-	({                                                                    \
-		register unsigned long a0 asm("a0") = (unsigned long)(__a0);  \
-		register unsigned long a1 asm("a1") = (unsigned long)(__a1);  \
-		register unsigned long a2 asm("a2") = (unsigned long)(__a2);  \
-		register unsigned long a6 asm("a6") = (unsigned long)(__fid); \
-		register unsigned long a7 asm("a7") = (unsigned long)(__eid); \
-		asm volatile("ecall"                                          \
-			     : "+r"(a0)                                       \
-			     : "r"(a1), "r"(a2), "r"(a6), "r"(a7)             \
-			     : "memory");                                     \
-		a0;                                                           \
-	})
+struct sbiret {
+	unsigned long error;
+	unsigned long value;
+};
 
-#define SBI_ECALL_0(__eid, __fid) SBI_ECALL(__eid, __fid, 0, 0, 0)
-#define SBI_ECALL_1(__eid, __fid, __a0) SBI_ECALL(__eid, __fid, __a0, 0, 0)
-#define SBI_ECALL_2(__eid, __fid, __a0, __a1) SBI_ECALL(__eid, __fid, __a0, __a1, 0)
+struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0,
+			unsigned long arg1, unsigned long arg2,
+			unsigned long arg3, unsigned long arg4,
+			unsigned long arg5)
+{
+	struct sbiret ret;
 
-#define sbi_ecall_console_putc(c) SBI_ECALL_1(SBI_EXT_0_1_CONSOLE_PUTCHAR, 0, (c))
+	register unsigned long a0 asm ("a0") = (unsigned long)(arg0);
+	register unsigned long a1 asm ("a1") = (unsigned long)(arg1);
+	register unsigned long a2 asm ("a2") = (unsigned long)(arg2);
+	register unsigned long a3 asm ("a3") = (unsigned long)(arg3);
+	register unsigned long a4 asm ("a4") = (unsigned long)(arg4);
+	register unsigned long a5 asm ("a5") = (unsigned long)(arg5);
+	register unsigned long a6 asm ("a6") = (unsigned long)(fid);
+	register unsigned long a7 asm ("a7") = (unsigned long)(ext);
+	asm volatile ("ecall"
+		      : "+r" (a0), "+r" (a1)
+		      : "r" (a2), "r" (a3), "r" (a4), "r" (a5), "r" (a6), "r" (a7)
+		      : "memory");
+	ret.error = a0;
+	ret.value = a1;
+
+	return ret;
+}
 
 static inline void sbi_ecall_console_puts(const char *str)
 {
-	while (str && *str)
-		sbi_ecall_console_putc(*str++);
+	sbi_ecall(SBI_EXT_DBCN, SBI_EXT_DBCN_CONSOLE_WRITE,
+		  sbi_strlen(str), (unsigned long)str, 0, 0, 0, 0);
 }
 
 #define wfi()                                             \

include/sbi/riscv_asm.h

@@ -181,6 +181,12 @@ int misa_xlen(void);
 /* Get RISC-V ISA string representation */
 void misa_string(int xlen, char *out, unsigned int out_sz);
 
+/* Disable pmp entry at a given index */
+int pmp_disable(unsigned int n);
+
+/* Check if the matching field is set */
+int is_pmp_entry_mapped(unsigned long entry);
+
 int pmp_set(unsigned int n, unsigned long prot, unsigned long addr,
 	    unsigned long log2len);
 

include/sbi/riscv_atomic.h

@@ -39,14 +39,14 @@ unsigned int atomic_raw_xchg_uint(volatile unsigned int *ptr,
 unsigned long atomic_raw_xchg_ulong(volatile unsigned long *ptr,
 				    unsigned long newval);
 /**
- * Set a bit in an atomic variable and return the new value.
+ * Set a bit in an atomic variable and return the value of bit before modify.
  * @nr : Bit to set.
  * @atom: atomic variable to modify
  */
 int atomic_set_bit(int nr, atomic_t *atom);
 
 /**
- * Clear a bit in an atomic variable and return the new value.
+ * Clear a bit in an atomic variable and return the value of bit before modify.
  * @nr : Bit to set.
  * @atom: atomic variable to modify
  */
@@ -54,14 +54,14 @@ int atomic_set_bit(int nr, atomic_t *atom);
 int atomic_clear_bit(int nr, atomic_t *atom);
 
 /**
- * Set a bit in any address and return the new value .
+ * Set a bit in any address and return the value of bit before modify.
  * @nr : Bit to set.
  * @addr: Address to modify
  */
 int atomic_raw_set_bit(int nr, volatile unsigned long *addr);
 
 /**
- * Clear a bit in any address and return the new value .
+ * Clear a bit in any address and return the value of bit before modify.
  * @nr : Bit to set.
  * @addr: Address to modify
  */

include/sbi/riscv_barrier.h

@@ -40,7 +40,11 @@
 #define smp_wmb()		RISCV_FENCE(w,w)
 
 /* CPU relax for busy loop */
-#define cpu_relax()		asm volatile ("" : : : "memory")
+#define cpu_relax()						\
+do {								\
+	unsigned long __t;					\
+	__asm__ __volatile__ ("div %0, %0, zero" : "=r" (__t));	\
+} while (0)
 
 /* clang-format on */
 

include/sbi/riscv_dbtr.h

@@ -0,0 +1,249 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Ventana Micro System, Inc.
+ *
+ * Authors:
+ *   Himanshu Chauhan <hchauhan@ventanamicro.com>
+ */
+
+#ifndef __RISCV_DBTR_H__
+#define __RISCV_DBTR_H__
+
+#define RV_MAX_TRIGGERS	32
+
+enum {
+	RISCV_DBTR_TRIG_NONE = 0,
+	RISCV_DBTR_TRIG_LEGACY,
+	RISCV_DBTR_TRIG_MCONTROL,
+	RISCV_DBTR_TRIG_ICOUNT,
+	RISCV_DBTR_TRIG_ITRIGGER,
+	RISCV_DBTR_TRIG_ETRIGGER,
+	RISCV_DBTR_TRIG_MCONTROL6,
+};
+
+#define RV_DBTR_BIT(_prefix, _name)		\
+	RV_DBTR_##_prefix##_##_name##_BIT
+
+#define RV_DBTR_BIT_MASK(_prefix, _name)	\
+	RV_DBTR_##_prefix##_name##_BIT_MASK
+
+#define RV_DBTR_DECLARE_BIT(_prefix, _name, _val)	\
+	RV_DBTR_BIT(_prefix, _name) = _val
+
+#define RV_DBTR_DECLARE_BIT_MASK(_prefix, _name, _width)		\
+	RV_DBTR_BIT_MASK(_prefix, _name) =				\
+		(((1UL << _width) - 1) << RV_DBTR_BIT(_prefix, _name))
+
+#define CLEAR_DBTR_BIT(_target, _prefix, _bit_name)		\
+	__clear_bit(RV_DBTR_BIT(_prefix, _bit_name), &_target)
+
+#define SET_DBTR_BIT(_target, _prefix, _bit_name)		\
+	__set_bit(RV_DBTR_BIT(_prefix, _bit_name), &_target)
+
+/* Trigger Data 1 */
+enum {
+	RV_DBTR_DECLARE_BIT(TDATA1, DATA,   0),
+#if __riscv_xlen == 64
+	RV_DBTR_DECLARE_BIT(TDATA1, DMODE,  59),
+	RV_DBTR_DECLARE_BIT(TDATA1, TYPE,   60),
+#elif __riscv_xlen == 32
+	RV_DBTR_DECLARE_BIT(TDATA1, DMODE,  27),
+	RV_DBTR_DECLARE_BIT(TDATA1, TYPE,   28),
+#else
+	#error "Unknown __riscv_xlen"
+#endif
+};
+
+enum {
+#if __riscv_xlen == 64
+	RV_DBTR_DECLARE_BIT_MASK(TDATA1, DATA,  59),
+#elif __riscv_xlen == 32
+	RV_DBTR_DECLARE_BIT_MASK(TDATA1, DATA,  27),
+#else
+	#error "Unknown __riscv_xlen"
+#endif
+	RV_DBTR_DECLARE_BIT_MASK(TDATA1, DMODE, 1),
+	RV_DBTR_DECLARE_BIT_MASK(TDATA1, TYPE,  4),
+};
+
+/* MC - Match Control Type Register */
+enum {
+	RV_DBTR_DECLARE_BIT(MC, LOAD,    0),
+	RV_DBTR_DECLARE_BIT(MC, STORE,   1),
+	RV_DBTR_DECLARE_BIT(MC, EXEC,    2),
+	RV_DBTR_DECLARE_BIT(MC, U,       3),
+	RV_DBTR_DECLARE_BIT(MC, S,       4),
+	RV_DBTR_DECLARE_BIT(MC, RES2,    5),
+	RV_DBTR_DECLARE_BIT(MC, M,       6),
+	RV_DBTR_DECLARE_BIT(MC, MATCH,   7),
+	RV_DBTR_DECLARE_BIT(MC, CHAIN,   11),
+	RV_DBTR_DECLARE_BIT(MC, ACTION,  12),
+	RV_DBTR_DECLARE_BIT(MC, SIZELO,  16),
+	RV_DBTR_DECLARE_BIT(MC, TIMING,  18),
+	RV_DBTR_DECLARE_BIT(MC, SELECT,  19),
+	RV_DBTR_DECLARE_BIT(MC, HIT,     20),
+#if __riscv_xlen >= 64
+	RV_DBTR_DECLARE_BIT(MC, SIZEHI,  21),
+#endif
+#if __riscv_xlen == 64
+	RV_DBTR_DECLARE_BIT(MC, MASKMAX, 53),
+	RV_DBTR_DECLARE_BIT(MC, DMODE,   59),
+	RV_DBTR_DECLARE_BIT(MC, TYPE,    60),
+#elif __riscv_xlen == 32
+	RV_DBTR_DECLARE_BIT(MC, MASKMAX, 21),
+	RV_DBTR_DECLARE_BIT(MC, DMODE,   27),
+	RV_DBTR_DECLARE_BIT(MC, TYPE,    28),
+#else
+	#error "Unknown __riscv_xlen"
+#endif
+};
+
+enum {
+	RV_DBTR_DECLARE_BIT_MASK(MC, LOAD,    1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, STORE,   1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, EXEC,    1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, U,       1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, S,       1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, RES2,    1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, M,       1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, MATCH,   4),
+	RV_DBTR_DECLARE_BIT_MASK(MC, CHAIN,   1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, ACTION,  4),
+	RV_DBTR_DECLARE_BIT_MASK(MC, SIZELO,  2),
+	RV_DBTR_DECLARE_BIT_MASK(MC, TIMING,  1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, SELECT,  1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, HIT,     1),
+#if __riscv_xlen >= 64
+	RV_DBTR_DECLARE_BIT_MASK(MC, SIZEHI,  2),
+#endif
+	RV_DBTR_DECLARE_BIT_MASK(MC, MASKMAX, 6),
+	RV_DBTR_DECLARE_BIT_MASK(MC, DMODE,   1),
+	RV_DBTR_DECLARE_BIT_MASK(MC, TYPE,    4),
+};
+
+/* MC6 - Match Control 6 Type Register */
+enum {
+	RV_DBTR_DECLARE_BIT(MC6, LOAD,   0),
+	RV_DBTR_DECLARE_BIT(MC6, STORE,  1),
+	RV_DBTR_DECLARE_BIT(MC6, EXEC,   2),
+	RV_DBTR_DECLARE_BIT(MC6, U,      3),
+	RV_DBTR_DECLARE_BIT(MC6, S,      4),
+	RV_DBTR_DECLARE_BIT(MC6, RES2,   5),
+	RV_DBTR_DECLARE_BIT(MC6, M,      6),
+	RV_DBTR_DECLARE_BIT(MC6, MATCH,  7),
+	RV_DBTR_DECLARE_BIT(MC6, CHAIN,  11),
+	RV_DBTR_DECLARE_BIT(MC6, ACTION, 12),
+	RV_DBTR_DECLARE_BIT(MC6, SIZE,   16),
+	RV_DBTR_DECLARE_BIT(MC6, TIMING, 20),
+	RV_DBTR_DECLARE_BIT(MC6, SELECT, 21),
+	RV_DBTR_DECLARE_BIT(MC6, HIT,    22),
+	RV_DBTR_DECLARE_BIT(MC6, VU,     23),
+	RV_DBTR_DECLARE_BIT(MC6, VS,     24),
+#if __riscv_xlen == 64
+	RV_DBTR_DECLARE_BIT(MC6, DMODE,  59),
+	RV_DBTR_DECLARE_BIT(MC6, TYPE,   60),
+#elif __riscv_xlen == 32
+	RV_DBTR_DECLARE_BIT(MC6, DMODE,  27),
+	RV_DBTR_DECLARE_BIT(MC6, TYPE,   28),
+#else
+	#error "Unknown __riscv_xlen"
+#endif
+};
+
+enum {
+	RV_DBTR_DECLARE_BIT_MASK(MC6, LOAD,   1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, STORE,  1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, EXEC,   1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, U,      1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, S,      1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, RES2,   1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, M,      1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, MATCH,  4),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, CHAIN,  1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, ACTION, 4),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, SIZE,   4),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, TIMING, 1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, SELECT, 1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, HIT,    1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, VU,     1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, VS,     1),
+#if __riscv_xlen == 64
+	RV_DBTR_DECLARE_BIT_MASK(MC6, DMODE,  1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, TYPE,   4),
+#elif __riscv_xlen == 32
+	RV_DBTR_DECLARE_BIT_MASK(MC6, DMODE,  1),
+	RV_DBTR_DECLARE_BIT_MASK(MC6, TYPE,   4),
+#else
+	#error "Unknown __riscv_xlen"
+#endif
+};
+
+#define RV_DBTR_SET_TDATA1_TYPE(_t1, _type)				\
+	do {								\
+		_t1 &= ~RV_DBTR_BIT_MASK(TDATA1, TYPE);			\
+		_t1 |= (((unsigned long)_type				\
+			 << RV_DBTR_BIT(TDATA1, TYPE))			\
+			& RV_DBTR_BIT_MASK(TDATA1, TYPE));		\
+	}while (0);
+
+#define RV_DBTR_SET_MC_TYPE(_t1, _type)				\
+	do {							\
+		_t1 &= ~RV_DBTR_BIT_MASK(MC, TYPE);		\
+		_t1 |= (((unsigned long)_type			\
+			 << RV_DBTR_BIT(MC, TYPE))		\
+			& RV_DBTR_BIT_MASK(MC, TYPE));		\
+	}while (0);
+
+#define RV_DBTR_SET_MC6_TYPE(_t1, _type)			\
+	do {							\
+		_t1 &= ~RV_DBTR_BIT_MASK(MC6, TYPE);		\
+		_t1 |= (((unsigned long)_type			\
+			 << RV_DBTR_BIT(MC6, TYPE))		\
+			& RV_DBTR_BIT_MASK(MC6, TYPE));		\
+	}while (0);
+
+#define RV_DBTR_SET_MC_EXEC(_t1)		\
+	SET_DBTR_BIT(_t1, MC, EXEC)
+
+#define RV_DBTR_SET_MC_LOAD(_t1)		\
+	SET_DBTR_BIT(_t1, MC, LOAD)
+
+#define RV_DBTR_SET_MC_STORE(_t1)		\
+	SET_DBTR_BIT(_t1, MC, STORE)
+
+#define RV_DBTR_SET_MC_SIZELO(_t1, _val)			\
+	do {							\
+		_t1 &= ~RV_DBTR_BIT_MASK(MC, SIZELO);		\
+		_t1 |= ((_val << RV_DBTR_BIT(MC, SIZELO))	\
+			& RV_DBTR_BIT_MASK(MC, SIZELO));	\
+	} while(0);
+
+#define RV_DBTR_SET_MC_SIZEHI(_t1, _val)			\
+	do {							\
+		_t1 &= ~RV_DBTR_BIT_MASK(MC, SIZEHI);		\
+		_t1 |= ((_val << RV_DBTR_BIT(MC, SIZEHI))	\
+			& RV_DBTR_BIT_MASK(MC, SIZEHI));	\
+	} while(0);
+
+#define RV_DBTR_SET_MC6_EXEC(_t1)		\
+	SET_DBTR_BIT(_t1, MC6, EXEC)
+
+#define RV_DBTR_SET_MC6_LOAD(_t1)		\
+	SET_DBTR_BIT(_t1, MC6, LOAD)
+
+#define RV_DBTR_SET_MC6_STORE(_t1)		\
+	SET_DBTR_BIT(_t1, MC6, STORE)
+
+#define RV_DBTR_SET_MC6_SIZE(_t1, _val)				\
+	do {							\
+		_t1 &= ~RV_DBTR_BIT_MASK(MC6, SIZE);		\
+		_t1 |= ((_val << RV_DBTR_BIT(MC6, SIZE))	\
+			& RV_DBTR_BIT_MASK(MC6, SIZE));		\
+	} while(0);
+
+typedef unsigned long riscv_dbtr_tdata1_mcontrol_t;
+typedef unsigned long riscv_dbtr_tdata1_mcontrol6_t;
+typedef unsigned long riscv_dbtr_tdata1_t;
+
+#endif /* __RISCV_DBTR_H__ */

include/sbi/riscv_elf.h

@@ -1,14 +1,6 @@
 #ifndef __RISCV_ELF_H__
 #define __RISCV_ELF_H__
 
-#include <sbi/riscv_asm.h>
-
-#define R_RISCV_32		1
-#define R_RISCV_64		2
 #define R_RISCV_RELATIVE	3
 
-#define RELOC_TYPE		__REG_SEL(R_RISCV_64, R_RISCV_32)
-#define SYM_INDEX		__REG_SEL(0x20,	0x8)
-#define SYM_SIZE		__REG_SEL(0x18,0x10)
-
 #endif

include/sbi/riscv_encoding.h

@@ -80,6 +80,8 @@
 #define HSTATUS_GVA			_UL(0x00000040)
 #define HSTATUS_VSBE			_UL(0x00000020)
 
+#define MCAUSE_IRQ_MASK			(_UL(1) << (__riscv_xlen - 1))
+
 #define IRQ_S_SOFT			1
 #define IRQ_VS_SOFT			2
 #define IRQ_M_SOFT			3
@@ -207,13 +209,10 @@
 
 #define MHPMEVENT_SSCOF_MASK		_ULL(0xFFFF000000000000)
 
-#if __riscv_xlen > 32
 #define ENVCFG_STCE			(_ULL(1) << 63)
 #define ENVCFG_PBMTE			(_ULL(1) << 62)
-#else
-#define ENVCFGH_STCE			(_UL(1) << 31)
-#define ENVCFGH_PBMTE			(_UL(1) << 30)
-#endif
+#define ENVCFG_ADUE			(_ULL(1) << 61)
+#define ENVCFG_CDE			(_ULL(1) << 60)
 #define ENVCFG_CBZE			(_UL(1) << 7)
 #define ENVCFG_CBCFE			(_UL(1) << 6)
 #define ENVCFG_CBIE_SHIFT		4
@@ -319,6 +318,9 @@
 /* Supervisor Configuration */
 #define CSR_SENVCFG			0x10a
 
+/* Supervisor Conter Inhibit */
+#define CSR_SCOUNTINHIBIT		0x120
+
 /* Supervisor Trap Handling */
 #define CSR_SSCRATCH			0x140
 #define CSR_SEPC			0x141
@@ -333,9 +335,14 @@
 /* Supervisor Protection and Translation */
 #define CSR_SATP			0x180
 
-/* Supervisor-Level Window to Indirectly Accessed Registers (AIA) */
+/* Supervisor Indirect Register Alias */
 #define CSR_SISELECT			0x150
 #define CSR_SIREG			0x151
+#define CSR_SIREG2          		0x152
+#define CSR_SIREG3          		0x153
+#define CSR_SIREG4          		0x155
+#define CSR_SIREG5          		0x156
+#define CSR_SIREG6          		0x157
 
 /* Supervisor-Level Interrupts (AIA) */
 #define CSR_STOPEI			0x15c
@@ -396,9 +403,14 @@
 #define CSR_HVIPRIO1			0x646
 #define CSR_HVIPRIO2			0x647
 
-/* VS-Level Window to Indirectly Accessed Registers (H-extension with AIA) */
+/* Virtual Supervisor Indirect Alias */
 #define CSR_VSISELECT			0x250
 #define CSR_VSIREG			0x251
+#define CSR_VSIREG2         		0x252
+#define CSR_VSIREG3         		0x253
+#define CSR_VSIREG4         		0x255
+#define CSR_VSIREG5         		0x256
+#define CSR_VSIREG6         		0x257
 
 /* VS-Level Interrupts (H-extension with AIA) */
 #define CSR_VSTOPEI			0x25c
@@ -430,6 +442,7 @@
 #define CSR_MARCHID			0xf12
 #define CSR_MIMPID			0xf13
 #define CSR_MHARTID			0xf14
+#define CSR_MCONFIGPTR			0xf15
 
 /* Machine Trap Setup */
 #define CSR_MSTATUS			0x300
@@ -602,6 +615,8 @@
 
 /* Machine Counter Setup */
 #define CSR_MCOUNTINHIBIT		0x320
+#define CSR_MCYCLECFG			0x321
+#define CSR_MINSTRETCFG			0x322
 #define CSR_MHPMEVENT3			0x323
 #define CSR_MHPMEVENT4			0x324
 #define CSR_MHPMEVENT5			0x325
@@ -633,6 +648,8 @@
 #define CSR_MHPMEVENT31			0x33f
 
 /* For RV32 */
+#define CSR_MCYCLECFGH			0x721
+#define CSR_MINSTRETCFGH		0x722
 #define CSR_MHPMEVENT3H			0x723
 #define CSR_MHPMEVENT4H			0x724
 #define CSR_MHPMEVENT5H			0x725
@@ -663,6 +680,21 @@
 #define CSR_MHPMEVENT30H		0x73e
 #define CSR_MHPMEVENT31H		0x73f
 
+/* Machine Security Configuration CSR (mseccfg) */
+#define CSR_MSECCFG			0x747
+#define CSR_MSECCFGH			0x757
+
+#define MSECCFG_MML_SHIFT		(0)
+#define MSECCFG_MML			(_UL(1) << MSECCFG_MML_SHIFT)
+#define MSECCFG_MMWP_SHIFT		(1)
+#define MSECCFG_MMWP			(_UL(1) << MSECCFG_MMWP_SHIFT)
+#define MSECCFG_RLB_SHIFT		(2)
+#define MSECCFG_RLB			(_UL(1) << MSECCFG_RLB_SHIFT)
+#define MSECCFG_USEED_SHIFT		(8)
+#define MSECCFG_USEED			(_UL(1) << MSECCFG_USEED_SHIFT)
+#define MSECCFG_SSEED_SHIFT		(9)
+#define MSECCFG_SSEED			(_UL(1) << MSECCFG_SSEED_SHIFT)
+
 /* Counter Overflow CSR */
 #define CSR_SCOUNTOVF			0xda0
 
@@ -671,6 +703,7 @@
 #define CSR_TDATA1			0x7a1
 #define CSR_TDATA2			0x7a2
 #define CSR_TDATA3			0x7a3
+#define CSR_TINFO			0x7a4
 
 /* Debug Mode Registers */
 #define CSR_DCSR			0x7b0
@@ -678,9 +711,14 @@
 #define CSR_DSCRATCH0			0x7b2
 #define CSR_DSCRATCH1			0x7b3
 
-/* Machine-Level Window to Indirectly Accessed Registers (AIA) */
+/* Machine Indirect Register Alias */
 #define CSR_MISELECT			0x350
 #define CSR_MIREG			0x351
+#define CSR_MIREG2          		0x352
+#define CSR_MIREG3          		0x353
+#define CSR_MIREG4          		0x355
+#define CSR_MIREG5          		0x356
+#define CSR_MIREG6          		0x357
 
 /* Machine-Level Interrupts (AIA) */
 #define CSR_MTOPEI			0x35c
@@ -821,6 +859,13 @@
 #define INSN_MATCH_C_FSWSP		0xe002
 #define INSN_MASK_C_FSWSP		0xe003
 
+#define INSN_MATCH_C_LHU		0x8400
+#define INSN_MASK_C_LHU		0xfc43
+#define INSN_MATCH_C_LH		0x8440
+#define INSN_MASK_C_LH			0xfc43
+#define INSN_MATCH_C_SH		0x8c00
+#define INSN_MASK_C_SH			0xfc43
+
 #define INSN_MASK_WFI			0xffffff00
 #define INSN_MATCH_WFI			0x10500000
 

include/sbi/riscv_fp.h

@@ -84,7 +84,7 @@
 #define GET_FFLAGS() csr_read(CSR_FFLAGS)
 #define SET_FFLAGS(value) csr_write(CSR_FFLAGS, (value))
 
-#define SET_FS_DIRTY() ((void)0)
+#define SET_FS_DIRTY(regs) (regs->mstatus |= MSTATUS_FS)
 
 #define GET_F32_RS1(insn, regs) (GET_F32_REG(insn, 15, regs))
 #define GET_F32_RS2(insn, regs) (GET_F32_REG(insn, 20, regs))
@@ -93,9 +93,9 @@
 #define GET_F64_RS2(insn, regs) (GET_F64_REG(insn, 20, regs))
 #define GET_F64_RS3(insn, regs) (GET_F64_REG(insn, 27, regs))
 #define SET_F32_RD(insn, regs, val) \
-	(SET_F32_REG(insn, 7, regs, val), SET_FS_DIRTY())
+	(SET_F32_REG(insn, 7, regs, val), SET_FS_DIRTY(regs))
 #define SET_F64_RD(insn, regs, val) \
-	(SET_F64_REG(insn, 7, regs, val), SET_FS_DIRTY())
+	(SET_F64_REG(insn, 7, regs, val), SET_FS_DIRTY(regs))
 
 #define GET_F32_RS2C(insn, regs) (GET_F32_REG(insn, 2, regs))
 #define GET_F32_RS2S(insn, regs) (GET_F32_REG(RVC_RS2S(insn), 0, regs))

include/sbi/sbi_bitmap.h

@@ -62,6 +62,11 @@ static inline void bitmap_zero(unsigned long *dst, int nbits)
 	}
 }
 
+static inline int bitmap_test(unsigned long *bmap, int bit)
+{
+	return __test_bit(bit, bmap);
+}
+
 static inline void bitmap_zero_except(unsigned long *dst,
 				      int exception, int nbits)
 {

include/sbi/sbi_bitops.h

@@ -26,6 +26,7 @@
 #define BIT_MASK(nr)		(1UL << ((nr) % BITS_PER_LONG))
 #define BIT_WORD(bit)		((bit) / BITS_PER_LONG)
 #define BIT_WORD_OFFSET(bit)	((bit) & (BITS_PER_LONG - 1))
+#define BIT_ALIGN(bit, align)	(((bit) + ((align) - 1)) & ~((align) - 1))
 
 #define GENMASK(h, l) \
 	(((~0UL) - (1UL << (l)) + 1) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
@@ -112,6 +113,22 @@ static inline unsigned long sbi_fls(unsigned long word)
 	return num;
 }
 
+/**
+ * sbi_popcount - find the number of set bit in a long word
+ * @word: the word to search
+ */
+static inline unsigned long sbi_popcount(unsigned long word)
+{
+	unsigned long count = 0;
+
+	while (word) {
+		word &= word - 1;
+		count++;
+	}
+
+	return count;
+}
+
 #define for_each_set_bit(bit, addr, size) \
 	for ((bit) = find_first_bit((addr), (size));		\
 	     (bit) < (size);					\

include/sbi/sbi_byteorder.h

@@ -7,7 +7,12 @@
 #ifndef __SBI_BYTEORDER_H__
 #define __SBI_BYTEORDER_H__
 
-#include <sbi/sbi_types.h>
+#ifdef __ASSEMBLER__
+# define _conv_cast(type, val) (val)
+#else
+# include <sbi/sbi_types.h>
+# define _conv_cast(type, val) ((type)(val))
+#endif
 
 #define BSWAP16(x)	((((x) & 0x00ff) << 8) | \
 			 (((x) & 0xff00) >> 8))
@@ -25,37 +30,47 @@
 			 (((x) & 0xff00000000000000ULL) >> 56))
 
 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__  /* CPU(little-endian) */
-#define cpu_to_be16(x)		((uint16_t)BSWAP16(x))
-#define cpu_to_be32(x)		((uint32_t)BSWAP32(x))
-#define cpu_to_be64(x)		((uint64_t)BSWAP64(x))
+#define cpu_to_be16(x)		_conv_cast(uint16_t, BSWAP16(x))
+#define cpu_to_be32(x)		_conv_cast(uint32_t, BSWAP32(x))
+#define cpu_to_be64(x)		_conv_cast(uint64_t, BSWAP64(x))
 
-#define be16_to_cpu(x)		((uint16_t)BSWAP16(x))
-#define be32_to_cpu(x)		((uint32_t)BSWAP32(x))
-#define be64_to_cpu(x)		((uint64_t)BSWAP64(x))
+#define be16_to_cpu(x)		_conv_cast(uint16_t, BSWAP16(x))
+#define be32_to_cpu(x)		_conv_cast(uint32_t, BSWAP32(x))
+#define be64_to_cpu(x)		_conv_cast(uint64_t, BSWAP64(x))
 
-#define cpu_to_le16(x)		((uint16_t)(x))
-#define cpu_to_le32(x)		((uint32_t)(x))
-#define cpu_to_le64(x)		((uint64_t)(x))
+#define cpu_to_le16(x)		_conv_cast(uint16_t, (x))
+#define cpu_to_le32(x)		_conv_cast(uint32_t, (x))
+#define cpu_to_le64(x)		_conv_cast(uint64_t, (x))
 
-#define le16_to_cpu(x)		((uint16_t)(x))
-#define le32_to_cpu(x)		((uint32_t)(x))
-#define le64_to_cpu(x)		((uint64_t)(x))
+#define le16_to_cpu(x)		_conv_cast(uint16_t, (x))
+#define le32_to_cpu(x)		_conv_cast(uint32_t, (x))
+#define le64_to_cpu(x)		_conv_cast(uint64_t, (x))
 #else /* CPU(big-endian) */
-#define cpu_to_be16(x)		((uint16_t)(x))
-#define cpu_to_be32(x)		((uint32_t)(x))
-#define cpu_to_be64(x)		((uint64_t)(x))
+#define cpu_to_be16(x)		_conv_cast(uint16_t, (x))
+#define cpu_to_be32(x)		_conv_cast(uint32_t, (x))
+#define cpu_to_be64(x)		_conv_cast(uint64_t, (x))
 
-#define be16_to_cpu(x)		((uint16_t)(x))
-#define be32_to_cpu(x)		((uint32_t)(x))
-#define be64_to_cpu(x)		((uint64_t)(x))
+#define be16_to_cpu(x)		_conv_cast(uint16_t, (x))
+#define be32_to_cpu(x)		_conv_cast(uint32_t, (x))
+#define be64_to_cpu(x)		_conv_cast(uint64_t, (x))
 
-#define cpu_to_le16(x)		((uint16_t)BSWAP16(x))
-#define cpu_to_le32(x)		((uint32_t)BSWAP32(x))
-#define cpu_to_le64(x)		((uint64_t)BSWAP64(x))
+#define cpu_to_le16(x)		_conv_cast(uint16_t, BSWAP16(x))
+#define cpu_to_le32(x)		_conv_cast(uint32_t, BSWAP32(x))
+#define cpu_to_le64(x)		_conv_cast(uint64_t, BSWAP64(x))
 
-#define le16_to_cpu(x)		((uint16_t)BSWAP16(x))
-#define le32_to_cpu(x)		((uint32_t)BSWAP32(x))
-#define le64_to_cpu(x)		((uint64_t)BSWAP64(x))
+#define le16_to_cpu(x)		_conv_cast(uint16_t, BSWAP16(x))
+#define le32_to_cpu(x)		_conv_cast(uint32_t, BSWAP32(x))
+#define le64_to_cpu(x)		_conv_cast(uint64_t, BSWAP64(x))
+#endif
+
+#if __riscv_xlen == 64
+#define cpu_to_lle		cpu_to_le64
+#define lle_to_cpu		le64_to_cpu
+#elif __riscv_xlen == 32
+#define cpu_to_lle		cpu_to_le32
+#define lle_to_cpu		le32_to_cpu
+#else
+#error "Unknown __riscv_xlen"
 #endif
 
 #endif /* __SBI_BYTEORDER_H__ */

include/sbi/sbi_dbtr.h

@@ -0,0 +1,125 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Ventana Micro Systems, Inc.
+ *
+ * Authors:
+ *   Himanshu Chauhan <hchauhan@ventanamicro.com>
+ */
+
+#ifndef __SBI_DBTR_H__
+#define __SBI_DBTR_H__
+
+#include <sbi/riscv_dbtr.h>
+#include <sbi/sbi_types.h>
+
+struct sbi_domain;
+
+enum {
+	RV_DBTR_DECLARE_BIT(TS, MAPPED, 0), /* trigger mapped to hw trigger */
+	RV_DBTR_DECLARE_BIT(TS, U, 1),
+	RV_DBTR_DECLARE_BIT(TS, S, 2),
+	RV_DBTR_DECLARE_BIT(TS, VU, 3),
+	RV_DBTR_DECLARE_BIT(TS, VS, 4),
+	RV_DBTR_DECLARE_BIT(TS, HAVE_TRIG, 5), /* H/w dbtr details available */
+	RV_DBTR_DECLARE_BIT(TS, HW_IDX, 8), /* Hardware index of trigger */
+};
+
+enum {
+	RV_DBTR_DECLARE_BIT_MASK(TS, MAPPED, 1),
+	RV_DBTR_DECLARE_BIT_MASK(TS, U, 1),
+	RV_DBTR_DECLARE_BIT_MASK(TS, S, 1),
+	RV_DBTR_DECLARE_BIT_MASK(TS, VU, 1),
+	RV_DBTR_DECLARE_BIT_MASK(TS, VS, 1),
+	RV_DBTR_DECLARE_BIT_MASK(TS, HAVE_TRIG, 1),
+	RV_DBTR_DECLARE_BIT_MASK(TS, HW_IDX, (__riscv_xlen-9)),
+};
+
+#if __riscv_xlen == 64
+#define SBI_DBTR_SHMEM_INVALID_ADDR	0xFFFFFFFFFFFFFFFFUL
+#elif __riscv_xlen == 32
+#define SBI_DBTR_SHMEM_INVALID_ADDR	0xFFFFFFFFUL
+#else
+#error "Unexpected __riscv_xlen"
+#endif
+
+struct sbi_dbtr_shmem {
+	unsigned long phys_lo;
+	unsigned long phys_hi;
+};
+
+struct sbi_dbtr_trigger {
+	unsigned long index;
+	unsigned long type_mask;
+	unsigned long state;
+	unsigned long tdata1;
+	unsigned long tdata2;
+	unsigned long tdata3;
+};
+
+struct sbi_dbtr_data_msg {
+	unsigned long tstate;
+	unsigned long tdata1;
+	unsigned long tdata2;
+	unsigned long tdata3;
+};
+
+struct sbi_dbtr_id_msg {
+	unsigned long idx;
+};
+
+struct sbi_dbtr_hart_triggers_state {
+	struct sbi_dbtr_trigger triggers[RV_MAX_TRIGGERS];
+	struct sbi_dbtr_shmem shmem;
+	u32 total_trigs;
+	u32 available_trigs;
+	u32 hartid;
+	u32 probed;
+};
+
+#define TDATA1_GET_TYPE(_t1)					\
+	EXTRACT_FIELD(_t1, RV_DBTR_BIT_MASK(TDATA1, TYPE))
+
+/* Set the hardware index of trigger in logical trigger state */
+#define SET_TRIG_HW_INDEX(_state, _idx)				\
+	do {							\
+		_state &= ~RV_DBTR_BIT_MASK(TS, HW_IDX);	\
+		_state |= (((unsigned long)_idx			\
+			    << RV_DBTR_BIT(TS, HW_IDX))		\
+			   & RV_DBTR_BIT_MASK(TS, HW_IDX));	\
+	}while (0);
+
+/** SBI shared mem messages layout */
+struct sbi_dbtr_shmem_entry {
+	struct sbi_dbtr_data_msg data;
+	struct sbi_dbtr_id_msg id;
+};
+
+#define SBI_DBTR_SHMEM_ALIGN_MASK	((__riscv_xlen / 8) - 1)
+
+/** Initialize debug triggers */
+int sbi_dbtr_init(struct sbi_scratch *scratch, bool coldboot);
+
+/** SBI DBTR extension functions */
+int sbi_dbtr_supported(void);
+int sbi_dbtr_setup_shmem(const struct sbi_domain *dom, unsigned long smode,
+			 unsigned long shmem_phys_lo,
+			 unsigned long shmem_phys_hi);
+int sbi_dbtr_num_trig(unsigned long trig_tdata1, unsigned long *out);
+int sbi_dbtr_read_trig(unsigned long smode,
+		       unsigned long trig_idx_base, unsigned long trig_count);
+int sbi_dbtr_install_trig(unsigned long smode,
+			  unsigned long trig_count, unsigned long *out);
+int sbi_dbtr_uninstall_trig(unsigned long trig_idx_base,
+			    unsigned long trig_idx_mask);
+int sbi_dbtr_enable_trig(unsigned long trig_idx_base,
+			 unsigned long trig_idx_mask);
+int sbi_dbtr_update_trig(unsigned long smode,
+			 unsigned long trig_idx_base,
+			 unsigned long trig_idx_mask);
+int sbi_dbtr_disable_trig(unsigned long trig_idx_base,
+			  unsigned long trig_idx_mask);
+
+int sbi_dbtr_get_total_triggers(void);
+
+#endif

include/sbi/sbi_domain.h

@@ -10,8 +10,10 @@
 #ifndef __SBI_DOMAIN_H__
 #define __SBI_DOMAIN_H__
 
+#include <sbi/riscv_locks.h>
 #include <sbi/sbi_types.h>
 #include <sbi/sbi_hartmask.h>
+#include <sbi/sbi_domain_context.h>
 
 struct sbi_scratch;
 
@@ -43,6 +45,80 @@ struct sbi_domain_memregion {
 #define SBI_DOMAIN_MEMREGION_SU_WRITABLE	(1UL << 4)
 #define SBI_DOMAIN_MEMREGION_SU_EXECUTABLE	(1UL << 5)
 
+#define SBI_DOMAIN_MEMREGION_ACCESS_MASK	(0x3fUL)
+#define SBI_DOMAIN_MEMREGION_M_ACCESS_MASK	(0x7UL)
+#define SBI_DOMAIN_MEMREGION_SU_ACCESS_MASK	(0x38UL)
+
+#define SBI_DOMAIN_MEMREGION_SU_ACCESS_SHIFT	(3)
+
+#define SBI_DOMAIN_MEMREGION_SHARED_RDONLY		\
+		(SBI_DOMAIN_MEMREGION_M_READABLE |	\
+		 SBI_DOMAIN_MEMREGION_SU_READABLE)
+
+#define SBI_DOMAIN_MEMREGION_SHARED_SUX_MRX		\
+		(SBI_DOMAIN_MEMREGION_M_READABLE   |	\
+		 SBI_DOMAIN_MEMREGION_M_EXECUTABLE |	\
+		 SBI_DOMAIN_MEMREGION_SU_EXECUTABLE)
+
+#define SBI_DOMAIN_MEMREGION_SHARED_SUX_MX		\
+		(SBI_DOMAIN_MEMREGION_M_EXECUTABLE |	\
+		 SBI_DOMAIN_MEMREGION_SU_EXECUTABLE)
+
+#define SBI_DOMAIN_MEMREGION_SHARED_SURW_MRW		\
+		(SBI_DOMAIN_MEMREGION_M_READABLE |	\
+		 SBI_DOMAIN_MEMREGION_M_WRITABLE |	\
+		 SBI_DOMAIN_MEMREGION_SU_READABLE|	\
+		 SBI_DOMAIN_MEMREGION_SU_WRITABLE)
+
+#define SBI_DOMAIN_MEMREGION_SHARED_SUR_MRW		\
+		(SBI_DOMAIN_MEMREGION_M_READABLE |	\
+		 SBI_DOMAIN_MEMREGION_M_WRITABLE |	\
+		 SBI_DOMAIN_MEMREGION_SU_READABLE)
+
+	/* Shared read-only region between M and SU mode */
+#define SBI_DOMAIN_MEMREGION_IS_SUR_MR(__flags)			 \
+		((__flags & SBI_DOMAIN_MEMREGION_ACCESS_MASK) == \
+		 SBI_DOMAIN_MEMREGION_SHARED_RDONLY)
+
+	/* Shared region: SU execute-only and M read/execute */
+#define SBI_DOMAIN_MEMREGION_IS_SUX_MRX(__flags)		 \
+		((__flags & SBI_DOMAIN_MEMREGION_ACCESS_MASK) == \
+		 SBI_DOMAIN_MEMREGION_SHARED_SUX_MRX)
+
+	/* Shared region: SU and M execute-only */
+#define SBI_DOMAIN_MEMREGION_IS_SUX_MX(__flags)			 \
+		((__flags & SBI_DOMAIN_MEMREGION_ACCESS_MASK) == \
+		 SBI_DOMAIN_MEMREGION_SHARED_SUX_MX)
+
+	/* Shared region: SU and M read/write */
+#define SBI_DOMAIN_MEMREGION_IS_SURW_MRW(__flags)		 \
+		((__flags & SBI_DOMAIN_MEMREGION_ACCESS_MASK) == \
+		 SBI_DOMAIN_MEMREGION_SHARED_SURW_MRW)
+
+	/* Shared region: SU read-only and M read/write */
+#define SBI_DOMAIN_MEMREGION_IS_SUR_MRW(__flags)		 \
+		((__flags & SBI_DOMAIN_MEMREGION_ACCESS_MASK) == \
+		 SBI_DOMAIN_MEMREGION_SHARED_SUR_MRW)
+
+	/*
+	 * Check if region flags match with any of the above
+	 * mentioned shared region type
+	 */
+#define SBI_DOMAIN_MEMREGION_IS_SHARED(_flags)			\
+		(SBI_DOMAIN_MEMREGION_IS_SUR_MR(_flags)  ||	\
+		 SBI_DOMAIN_MEMREGION_IS_SUX_MRX(_flags) ||	\
+		 SBI_DOMAIN_MEMREGION_IS_SUX_MX(_flags)  ||	\
+		 SBI_DOMAIN_MEMREGION_IS_SURW_MRW(_flags)||	\
+		 SBI_DOMAIN_MEMREGION_IS_SUR_MRW(_flags))
+
+#define SBI_DOMAIN_MEMREGION_M_ONLY_ACCESS(__flags)			\
+		((__flags & SBI_DOMAIN_MEMREGION_M_ACCESS_MASK) &&	\
+		 !(__flags & SBI_DOMAIN_MEMREGION_SU_ACCESS_MASK))
+
+#define SBI_DOMAIN_MEMREGION_SU_ONLY_ACCESS(__flags)			\
+		((__flags & SBI_DOMAIN_MEMREGION_SU_ACCESS_MASK)  &&	\
+		 !(__flags & SBI_DOMAIN_MEMREGION_M_ACCESS_MASK))
+
 /** Bit to control if permissions are enforced on all modes */
 #define SBI_DOMAIN_MEMREGION_ENF_PERMISSIONS	(1UL << 6)
 
@@ -78,12 +154,6 @@ struct sbi_domain_memregion {
 				(SBI_DOMAIN_MEMREGION_SU_EXECUTABLE | \
 				 SBI_DOMAIN_MEMREGION_M_EXECUTABLE)
 
-#define SBI_DOMAIN_MEMREGION_ACCESS_MASK	(0x3fUL)
-#define SBI_DOMAIN_MEMREGION_M_ACCESS_MASK	(0x7UL)
-#define SBI_DOMAIN_MEMREGION_SU_ACCESS_MASK	(0x38UL)
-
-#define SBI_DOMAIN_MEMREGION_SU_ACCESS_SHIFT	(3)
-
 #define SBI_DOMAIN_MEMREGION_MMIO		(1UL << 31)
 	unsigned long flags;
 };
@@ -104,10 +174,14 @@ struct sbi_domain {
 	 * in the coldboot path
 	 */
 	struct sbi_hartmask assigned_harts;
+	/** Spinlock for accessing assigned_harts */
+	spinlock_t assigned_harts_lock;
 	/** Name of this domain */
 	char name[64];
 	/** Possible HARTs in this domain */
 	const struct sbi_hartmask *possible_harts;
+	/** Contexts for possible HARTs indexed by hartindex */
+	struct sbi_context *hartindex_to_context_table[SBI_HARTMASK_MAX_BITS];
 	/** Array of memory regions terminated by a region with order zero */
 	struct sbi_domain_memregion *regions;
 	/** HART id of the HART booting this domain */
@@ -129,12 +203,15 @@ struct sbi_domain {
 /** The root domain instance */
 extern struct sbi_domain root;
 
-/** Get pointer to sbi_domain from HART id */
-struct sbi_domain *sbi_hartid_to_domain(u32 hartid);
+/** Get pointer to sbi_domain from HART index */
+struct sbi_domain *sbi_hartindex_to_domain(u32 hartindex);
+
+/** Update HART local pointer to point to specified domain */
+void sbi_update_hartindex_to_domain(u32 hartindex, struct sbi_domain *dom);
 
 /** Get pointer to sbi_domain for current HART */
 #define sbi_domain_thishart_ptr() \
-	sbi_hartid_to_domain(current_hartid())
+	sbi_hartindex_to_domain(sbi_hartid_to_hartindex(current_hartid()))
 
 /** Index to domain table */
 extern struct sbi_domain *domidx_to_domain_table[];

include/sbi/sbi_domain_context.h

@@ -0,0 +1,77 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) IPADS@SJTU 2023. All rights reserved.
+ */
+
+#ifndef __SBI_DOMAIN_CONTEXT_H__
+#define __SBI_DOMAIN_CONTEXT_H__
+
+#include <sbi/sbi_types.h>
+#include <sbi/sbi_trap.h>
+#include <sbi/sbi_domain.h>
+
+/** Context representation for a hart within a domain */
+struct sbi_context {
+	/** Trap-related states such as GPRs, mepc, and mstatus */
+	struct sbi_trap_context trap_ctx;
+
+	/** Supervisor status register */
+	unsigned long sstatus;
+	/** Supervisor interrupt enable register */
+	unsigned long sie;
+	/** Supervisor trap vector base address register */
+	unsigned long stvec;
+	/** Supervisor scratch register for temporary storage */
+	unsigned long sscratch;
+	/** Supervisor exception program counter register */
+	unsigned long sepc;
+	/** Supervisor cause register */
+	unsigned long scause;
+	/** Supervisor trap value register */
+	unsigned long stval;
+	/** Supervisor interrupt pending register */
+	unsigned long sip;
+	/** Supervisor address translation and protection register */
+	unsigned long satp;
+	/** Counter-enable register */
+	unsigned long scounteren;
+	/** Supervisor environment configuration register */
+	unsigned long senvcfg;
+
+	/** Reference to the owning domain */
+	struct sbi_domain *dom;
+	/** Previous context (caller) to jump to during context exits */
+	struct sbi_context *prev_ctx;
+	/** Is context initialized and runnable */
+	bool initialized;
+};
+
+/** Get the context pointer for a given hart index and domain */
+#define sbi_hartindex_to_domain_context(__hartindex, __d) \
+	(__d)->hartindex_to_context_table[__hartindex]
+
+/** Macro to obtain the current hart's context pointer */
+#define sbi_domain_context_thishart_ptr()                  \
+	sbi_hartindex_to_domain_context(                   \
+		sbi_hartid_to_hartindex(current_hartid()), \
+		sbi_domain_thishart_ptr())
+
+/**
+ * Enter a specific domain context synchronously
+ * @param dom pointer to domain
+ *
+ * @return 0 on success and negative error code on failure
+ */
+int sbi_domain_context_enter(struct sbi_domain *dom);
+
+/**
+ * Exit the current domain context, and then return to the caller
+ * of sbi_domain_context_enter or attempt to start the next domain
+ * context to be initialized
+ *
+ * @return 0 on success and negative error code on failure
+ */
+int sbi_domain_context_exit(void);
+
+#endif // __SBI_DOMAIN_CONTEXT_H__

include/sbi/sbi_ecall.h

@@ -13,12 +13,19 @@
 #include <sbi/sbi_types.h>
 #include <sbi/sbi_list.h>
 
-#define SBI_ECALL_VERSION_MAJOR		1
+#define SBI_ECALL_VERSION_MAJOR		2
 #define SBI_ECALL_VERSION_MINOR		0
 #define SBI_OPENSBI_IMPID		1
 
 struct sbi_trap_regs;
-struct sbi_trap_info;
+struct sbi_trap_context;
+
+struct sbi_ecall_return {
+	/* Return flag to skip register update */
+	bool skip_regs_update;
+	/* Return value */
+	unsigned long value;
+};
 
 struct sbi_ecall_extension {
 	/* head is used by the extension list */
@@ -62,9 +69,8 @@ struct sbi_ecall_extension {
 	 * never invoked with an invalid or unavailable extension ID.
 	 */
 	int (* handle)(unsigned long extid, unsigned long funcid,
-		       const struct sbi_trap_regs *regs,
-		       unsigned long *out_val,
-		       struct sbi_trap_info *out_trap);
+		       struct sbi_trap_regs *regs,
+		       struct sbi_ecall_return *out);
 };
 
 u16 sbi_ecall_version_major(void);
@@ -81,7 +87,7 @@ int sbi_ecall_register_extension(struct sbi_ecall_extension *ext);
 
 void sbi_ecall_unregister_extension(struct sbi_ecall_extension *ext);
 
-int sbi_ecall_handler(struct sbi_trap_regs *regs);
+int sbi_ecall_handler(struct sbi_trap_context *tcntx);
 
 int sbi_ecall_init(void);
 

include/sbi/sbi_ecall_interface.h

@@ -32,6 +32,9 @@
 #define SBI_EXT_DBCN				0x4442434E
 #define SBI_EXT_SUSP				0x53555350
 #define SBI_EXT_CPPC				0x43505043
+#define SBI_EXT_DBTR				0x44425452
+#define SBI_EXT_SSE				0x535345
+#define SBI_EXT_FWFT				0x46574654
 
 /* SBI function IDs for BASE extension*/
 #define SBI_EXT_BASE_GET_SPEC_VERSION		0x0
@@ -103,6 +106,43 @@
 #define SBI_EXT_PMU_COUNTER_STOP	0x4
 #define SBI_EXT_PMU_COUNTER_FW_READ	0x5
 #define SBI_EXT_PMU_COUNTER_FW_READ_HI	0x6
+#define SBI_EXT_PMU_SNAPSHOT_SET_SHMEM	0x7
+
+/* SBI function IDs for DBTR extension */
+#define SBI_EXT_DBTR_NUM_TRIGGERS	0x0
+#define SBI_EXT_DBTR_SETUP_SHMEM	0x1
+#define SBI_EXT_DBTR_TRIGGER_READ	0x2
+#define SBI_EXT_DBTR_TRIGGER_INSTALL	0x3
+#define SBI_EXT_DBTR_TRIGGER_UPDATE	0x4
+#define SBI_EXT_DBTR_TRIGGER_UNINSTALL	0x5
+#define SBI_EXT_DBTR_TRIGGER_ENABLE	0x6
+#define SBI_EXT_DBTR_TRIGGER_DISABLE	0x7
+
+/* SBI function IDs for FW feature extension */
+#define SBI_EXT_FWFT_SET		0x0
+#define SBI_EXT_FWFT_GET		0x1
+
+enum sbi_fwft_feature_t {
+	SBI_FWFT_MISALIGNED_EXC_DELEG		= 0x0,
+	SBI_FWFT_LANDING_PAD			= 0x1,
+	SBI_FWFT_SHADOW_STACK			= 0x2,
+	SBI_FWFT_DOUBLE_TRAP			= 0x3,
+	SBI_FWFT_PTE_AD_HW_UPDATING		= 0x4,
+	SBI_FWFT_LOCAL_RESERVED_START		= 0x5,
+	SBI_FWFT_LOCAL_RESERVED_END		= 0x3fffffff,
+	SBI_FWFT_LOCAL_PLATFORM_START		= 0x40000000,
+	SBI_FWFT_LOCAL_PLATFORM_END		= 0x7fffffff,
+
+	SBI_FWFT_GLOBAL_RESERVED_START		= 0x80000000,
+	SBI_FWFT_GLOBAL_RESERVED_END		= 0xbfffffff,
+	SBI_FWFT_GLOBAL_PLATFORM_START		= 0xc0000000,
+	SBI_FWFT_GLOBAL_PLATFORM_END		= 0xffffffff,
+};
+
+#define SBI_FWFT_GLOBAL_FEATURE_BIT		(1 << 31)
+#define SBI_FWFT_PLATFORM_FEATURE_BIT		(1 << 30)
+
+#define SBI_FWFT_SET_FLAG_LOCK			(1 << 0)
 
 /** General pmu event codes specified in SBI PMU extension */
 enum sbi_pmu_hw_generic_events_t {
@@ -241,9 +281,11 @@ enum sbi_pmu_ctr_type {
 
 /* Flags defined for counter start function */
 #define SBI_PMU_START_FLAG_SET_INIT_VALUE (1 << 0)
+#define SBI_PMU_START_FLAG_INIT_FROM_SNAPSHOT (1 << 1)
 
 /* Flags defined for counter stop function */
 #define SBI_PMU_STOP_FLAG_RESET (1 << 0)
+#define SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT (1 << 1)
 
 /* SBI function IDs for DBCN extension */
 #define SBI_EXT_DBCN_CONSOLE_WRITE		0x0
@@ -290,6 +332,80 @@ enum sbi_cppc_reg_id {
 	SBI_CPPC_NON_ACPI_LAST		= SBI_CPPC_TRANSITION_LATENCY,
 };
 
+/* SBI Function IDs for SSE extension */
+#define SBI_EXT_SSE_READ_ATTR		0x00000000
+#define SBI_EXT_SSE_WRITE_ATTR		0x00000001
+#define SBI_EXT_SSE_REGISTER		0x00000002
+#define SBI_EXT_SSE_UNREGISTER		0x00000003
+#define SBI_EXT_SSE_ENABLE		0x00000004
+#define SBI_EXT_SSE_DISABLE		0x00000005
+#define SBI_EXT_SSE_COMPLETE		0x00000006
+#define SBI_EXT_SSE_INJECT		0x00000007
+
+/* SBI SSE Event Attributes. */
+enum sbi_sse_attr_id {
+	SBI_SSE_ATTR_STATUS		= 0x00000000,
+	SBI_SSE_ATTR_PRIO		= 0x00000001,
+	SBI_SSE_ATTR_CONFIG		= 0x00000002,
+	SBI_SSE_ATTR_PREFERRED_HART	= 0x00000003,
+	SBI_SSE_ATTR_ENTRY_PC		= 0x00000004,
+	SBI_SSE_ATTR_ENTRY_ARG		= 0x00000005,
+	SBI_SSE_ATTR_INTERRUPTED_SEPC	= 0x00000006,
+	SBI_SSE_ATTR_INTERRUPTED_FLAGS	= 0x00000007,
+	SBI_SSE_ATTR_INTERRUPTED_A6	= 0x00000008,
+	SBI_SSE_ATTR_INTERRUPTED_A7	= 0x00000009,
+
+	SBI_SSE_ATTR_MAX		= 0x0000000A
+};
+
+#define SBI_SSE_ATTR_STATUS_STATE_OFFSET	0
+#define SBI_SSE_ATTR_STATUS_STATE_MASK		0x3
+#define SBI_SSE_ATTR_STATUS_PENDING_OFFSET	2
+#define SBI_SSE_ATTR_STATUS_INJECT_OFFSET	3
+
+#define SBI_SSE_ATTR_CONFIG_ONESHOT	(1 << 0)
+
+#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_STATUS_SPP	BIT(0)
+#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_STATUS_SPIE	BIT(1)
+#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_HSTATUS_SPV	BIT(2)
+#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_HSTATUS_SPVP	BIT(3)
+
+enum sbi_sse_state {
+	SBI_SSE_STATE_UNUSED		= 0,
+	SBI_SSE_STATE_REGISTERED	= 1,
+	SBI_SSE_STATE_ENABLED		= 2,
+	SBI_SSE_STATE_RUNNING		= 3,
+};
+
+/* SBI SSE Event IDs. */
+#define SBI_SSE_EVENT_LOCAL_RAS			0x00000000
+#define SBI_SSE_EVENT_LOCAL_PLAT_0_START	0x00004000
+#define SBI_SSE_EVENT_LOCAL_PLAT_0_END		0x00007fff
+#define SBI_SSE_EVENT_GLOBAL_RAS		0x00008000
+#define SBI_SSE_EVENT_GLOBAL_PLAT_0_START	0x00004000
+#define SBI_SSE_EVENT_GLOBAL_PLAT_0_END		0x00007fff
+
+#define SBI_SSE_EVENT_LOCAL_PMU			0x00010000
+#define SBI_SSE_EVENT_LOCAL_PLAT_1_START	0x00014000
+#define SBI_SSE_EVENT_LOCAL_PLAT_1_END		0x00017fff
+#define SBI_SSE_EVENT_GLOBAL_PLAT_1_START	0x0001c000
+#define SBI_SSE_EVENT_GLOBAL_PLAT_1_END		0x0001ffff
+
+#define SBI_SSE_EVENT_LOCAL_PLAT_2_START	0x00024000
+#define SBI_SSE_EVENT_LOCAL_PLAT_2_END		0x00027fff
+#define SBI_SSE_EVENT_GLOBAL_PLAT_2_START	0x0002c000
+#define SBI_SSE_EVENT_GLOBAL_PLAT_2_END		0x0002ffff
+
+#define SBI_SSE_EVENT_LOCAL_SOFTWARE		0xffff0000
+#define SBI_SSE_EVENT_LOCAL_PLAT_3_START	0xffff4000
+#define SBI_SSE_EVENT_LOCAL_PLAT_3_END		0xffff7fff
+#define SBI_SSE_EVENT_GLOBAL_SOFTWARE		0xffff8000
+#define SBI_SSE_EVENT_GLOBAL_PLAT_3_START	0xffffc000
+#define SBI_SSE_EVENT_GLOBAL_PLAT_3_END		0xffffffff
+
+#define SBI_SSE_EVENT_GLOBAL_BIT		(1 << 15)
+#define SBI_SSE_EVENT_PLATFORM_BIT		(1 << 14)
+
 /* SBI base specification related macros */
 #define SBI_SPEC_VERSION_MAJOR_OFFSET		24
 #define SBI_SPEC_VERSION_MAJOR_MASK		0x7f
@@ -309,8 +425,11 @@ enum sbi_cppc_reg_id {
 #define SBI_ERR_ALREADY_AVAILABLE		-6
 #define SBI_ERR_ALREADY_STARTED			-7
 #define SBI_ERR_ALREADY_STOPPED			-8
+#define SBI_ERR_NO_SHMEM			-9
+#define SBI_ERR_INVALID_STATE			-10
+#define SBI_ERR_BAD_RANGE			-11
 
-#define SBI_LAST_ERR				SBI_ERR_ALREADY_STOPPED
+#define SBI_LAST_ERR				SBI_ERR_BAD_RANGE
 
 /* clang-format on */
 

include/sbi/sbi_error.h

@@ -23,6 +23,9 @@
 #define SBI_EALREADY		SBI_ERR_ALREADY_AVAILABLE
 #define SBI_EALREADY_STARTED	SBI_ERR_ALREADY_STARTED
 #define SBI_EALREADY_STOPPED	SBI_ERR_ALREADY_STOPPED
+#define SBI_ENO_SHMEM		SBI_ERR_NO_SHMEM
+#define SBI_EINVALID_STATE	SBI_ERR_INVALID_STATE
+#define SBI_EBAD_RANGE		SBI_ERR_BAD_RANGE
 
 #define SBI_ENODEV		-1000
 #define SBI_ENOSYS		-1001
@@ -31,9 +34,8 @@
 #define SBI_EILL		-1004
 #define SBI_ENOSPC		-1005
 #define SBI_ENOMEM		-1006
-#define SBI_ETRAP		-1007
-#define SBI_EUNKNOWN		-1008
-#define SBI_ENOENT		-1009
+#define SBI_EUNKNOWN		-1007
+#define SBI_ENOENT		-1008
 
 /* clang-format on */
 

include/sbi/sbi_fwft.h

@@ -0,0 +1,23 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Rivos Inc.
+ *
+ * Authors:
+ *   Clément Léger <cleger@rivosinc.com>
+ */
+
+#ifndef __SBI_FW_FEATURE_H__
+#define __SBI_FW_FEATURE_H__
+
+#include <sbi/sbi_ecall_interface.h>
+
+struct sbi_scratch;
+
+int sbi_fwft_set(enum sbi_fwft_feature_t feature, unsigned long value,
+		 unsigned long flags);
+int sbi_fwft_get(enum sbi_fwft_feature_t feature, unsigned long *out_val);
+
+int sbi_fwft_init(struct sbi_scratch *scratch, bool cold_boot);
+
+#endif

include/sbi/sbi_hart.h

@@ -11,6 +11,7 @@
 #define __SBI_HART_H__
 
 #include <sbi/sbi_types.h>
+#include <sbi/sbi_bitops.h>
 
 /** Possible privileged specification versions of a hart */
 enum sbi_hart_priv_versions {
@@ -26,29 +27,81 @@ enum sbi_hart_priv_versions {
 
 /** Possible ISA extensions of a hart */
 enum sbi_hart_extensions {
-	/** Hart has Sscofpmt extension */
-	SBI_HART_EXT_SSCOFPMF = 0,
-	/** HART has HW time CSR (extension name not available) */
-	SBI_HART_EXT_TIME,
 	/** HART has AIA M-mode CSRs */
-	SBI_HART_EXT_SMAIA,
+	SBI_HART_EXT_SMAIA = 0,
+	/** HART has Smepmp */
+	SBI_HART_EXT_SMEPMP,
 	/** HART has Smstateen CSR **/
 	SBI_HART_EXT_SMSTATEEN,
+	/** Hart has Sscofpmt extension */
+	SBI_HART_EXT_SSCOFPMF,
 	/** HART has Sstc extension */
 	SBI_HART_EXT_SSTC,
+	/** HART has Zicntr extension (i.e. HW cycle, time & instret CSRs) */
+	SBI_HART_EXT_ZICNTR,
+	/** HART has Zihpm extension */
+	SBI_HART_EXT_ZIHPM,
+	/** HART has Zkr extension */
+	SBI_HART_EXT_ZKR,
+	/** Hart has Smcntrpmf extension */
+	SBI_HART_EXT_SMCNTRPMF,
+	/** Hart has Xandespmu extension */
+	SBI_HART_EXT_XANDESPMU,
+	/** Hart has Zicboz extension */
+	SBI_HART_EXT_ZICBOZ,
+	/** Hart has Zicbom extension */
+	SBI_HART_EXT_ZICBOM,
+	/** Hart has Svpbmt extension */
+	SBI_HART_EXT_SVPBMT,
+	/** Hart has debug trigger extension */
+	SBI_HART_EXT_SDTRIG,
+	/** Hart has Smcsrind extension */
+	SBI_HART_EXT_SMCSRIND,
+	/** Hart has Smcdeleg extension */
+	SBI_HART_EXT_SMCDELEG,
+	/** Hart has Sscsrind extension */
+	SBI_HART_EXT_SSCSRIND,
+	/** Hart has Ssccfg extension */
+	SBI_HART_EXT_SSCCFG,
+	/** Hart has Svade extension */
+	SBI_HART_EXT_SVADE,
+	/** Hart has Svadu extension */
+	SBI_HART_EXT_SVADU,
 
 	/** Maximum index of Hart extension */
 	SBI_HART_EXT_MAX,
 };
 
+struct sbi_hart_ext_data {
+	const unsigned int id;
+	const char *name;
+};
+
+extern const struct sbi_hart_ext_data sbi_hart_ext[];
+
+/*
+ * Smepmp enforces access boundaries between M-mode and
+ * S/U-mode. When it is enabled, the PMPs are programmed
+ * such that M-mode doesn't have access to S/U-mode memory.
+ *
+ * To give M-mode R/W access to the shared memory between M and
+ * S/U-mode, first entry is reserved. It is disabled at boot.
+ * When shared memory access is required, the physical address
+ * should be programmed into the first PMP entry with R/W
+ * permissions to the M-mode. Once the work is done, it should be
+ * unmapped. sbi_hart_map_saddr/sbi_hart_unmap_saddr function
+ * pair should be used to map/unmap the shared memory.
+ */
+#define SBI_SMEPMP_RESV_ENTRY		0
+
 struct sbi_hart_features {
 	bool detected;
 	int priv_version;
-	unsigned long extensions;
+	unsigned long extensions[BITS_TO_LONGS(SBI_HART_EXT_MAX)];
 	unsigned int pmp_count;
 	unsigned int pmp_addr_bits;
-	unsigned long pmp_gran;
-	unsigned int mhpm_count;
+	unsigned int pmp_log2gran;
+	unsigned int mhpm_mask;
 	unsigned int mhpm_bits;
 };
 
@@ -63,14 +116,16 @@ static inline ulong sbi_hart_expected_trap_addr(void)
 	return (ulong)sbi_hart_expected_trap;
 }
 
-unsigned int sbi_hart_mhpm_count(struct sbi_scratch *scratch);
+unsigned int sbi_hart_mhpm_mask(struct sbi_scratch *scratch);
 void sbi_hart_delegation_dump(struct sbi_scratch *scratch,
 			      const char *prefix, const char *suffix);
 unsigned int sbi_hart_pmp_count(struct sbi_scratch *scratch);
-unsigned long sbi_hart_pmp_granularity(struct sbi_scratch *scratch);
+unsigned int sbi_hart_pmp_log2gran(struct sbi_scratch *scratch);
 unsigned int sbi_hart_pmp_addrbits(struct sbi_scratch *scratch);
 unsigned int sbi_hart_mhpm_bits(struct sbi_scratch *scratch);
 int sbi_hart_pmp_configure(struct sbi_scratch *scratch);
+int sbi_hart_map_saddr(unsigned long base, unsigned long size);
+int sbi_hart_unmap_saddr(void);
 int sbi_hart_priv_version(struct sbi_scratch *scratch);
 void sbi_hart_get_priv_version_str(struct sbi_scratch *scratch,
 				   char *version_str, int nvstr);

include/sbi/sbi_hartmask.h

@@ -11,6 +11,7 @@
 #define __SBI_HARTMASK_H__
 
 #include <sbi/sbi_bitmap.h>
+#include <sbi/sbi_scratch.h>
 
 /**
  * Maximum number of bits in a hartmask
@@ -32,7 +33,10 @@ struct sbi_hartmask {
 
 /** Initialize hartmask to zero except a particular HART id */
 #define SBI_HARTMASK_INIT_EXCEPT(__m, __h)	\
-	bitmap_zero_except(((__m)->bits), (__h), SBI_HARTMASK_MAX_BITS)
+	do { \
+		u32 __i = sbi_hartid_to_hartindex(__h); \
+		bitmap_zero_except(((__m)->bits), __i, SBI_HARTMASK_MAX_BITS); \
+	} while(0)
 
 /**
  * Get underlying bitmap of hartmask
@@ -41,37 +45,68 @@ struct sbi_hartmask {
 #define sbi_hartmask_bits(__m)		((__m)->bits)
 
 /**
- * Set a HART in hartmask
+ * Set a HART index in hartmask
+ * @param i HART index to set
+ * @param m the hartmask pointer
+ */
+static inline void sbi_hartmask_set_hartindex(u32 i, struct sbi_hartmask *m)
+{
+	if (i < SBI_HARTMASK_MAX_BITS)
+		__set_bit(i, m->bits);
+}
+
+/**
+ * Set a HART id in hartmask
  * @param h HART id to set
  * @param m the hartmask pointer
  */
-static inline void sbi_hartmask_set_hart(u32 h, struct sbi_hartmask *m)
+static inline void sbi_hartmask_set_hartid(u32 h, struct sbi_hartmask *m)
 {
-	if (h < SBI_HARTMASK_MAX_BITS)
-		__set_bit(h, m->bits);
+	sbi_hartmask_set_hartindex(sbi_hartid_to_hartindex(h), m);
 }
 
 /**
- * Clear a HART in hartmask
+ * Clear a HART index in hartmask
+ * @param i HART index to clear
+ * @param m the hartmask pointer
+ */
+static inline void sbi_hartmask_clear_hartindex(u32 i, struct sbi_hartmask *m)
+{
+	if (i < SBI_HARTMASK_MAX_BITS)
+		__clear_bit(i, m->bits);
+}
+
+/**
+ * Clear a HART id in hartmask
  * @param h HART id to clear
  * @param m the hartmask pointer
  */
-static inline void sbi_hartmask_clear_hart(u32 h, struct sbi_hartmask *m)
+static inline void sbi_hartmask_clear_hartid(u32 h, struct sbi_hartmask *m)
 {
-	if (h < SBI_HARTMASK_MAX_BITS)
-		__clear_bit(h, m->bits);
+	sbi_hartmask_clear_hartindex(sbi_hartid_to_hartindex(h), m);
 }
 
 /**
- * Test a HART in hartmask
+ * Test a HART index in hartmask
+ * @param i HART index to test
+ * @param m the hartmask pointer
+ */
+static inline int sbi_hartmask_test_hartindex(u32 i,
+					      const struct sbi_hartmask *m)
+{
+	if (i < SBI_HARTMASK_MAX_BITS)
+		return __test_bit(i, m->bits);
+	return 0;
+}
+
+/**
+ * Test a HART id in hartmask
  * @param h HART id to test
  * @param m the hartmask pointer
  */
-static inline int sbi_hartmask_test_hart(u32 h, const struct sbi_hartmask *m)
+static inline int sbi_hartmask_test_hartid(u32 h, const struct sbi_hartmask *m)
 {
-	if (h < SBI_HARTMASK_MAX_BITS)
-		return __test_bit(h, m->bits);
-	return 0;
+	return sbi_hartmask_test_hartindex(sbi_hartid_to_hartindex(h), m);
 }
 
 /**
@@ -134,8 +169,14 @@ static inline void sbi_hartmask_xor(struct sbi_hartmask *dstp,
 		   sbi_hartmask_bits(src2p), SBI_HARTMASK_MAX_BITS);
 }
 
-/** Iterate over each HART in hartmask */
-#define sbi_hartmask_for_each_hart(__h, __m)	\
-	for_each_set_bit(__h, (__m)->bits, SBI_HARTMASK_MAX_BITS)
+/**
+ * Iterate over each HART index in hartmask
+ * __i hart index
+ * __m hartmask
+*/
+#define sbi_hartmask_for_each_hartindex(__i, __m) \
+	for((__i) = find_first_bit((__m)->bits, SBI_HARTMASK_MAX_BITS); \
+		(__i) < SBI_HARTMASK_MAX_BITS; \
+		(__i) = find_next_bit((__m)->bits, SBI_HARTMASK_MAX_BITS, (__i) + 1))
 
 #endif

include/sbi/sbi_heap.h

@@ -12,6 +12,9 @@
 
 #include <sbi/sbi_types.h>
 
+/* Alignment of heap base address and size */
+#define HEAP_BASE_ALIGN			1024
+
 struct sbi_scratch;
 
 /** Allocate from heap area */

include/sbi/sbi_illegal_insn.h

@@ -12,8 +12,8 @@
 
 #include <sbi/sbi_types.h>
 
-struct sbi_trap_regs;
+struct sbi_trap_context;
 
-int sbi_illegal_insn_handler(ulong insn, struct sbi_trap_regs *regs);
+int sbi_illegal_insn_handler(struct sbi_trap_context *tcntx);
 
 #endif

include/sbi/sbi_ipi.h

@@ -14,7 +14,7 @@
 
 /* clang-format off */
 
-#define SBI_IPI_EVENT_MAX			__riscv_xlen
+#define SBI_IPI_EVENT_MAX			(8 * __SIZEOF_LONG__)
 
 /* clang-format on */
 
@@ -23,11 +23,11 @@ struct sbi_ipi_device {
 	/** Name of the IPI device */
 	char name[32];
 
-	/** Send IPI to a target HART */
-	void (*ipi_send)(u32 target_hart);
+	/** Send IPI to a target HART index */
+	void (*ipi_send)(u32 hart_index);
 
-	/** Clear IPI for a target HART */
-	void (*ipi_clear)(u32 target_hart);
+	/** Clear IPI for a target HART index */
+	void (*ipi_clear)(u32 hart_index);
 };
 
 enum sbi_ipi_update_type {
@@ -54,7 +54,7 @@ struct sbi_ipi_event_ops {
 	 */
 	int (* update)(struct sbi_scratch *scratch,
 			struct sbi_scratch *remote_scratch,
-			u32 remote_hartid, void *data);
+			u32 remote_hartindex, void *data);
 
 	/**
 	 * Sync callback to wait for remote HART
@@ -85,9 +85,9 @@ int sbi_ipi_send_halt(ulong hmask, ulong hbase);
 
 void sbi_ipi_process(void);
 
-int sbi_ipi_raw_send(u32 target_hart);
+int sbi_ipi_raw_send(u32 hartindex);
 
-void sbi_ipi_raw_clear(u32 target_hart);
+void sbi_ipi_raw_clear(u32 hartindex);
 
 const struct sbi_ipi_device *sbi_ipi_get_device(void);
 

include/sbi/sbi_irqchip.h

@@ -13,7 +13,6 @@
 #include <sbi/sbi_types.h>
 
 struct sbi_scratch;
-struct sbi_trap_regs;
 
 /**
  * Set external interrupt handling function
@@ -23,7 +22,7 @@ struct sbi_trap_regs;
  *
  * @param fn function pointer for handling external irqs
  */
-void sbi_irqchip_set_irqfn(int (*fn)(struct sbi_trap_regs *regs));
+void sbi_irqchip_set_irqfn(int (*fn)(void));
 
 /**
  * Process external interrupts
@@ -33,7 +32,7 @@ void sbi_irqchip_set_irqfn(int (*fn)(struct sbi_trap_regs *regs));
  *
  * @param regs pointer for trap registers
  */
-int sbi_irqchip_process(struct sbi_trap_regs *regs);
+int sbi_irqchip_process(void);
 
 /** Initialize interrupt controllers */
 int sbi_irqchip_init(struct sbi_scratch *scratch, bool cold_boot);

include/sbi/sbi_misaligned_ldst.h

@@ -1,23 +0,0 @@
-/*
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2019 Western Digital Corporation or its affiliates.
- *
- * Authors:
- *   Anup Patel <anup.patel@wdc.com>
- */
-
-#ifndef __SBI_MISALIGNED_LDST_H__
-#define __SBI_MISALIGNED_LDST_H__
-
-#include <sbi/sbi_types.h>
-
-struct sbi_trap_regs;
-
-int sbi_misaligned_load_handler(ulong addr, ulong tval2, ulong tinst,
-				struct sbi_trap_regs *regs);
-
-int sbi_misaligned_store_handler(ulong addr, ulong tval2, ulong tinst,
-				 struct sbi_trap_regs *regs);
-
-#endif

include/sbi/sbi_platform.h

@@ -48,11 +48,13 @@
 #include <sbi/sbi_error.h>
 #include <sbi/sbi_scratch.h>
 #include <sbi/sbi_version.h>
+#include <sbi/sbi_trap_ldst.h>
 
 struct sbi_domain_memregion;
-struct sbi_trap_info;
+struct sbi_ecall_return;
 struct sbi_trap_regs;
 struct sbi_hart_features;
+union sbi_ldst_data;
 
 /** Possible feature flags of a platform */
 enum sbi_platform_features {
@@ -125,6 +127,9 @@ struct sbi_platform_operations {
 	/** Get tlb flush limit value **/
 	u64 (*get_tlbr_flush_limit)(void);
 
+	/** Get tlb fifo num entries*/
+	u32 (*get_tlb_num_entries)(void);
+
 	/** Initialize platform timer for current HART */
 	int (*timer_init)(bool cold_boot);
 	/** Exit platform timer for current HART */
@@ -134,9 +139,15 @@ struct sbi_platform_operations {
 	bool (*vendor_ext_check)(void);
 	/** platform specific SBI extension implementation provider */
 	int (*vendor_ext_provider)(long funcid,
-				   const struct sbi_trap_regs *regs,
-				   unsigned long *out_value,
-				   struct sbi_trap_info *out_trap);
+				   struct sbi_trap_regs *regs,
+				   struct sbi_ecall_return *out);
+
+	/** platform specific handler to fixup load fault */
+	int (*emulate_load)(int rlen, unsigned long addr,
+			    union sbi_ldst_data *out_val);
+	/** platform specific handler to fixup store fault */
+	int (*emulate_store)(int wlen, unsigned long addr,
+			     union sbi_ldst_data in_val);
 };
 
 /** Platform default per-HART stack size for exception/interrupt handling */
@@ -258,16 +269,6 @@ _Static_assert(
 #define sbi_platform_has_mfaults_delegation(__p) \
 	((__p)->features & SBI_PLATFORM_HAS_MFAULTS_DELEGATION)
 
-/**
- * Get HART index for the given HART
- *
- * @param plat pointer to struct sbi_platform
- * @param hartid HART ID
- *
- * @return 0 <= value < hart_count for valid HART otherwise -1U
- */
-u32 sbi_platform_hart_index(const struct sbi_platform *plat, u32 hartid);
-
 /**
  * Get the platform features in string format
  *
@@ -325,6 +326,20 @@ static inline u64 sbi_platform_tlbr_flush_limit(const struct sbi_platform *plat)
 	return SBI_PLATFORM_TLB_RANGE_FLUSH_LIMIT_DEFAULT;
 }
 
+/**
+ * Get platform specific tlb fifo num entries.
+ *
+ * @param plat pointer to struct sbi_platform
+ *
+ * @return number of tlb fifo entries
+*/
+static inline u32 sbi_platform_tlb_fifo_num_entries(const struct sbi_platform *plat)
+{
+	if (plat && sbi_platform_ops(plat)->get_tlb_num_entries)
+		return sbi_platform_ops(plat)->get_tlb_num_entries();
+	return sbi_scratch_last_hartindex() + 1;
+}
+
 /**
  * Get total number of HARTs supported by the platform
  *
@@ -353,24 +368,6 @@ static inline u32 sbi_platform_hart_stack_size(const struct sbi_platform *plat)
 	return 0;
 }
 
-/**
- * Check whether given HART is invalid
- *
- * @param plat pointer to struct sbi_platform
- * @param hartid HART ID
- *
- * @return true if HART is invalid and false otherwise
- */
-static inline bool sbi_platform_hart_invalid(const struct sbi_platform *plat,
-					     u32 hartid)
-{
-	if (!plat)
-		return true;
-	if (plat->hart_count <= sbi_platform_hart_index(plat, hartid))
-		return true;
-	return false;
-}
-
 /**
  * Check whether given HART is allowed to do cold boot
  *
@@ -677,20 +674,60 @@ static inline bool sbi_platform_vendor_ext_check(
 static inline int sbi_platform_vendor_ext_provider(
 					const struct sbi_platform *plat,
 					long funcid,
-					const struct sbi_trap_regs *regs,
-					unsigned long *out_value,
-					struct sbi_trap_info *out_trap)
+					struct sbi_trap_regs *regs,
+					struct sbi_ecall_return *out)
 {
-	if (plat && sbi_platform_ops(plat)->vendor_ext_provider) {
+	if (plat && sbi_platform_ops(plat)->vendor_ext_provider)
 		return sbi_platform_ops(plat)->vendor_ext_provider(funcid,
-								regs,
-								out_value,
-								out_trap);
-	}
+								regs, out);
 
 	return SBI_ENOTSUPP;
 }
 
+/**
+ * Ask platform to emulate the trapped load
+ *
+ * @param plat pointer to struct sbi_platform
+ * @param rlen length of the load: 1/2/4/8...
+ * @param addr virtual address of the load. Platform needs to page-walk and
+ *        find the physical address if necessary
+ * @param out_val value loaded
+ *
+ * @return 0 on success and negative error code on failure
+ */
+static inline int sbi_platform_emulate_load(const struct sbi_platform *plat,
+					    int rlen, unsigned long addr,
+					    union sbi_ldst_data *out_val)
+{
+	if (plat && sbi_platform_ops(plat)->emulate_load) {
+		return sbi_platform_ops(plat)->emulate_load(rlen, addr,
+							    out_val);
+	}
+	return SBI_ENOTSUPP;
+}
+
+/**
+ * Ask platform to emulate the trapped store
+ *
+ * @param plat pointer to struct sbi_platform
+ * @param wlen length of the store: 1/2/4/8...
+ * @param addr virtual address of the store. Platform needs to page-walk and
+ *        find the physical address if necessary
+ * @param in_val value to store
+ *
+ * @return 0 on success and negative error code on failure
+ */
+static inline int sbi_platform_emulate_store(const struct sbi_platform *plat,
+					     int wlen, unsigned long addr,
+					     union sbi_ldst_data in_val)
+{
+	if (plat && sbi_platform_ops(plat)->emulate_store) {
+		return sbi_platform_ops(plat)->emulate_store(wlen, addr,
+							     in_val);
+	}
+	return SBI_ENOTSUPP;
+}
+
 #endif
 
 #endif

include/sbi/sbi_pmu.h

@@ -11,6 +11,7 @@
 #define __SBI_PMU_H__
 
 #include <sbi/sbi_types.h>
+#include <sbi/sbi_trap.h>
 
 struct sbi_scratch;
 
@@ -23,6 +24,7 @@ struct sbi_scratch;
 #define SBI_PMU_HW_CTR_MAX 32
 #define SBI_PMU_CTR_MAX	   (SBI_PMU_HW_CTR_MAX + SBI_PMU_FW_CTR_MAX)
 #define SBI_PMU_FIXED_CTR_MASK 0x07
+#define SBI_PMU_CY_IR_MASK	0x05
 
 struct sbi_pmu_device {
 	/** Name of the PMU platform device */
@@ -89,6 +91,12 @@ struct sbi_pmu_device {
 	 * Custom function returning the machine-specific irq-bit.
 	 */
 	int (*hw_counter_irq_bit)(void);
+
+	/**
+	 * Custom function to inhibit counting of events while in
+	 * specified mode.
+	 */
+	void (*hw_counter_filter_mode)(unsigned long flags, int counter_index);
 };
 
 /** Get the PMU platform device */
@@ -143,4 +151,6 @@ int sbi_pmu_ctr_cfg_match(unsigned long cidx_base, unsigned long cidx_mask,
 
 int sbi_pmu_ctr_incr_fw(enum sbi_pmu_fw_event_code_id fw_id);
 
+void sbi_pmu_ovf_irq();
+
 #endif

include/sbi/sbi_scratch.h

@@ -36,8 +36,8 @@
 #define SBI_SCRATCH_PLATFORM_ADDR_OFFSET	(9 * __SIZEOF_POINTER__)
 /** Offset of hartid_to_scratch member in sbi_scratch */
 #define SBI_SCRATCH_HARTID_TO_SCRATCH_OFFSET	(10 * __SIZEOF_POINTER__)
-/** Offset of trap_exit member in sbi_scratch */
-#define SBI_SCRATCH_TRAP_EXIT_OFFSET		(11 * __SIZEOF_POINTER__)
+/** Offset of trap_context member in sbi_scratch */
+#define SBI_SCRATCH_TRAP_CONTEXT_OFFSET		(11 * __SIZEOF_POINTER__)
 /** Offset of tmp0 member in sbi_scratch */
 #define SBI_SCRATCH_TMP0_OFFSET			(12 * __SIZEOF_POINTER__)
 /** Offset of options member in sbi_scratch */
@@ -77,8 +77,8 @@ struct sbi_scratch {
 	unsigned long platform_addr;
 	/** Address of HART ID to sbi_scratch conversion function */
 	unsigned long hartid_to_scratch;
-	/** Address of trap exit function */
-	unsigned long trap_exit;
+	/** Address of current trap context */
+	unsigned long trap_context;
 	/** Temporary storage */
 	unsigned long tmp0;
 	/** Options for OpenSBI library */
@@ -130,10 +130,10 @@ _Static_assert(
 	"struct sbi_scratch definition has changed, please redefine "
 	"SBI_SCRATCH_HARTID_TO_SCRATCH_OFFSET");
 _Static_assert(
-	offsetof(struct sbi_scratch, trap_exit)
-		== SBI_SCRATCH_TRAP_EXIT_OFFSET,
+	offsetof(struct sbi_scratch, trap_context)
+		== SBI_SCRATCH_TRAP_CONTEXT_OFFSET,
 	"struct sbi_scratch definition has changed, please redefine "
-	"SBI_SCRATCH_TRAP_EXIT_OFFSET");
+	"SBI_SCRATCH_TRAP_CONTEXT_OFFSET");
 _Static_assert(
 	offsetof(struct sbi_scratch, tmp0)
 		== SBI_SCRATCH_TMP0_OFFSET,
@@ -202,18 +202,51 @@ do {									\
 					= (__type)(__ptr);		\
 } while (0)
 
-/** HART id to scratch table */
-extern struct sbi_scratch *hartid_to_scratch_table[];
+/** Last HART index having a sbi_scratch pointer */
+extern u32 last_hartindex_having_scratch;
+
+/** Get last HART index having a sbi_scratch pointer */
+#define sbi_scratch_last_hartindex()	last_hartindex_having_scratch
+
+/** Check whether a particular HART index is valid or not */
+#define sbi_hartindex_valid(__hartindex) \
+(((__hartindex) <= sbi_scratch_last_hartindex()) ? true : false)
+
+/** HART index to HART id table */
+extern u32 hartindex_to_hartid_table[];
+
+/** Get sbi_scratch from HART index */
+#define sbi_hartindex_to_hartid(__hartindex)		\
+({							\
+	((__hartindex) <= sbi_scratch_last_hartindex()) ?\
+	hartindex_to_hartid_table[__hartindex] : -1U;	\
+})
+
+/** HART index to scratch table */
+extern struct sbi_scratch *hartindex_to_scratch_table[];
+
+/** Get sbi_scratch from HART index */
+#define sbi_hartindex_to_scratch(__hartindex)		\
+({							\
+	((__hartindex) <= sbi_scratch_last_hartindex()) ?\
+	hartindex_to_scratch_table[__hartindex] : NULL;\
+})
+
+/**
+ * Get logical index for given HART id
+ * @param hartid physical HART id
+ * @returns value between 0 to SBI_HARTMASK_MAX_BITS upon success and
+ *	    SBI_HARTMASK_MAX_BITS upon failure.
+ */
+u32 sbi_hartid_to_hartindex(u32 hartid);
 
 /** Get sbi_scratch from HART id */
 #define sbi_hartid_to_scratch(__hartid) \
-	hartid_to_scratch_table[__hartid]
+	sbi_hartindex_to_scratch(sbi_hartid_to_hartindex(__hartid))
 
-/** Last HART id having a sbi_scratch pointer */
-extern u32 last_hartid_having_scratch;
-
-/** Get last HART id having a sbi_scratch pointer */
-#define sbi_scratch_last_hartid()	last_hartid_having_scratch
+/** Check whether particular HART id is valid or not */
+#define sbi_hartid_valid(__hartid)	\
+	sbi_hartindex_valid(sbi_hartid_to_hartindex(__hartid))
 
 #endif
 

include/sbi/sbi_sse.h

@@ -0,0 +1,93 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Rivos Systems.
+ */
+
+#ifndef __SBI_SSE_H__
+#define __SBI_SSE_H__
+
+#include <sbi/sbi_types.h>
+#include <sbi/sbi_list.h>
+#include <sbi/riscv_locks.h>
+
+struct sbi_scratch;
+struct sbi_trap_regs;
+struct sbi_ecall_return;
+
+#define EXC_MODE_PP_SHIFT		0
+#define EXC_MODE_PP			BIT(EXC_MODE_PP_SHIFT)
+#define EXC_MODE_PV_SHIFT		1
+#define EXC_MODE_PV			BIT(EXC_MODE_PV_SHIFT)
+#define EXC_MODE_SSTATUS_SPIE_SHIFT	2
+#define EXC_MODE_SSTATUS_SPIE		BIT(EXC_MODE_SSTATUS_SPIE_SHIFT)
+
+struct sbi_sse_cb_ops {
+	/**
+	 * Called when hart_id is changed on the event.
+	 */
+	void (*set_hartid_cb)(uint32_t event_id, unsigned long hart_id);
+
+	/**
+	 * Called when the SBI_EXT_SSE_COMPLETE is invoked on the event.
+	 */
+	void (*complete_cb)(uint32_t event_id);
+
+	/**
+	 * Called when the SBI_EXT_SSE_REGISTER is invoked on the event.
+	 */
+	void (*register_cb)(uint32_t event_id);
+
+	/**
+	 * Called when the SBI_EXT_SSE_UNREGISTER is invoked on the event.
+	 */
+	void (*unregister_cb)(uint32_t event_id);
+
+	/**
+	 * Called when the SBI_EXT_SSE_ENABLE is invoked on the event.
+	 */
+	void (*enable_cb)(uint32_t event_id);
+
+	/**
+	 * Called when the SBI_EXT_SSE_DISABLE is invoked on the event.
+	 */
+	void (*disable_cb)(uint32_t event_id);
+};
+
+/* Set the callback operations for an event
+ * @param event_id Event identifier (SBI_SSE_EVENT_*)
+ * @param cb_ops Callback operations
+ * @return 0 on success, error otherwise
+ */
+int sbi_sse_set_cb_ops(uint32_t event_id, const struct sbi_sse_cb_ops *cb_ops);
+
+/* Inject an event to the current hard
+ * @param event_id Event identifier (SBI_SSE_EVENT_*)
+ * @param regs Registers that were used on SBI entry
+ * @return 0 on success, error otherwise
+ */
+int sbi_sse_inject_event(uint32_t event_id);
+
+void sbi_sse_process_pending_events(struct sbi_trap_regs *regs);
+
+
+int sbi_sse_init(struct sbi_scratch *scratch, bool cold_boot);
+void sbi_sse_exit(struct sbi_scratch *scratch);
+
+/* Interface called from sbi_ecall_sse.c */
+int sbi_sse_register(uint32_t event_id, unsigned long handler_entry_pc,
+		     unsigned long handler_entry_arg);
+int sbi_sse_unregister(uint32_t event_id);
+int sbi_sse_enable(uint32_t event_id);
+int sbi_sse_disable(uint32_t event_id);
+int sbi_sse_complete(struct sbi_trap_regs *regs, struct sbi_ecall_return *out);
+int sbi_sse_inject_from_ecall(uint32_t event_id, unsigned long hart_id,
+			      struct sbi_ecall_return *out);
+int sbi_sse_read_attrs(uint32_t event_id, uint32_t base_attr_id,
+		       uint32_t attr_count, unsigned long output_phys_lo,
+		       unsigned long output_phys_hi);
+int sbi_sse_write_attrs(uint32_t event_id, uint32_t base_attr_id,
+			uint32_t attr_count, unsigned long input_phys_lo,
+			unsigned long input_phys_hi);
+
+#endif

include/sbi/sbi_tlb.h

@@ -20,34 +20,35 @@
 
 /* clang-format on */
 
-#define SBI_TLB_FIFO_NUM_ENTRIES		8
-
 struct sbi_scratch;
 
+enum sbi_tlb_type {
+	SBI_TLB_FENCE_I = 0,
+	SBI_TLB_SFENCE_VMA,
+	SBI_TLB_SFENCE_VMA_ASID,
+	SBI_TLB_HFENCE_GVMA_VMID,
+	SBI_TLB_HFENCE_GVMA,
+	SBI_TLB_HFENCE_VVMA_ASID,
+	SBI_TLB_HFENCE_VVMA,
+	SBI_TLB_TYPE_MAX,
+};
+
 struct sbi_tlb_info {
 	unsigned long start;
 	unsigned long size;
-	unsigned long asid;
-	unsigned long vmid;
-	void (*local_fn)(struct sbi_tlb_info *tinfo);
+	uint16_t asid;
+	uint16_t vmid;
+	enum sbi_tlb_type type;
 	struct sbi_hartmask smask;
 };
 
-void sbi_tlb_local_hfence_vvma(struct sbi_tlb_info *tinfo);
-void sbi_tlb_local_hfence_gvma(struct sbi_tlb_info *tinfo);
-void sbi_tlb_local_sfence_vma(struct sbi_tlb_info *tinfo);
-void sbi_tlb_local_hfence_vvma_asid(struct sbi_tlb_info *tinfo);
-void sbi_tlb_local_hfence_gvma_vmid(struct sbi_tlb_info *tinfo);
-void sbi_tlb_local_sfence_vma_asid(struct sbi_tlb_info *tinfo);
-void sbi_tlb_local_fence_i(struct sbi_tlb_info *tinfo);
-
-#define SBI_TLB_INFO_INIT(__p, __start, __size, __asid, __vmid, __lfn, __src) \
+#define SBI_TLB_INFO_INIT(__p, __start, __size, __asid, __vmid, __type, __src) \
 do { \
 	(__p)->start = (__start); \
 	(__p)->size = (__size); \
 	(__p)->asid = (__asid); \
 	(__p)->vmid = (__vmid); \
-	(__p)->local_fn = (__lfn); \
+	(__p)->type = (__type); \
 	SBI_HARTMASK_INIT_EXCEPT(&(__p)->smask, (__src)); \
 } while (0)
 

include/sbi/sbi_trap.h

@@ -87,20 +87,18 @@
 /** Last member index in sbi_trap_regs */
 #define SBI_TRAP_REGS_last			35
 
-/** Index of epc member in sbi_trap_info */
-#define SBI_TRAP_INFO_epc			0
 /** Index of cause member in sbi_trap_info */
-#define SBI_TRAP_INFO_cause			1
+#define SBI_TRAP_INFO_cause			0
 /** Index of tval member in sbi_trap_info */
-#define SBI_TRAP_INFO_tval			2
+#define SBI_TRAP_INFO_tval			1
 /** Index of tval2 member in sbi_trap_info */
-#define SBI_TRAP_INFO_tval2			3
+#define SBI_TRAP_INFO_tval2			2
 /** Index of tinst member in sbi_trap_info */
-#define SBI_TRAP_INFO_tinst			4
+#define SBI_TRAP_INFO_tinst			3
 /** Index of gva member in sbi_trap_info */
-#define SBI_TRAP_INFO_gva			5
+#define SBI_TRAP_INFO_gva			4
 /** Last member index in sbi_trap_info */
-#define SBI_TRAP_INFO_last			6
+#define SBI_TRAP_INFO_last			5
 
 /* clang-format on */
 
@@ -114,9 +112,15 @@
 /** Size (in bytes) of sbi_trap_info */
 #define SBI_TRAP_INFO_SIZE SBI_TRAP_INFO_OFFSET(last)
 
+/** Size (in bytes) of sbi_trap_context */
+#define SBI_TRAP_CONTEXT_SIZE (SBI_TRAP_REGS_SIZE + \
+			       SBI_TRAP_INFO_SIZE + \
+			       __SIZEOF_POINTER__)
+
 #ifndef __ASSEMBLER__
 
 #include <sbi/sbi_types.h>
+#include <sbi/sbi_scratch.h>
 
 /** Representation of register state at time of trap/interrupt */
 struct sbi_trap_regs {
@@ -194,8 +198,6 @@ struct sbi_trap_regs {
 
 /** Representation of trap details */
 struct sbi_trap_info {
-	/** epc Trap program counter */
-	unsigned long epc;
 	/** cause Trap exception cause */
 	unsigned long cause;
 	/** tval Trap value */
@@ -208,6 +210,16 @@ struct sbi_trap_info {
 	unsigned long gva;
 };
 
+/** Representation of trap context saved on stack */
+struct sbi_trap_context {
+	/** Register state */
+	struct sbi_trap_regs regs;
+	/** Trap details */
+	struct sbi_trap_info trap;
+	/** Pointer to previous trap context */
+	struct sbi_trap_context *prev_context;
+};
+
 static inline unsigned long sbi_regs_gva(const struct sbi_trap_regs *regs)
 {
 	/*
@@ -225,11 +237,20 @@ static inline unsigned long sbi_regs_gva(const struct sbi_trap_regs *regs)
 }
 
 int sbi_trap_redirect(struct sbi_trap_regs *regs,
-		      struct sbi_trap_info *trap);
+		      const struct sbi_trap_info *trap);
 
-struct sbi_trap_regs *sbi_trap_handler(struct sbi_trap_regs *regs);
+static inline struct sbi_trap_context *sbi_trap_get_context(struct sbi_scratch *scratch)
+{
+	return (scratch) ? (void *)scratch->trap_context : NULL;
+}
 
-void __noreturn sbi_trap_exit(const struct sbi_trap_regs *regs);
+static inline void sbi_trap_set_context(struct sbi_scratch *scratch,
+					struct sbi_trap_context *tcntx)
+{
+	scratch->trap_context = (unsigned long)tcntx;
+}
+
+struct sbi_trap_context *sbi_trap_handler(struct sbi_trap_context *tcntx);
 
 #endif
 

include/sbi/sbi_trap_ldst.h

@@ -0,0 +1,31 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2019 Western Digital Corporation or its affiliates.
+ *
+ * Authors:
+ *   Anup Patel <anup.patel@wdc.com>
+ */
+
+#ifndef __SBI_TRAP_LDST_H__
+#define __SBI_TRAP_LDST_H__
+
+#include <sbi/sbi_types.h>
+#include <sbi/sbi_trap.h>
+
+union sbi_ldst_data {
+	u64 data_u64;
+	u32 data_u32;
+	u8 data_bytes[8];
+	ulong data_ulong;
+};
+
+int sbi_misaligned_load_handler(struct sbi_trap_context *tcntx);
+
+int sbi_misaligned_store_handler(struct sbi_trap_context *tcntx);
+
+int sbi_load_access_handler(struct sbi_trap_context *tcntx);
+
+int sbi_store_access_handler(struct sbi_trap_context *tcntx);
+
+#endif

include/sbi/sbi_unit_test.h

@@ -0,0 +1,73 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Author: Ivan Orlov <ivan.orlov0322@gmail.com>
+ */
+#ifdef CONFIG_SBIUNIT
+#ifndef __SBI_UNIT_H__
+#define __SBI_UNIT_H__
+
+#include <sbi/sbi_types.h>
+#include <sbi/sbi_console.h>
+#include <sbi/sbi_string.h>
+
+struct sbiunit_test_case {
+	const char *name;
+	bool failed;
+	void (*test_func)(struct sbiunit_test_case *test);
+};
+
+struct sbiunit_test_suite {
+	const char *name;
+	void (*init)(void);
+	struct sbiunit_test_case *cases;
+};
+
+#define SBIUNIT_TEST_CASE(func)		\
+	{				\
+		.name = #func,		\
+		.failed = false,	\
+		.test_func = (func)	\
+	}
+
+#define SBIUNIT_END_CASE { }
+
+#define SBIUNIT_TEST_SUITE(suite_name, cases_arr)		\
+	struct sbiunit_test_suite suite_name = {		\
+		.name = #suite_name,				\
+		.init = NULL,					\
+		.cases = cases_arr				\
+	}
+
+#define _sbiunit_msg(test, msg) "[SBIUnit] [%s:%d]: %s: %s", __FILE__,	\
+				__LINE__, test->name, msg
+
+#define SBIUNIT_INFO(test, msg) sbi_printf(_sbiunit_msg(test, msg))
+#define SBIUNIT_PANIC(test, msg) sbi_panic(_sbiunit_msg(test, msg))
+
+#define SBIUNIT_EXPECT(test, cond) do {							\
+	if (!(cond)) {									\
+		test->failed = true;							\
+		SBIUNIT_INFO(test, "Condition \"" #cond "\" expected to be true!\n");	\
+	}										\
+} while (0)
+
+#define SBIUNIT_ASSERT(test, cond) do {						\
+	if (!(cond))								\
+		SBIUNIT_PANIC(test, "Condition \"" #cond "\" must be true!\n");	\
+} while (0)
+
+#define SBIUNIT_EXPECT_EQ(test, a, b) SBIUNIT_EXPECT(test, (a) == (b))
+#define SBIUNIT_ASSERT_EQ(test, a, b) SBIUNIT_ASSERT(test, (a) == (b))
+#define SBIUNIT_EXPECT_NE(test, a, b) SBIUNIT_EXPECT(test, (a) != (b))
+#define SBIUNIT_ASSERT_NE(test, a, b) SBIUNIT_ASSERT(test, (a) != (b))
+#define SBIUNIT_EXPECT_MEMEQ(test, a, b, len) SBIUNIT_EXPECT(test, !sbi_memcmp(a, b, len))
+#define SBIUNIT_ASSERT_MEMEQ(test, a, b, len) SBIUNIT_ASSERT(test, !sbi_memcmp(a, b, len))
+#define SBIUNIT_EXPECT_STREQ(test, a, b, len) SBIUNIT_EXPECT(test, !sbi_strncmp(a, b, len))
+#define SBIUNIT_ASSERT_STREQ(test, a, b, len) SBIUNIT_ASSERT(test, !sbi_strncmp(a, b, len))
+
+void run_all_tests(void);
+#endif
+#else
+#define run_all_tests()
+#endif

include/sbi/sbi_version.h

@@ -11,7 +11,7 @@
 #define __SBI_VERSION_H__
 
 #define OPENSBI_VERSION_MAJOR 1
-#define OPENSBI_VERSION_MINOR 3
+#define OPENSBI_VERSION_MINOR 5
 
 /**
  * OpenSBI 32-bit version with:

include/sbi_utils/fdt/fdt_helper.h

@@ -48,6 +48,9 @@ int fdt_parse_phandle_with_args(void *fdt, int nodeoff,
 int fdt_get_node_addr_size(void *fdt, int node, int index,
 			   uint64_t *addr, uint64_t *size);
 
+int fdt_get_node_addr_size_by_name(void *fdt, int node, const char *name,
+				   uint64_t *addr, uint64_t *size);
+
 bool fdt_node_is_enabled(void *fdt, int nodeoff);
 
 int fdt_parse_hart_id(void *fdt, int cpu_offset, u32 *hartid);
@@ -56,6 +59,9 @@ int fdt_parse_max_enabled_hart_id(void *fdt, u32 *max_hartid);
 
 int fdt_parse_timebase_frequency(void *fdt, unsigned long *freq);
 
+int fdt_parse_isa_extensions(void *fdt, unsigned int hard_id,
+			     unsigned long *extensions);
+
 int fdt_parse_gaisler_uart_node(void *fdt, int nodeoffset,
 				struct platform_uart_data *uart);
 
@@ -93,7 +99,8 @@ int fdt_parse_plic_node(void *fdt, int nodeoffset, struct plic_data *plic);
 
 int fdt_parse_plic(void *fdt, struct plic_data *plic, const char *compat);
 
-int fdt_parse_aclint_node(void *fdt, int nodeoffset, bool for_timer,
+int fdt_parse_aclint_node(void *fdt, int nodeoffset,
+			  bool for_timer, bool allow_regname,
 			  unsigned long *out_addr1, unsigned long *out_size1,
 			  unsigned long *out_addr2, unsigned long *out_size2,
 			  u32 *out_first_hartid, u32 *out_hart_count);

include/sbi_utils/fdt/fdt_pmu.h

@@ -13,6 +13,23 @@
 
 #include <sbi/sbi_types.h>
 
+struct fdt_pmu_hw_event_select_map {
+	uint32_t eidx;
+	uint64_t select;
+};
+
+struct fdt_pmu_hw_event_counter_map {
+	uint32_t eidx_start;
+	uint32_t eidx_end;
+	uint32_t ctr_map;
+};
+
+struct fdt_pmu_raw_event_counter_map {
+	uint64_t select;
+	uint64_t select_mask;
+	uint32_t ctr_map;
+};
+
 #ifdef CONFIG_FDT_PMU
 
 /**
@@ -26,7 +43,7 @@
  *
  * @param fdt device tree blob
  */
-void fdt_pmu_fixup(void *fdt);
+int fdt_pmu_fixup(void *fdt);
 
 /**
  * Setup PMU data from device tree
@@ -45,6 +62,11 @@ int fdt_pmu_setup(void *fdt);
  */
 uint64_t fdt_pmu_get_select_value(uint32_t event_idx);
 
+/** The event index to selector value table instance */
+extern struct fdt_pmu_hw_event_select_map fdt_pmu_evt_select[];
+/** The number of valid entries in fdt_pmu_evt_select[] */
+extern uint32_t hw_event_count;
+
 #else
 
 static inline void fdt_pmu_fixup(void *fdt) { }

include/sbi_utils/i2c/i2c.h

@@ -15,9 +15,6 @@
 
 /** Representation of a I2C adapter */
 struct i2c_adapter {
-	/** Pointer to I2C driver owning this I2C adapter */
-	void *driver;
-
 	/** Unique ID of the I2C adapter assigned by the driver */
 	int id;
 

include/sbi_utils/ipi/andes_plicsw.h

@@ -13,30 +13,23 @@
 #ifndef _IPI_ANDES_PLICSW_H_
 #define _IPI_ANDES_PLICSW_H_
 
-#define PLICSW_PRIORITY_BASE 0x4
+#define PLICSW_PRIORITY_BASE	0x4
 
-#define PLICSW_PENDING_BASE 0x1000
-#define PLICSW_PENDING_STRIDE 0x8
+#define PLICSW_PENDING_BASE	0x1000
 
-#define PLICSW_ENABLE_BASE 0x2000
-#define PLICSW_ENABLE_STRIDE 0x80
+#define PLICSW_ENABLE_BASE	0x2000
+#define PLICSW_ENABLE_STRIDE	0x80
 
-#define PLICSW_CONTEXT_BASE 0x200000
-#define PLICSW_CONTEXT_STRIDE 0x1000
-#define PLICSW_CONTEXT_CLAIM 0x4
+#define PLICSW_CONTEXT_BASE	0x200000
+#define PLICSW_CONTEXT_STRIDE	0x1000
+#define PLICSW_CONTEXT_CLAIM	0x4
 
-#define PLICSW_HART_MASK 0x01010101
-
-#define PLICSW_HART_MAX_NR 8
-
-#define PLICSW_REGION_ALIGN 0x1000
+#define PLICSW_REGION_ALIGN	0x1000
 
 struct plicsw_data {
 	unsigned long addr;
 	unsigned long size;
 	uint32_t hart_count;
-	/* hart id to source id table */
-	uint32_t source_id[PLICSW_HART_MAX_NR];
 };
 
 int plicsw_warm_ipi_init(void);

include/sbi_utils/irqchip/plic.h

@@ -14,6 +14,7 @@
 
 struct plic_data {
 	unsigned long addr;
+	unsigned long size;
 	unsigned long num_src;
 };
 

include/sbi_utils/regmap/fdt_regmap.h

@@ -0,0 +1,31 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Ventana Micro Systems Inc.
+ *
+ * Authors:
+ *   Anup Patel <apatel@ventanamicro.com>
+ */
+
+#ifndef __FDT_REGMAP_H__
+#define __FDT_REGMAP_H__
+
+#include <sbi_utils/regmap/regmap.h>
+
+struct fdt_phandle_args;
+
+/** FDT based regmap driver */
+struct fdt_regmap {
+	const struct fdt_match *match_table;
+	int (*init)(void *fdt, int nodeoff, u32 phandle,
+		    const struct fdt_match *match);
+};
+
+/** Get regmap instance based on phandle */
+int fdt_regmap_get_by_phandle(void *fdt, u32 phandle,
+			      struct regmap **out_rmap);
+
+/** Get regmap instance based on "regmap' property of the specified DT node */
+int fdt_regmap_get(void *fdt, int nodeoff, struct regmap **out_rmap);
+
+#endif

include/sbi_utils/regmap/regmap.h

@@ -0,0 +1,67 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Ventana Micro Systems Inc.
+ *
+ * Authors:
+ *   Anup Patel <apatel@ventanamicro.com>
+ */
+
+#ifndef __REGMAP_H__
+#define __REGMAP_H__
+
+#include <sbi/sbi_types.h>
+#include <sbi/sbi_list.h>
+
+/** Representation of a regmap instance */
+struct regmap {
+	/** Uniquie ID of the regmap instance assigned by the driver */
+	unsigned int id;
+
+	/** Configuration of regmap registers */
+	int reg_shift;
+	int reg_stride;
+	unsigned int reg_base;
+	unsigned int reg_max;
+
+	/** Read a regmap register */
+	int (*reg_read)(struct regmap *rmap, unsigned int reg,
+			unsigned int *val);
+
+	/** Write a regmap register */
+	int (*reg_write)(struct regmap *rmap, unsigned int reg,
+			 unsigned int val);
+
+	/** Read-modify-write a regmap register */
+	int (*reg_update_bits)(struct regmap *rmap, unsigned int reg,
+			       unsigned int mask, unsigned int val);
+
+	/** List */
+	struct sbi_dlist node;
+};
+
+static inline struct regmap *to_regmap(struct sbi_dlist *node)
+{
+	return container_of(node, struct regmap, node);
+}
+
+/** Find a registered regmap instance */
+struct regmap *regmap_find(unsigned int id);
+
+/** Register a regmap instance */
+int regmap_add(struct regmap *rmap);
+
+/** Un-register a regmap instance */
+void regmap_remove(struct regmap *rmap);
+
+/** Read a register in a regmap instance */
+int regmap_read(struct regmap *rmap, unsigned int reg, unsigned int *val);
+
+/** Write a register in a regmap instance */
+int regmap_write(struct regmap *rmap, unsigned int reg, unsigned int val);
+
+/** Read-modify-write a register in a regmap instance */
+int regmap_update_bits(struct regmap *rmap, unsigned int reg,
+		       unsigned int mask, unsigned int val);
+
+#endif

include/sbi_utils/sys/sifive_test.h

@@ -1,17 +0,0 @@
-/*
- * SPDX-License-Identifier: BSD-3-Clause
- *
- * Copyright (c) 2020 Western Digital Corporation or its affiliates.
- *
- * Authors:
- *   Anup Patel <anup.patel@wdc.com>
- */
-
-#ifndef __SYS_SIFIVE_TEST_H__
-#define __SYS_SIFIVE_TEST_H__
-
-#include <sbi/sbi_types.h>
-
-int sifive_test_init(unsigned long base);
-
-#endif

lib/sbi/Kconfig

@@ -38,6 +38,10 @@ config SBI_ECALL_CPPC
 	bool "CPPC extension"
 	default y
 
+config SBI_ECALL_FWFT
+	bool "Firmware Feature extension"
+	default y
+
 config SBI_ECALL_LEGACY
 	bool "SBI v0.1 legacy extensions"
 	default y
@@ -46,4 +50,16 @@ config SBI_ECALL_VENDOR
 	bool "Platform-defined vendor extensions"
 	default y
 
+config SBI_ECALL_DBTR
+	bool "Debug Trigger Extension"
+	default y
+
+config SBIUNIT
+	bool "Enable SBIUNIT tests"
+	default n
+
+config SBI_ECALL_SSE
+	bool "SSE extension"
+	default y
+
 endmenu

lib/sbi/objects.mk

@@ -46,18 +46,29 @@ libsbi-objs-$(CONFIG_SBI_ECALL_DBCN) += sbi_ecall_dbcn.o
 carray-sbi_ecall_exts-$(CONFIG_SBI_ECALL_CPPC) += ecall_cppc
 libsbi-objs-$(CONFIG_SBI_ECALL_CPPC) += sbi_ecall_cppc.o
 
+carray-sbi_ecall_exts-$(CONFIG_SBI_ECALL_FWFT) += ecall_fwft
+libsbi-objs-$(CONFIG_SBI_ECALL_FWFT) += sbi_ecall_fwft.o
+
 carray-sbi_ecall_exts-$(CONFIG_SBI_ECALL_LEGACY) += ecall_legacy
 libsbi-objs-$(CONFIG_SBI_ECALL_LEGACY) += sbi_ecall_legacy.o
 
 carray-sbi_ecall_exts-$(CONFIG_SBI_ECALL_VENDOR) += ecall_vendor
 libsbi-objs-$(CONFIG_SBI_ECALL_VENDOR) += sbi_ecall_vendor.o
 
+carray-sbi_ecall_exts-$(CONFIG_SBI_ECALL_DBTR) += ecall_dbtr
+libsbi-objs-$(CONFIG_SBI_ECALL_DBTR) += sbi_ecall_dbtr.o
+
+carray-sbi_ecall_exts-$(CONFIG_SBI_ECALL_SSE) += ecall_sse
+libsbi-objs-$(CONFIG_SBI_ECALL_SSE) += sbi_ecall_sse.o
+
 libsbi-objs-y += sbi_bitmap.o
 libsbi-objs-y += sbi_bitops.o
 libsbi-objs-y += sbi_console.o
+libsbi-objs-y += sbi_domain_context.o
 libsbi-objs-y += sbi_domain.o
 libsbi-objs-y += sbi_emulate_csr.o
 libsbi-objs-y += sbi_fifo.o
+libsbi-objs-y += sbi_fwft.o
 libsbi-objs-y += sbi_hart.o
 libsbi-objs-y += sbi_heap.o
 libsbi-objs-y += sbi_math.o
@@ -67,15 +78,17 @@ libsbi-objs-y += sbi_illegal_insn.o
 libsbi-objs-y += sbi_init.o
 libsbi-objs-y += sbi_ipi.o
 libsbi-objs-y += sbi_irqchip.o
-libsbi-objs-y += sbi_misaligned_ldst.o
 libsbi-objs-y += sbi_platform.o
 libsbi-objs-y += sbi_pmu.o
+libsbi-objs-y += sbi_dbtr.o
 libsbi-objs-y += sbi_scratch.o
+libsbi-objs-y += sbi_sse.o
 libsbi-objs-y += sbi_string.o
 libsbi-objs-y += sbi_system.o
 libsbi-objs-y += sbi_timer.o
 libsbi-objs-y += sbi_tlb.o
 libsbi-objs-y += sbi_trap.o
+libsbi-objs-y += sbi_trap_ldst.o
 libsbi-objs-y += sbi_unpriv.o
 libsbi-objs-y += sbi_expected_trap.o
 libsbi-objs-y += sbi_cppc.o

lib/sbi/riscv_asm.c

@@ -128,6 +128,8 @@ unsigned long csr_read_num(int csr_num)
 	switchcase_csr_read_8(CSR_MHPMCOUNTER8, ret)
 	switchcase_csr_read_16(CSR_MHPMCOUNTER16, ret)
 	switchcase_csr_read(CSR_MCOUNTINHIBIT, ret)
+	switchcase_csr_read(CSR_MCYCLECFG, ret)
+	switchcase_csr_read(CSR_MINSTRETCFG, ret)
 	switchcase_csr_read(CSR_MHPMEVENT3, ret)
 	switchcase_csr_read_4(CSR_MHPMEVENT4, ret)
 	switchcase_csr_read_8(CSR_MHPMEVENT8, ret)
@@ -139,6 +141,12 @@ unsigned long csr_read_num(int csr_num)
 	switchcase_csr_read_4(CSR_MHPMCOUNTER4H, ret)
 	switchcase_csr_read_8(CSR_MHPMCOUNTER8H, ret)
 	switchcase_csr_read_16(CSR_MHPMCOUNTER16H, ret)
+	/**
+	 * The CSR range M[CYCLE, INSTRET]CFGH are available only if smcntrpmf
+	 * extension is present. The caller must ensure that.
+	 */
+	switchcase_csr_read(CSR_MCYCLECFGH, ret)
+	switchcase_csr_read(CSR_MINSTRETCFGH, ret)
 	/**
 	 * The CSR range MHPMEVENT[3-16]H are available only if sscofpmf
 	 * extension is present. The caller must ensure that.
@@ -206,12 +214,16 @@ void csr_write_num(int csr_num, unsigned long val)
 	switchcase_csr_write_4(CSR_MHPMCOUNTER4H, val)
 	switchcase_csr_write_8(CSR_MHPMCOUNTER8H, val)
 	switchcase_csr_write_16(CSR_MHPMCOUNTER16H, val)
+	switchcase_csr_write(CSR_MCYCLECFGH, val)
+	switchcase_csr_write(CSR_MINSTRETCFGH, val)
 	switchcase_csr_write(CSR_MHPMEVENT3H, val)
 	switchcase_csr_write_4(CSR_MHPMEVENT4H, val)
 	switchcase_csr_write_8(CSR_MHPMEVENT8H, val)
 	switchcase_csr_write_16(CSR_MHPMEVENT16H, val)
 #endif
 	switchcase_csr_write(CSR_MCOUNTINHIBIT, val)
+	switchcase_csr_write(CSR_MCYCLECFG, val)
+	switchcase_csr_write(CSR_MINSTRETCFG, val)
 	switchcase_csr_write(CSR_MHPMEVENT3, val)
 	switchcase_csr_write_4(CSR_MHPMEVENT4, val)
 	switchcase_csr_write_8(CSR_MHPMEVENT8, val)
@@ -246,6 +258,48 @@ static unsigned long ctz(unsigned long x)
 	return ret;
 }
 
+int pmp_disable(unsigned int n)
+{
+	int pmpcfg_csr, pmpcfg_shift;
+	unsigned long cfgmask, pmpcfg;
+
+	if (n >= PMP_COUNT)
+		return SBI_EINVAL;
+
+#if __riscv_xlen == 32
+	pmpcfg_csr   = CSR_PMPCFG0 + (n >> 2);
+	pmpcfg_shift = (n & 3) << 3;
+#elif __riscv_xlen == 64
+	pmpcfg_csr   = (CSR_PMPCFG0 + (n >> 2)) & ~1;
+	pmpcfg_shift = (n & 7) << 3;
+#else
+# error "Unexpected __riscv_xlen"
+#endif
+
+	/* Clear the address matching bits to disable the pmp entry */
+	cfgmask = ~(0xffUL << pmpcfg_shift);
+	pmpcfg	= (csr_read_num(pmpcfg_csr) & cfgmask);
+
+	csr_write_num(pmpcfg_csr, pmpcfg);
+
+	return SBI_OK;
+}
+
+int is_pmp_entry_mapped(unsigned long entry)
+{
+	unsigned long prot;
+	unsigned long addr;
+	unsigned long log2len;
+
+	pmp_get(entry, &prot, &addr, &log2len);
+
+	/* If address matching bits are non-zero, the entry is enable */
+	if (prot & PMP_A)
+		return true;
+
+	return false;
+}
+
 int pmp_set(unsigned int n, unsigned long prot, unsigned long addr,
 	    unsigned long log2len)
 {

lib/sbi/riscv_atomic.c

@@ -12,6 +12,10 @@
 #include <sbi/riscv_atomic.h>
 #include <sbi/riscv_barrier.h>
 
+#ifndef __riscv_atomic
+#error "opensbi strongly relies on the A extension of RISC-V"
+#endif
+
 long atomic_read(atomic_t *atom)
 {
 	long ret = atom->counter;
@@ -79,175 +83,51 @@ long atomic_sub_return(atomic_t *atom, long value)
 		(__typeof__(*(ptr))) __axchg((ptr), _x_, sizeof(*(ptr)));	\
 	})
 
-
-#define __xchg(ptr, new, size)                                            \
-	({                                                                \
-		__typeof__(ptr) __ptr	 = (ptr);                         \
-		__typeof__(*(ptr)) __new = (new);                         \
-		__typeof__(*(ptr)) __ret;                                 \
-		register unsigned int __rc;                               \
-		switch (size) {                                           \
-		case 4:                                                   \
-			__asm__ __volatile__("0:	lr.w %0, %2\n"    \
-					     "	sc.w.rl %1, %z3, %2\n"    \
-					     "	bnez %1, 0b\n"            \
-					     "	fence rw, rw\n"           \
-					     : "=&r"(__ret), "=&r"(__rc), \
-					       "+A"(*__ptr)               \
-					     : "rJ"(__new)                \
-					     : "memory");                 \
-			break;                                            \
-		case 8:                                                   \
-			__asm__ __volatile__("0:	lr.d %0, %2\n"    \
-					     "	sc.d.rl %1, %z3, %2\n"    \
-					     "	bnez %1, 0b\n"            \
-					     "	fence rw, rw\n"           \
-					     : "=&r"(__ret), "=&r"(__rc), \
-					       "+A"(*__ptr)               \
-					     : "rJ"(__new)                \
-					     : "memory");                 \
-			break;                                            \
-		default:                                                  \
-			break;                                            \
-		}                                                         \
-		__ret;                                                    \
-	})
-
-#define xchg(ptr, n)                                                     \
-	({                                                               \
-		__typeof__(*(ptr)) _n_ = (n);                            \
-		(__typeof__(*(ptr))) __xchg((ptr), _n_, sizeof(*(ptr))); \
-	})
-
-#define __cmpxchg(ptr, old, new, size)                                    \
-	({                                                                \
-		__typeof__(ptr) __ptr	 = (ptr);                         \
-		__typeof__(*(ptr)) __old = (old);                         \
-		__typeof__(*(ptr)) __new = (new);                         \
-		__typeof__(*(ptr)) __ret;                                 \
-		register unsigned int __rc;                               \
-		switch (size) {                                           \
-		case 4:                                                   \
-			__asm__ __volatile__("0:	lr.w %0, %2\n"    \
-					     "	bne  %0, %z3, 1f\n"       \
-					     "	sc.w.rl %1, %z4, %2\n"    \
-					     "	bnez %1, 0b\n"            \
-					     "	fence rw, rw\n"           \
-					     "1:\n"                       \
-					     : "=&r"(__ret), "=&r"(__rc), \
-					       "+A"(*__ptr)               \
-					     : "rJ"(__old), "rJ"(__new)   \
-					     : "memory");                 \
-			break;                                            \
-		case 8:                                                   \
-			__asm__ __volatile__("0:	lr.d %0, %2\n"    \
-					     "	bne %0, %z3, 1f\n"        \
-					     "	sc.d.rl %1, %z4, %2\n"    \
-					     "	bnez %1, 0b\n"            \
-					     "	fence rw, rw\n"           \
-					     "1:\n"                       \
-					     : "=&r"(__ret), "=&r"(__rc), \
-					       "+A"(*__ptr)               \
-					     : "rJ"(__old), "rJ"(__new)   \
-					     : "memory");                 \
-			break;                                            \
-		default:                                                  \
-			break;                                            \
-		}                                                         \
-		__ret;                                                    \
-	})
-
-#define cmpxchg(ptr, o, n)                                          \
-	({                                                          \
-		__typeof__(*(ptr)) _o_ = (o);                       \
-		__typeof__(*(ptr)) _n_ = (n);                       \
-		(__typeof__(*(ptr)))                                \
-			__cmpxchg((ptr), _o_, _n_, sizeof(*(ptr))); \
-	})
-
 long atomic_cmpxchg(atomic_t *atom, long oldval, long newval)
 {
-#ifdef __riscv_atomic
 	return __sync_val_compare_and_swap(&atom->counter, oldval, newval);
-#else
-	return cmpxchg(&atom->counter, oldval, newval);
-#endif
 }
 
 long atomic_xchg(atomic_t *atom, long newval)
 {
 	/* Atomically set new value and return old value. */
-#ifdef __riscv_atomic
 	return axchg(&atom->counter, newval);
-#else
-	return xchg(&atom->counter, newval);
-#endif
 }
 
 unsigned int atomic_raw_xchg_uint(volatile unsigned int *ptr,
 				  unsigned int newval)
 {
 	/* Atomically set new value and return old value. */
-#ifdef __riscv_atomic
 	return axchg(ptr, newval);
-#else
-	return xchg(ptr, newval);
-#endif
 }
 
 unsigned long atomic_raw_xchg_ulong(volatile unsigned long *ptr,
 				    unsigned long newval)
 {
 	/* Atomically set new value and return old value. */
-#ifdef __riscv_atomic
 	return axchg(ptr, newval);
-#else
-	return xchg(ptr, newval);
-#endif
 }
 
-#if (__SIZEOF_POINTER__ == 8)
-#define __AMO(op) "amo" #op ".d"
-#elif (__SIZEOF_POINTER__ == 4)
-#define __AMO(op) "amo" #op ".w"
-#else
-#error "Unexpected __SIZEOF_POINTER__"
-#endif
-
-#define __atomic_op_bit_ord(op, mod, nr, addr, ord)                          \
-	({                                                                   \
-		unsigned long __res, __mask;                                 \
-		__mask = BIT_MASK(nr);                                       \
-		__asm__ __volatile__(__AMO(op) #ord " %0, %2, %1"            \
-				     : "=r"(__res), "+A"(addr[BIT_WORD(nr)]) \
-				     : "r"(mod(__mask))                      \
-				     : "memory");                            \
-		__res;                                                       \
-	})
-
-#define __atomic_op_bit(op, mod, nr, addr) \
-	__atomic_op_bit_ord(op, mod, nr, addr, .aqrl)
-
-/* Bitmask modifiers */
-#define __NOP(x) (x)
-#define __NOT(x) (~(x))
-
-inline int atomic_raw_set_bit(int nr, volatile unsigned long *addr)
+int atomic_raw_set_bit(int nr, volatile unsigned long *addr)
 {
-	return __atomic_op_bit(or, __NOP, nr, addr);
+	unsigned long res, mask = BIT_MASK(nr);
+	res = __atomic_fetch_or(&addr[BIT_WORD(nr)], mask, __ATOMIC_RELAXED);
+	return res & mask ? 1 : 0;
 }
 
-inline int atomic_raw_clear_bit(int nr, volatile unsigned long *addr)
+int atomic_raw_clear_bit(int nr, volatile unsigned long *addr)
 {
-	return __atomic_op_bit(and, __NOT, nr, addr);
+	unsigned long res, mask = BIT_MASK(nr);
+	res = __atomic_fetch_and(&addr[BIT_WORD(nr)], ~mask, __ATOMIC_RELAXED);
+	return res & mask ? 1 : 0;
 }
 
-inline int atomic_set_bit(int nr, atomic_t *atom)
+int atomic_set_bit(int nr, atomic_t *atom)
 {
 	return atomic_raw_set_bit(nr, (unsigned long *)&atom->counter);
 }
 
-inline int atomic_clear_bit(int nr, atomic_t *atom)
+int atomic_clear_bit(int nr, atomic_t *atom)
 {
 	return atomic_raw_clear_bit(nr, (unsigned long *)&atom->counter);
 }

lib/sbi/sbi_console.c

@@ -37,28 +37,22 @@ int sbi_getc(void)
 	return -1;
 }
 
-void sbi_putc(char ch)
-{
-	if (console_dev && console_dev->console_putc) {
-		if (ch == '\n')
-			console_dev->console_putc('\r');
-		console_dev->console_putc(ch);
-	}
-}
-
 static unsigned long nputs(const char *str, unsigned long len)
 {
-	unsigned long i, ret;
+	unsigned long i;
 
-	if (console_dev && console_dev->console_puts) {
-		ret = console_dev->console_puts(str, len);
-	} else {
-		for (i = 0; i < len; i++)
-			sbi_putc(str[i]);
-		ret = len;
+	if (console_dev) {
+		if (console_dev->console_puts)
+			return console_dev->console_puts(str, len);
+		else if (console_dev->console_putc) {
+			for (i = 0; i < len; i++) {
+				if (str[i] == '\n')
+					console_dev->console_putc('\r');
+				console_dev->console_putc(str[i]);
+			}
+		}
 	}
-
-	return ret;
+	return len;
 }
 
 static void nputs_all(const char *str, unsigned long len)
@@ -69,6 +63,11 @@ static void nputs_all(const char *str, unsigned long len)
 		p += nputs(&str[p], len - p);
 }
 
+void sbi_putc(char ch)
+{
+	nputs_all(&ch, 1);
+}
+
 void sbi_puts(const char *str)
 {
 	unsigned long len = sbi_strlen(str);
@@ -120,6 +119,8 @@ unsigned long sbi_ngets(char *str, unsigned long len)
 #define PAD_RIGHT 1
 #define PAD_ZERO 2
 #define PAD_ALTERNATE 4
+#define PAD_SIGN 8
+#define USE_TBUF 16
 #define PRINT_BUF_LEN 64
 
 #define va_start(v, l) __builtin_va_start((v), l)
@@ -127,7 +128,7 @@ unsigned long sbi_ngets(char *str, unsigned long len)
 #define va_arg __builtin_va_arg
 typedef __builtin_va_list va_list;
 
-static void printc(char **out, u32 *out_len, char ch)
+static void printc(char **out, u32 *out_len, char ch, int flags)
 {
 	if (!out) {
 		sbi_putc(ch);
@@ -141,60 +142,66 @@ static void printc(char **out, u32 *out_len, char ch)
 	if (!out_len || *out_len > 1) {
 		*(*out)++ = ch;
 		**out = '\0';
+		if (out_len) {
+			--(*out_len);
+			if ((flags & USE_TBUF) && *out_len == 1) {
+				nputs_all(console_tbuf, CONSOLE_TBUF_MAX - *out_len);
+				*out = console_tbuf;
+				*out_len = CONSOLE_TBUF_MAX;
+			}
+		}
 	}
-
-	if (out_len && *out_len > 0)
-		--(*out_len);
 }
 
 static int prints(char **out, u32 *out_len, const char *string, int width,
 		  int flags)
 {
-	int pc	     = 0;
-	char padchar = ' ';
-
-	if (width > 0) {
-		int len = 0;
-		const char *ptr;
-		for (ptr = string; *ptr; ++ptr)
-			++len;
-		if (len >= width)
-			width = 0;
-		else
-			width -= len;
-		if (flags & PAD_ZERO)
-			padchar = '0';
-	}
+	int pc = 0;
+	width -= sbi_strlen(string);
 	if (!(flags & PAD_RIGHT)) {
 		for (; width > 0; --width) {
-			printc(out, out_len, padchar);
+			printc(out, out_len, flags & PAD_ZERO ? '0' : ' ', flags);
 			++pc;
 		}
 	}
 	for (; *string; ++string) {
-		printc(out, out_len, *string);
+		printc(out, out_len, *string, flags);
 		++pc;
 	}
 	for (; width > 0; --width) {
-		printc(out, out_len, padchar);
+		printc(out, out_len, ' ', flags);
 		++pc;
 	}
 
 	return pc;
 }
 
-static int printi(char **out, u32 *out_len, long long i, int b, int sg,
-		  int width, int flags, int letbase)
+static int printi(char **out, u32 *out_len, long long i,
+		  int width, int flags, int type)
 {
-	char print_buf[PRINT_BUF_LEN];
-	char *s;
-	int neg = 0, pc = 0;
-	u64 t;
-	unsigned long long u = i;
+	int pc = 0;
+	char *s, sign = 0, letbase, print_buf[PRINT_BUF_LEN];
+	unsigned long long u, b, t;
 
-	if (sg && b == 10 && i < 0) {
-		neg = 1;
-		u   = -i;
+	b = 10;
+	letbase = 'a';
+	if (type == 'o')
+		b = 8;
+	else if (type == 'x' || type == 'X' || type == 'p' || type == 'P') {
+		b = 16;
+		letbase &= ~0x20;
+		letbase |= type & 0x20;
+	}
+
+	u = i;
+	sign = 0;
+	if (type == 'i' || type == 'd') {
+		if ((flags & PAD_SIGN) && i > 0)
+			sign = '+';
+		if (i < 0) {
+			sign = '-';
+			u = -i;
+		}
 	}
 
 	s  = print_buf + PRINT_BUF_LEN - 1;
@@ -212,23 +219,33 @@ static int printi(char **out, u32 *out_len, long long i, int b, int sg,
 		}
 	}
 
-	if (flags & PAD_ALTERNATE) {
-		if ((b == 16) && (letbase == 'A')) {
-			*--s = 'X';
-		} else if ((b == 16) && (letbase == 'a')) {
-			*--s = 'x';
-		}
-		*--s = '0';
-	}
-
-	if (neg) {
-		if (width && (flags & PAD_ZERO)) {
-			printc(out, out_len, '-');
+	if (flags & PAD_ZERO) {
+		if (sign) {
+			printc(out, out_len, sign, flags);
 			++pc;
 			--width;
-		} else {
-			*--s = '-';
 		}
+		if (i && (flags & PAD_ALTERNATE)) {
+			if (b == 16 || b == 8) {
+				printc(out, out_len, '0', flags);
+				++pc;
+				--width;
+			}
+			if (b == 16) {
+				printc(out, out_len, 'x' - 'a' + letbase, flags);
+				++pc;
+				--width;
+			}
+		}
+	} else {
+		if (i && (flags & PAD_ALTERNATE)) {
+			if (b == 16)
+				*--s = 'x' - 'a' + letbase;
+			if (b == 16 || b == 8)
+				*--s = '0';
+		}
+		if (sign)
+			*--s = sign;
 	}
 
 	return pc + prints(out, out_len, s, width, flags);
@@ -236,10 +253,10 @@ static int printi(char **out, u32 *out_len, long long i, int b, int sg,
 
 static int print(char **out, u32 *out_len, const char *format, va_list args)
 {
+	bool flags_done;
 	int width, flags, pc = 0;
-	char scr[2], *tout;
+	char type, scr[2], *tout;
 	bool use_tbuf = (!out) ? true : false;
-	unsigned long long tmp;
 
 	/*
 	 * The console_tbuf is protected by console_out_lock and
@@ -253,33 +270,51 @@ static int print(char **out, u32 *out_len, const char *format, va_list args)
 		out_len = &console_tbuf_len;
 	}
 
-	for (; *format != 0; ++format) {
-		if (use_tbuf && !console_tbuf_len) {
-			nputs_all(console_tbuf, CONSOLE_TBUF_MAX);
-			console_tbuf_len = CONSOLE_TBUF_MAX;
-			tout = console_tbuf;
-		}
+	/* handle special case: *out_len == 1*/
+	if (out) {
+		if(!out_len || *out_len)
+			**out = '\0';
+	}
 
+	for (; *format != 0; ++format) {
+		width = flags = 0;
+		if (use_tbuf)
+			flags |= USE_TBUF;
 		if (*format == '%') {
 			++format;
-			width = flags = 0;
 			if (*format == '\0')
 				break;
 			if (*format == '%')
 				goto literal;
 			/* Get flags */
-			if (*format == '-') {
-				++format;
-				flags = PAD_RIGHT;
-			}
-			if (*format == '#') {
-				++format;
-				flags |= PAD_ALTERNATE;
-			}
-			while (*format == '0') {
-				++format;
-				flags |= PAD_ZERO;
+			flags_done = false;
+			while (!flags_done) {
+				switch (*format) {
+				case '-':
+					flags |= PAD_RIGHT;
+					break;
+				case '+':
+					flags |= PAD_SIGN;
+					break;
+				case '#':
+					flags |= PAD_ALTERNATE;
+					break;
+				case '0':
+					flags |= PAD_ZERO;
+					break;
+				case ' ':
+				case '\'':
+					/* Ignored flags, do nothing */
+					break;
+				default:
+					flags_done = true;
+					break;
+				}
+				if (!flags_done)
+					++format;
 			}
+			if (flags & PAD_RIGHT)
+				flags &= ~PAD_ZERO;
 			/* Get width */
 			for (; *format >= '0' && *format <= '9'; ++format) {
 				width *= 10;
@@ -293,83 +328,47 @@ static int print(char **out, u32 *out_len, const char *format, va_list args)
 			}
 			if ((*format == 'd') || (*format == 'i')) {
 				pc += printi(out, out_len, va_arg(args, int),
-					     10, 1, width, flags, '0');
+					     width, flags, *format);
 				continue;
 			}
-			if (*format == 'x') {
-				pc += printi(out, out_len,
-					     va_arg(args, unsigned int), 16, 0,
-					     width, flags, 'a');
+			if ((*format == 'u') || (*format == 'o')
+					 || (*format == 'x') || (*format == 'X')) {
+				pc += printi(out, out_len, va_arg(args, unsigned int),
+					     width, flags, *format);
 				continue;
 			}
-			if (*format == 'X') {
-				pc += printi(out, out_len,
-					     va_arg(args, unsigned int), 16, 0,
-					     width, flags, 'A');
+			if ((*format == 'p') || (*format == 'P')) {
+				pc += printi(out, out_len, (uintptr_t)va_arg(args, void*),
+					     width, flags, *format);
 				continue;
 			}
-			if (*format == 'u') {
-				pc += printi(out, out_len,
-					     va_arg(args, unsigned int), 10, 0,
-					     width, flags, 'a');
-				continue;
-			}
-			if (*format == 'p') {
-				pc += printi(out, out_len,
-					     va_arg(args, unsigned long), 16, 0,
-					     width, flags, 'a');
-				continue;
-			}
-			if (*format == 'P') {
-				pc += printi(out, out_len,
-					     va_arg(args, unsigned long), 16, 0,
-					     width, flags, 'A');
-				continue;
-			}
-			if (*format == 'l' && *(format + 1) == 'l') {
-				tmp = va_arg(args, unsigned long long);
-				if (*(format + 2) == 'u') {
-					format += 2;
-					pc += printi(out, out_len, tmp, 10, 0,
-						     width, flags, 'a');
-				} else if (*(format + 2) == 'x') {
-					format += 2;
-					pc += printi(out, out_len, tmp, 16, 0,
-						     width, flags, 'a');
-				} else if (*(format + 2) == 'X') {
-					format += 2;
-					pc += printi(out, out_len, tmp, 16, 0,
-						     width, flags, 'A');
-				} else {
-					format += 1;
-					pc += printi(out, out_len, tmp, 10, 1,
-						     width, flags, '0');
+			if (*format == 'l') {
+				type = 'i';
+				if (format[1] == 'l') {
+					++format;
+					if ((format[1] == 'u') || (format[1] == 'o')
+							|| (format[1] == 'd') || (format[1] == 'i')
+							|| (format[1] == 'x') || (format[1] == 'X')) {
+						++format;
+						type = *format;
+					}
+					pc += printi(out, out_len, va_arg(args, long long),
+						width, flags, type);
+					continue;
 				}
-				continue;
-			} else if (*format == 'l') {
-				if (*(format + 1) == 'u') {
-					format += 1;
-					pc += printi(
-						out, out_len,
-						va_arg(args, unsigned long), 10,
-						0, width, flags, 'a');
-				} else if (*(format + 1) == 'x') {
-					format += 1;
-					pc += printi(
-						out, out_len,
-						va_arg(args, unsigned long), 16,
-						0, width, flags, 'a');
-				} else if (*(format + 1) == 'X') {
-					format += 1;
-					pc += printi(
-						out, out_len,
-						va_arg(args, unsigned long), 16,
-						0, width, flags, 'A');
-				} else {
-					pc += printi(out, out_len,
-						     va_arg(args, long), 10, 1,
-						     width, flags, '0');
+				if ((format[1] == 'u') || (format[1] == 'o')
+						|| (format[1] == 'd') || (format[1] == 'i')
+						|| (format[1] == 'x') || (format[1] == 'X')) {
+					++format;
+					type = *format;
 				}
+				if ((type == 'd') || (type == 'i'))
+					pc += printi(out, out_len, va_arg(args, long),
+					     width, flags, type);
+				else
+					pc += printi(out, out_len, va_arg(args, unsigned long),
+					     width, flags, type);
+				continue;
 			}
 			if (*format == 'c') {
 				/* char are converted to int then pushed on the stack */
@@ -380,7 +379,7 @@ static int print(char **out, u32 *out_len, const char *format, va_list args)
 			}
 		} else {
 literal:
-			printc(out, out_len, *format);
+			printc(out, out_len, *format, flags);
 			++pc;
 		}
 	}
@@ -473,7 +472,7 @@ const struct sbi_console_device *sbi_console_get_device(void)
 
 void sbi_console_set_device(const struct sbi_console_device *dev)
 {
-	if (!dev || console_dev)
+	if (!dev)
 		return;
 
 	console_dev = dev;

lib/sbi/sbi_dbtr.c

@@ -0,0 +1,711 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Ventana Micro Systems, Inc.
+ *
+ * Author(s):
+ *   Himanshu Chauhan <hchauhan@ventanamicro.com>
+ */
+
+#include <sbi/sbi_ecall_interface.h>
+#include <sbi/sbi_csr_detect.h>
+#include <sbi/sbi_platform.h>
+#include <sbi/sbi_byteorder.h>
+#include <sbi/sbi_console.h>
+#include <sbi/sbi_domain.h>
+#include <sbi/sbi_trap.h>
+#include <sbi/sbi_dbtr.h>
+#include <sbi/sbi_heap.h>
+#include <sbi/riscv_encoding.h>
+#include <sbi/riscv_asm.h>
+
+
+/** Offset of pointer to HART's debug triggers info in scratch space */
+static unsigned long hart_state_ptr_offset;
+
+#define dbtr_get_hart_state_ptr(__scratch)				\
+	sbi_scratch_read_type((__scratch), void *, hart_state_ptr_offset)
+
+#define dbtr_thishart_state_ptr()				\
+	dbtr_get_hart_state_ptr(sbi_scratch_thishart_ptr())
+
+#define dbtr_set_hart_state_ptr(__scratch, __hart_state)		\
+	sbi_scratch_write_type((__scratch), void *, hart_state_ptr_offset, \
+			       (__hart_state))
+
+#define INDEX_TO_TRIGGER(_index)					\
+	({								\
+		struct sbi_dbtr_trigger *__trg = NULL;			\
+		struct sbi_dbtr_hart_triggers_state *__hs = NULL;	\
+		__hs = dbtr_get_hart_state_ptr(sbi_scratch_thishart_ptr()); \
+		__trg = &__hs->triggers[_index];			\
+		(__trg);						\
+	})
+
+#define for_each_trig_entry(_base, _max, _etype, _entry)		\
+	for (int _idx = 0; _entry = ((_etype *)_base + _idx),		\
+	     _idx < _max;						\
+	     _idx++, _entry = ((_etype *)_base + _idx))
+
+#define DBTR_SHMEM_MAKE_PHYS(_p_hi, _p_lo) (_p_lo)
+
+/* must call with hs != NULL */
+static inline bool sbi_dbtr_shmem_disabled(
+	struct sbi_dbtr_hart_triggers_state *hs)
+{
+	return (hs->shmem.phys_lo == SBI_DBTR_SHMEM_INVALID_ADDR &&
+		hs->shmem.phys_hi == SBI_DBTR_SHMEM_INVALID_ADDR
+		? true : false);
+}
+
+/* must call with hs != NULL */
+static inline void sbi_dbtr_disable_shmem(
+	struct sbi_dbtr_hart_triggers_state *hs)
+{
+	hs->shmem.phys_lo = SBI_DBTR_SHMEM_INVALID_ADDR;
+	hs->shmem.phys_hi = SBI_DBTR_SHMEM_INVALID_ADDR;
+}
+
+/* must call with hs which is not disabled */
+static inline void *hart_shmem_base(
+	struct sbi_dbtr_hart_triggers_state *hs)
+{
+	return ((void *)(unsigned long)DBTR_SHMEM_MAKE_PHYS(
+			hs->shmem.phys_hi, hs->shmem.phys_lo));
+}
+
+static void sbi_trigger_init(struct sbi_dbtr_trigger *trig,
+			     unsigned long type_mask, unsigned long idx)
+{
+	trig->type_mask = type_mask;
+	trig->state = 0;
+	trig->tdata1 = 0;
+	trig->tdata2 = 0;
+	trig->tdata3 = 0;
+	trig->index = idx;
+}
+
+static inline struct sbi_dbtr_trigger *sbi_alloc_trigger(void)
+{
+	int i;
+	struct sbi_dbtr_trigger *f_trig = NULL;
+	struct sbi_dbtr_hart_triggers_state *hart_state;
+
+	hart_state = dbtr_thishart_state_ptr();
+	if (!hart_state)
+		return NULL;
+
+	if (hart_state->available_trigs <= 0)
+		return NULL;
+
+	for (i = 0; i < hart_state->total_trigs; i++) {
+		f_trig = INDEX_TO_TRIGGER(i);
+		if (f_trig->state & RV_DBTR_BIT_MASK(TS, MAPPED))
+			continue;
+		hart_state->available_trigs--;
+		break;
+	}
+
+	if (i == hart_state->total_trigs)
+		return NULL;
+
+	__set_bit(RV_DBTR_BIT(TS, MAPPED), &f_trig->state);
+
+	return f_trig;
+}
+
+static inline void sbi_free_trigger(struct sbi_dbtr_trigger *trig)
+{
+	struct sbi_dbtr_hart_triggers_state *hart_state;
+
+	if (trig == NULL)
+		return;
+
+	hart_state = dbtr_thishart_state_ptr();
+	if (!hart_state)
+		return;
+
+	trig->state = 0;
+	trig->tdata1 = 0;
+	trig->tdata2 = 0;
+	trig->tdata3 = 0;
+
+	hart_state->available_trigs++;
+}
+
+int sbi_dbtr_init(struct sbi_scratch *scratch, bool coldboot)
+{
+	struct sbi_trap_info trap = {0};
+	unsigned long tdata1;
+	unsigned long val;
+	int i;
+	struct sbi_dbtr_hart_triggers_state *hart_state = NULL;
+
+	if (!sbi_hart_has_extension(scratch, SBI_HART_EXT_SDTRIG))
+		return SBI_SUCCESS;
+
+	if (coldboot) {
+		hart_state_ptr_offset = sbi_scratch_alloc_type_offset(void *);
+		if (!hart_state_ptr_offset)
+			return SBI_ENOMEM;
+	}
+
+	hart_state = dbtr_get_hart_state_ptr(scratch);
+	if (!hart_state) {
+		hart_state = sbi_zalloc(sizeof(*hart_state));
+		if (!hart_state)
+			return SBI_ENOMEM;
+		hart_state->hartid = current_hartid();
+		dbtr_set_hart_state_ptr(scratch, hart_state);
+	}
+
+	/* disable the shared memory */
+	sbi_dbtr_disable_shmem(hart_state);
+
+	/* Skip probing triggers if already probed */
+	if (hart_state->probed)
+		goto _probed;
+
+	for (i = 0; i < RV_MAX_TRIGGERS; i++) {
+		csr_write_allowed(CSR_TSELECT, (ulong)&trap, i);
+		if (trap.cause)
+			break;
+
+		val = csr_read_allowed(CSR_TSELECT, (ulong)&trap);
+		if (trap.cause)
+			break;
+
+		/*
+		 * Read back tselect and check that it contains the
+		 * written value
+		 */
+		if (val != i)
+			break;
+
+		val = csr_read_allowed(CSR_TINFO, (ulong)&trap);
+		if (trap.cause) {
+			/*
+			 * If reading tinfo caused an exception, the
+			 * debugger must read tdata1 to discover the
+			 * type.
+			 */
+			tdata1 = csr_read_allowed(CSR_TDATA1,
+						  (ulong)&trap);
+			if (trap.cause)
+				break;
+
+			if (TDATA1_GET_TYPE(tdata1) == 0)
+				break;
+
+			sbi_trigger_init(INDEX_TO_TRIGGER(i),
+					 BIT(TDATA1_GET_TYPE(tdata1)),
+					 i);
+			hart_state->total_trigs++;
+		} else {
+			if (val == 1)
+				break;
+
+			sbi_trigger_init(INDEX_TO_TRIGGER(i), val, i);
+			hart_state->total_trigs++;
+		}
+	}
+
+	hart_state->probed = 1;
+
+ _probed:
+	hart_state->available_trigs = hart_state->total_trigs;
+
+	return SBI_SUCCESS;
+}
+
+int sbi_dbtr_get_total_triggers(void)
+{
+	struct sbi_dbtr_hart_triggers_state *hs;
+	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
+
+	/*
+	 * This function may be used during ecall registration.
+	 * By that time the debug trigger module might not be
+	 * initialized. If the extension is not supported, report
+	 * number of triggers as 0.
+	 */
+	if (!sbi_hart_has_extension(scratch, SBI_HART_EXT_SDTRIG))
+		return 0;
+
+	hs = dbtr_thishart_state_ptr();
+	if (!hs)
+		return 0;
+
+	return hs->total_trigs;
+}
+
+int sbi_dbtr_setup_shmem(const struct sbi_domain *dom, unsigned long smode,
+			 unsigned long shmem_phys_lo,
+			 unsigned long shmem_phys_hi)
+{
+	u32 hartid = current_hartid();
+	struct sbi_dbtr_hart_triggers_state *hart_state;
+
+	if (dom && !sbi_domain_is_assigned_hart(dom, hartid)) {
+		sbi_dprintf("%s: calling hart not assigned to this domain\n",
+			   __func__);
+		return SBI_ERR_DENIED;
+	}
+
+	hart_state = dbtr_thishart_state_ptr();
+	if (!hart_state)
+		return SBI_ERR_FAILED;
+
+	/* call is to disable shared memory */
+	if (shmem_phys_lo == SBI_DBTR_SHMEM_INVALID_ADDR
+	    && shmem_phys_hi == SBI_DBTR_SHMEM_INVALID_ADDR) {
+		sbi_dbtr_disable_shmem(hart_state);
+		return SBI_SUCCESS;
+	}
+
+	/* the shared memory must be disabled on this hart */
+	if (!sbi_dbtr_shmem_disabled(hart_state))
+		return SBI_ERR_ALREADY_AVAILABLE;
+
+	/* lower physical address must be XLEN/8 bytes aligned */
+	if (shmem_phys_lo & SBI_DBTR_SHMEM_ALIGN_MASK)
+		return SBI_ERR_INVALID_PARAM;
+
+	/*
+	* On RV32, the M-mode can only access the first 4GB of
+	* the physical address space because M-mode does not have
+	* MMU to access full 34-bit physical address space.
+	* So fail if the upper 32 bits of the physical address
+	* is non-zero on RV32.
+	*
+	* On RV64, kernel sets upper 64bit address part to zero.
+	* So fail if the upper 64bit of the physical address
+	* is non-zero on RV64.
+	*/
+	if (shmem_phys_hi)
+		return SBI_EINVALID_ADDR;
+
+	if (dom && !sbi_domain_check_addr(dom,
+		  DBTR_SHMEM_MAKE_PHYS(shmem_phys_hi, shmem_phys_lo), smode,
+		  SBI_DOMAIN_READ | SBI_DOMAIN_WRITE))
+		return SBI_ERR_INVALID_ADDRESS;
+
+	hart_state->shmem.phys_lo = shmem_phys_lo;
+	hart_state->shmem.phys_hi = shmem_phys_hi;
+
+	return SBI_SUCCESS;
+}
+
+static void dbtr_trigger_setup(struct sbi_dbtr_trigger *trig,
+			       struct sbi_dbtr_data_msg *recv)
+{
+	unsigned long tdata1;
+
+	if (!trig)
+		return;
+
+	trig->tdata1 = lle_to_cpu(recv->tdata1);
+	trig->tdata2 = lle_to_cpu(recv->tdata2);
+	trig->tdata3 = lle_to_cpu(recv->tdata3);
+
+	tdata1 = lle_to_cpu(recv->tdata1);
+
+	trig->state = 0;
+
+	__set_bit(RV_DBTR_BIT(TS, MAPPED), &trig->state);
+
+	SET_TRIG_HW_INDEX(trig->state, trig->index);
+
+	switch (TDATA1_GET_TYPE(tdata1)) {
+	case RISCV_DBTR_TRIG_MCONTROL:
+		if (__test_bit(RV_DBTR_BIT(MC, U), &tdata1))
+			__set_bit(RV_DBTR_BIT(TS, U), &trig->state);
+
+		if (__test_bit(RV_DBTR_BIT(MC, S), &tdata1))
+			__set_bit(RV_DBTR_BIT(TS, S), &trig->state);
+		break;
+	case RISCV_DBTR_TRIG_MCONTROL6:
+		if (__test_bit(RV_DBTR_BIT(MC6, U), &tdata1))
+			__set_bit(RV_DBTR_BIT(TS, U), &trig->state);
+
+		if (__test_bit(RV_DBTR_BIT(MC6, S), &tdata1))
+			__set_bit(RV_DBTR_BIT(TS, S), &trig->state);
+
+		if (__test_bit(RV_DBTR_BIT(MC6, VU), &tdata1))
+			__set_bit(RV_DBTR_BIT(TS, VU), &trig->state);
+
+		if (__test_bit(RV_DBTR_BIT(MC6, VS), &tdata1))
+			__set_bit(RV_DBTR_BIT(TS, VS), &trig->state);
+		break;
+	default:
+		sbi_dprintf("%s: Unknown type (tdata1: 0x%lx Type: %ld)\n",
+			    __func__, tdata1, TDATA1_GET_TYPE(tdata1));
+		break;
+	}
+}
+
+static inline void update_bit(unsigned long new, int nr, volatile unsigned long *addr)
+{
+	if (new)
+		__set_bit(nr, addr);
+	else
+		__clear_bit(nr, addr);
+}
+
+static void dbtr_trigger_enable(struct sbi_dbtr_trigger *trig)
+{
+	unsigned long state;
+	unsigned long tdata1;
+
+	if (!trig && !(trig->state & RV_DBTR_BIT_MASK(TS, MAPPED)))
+		return;
+
+	state = trig->state;
+	tdata1 = trig->tdata1;
+
+	switch (TDATA1_GET_TYPE(tdata1)) {
+	case RISCV_DBTR_TRIG_MCONTROL:
+		update_bit(state & RV_DBTR_BIT_MASK(TS, U),
+			   RV_DBTR_BIT(MC, U), &trig->tdata1);
+		update_bit(state & RV_DBTR_BIT_MASK(TS, S),
+			   RV_DBTR_BIT(MC, S), &trig->tdata1);
+		break;
+	case RISCV_DBTR_TRIG_MCONTROL6:
+		update_bit(state & RV_DBTR_BIT_MASK(TS, VU),
+			   RV_DBTR_BIT(MC6, VU), &trig->tdata1);
+		update_bit(state & RV_DBTR_BIT_MASK(TS, VS),
+			   RV_DBTR_BIT(MC6, VS), &trig->tdata1);
+		update_bit(state & RV_DBTR_BIT_MASK(TS, U),
+			   RV_DBTR_BIT(MC6, U), &trig->tdata1);
+		update_bit(state & RV_DBTR_BIT_MASK(TS, S),
+			   RV_DBTR_BIT(MC6, S), &trig->tdata1);
+		break;
+	default:
+		break;
+	}
+
+	/*
+	 * RISC-V Debug Support v1.0.0 section 5.5:
+	 * Debugger cannot simply set a trigger by writing tdata1, then tdata2,
+	 * etc. The current value of tdata2 might not be legal with the new
+	 * value of tdata1. To help with this situation, it is guaranteed that
+	 * writing 0 to tdata1 disables the trigger, and leaves it in a state
+	 * where tdata2 and tdata3 can be written with any value that makes
+	 * sense for any trigger type supported by this trigger.
+	 */
+	csr_write(CSR_TSELECT, trig->index);
+	csr_write(CSR_TDATA1, 0x0);
+	csr_write(CSR_TDATA2, trig->tdata2);
+	csr_write(CSR_TDATA1, trig->tdata1);
+}
+
+static void dbtr_trigger_disable(struct sbi_dbtr_trigger *trig)
+{
+	unsigned long tdata1;
+
+	if (!trig && !(trig->state & RV_DBTR_BIT_MASK(TS, MAPPED)))
+		return;
+
+	tdata1 = trig->tdata1;
+
+	switch (TDATA1_GET_TYPE(tdata1)) {
+	case RISCV_DBTR_TRIG_MCONTROL:
+		__clear_bit(RV_DBTR_BIT(MC, U), &trig->tdata1);
+		__clear_bit(RV_DBTR_BIT(MC, S), &trig->tdata1);
+		break;
+	case RISCV_DBTR_TRIG_MCONTROL6:
+		__clear_bit(RV_DBTR_BIT(MC6, VU), &trig->tdata1);
+		__clear_bit(RV_DBTR_BIT(MC6, VS), &trig->tdata1);
+		__clear_bit(RV_DBTR_BIT(MC6, U), &trig->tdata1);
+		__clear_bit(RV_DBTR_BIT(MC6, S), &trig->tdata1);
+		break;
+	default:
+		break;
+	}
+
+	csr_write(CSR_TSELECT, trig->index);
+	csr_write(CSR_TDATA1, trig->tdata1);
+}
+
+static void dbtr_trigger_clear(struct sbi_dbtr_trigger *trig)
+{
+	if (!trig && !(trig->state & RV_DBTR_BIT_MASK(TS, MAPPED)))
+		return;
+
+	csr_write(CSR_TSELECT, trig->index);
+	csr_write(CSR_TDATA1, 0x0);
+	csr_write(CSR_TDATA2, 0x0);
+}
+
+static int dbtr_trigger_supported(unsigned long type)
+{
+	switch (type) {
+	case RISCV_DBTR_TRIG_MCONTROL:
+	case RISCV_DBTR_TRIG_MCONTROL6:
+		return 1;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int dbtr_trigger_valid(unsigned long type, unsigned long tdata)
+{
+	switch (type) {
+	case RISCV_DBTR_TRIG_MCONTROL:
+		if (!(tdata & RV_DBTR_BIT_MASK(MC, DMODE)) &&
+		    !(tdata & RV_DBTR_BIT_MASK(MC, M)))
+			return 1;
+		break;
+	case RISCV_DBTR_TRIG_MCONTROL6:
+		if (!(tdata & RV_DBTR_BIT_MASK(MC6, DMODE)) &&
+		    !(tdata & RV_DBTR_BIT_MASK(MC6, M)))
+			return 1;
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+int sbi_dbtr_num_trig(unsigned long data, unsigned long *out)
+{
+	unsigned long type = TDATA1_GET_TYPE(data);
+	u32 hartid = current_hartid();
+	unsigned long total = 0;
+	struct sbi_dbtr_trigger *trig;
+	int i;
+	struct sbi_dbtr_hart_triggers_state *hs;
+
+	hs = dbtr_thishart_state_ptr();
+	if (!hs)
+		return SBI_ERR_FAILED;
+
+	if (data == 0) {
+		*out = hs->total_trigs;
+		return SBI_SUCCESS;
+	}
+
+	for (i = 0; i < hs->total_trigs; i++) {
+		trig = INDEX_TO_TRIGGER(i);
+
+		if (__test_bit(type, &trig->type_mask))
+			total++;
+	}
+
+	sbi_dprintf("%s: hart%d: total triggers of type %lu: %lu\n",
+		    __func__, hartid, type, total);
+
+	*out = total;
+	return SBI_SUCCESS;
+}
+
+int sbi_dbtr_read_trig(unsigned long smode,
+		       unsigned long trig_idx_base, unsigned long trig_count)
+{
+	struct sbi_dbtr_data_msg *xmit;
+	struct sbi_dbtr_trigger *trig;
+	struct sbi_dbtr_shmem_entry *entry;
+	void *shmem_base = NULL;
+	struct sbi_dbtr_hart_triggers_state *hs = NULL;
+
+	hs = dbtr_thishart_state_ptr();
+	if (!hs)
+		return SBI_ERR_FAILED;
+
+	if (trig_idx_base >= hs->total_trigs ||
+	    trig_idx_base + trig_count >= hs->total_trigs)
+		return SBI_ERR_INVALID_PARAM;
+
+	if (sbi_dbtr_shmem_disabled(hs))
+		return SBI_ERR_NO_SHMEM;
+
+	shmem_base = hart_shmem_base(hs);
+
+	for_each_trig_entry(shmem_base, trig_count, typeof(*entry), entry) {
+		sbi_hart_map_saddr((unsigned long)entry, sizeof(*entry));
+		xmit = &entry->data;
+		trig = INDEX_TO_TRIGGER((_idx + trig_idx_base));
+		xmit->tstate = cpu_to_lle(trig->state);
+		xmit->tdata1 = cpu_to_lle(trig->tdata1);
+		xmit->tdata2 = cpu_to_lle(trig->tdata2);
+		xmit->tdata3 = cpu_to_lle(trig->tdata3);
+		sbi_hart_unmap_saddr();
+	}
+
+	return SBI_SUCCESS;
+}
+
+int sbi_dbtr_install_trig(unsigned long smode,
+			  unsigned long trig_count, unsigned long *out)
+{
+	void *shmem_base = NULL;
+	struct sbi_dbtr_shmem_entry *entry;
+	struct sbi_dbtr_data_msg *recv;
+	struct sbi_dbtr_id_msg *xmit;
+	unsigned long ctrl;
+	struct sbi_dbtr_trigger *trig;
+	struct sbi_dbtr_hart_triggers_state *hs = NULL;
+
+	hs = dbtr_thishart_state_ptr();
+	if (!hs)
+		return SBI_ERR_FAILED;
+
+	if (sbi_dbtr_shmem_disabled(hs))
+		return SBI_ERR_NO_SHMEM;
+
+	shmem_base = hart_shmem_base(hs);
+
+	/* Check requested triggers configuration */
+	for_each_trig_entry(shmem_base, trig_count, typeof(*entry), entry) {
+		sbi_hart_map_saddr((unsigned long)entry, sizeof(*entry));
+		recv = (struct sbi_dbtr_data_msg *)(&entry->data);
+		ctrl = recv->tdata1;
+
+		if (!dbtr_trigger_supported(TDATA1_GET_TYPE(ctrl))) {
+			*out = _idx;
+			sbi_hart_unmap_saddr();
+			return SBI_ERR_FAILED;
+		}
+
+		if (!dbtr_trigger_valid(TDATA1_GET_TYPE(ctrl), ctrl)) {
+			*out = _idx;
+			sbi_hart_unmap_saddr();
+			return SBI_ERR_FAILED;
+		}
+		sbi_hart_unmap_saddr();
+	}
+
+	if (hs->available_trigs < trig_count) {
+		*out = hs->available_trigs;
+		return SBI_ERR_FAILED;
+	}
+
+	/* Install triggers */
+	for_each_trig_entry(shmem_base, trig_count, typeof(*entry), entry) {
+		/*
+		 * Since we have already checked if enough triggers are
+		 * available, trigger allocation must succeed.
+		 */
+		trig = sbi_alloc_trigger();
+
+		sbi_hart_map_saddr((unsigned long)entry, sizeof(*entry));
+
+		recv = (struct sbi_dbtr_data_msg *)(&entry->data);
+		xmit = (struct sbi_dbtr_id_msg *)(&entry->id);
+
+		dbtr_trigger_setup(trig,  recv);
+		dbtr_trigger_enable(trig);
+		xmit->idx = cpu_to_lle(trig->index);
+		sbi_hart_unmap_saddr();
+	}
+
+	return SBI_SUCCESS;
+}
+
+int sbi_dbtr_uninstall_trig(unsigned long trig_idx_base,
+			    unsigned long trig_idx_mask)
+{
+	unsigned long trig_mask = trig_idx_mask << trig_idx_base;
+	unsigned long idx = trig_idx_base;
+	struct sbi_dbtr_trigger *trig;
+	struct sbi_dbtr_hart_triggers_state *hs;
+
+	hs = dbtr_thishart_state_ptr();
+	if (!hs)
+		return SBI_ERR_FAILED;
+
+	for_each_set_bit_from(idx, &trig_mask, hs->total_trigs) {
+		trig = INDEX_TO_TRIGGER(idx);
+		if (!(trig->state & RV_DBTR_BIT_MASK(TS, MAPPED)))
+			return SBI_ERR_INVALID_PARAM;
+
+		dbtr_trigger_clear(trig);
+
+		sbi_free_trigger(trig);
+	}
+
+	return SBI_SUCCESS;
+}
+
+int sbi_dbtr_enable_trig(unsigned long trig_idx_base,
+			 unsigned long trig_idx_mask)
+{
+	unsigned long trig_mask = trig_idx_mask << trig_idx_base;
+	unsigned long idx = trig_idx_base;
+	struct sbi_dbtr_trigger *trig;
+	struct sbi_dbtr_hart_triggers_state *hs;
+
+	hs = dbtr_thishart_state_ptr();
+	if (!hs)
+		return SBI_ERR_FAILED;
+
+	for_each_set_bit_from(idx, &trig_mask, hs->total_trigs) {
+		trig = INDEX_TO_TRIGGER(idx);
+		sbi_dprintf("%s: enable trigger %lu\n", __func__, idx);
+		dbtr_trigger_enable(trig);
+	}
+
+	return SBI_SUCCESS;
+}
+
+int sbi_dbtr_update_trig(unsigned long smode,
+			 unsigned long trig_idx_base,
+			 unsigned long trig_idx_mask)
+{
+	unsigned long trig_mask = trig_idx_mask << trig_idx_base;
+	unsigned long idx = trig_idx_base;
+	struct sbi_dbtr_data_msg *recv;
+	unsigned long uidx = 0;
+	struct sbi_dbtr_trigger *trig;
+	struct sbi_dbtr_shmem_entry *entry;
+	void *shmem_base = NULL;
+	struct sbi_dbtr_hart_triggers_state *hs = NULL;
+
+	hs = dbtr_thishart_state_ptr();
+	if (!hs)
+		return SBI_ERR_FAILED;
+
+	if (sbi_dbtr_shmem_disabled(hs))
+		return SBI_ERR_NO_SHMEM;
+
+	shmem_base = hart_shmem_base(hs);
+
+	for_each_set_bit_from(idx, &trig_mask, hs->total_trigs) {
+		trig = INDEX_TO_TRIGGER(idx);
+
+		if (!(trig->state & RV_DBTR_BIT_MASK(TS, MAPPED)))
+			return SBI_ERR_INVALID_PARAM;
+
+		entry = (shmem_base + uidx * sizeof(*entry));
+		recv = &entry->data;
+
+		trig->tdata2 = lle_to_cpu(recv->tdata2);
+		dbtr_trigger_enable(trig);
+		uidx++;
+	}
+
+	return SBI_SUCCESS;
+}
+
+int sbi_dbtr_disable_trig(unsigned long trig_idx_base,
+			  unsigned long trig_idx_mask)
+{
+	unsigned long trig_mask = trig_idx_mask << trig_idx_base;
+	unsigned long idx = trig_idx_base;
+	struct sbi_dbtr_trigger *trig;
+	struct sbi_dbtr_hart_triggers_state *hs;
+
+	hs = dbtr_thishart_state_ptr();
+	if (!hs)
+		return SBI_ERR_FAILED;
+
+	for_each_set_bit_from(idx, &trig_mask, hs->total_trigs) {
+		trig = INDEX_TO_TRIGGER(idx);
+		dbtr_trigger_disable(trig);
+	}
+
+	return SBI_SUCCESS;
+}

lib/sbi/sbi_domain.c

@@ -40,22 +40,22 @@ struct sbi_domain root = {
 
 static unsigned long domain_hart_ptr_offset;
 
-struct sbi_domain *sbi_hartid_to_domain(u32 hartid)
+struct sbi_domain *sbi_hartindex_to_domain(u32 hartindex)
 {
 	struct sbi_scratch *scratch;
 
-	scratch = sbi_hartid_to_scratch(hartid);
+	scratch = sbi_hartindex_to_scratch(hartindex);
 	if (!scratch || !domain_hart_ptr_offset)
 		return NULL;
 
 	return sbi_scratch_read_type(scratch, void *, domain_hart_ptr_offset);
 }
 
-static void update_hartid_to_domain(u32 hartid, struct sbi_domain *dom)
+void sbi_update_hartindex_to_domain(u32 hartindex, struct sbi_domain *dom)
 {
 	struct sbi_scratch *scratch;
 
-	scratch = sbi_hartid_to_scratch(hartid);
+	scratch = sbi_hartindex_to_scratch(hartindex);
 	if (!scratch)
 		return;
 
@@ -64,28 +64,34 @@ static void update_hartid_to_domain(u32 hartid, struct sbi_domain *dom)
 
 bool sbi_domain_is_assigned_hart(const struct sbi_domain *dom, u32 hartid)
 {
-	if (dom)
-		return sbi_hartmask_test_hart(hartid, &dom->assigned_harts);
+	bool ret;
+	struct sbi_domain *tdom = (struct sbi_domain *)dom;
 
-	return false;
+	if (!dom)
+		return false;
+
+	spin_lock(&tdom->assigned_harts_lock);
+	ret = sbi_hartmask_test_hartid(hartid, &tdom->assigned_harts);
+	spin_unlock(&tdom->assigned_harts_lock);
+
+	return ret;
 }
 
 ulong sbi_domain_get_assigned_hartmask(const struct sbi_domain *dom,
 				       ulong hbase)
 {
-	ulong ret, bword, boff;
+	ulong ret = 0;
+	struct sbi_domain *tdom = (struct sbi_domain *)dom;
 
 	if (!dom)
 		return 0;
 
-	bword = BIT_WORD(hbase);
-	boff = BIT_WORD_OFFSET(hbase);
-
-	ret = sbi_hartmask_bits(&dom->assigned_harts)[bword++] >> boff;
-	if (boff && bword < BIT_WORD(SBI_HARTMASK_MAX_BITS)) {
-		ret |= (sbi_hartmask_bits(&dom->assigned_harts)[bword] &
-			(BIT(boff) - 1UL)) << (BITS_PER_LONG - boff);
+	spin_lock(&tdom->assigned_harts_lock);
+	for (int i = 0; i < 8 * sizeof(ret); i++) {
+		if (sbi_hartmask_test_hartid(hbase + i, &tdom->assigned_harts))
+			ret |= 1UL << i;
 	}
+	spin_unlock(&tdom->assigned_harts_lock);
 
 	return ret;
 }
@@ -201,12 +207,11 @@ static bool is_region_subset(const struct sbi_domain_memregion *regA,
 	return false;
 }
 
-/** Check if regionA conflicts regionB */
-static bool is_region_conflict(const struct sbi_domain_memregion *regA,
-				const struct sbi_domain_memregion *regB)
+/** Check if regionA can be replaced by regionB */
+static bool is_region_compatible(const struct sbi_domain_memregion *regA,
+				 const struct sbi_domain_memregion *regB)
 {
-	if ((is_region_subset(regA, regB) || is_region_subset(regB, regA)) &&
-	    regA->flags == regB->flags)
+	if (is_region_subset(regA, regB) && regA->flags == regB->flags)
 		return true;
 
 	return false;
@@ -264,11 +269,26 @@ static const struct sbi_domain_memregion *find_next_subset_region(
 	return ret;
 }
 
-static int sanitize_domain(const struct sbi_platform *plat,
-			   struct sbi_domain *dom)
+static void swap_region(struct sbi_domain_memregion* reg1,
+			struct sbi_domain_memregion* reg2)
+{
+	struct sbi_domain_memregion treg;
+
+	sbi_memcpy(&treg, reg1, sizeof(treg));
+	sbi_memcpy(reg1, reg2, sizeof(treg));
+	sbi_memcpy(reg2, &treg, sizeof(treg));
+}
+
+static void clear_region(struct sbi_domain_memregion* reg)
+{
+	sbi_memset(reg, 0x0, sizeof(*reg));
+}
+
+static int sanitize_domain(struct sbi_domain *dom)
 {
 	u32 i, j, count;
-	struct sbi_domain_memregion treg, *reg, *reg1;
+	bool is_covered;
+	struct sbi_domain_memregion *reg, *reg1;
 
 	/* Check possible HARTs */
 	if (!dom->possible_harts) {
@@ -276,10 +296,11 @@ static int sanitize_domain(const struct sbi_platform *plat,
 			   __func__, dom->name);
 		return SBI_EINVAL;
 	}
-	sbi_hartmask_for_each_hart(i, dom->possible_harts) {
-		if (sbi_platform_hart_invalid(plat, i)) {
+	sbi_hartmask_for_each_hartindex(i, dom->possible_harts) {
+		if (!sbi_hartindex_valid(i)) {
 			sbi_printf("%s: %s possible HART mask has invalid "
-				   "hart %d\n", __func__, dom->name, i);
+				   "hart %d\n", __func__,
+				   dom->name, sbi_hartindex_to_hartid(i));
 			return SBI_EINVAL;
 		}
 	}
@@ -318,25 +339,38 @@ static int sanitize_domain(const struct sbi_platform *plat,
 		for (j = i + 1; j < count; j++) {
 			reg1 = &dom->regions[j];
 
-			if (is_region_conflict(reg1, reg)) {
-				sbi_printf("%s: %s conflict between regions "
-					"(base=0x%lx order=%lu flags=0x%lx) and "
-					"(base=0x%lx order=%lu flags=0x%lx)\n",
-					__func__, dom->name,
-					reg->base, reg->order, reg->flags,
-					reg1->base, reg1->order, reg1->flags);
-				return SBI_EINVAL;
-			}
-
 			if (!is_region_before(reg1, reg))
 				continue;
 
-			sbi_memcpy(&treg, reg1, sizeof(treg));
-			sbi_memcpy(reg1, reg, sizeof(treg));
-			sbi_memcpy(reg, &treg, sizeof(treg));
+			swap_region(reg, reg1);
 		}
 	}
 
+	/* Remove covered regions */
+	while(i < (count - 1)) {
+		is_covered = false;
+		reg = &dom->regions[i];
+
+		for (j = i + 1; j < count; j++) {
+			reg1 = &dom->regions[j];
+
+			if (is_region_compatible(reg, reg1)) {
+				is_covered = true;
+				break;
+			}
+		}
+
+		/* find a region is superset of reg, remove reg */
+		if (is_covered) {
+			for (j = i; j < (count - 1); j++)
+				swap_region(&dom->regions[j],
+					    &dom->regions[j + 1]);
+			clear_region(&dom->regions[count - 1]);
+			count--;
+		} else
+			i++;
+	}
+
 	/*
 	 * We don't need to check boot HART id of domain because if boot
 	 * HART id is not possible/assigned to this domain then it won't
@@ -400,7 +434,7 @@ bool sbi_domain_check_addr_range(const struct sbi_domain *dom,
 
 void sbi_domain_dump(const struct sbi_domain *dom, const char *suffix)
 {
-	u32 i, k;
+	u32 i, j, k;
 	unsigned long rstart, rend;
 	struct sbi_domain_memregion *reg;
 
@@ -412,9 +446,11 @@ void sbi_domain_dump(const struct sbi_domain *dom, const char *suffix)
 
 	k = 0;
 	sbi_printf("Domain%d HARTs       %s: ", dom->index, suffix);
-	sbi_hartmask_for_each_hart(i, dom->possible_harts)
+	sbi_hartmask_for_each_hartindex(i, dom->possible_harts) {
+		j = sbi_hartindex_to_hartid(i);
 		sbi_printf("%s%d%s", (k++) ? "," : "",
-			   i, sbi_domain_is_assigned_hart(dom, i) ? "*" : "");
+			   j, sbi_domain_is_assigned_hart(dom, j) ? "*" : "");
+	}
 	sbi_printf("\n");
 
 	i = 0;
@@ -499,7 +535,6 @@ int sbi_domain_register(struct sbi_domain *dom,
 	int rc;
 	struct sbi_domain *tdom;
 	u32 cold_hartid = current_hartid();
-	const struct sbi_platform *plat = sbi_platform_thishart_ptr();
 
 	/* Sanity checks */
 	if (!dom || !assign_mask || domain_finalized)
@@ -522,7 +557,7 @@ int sbi_domain_register(struct sbi_domain *dom,
 	}
 
 	/* Sanitize discovered domain */
-	rc = sanitize_domain(plat, dom);
+	rc = sanitize_domain(dom);
 	if (rc) {
 		sbi_printf("%s: sanity checks failed for"
 			   " %s (error %d)\n", __func__,
@@ -534,26 +569,29 @@ int sbi_domain_register(struct sbi_domain *dom,
 	dom->index = domain_count++;
 	domidx_to_domain_table[dom->index] = dom;
 
+	/* Initialize spinlock for dom->assigned_harts */
+	SPIN_LOCK_INIT(dom->assigned_harts_lock);
+
 	/* Clear assigned HARTs of domain */
 	sbi_hartmask_clear_all(&dom->assigned_harts);
 
 	/* Assign domain to HART if HART is a possible HART */
-	sbi_hartmask_for_each_hart(i, assign_mask) {
-		if (!sbi_hartmask_test_hart(i, dom->possible_harts))
+	sbi_hartmask_for_each_hartindex(i, assign_mask) {
+		if (!sbi_hartmask_test_hartindex(i, dom->possible_harts))
 			continue;
 
-		tdom = sbi_hartid_to_domain(i);
+		tdom = sbi_hartindex_to_domain(i);
 		if (tdom)
-			sbi_hartmask_clear_hart(i,
+			sbi_hartmask_clear_hartindex(i,
 					&tdom->assigned_harts);
-		update_hartid_to_domain(i, dom);
-		sbi_hartmask_set_hart(i, &dom->assigned_harts);
+		sbi_update_hartindex_to_domain(i, dom);
+		sbi_hartmask_set_hartindex(i, &dom->assigned_harts);
 
 		/*
 		 * If cold boot HART is assigned to this domain then
 		 * override boot HART of this domain.
 		 */
-		if (i == cold_hartid &&
+		if (sbi_hartindex_to_hartid(i) == cold_hartid &&
 		    dom->boot_hartid != cold_hartid) {
 			sbi_printf("Domain%d Boot HARTID forced to"
 				   " %d\n", dom->index, cold_hartid);
@@ -569,21 +607,16 @@ int sbi_domain_root_add_memregion(const struct sbi_domain_memregion *reg)
 	int rc;
 	bool reg_merged;
 	struct sbi_domain_memregion *nreg, *nreg1, *nreg2;
-	const struct sbi_platform *plat = sbi_platform_thishart_ptr();
 
 	/* Sanity checks */
 	if (!reg || domain_finalized || !root.regions ||
 	    (ROOT_REGION_MAX <= root_memregs_count))
 		return SBI_EINVAL;
 
-	/* Check for conflicts */
+	/* Check whether compatible region exists for the new one */
 	sbi_domain_for_each_memregion(&root, nreg) {
-		if (is_region_conflict(reg, nreg)) {
-			sbi_printf("%s: is_region_conflict check failed"
-			" 0x%lx conflicts existing 0x%lx\n", __func__,
-				   reg->base, nreg->base);
-			return SBI_EALREADY;
-		}
+		if (is_region_compatible(reg, nreg))
+			return 0;
 	}
 
 	/* Append the memregion to root memregions */
@@ -595,7 +628,7 @@ int sbi_domain_root_add_memregion(const struct sbi_domain_memregion *reg)
 	/* Sort and optimize root regions */
 	do {
 		/* Sanitize the root domain so that memregions are sorted */
-		rc = sanitize_domain(plat, &root);
+		rc = sanitize_domain(&root);
 		if (rc) {
 			sbi_printf("%s: sanity checks failed for"
 				   " %s (error %d)\n", __func__,
@@ -673,36 +706,43 @@ int sbi_domain_finalize(struct sbi_scratch *scratch, u32 cold_hartid)
 
 	/* Startup boot HART of domains */
 	sbi_domain_for_each(i, dom) {
-		/* Domain boot HART */
-		dhart = dom->boot_hartid;
+		/* Domain boot HART index */
+		dhart = sbi_hartid_to_hartindex(dom->boot_hartid);
 
 		/* Ignore of boot HART is off limits */
-		if (SBI_HARTMASK_MAX_BITS <= dhart)
+		if (!sbi_hartindex_valid(dhart))
 			continue;
 
 		/* Ignore if boot HART not possible for this domain */
-		if (!sbi_hartmask_test_hart(dhart, dom->possible_harts))
+		if (!sbi_hartmask_test_hartindex(dhart, dom->possible_harts))
 			continue;
 
 		/* Ignore if boot HART assigned different domain */
-		if (sbi_hartid_to_domain(dhart) != dom ||
-		    !sbi_hartmask_test_hart(dhart, &dom->assigned_harts))
+		if (sbi_hartindex_to_domain(dhart) != dom)
+			continue;
+
+		/* Ignore if boot HART is not part of the assigned HARTs */
+		spin_lock(&dom->assigned_harts_lock);
+		rc = sbi_hartmask_test_hartindex(dhart, &dom->assigned_harts);
+		spin_unlock(&dom->assigned_harts_lock);
+		if (!rc)
 			continue;
 
 		/* Startup boot HART of domain */
-		if (dhart == cold_hartid) {
+		if (dom->boot_hartid == cold_hartid) {
 			scratch->next_addr = dom->next_addr;
 			scratch->next_mode = dom->next_mode;
 			scratch->next_arg1 = dom->next_arg1;
 		} else {
-			rc = sbi_hsm_hart_start(scratch, NULL, dhart,
+			rc = sbi_hsm_hart_start(scratch, NULL,
+						dom->boot_hartid,
 						dom->next_addr,
 						dom->next_mode,
 						dom->next_arg1);
 			if (rc) {
 				sbi_printf("%s: failed to start boot HART %d"
 					   " for %s (error %d)\n", __func__,
-					   dhart, dom->name, rc);
+					   dom->boot_hartid, dom->name, rc);
 				return rc;
 			}
 		}
@@ -772,11 +812,17 @@ int sbi_domain_init(struct sbi_scratch *scratch, u32 cold_hartid)
 
 	root.fw_region_inited = true;
 
-	/* Root domain allow everything memory region */
+	/*
+	 * Allow SU RWX on rest of the memory region. Since pmp entries
+	 * have implicit priority on index, previous entries will
+	 * deny access to SU on M-mode region. Also, M-mode will not
+	 * have access to SU region while previous entries will allow
+	 * access to M-mode regions.
+	 */
 	sbi_domain_memregion_init(0, ~0UL,
-				  (SBI_DOMAIN_MEMREGION_READABLE |
-				   SBI_DOMAIN_MEMREGION_WRITEABLE |
-				   SBI_DOMAIN_MEMREGION_EXECUTABLE),
+				  (SBI_DOMAIN_MEMREGION_SU_READABLE |
+				   SBI_DOMAIN_MEMREGION_SU_WRITABLE |
+				   SBI_DOMAIN_MEMREGION_SU_EXECUTABLE),
 				  &root_memregs[root_memregs_count++]);
 
 	/* Root domain memory region end */
@@ -791,11 +837,8 @@ int sbi_domain_init(struct sbi_scratch *scratch, u32 cold_hartid)
 	root.next_mode = scratch->next_mode;
 
 	/* Root domain possible and assigned HARTs */
-	for (i = 0; i < SBI_HARTMASK_MAX_BITS; i++) {
-		if (sbi_platform_hart_invalid(plat, i))
-			continue;
-		sbi_hartmask_set_hart(i, root_hmask);
-	}
+	for (i = 0; i < plat->hart_count; i++)
+		sbi_hartmask_set_hartindex(i, root_hmask);
 
 	/* Finally register the root domain */
 	rc = sbi_domain_register(&root, root_hmask);

lib/sbi/sbi_domain_context.c

@@ -0,0 +1,162 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) IPADS@SJTU 2023. All rights reserved.
+ */
+
+#include <sbi/sbi_error.h>
+#include <sbi/riscv_locks.h>
+#include <sbi/riscv_asm.h>
+#include <sbi/sbi_console.h>
+#include <sbi/sbi_hsm.h>
+#include <sbi/sbi_hart.h>
+#include <sbi/sbi_heap.h>
+#include <sbi/sbi_scratch.h>
+#include <sbi/sbi_string.h>
+#include <sbi/sbi_domain_context.h>
+
+/**
+ * Switches the HART context from the current domain to the target domain.
+ * This includes changing domain assignments and reconfiguring PMP, as well
+ * as saving and restoring CSRs and trap states.
+ *
+ * @param ctx pointer to the current HART context
+ * @param dom_ctx pointer to the target domain context
+ */
+static void switch_to_next_domain_context(struct sbi_context *ctx,
+					  struct sbi_context *dom_ctx)
+{
+	u32 hartindex = sbi_hartid_to_hartindex(current_hartid());
+	struct sbi_trap_context *trap_ctx;
+	struct sbi_domain *current_dom = ctx->dom;
+	struct sbi_domain *target_dom = dom_ctx->dom;
+	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
+	unsigned int pmp_count = sbi_hart_pmp_count(scratch);
+
+	/* Assign current hart to target domain */
+	spin_lock(&current_dom->assigned_harts_lock);
+	sbi_hartmask_clear_hartindex(hartindex, &current_dom->assigned_harts);
+	spin_unlock(&current_dom->assigned_harts_lock);
+
+	sbi_update_hartindex_to_domain(hartindex, target_dom);
+
+	spin_lock(&target_dom->assigned_harts_lock);
+	sbi_hartmask_set_hartindex(hartindex, &target_dom->assigned_harts);
+	spin_unlock(&target_dom->assigned_harts_lock);
+
+	/* Reconfigure PMP settings for the new domain */
+	for (int i = 0; i < pmp_count; i++) {
+		pmp_disable(i);
+	}
+	sbi_hart_pmp_configure(scratch);
+
+	/* Save current CSR context and restore target domain's CSR context */
+	ctx->sstatus	= csr_swap(CSR_SSTATUS, dom_ctx->sstatus);
+	ctx->sie	= csr_swap(CSR_SIE, dom_ctx->sie);
+	ctx->stvec	= csr_swap(CSR_STVEC, dom_ctx->stvec);
+	ctx->sscratch	= csr_swap(CSR_SSCRATCH, dom_ctx->sscratch);
+	ctx->sepc	= csr_swap(CSR_SEPC, dom_ctx->sepc);
+	ctx->scause	= csr_swap(CSR_SCAUSE, dom_ctx->scause);
+	ctx->stval	= csr_swap(CSR_STVAL, dom_ctx->stval);
+	ctx->sip	= csr_swap(CSR_SIP, dom_ctx->sip);
+	ctx->satp	= csr_swap(CSR_SATP, dom_ctx->satp);
+	if (sbi_hart_priv_version(scratch) >= SBI_HART_PRIV_VER_1_10)
+		ctx->scounteren = csr_swap(CSR_SCOUNTEREN, dom_ctx->scounteren);
+	if (sbi_hart_priv_version(scratch) >= SBI_HART_PRIV_VER_1_12)
+		ctx->senvcfg	= csr_swap(CSR_SENVCFG, dom_ctx->senvcfg);
+
+	/* Save current trap state and restore target domain's trap state */
+	trap_ctx = sbi_trap_get_context(scratch);
+	sbi_memcpy(&ctx->trap_ctx, trap_ctx, sizeof(*trap_ctx));
+	sbi_memcpy(trap_ctx, &dom_ctx->trap_ctx, sizeof(*trap_ctx));
+
+	/* Mark current context structure initialized because context saved */
+	ctx->initialized = true;
+
+	/* If target domain context is not initialized or runnable */
+	if (!dom_ctx->initialized) {
+		/* Startup boot HART of target domain */
+		if (current_hartid() == target_dom->boot_hartid)
+			sbi_hart_switch_mode(target_dom->boot_hartid,
+					     target_dom->next_arg1,
+					     target_dom->next_addr,
+					     target_dom->next_mode,
+					     false);
+		else
+			sbi_hsm_hart_stop(scratch, true);
+	}
+}
+
+int sbi_domain_context_enter(struct sbi_domain *dom)
+{
+	struct sbi_context *ctx = sbi_domain_context_thishart_ptr();
+	struct sbi_context *dom_ctx = sbi_hartindex_to_domain_context(
+		sbi_hartid_to_hartindex(current_hartid()), dom);
+
+	/* Validate the domain context existence */
+	if (!dom_ctx)
+		return SBI_EINVAL;
+
+	/* Update target context's previous context to indicate the caller */
+	dom_ctx->prev_ctx = ctx;
+
+	switch_to_next_domain_context(ctx, dom_ctx);
+
+	return 0;
+}
+
+int sbi_domain_context_exit(void)
+{
+	u32 i, hartindex = sbi_hartid_to_hartindex(current_hartid());
+	struct sbi_domain *dom;
+	struct sbi_context *ctx = sbi_domain_context_thishart_ptr();
+	struct sbi_context *dom_ctx, *tmp;
+
+	/*
+	 * If it's first time to call `exit` on the current hart, no
+	 * context allocated before. Loop through each domain to allocate
+	 * its context on the current hart if valid.
+	 */
+	if (!ctx) {
+		sbi_domain_for_each(i, dom) {
+			if (!sbi_hartmask_test_hartindex(hartindex,
+							 dom->possible_harts))
+				continue;
+
+			dom_ctx = sbi_zalloc(sizeof(struct sbi_context));
+			if (!dom_ctx)
+				return SBI_ENOMEM;
+
+			/* Bind context and domain */
+			dom_ctx->dom				   = dom;
+			dom->hartindex_to_context_table[hartindex] = dom_ctx;
+		}
+
+		ctx = sbi_domain_context_thishart_ptr();
+	}
+
+	dom_ctx = ctx->prev_ctx;
+
+	/* If no previous caller context */
+	if (!dom_ctx) {
+		/* Try to find next uninitialized user-defined domain's context */
+		sbi_domain_for_each(i, dom) {
+			if (dom == &root || dom == sbi_domain_thishart_ptr())
+				continue;
+
+			tmp = sbi_hartindex_to_domain_context(hartindex, dom);
+			if (tmp && !tmp->initialized) {
+				dom_ctx = tmp;
+				break;
+			}
+		}
+	}
+
+	/* Take the root domain context if fail to find */
+	if (!dom_ctx)
+		dom_ctx = sbi_hartindex_to_domain_context(hartindex, &root);
+
+	switch_to_next_domain_context(ctx, dom_ctx);
+
+	return 0;
+}

lib/sbi/sbi_ecall.c

@@ -95,20 +95,19 @@ void sbi_ecall_unregister_extension(struct sbi_ecall_extension *ext)
 		sbi_list_del_init(&ext->head);
 }
 
-int sbi_ecall_handler(struct sbi_trap_regs *regs)
+int sbi_ecall_handler(struct sbi_trap_context *tcntx)
 {
 	int ret = 0;
+	struct sbi_trap_regs *regs = &tcntx->regs;
 	struct sbi_ecall_extension *ext;
 	unsigned long extension_id = regs->a7;
 	unsigned long func_id = regs->a6;
-	struct sbi_trap_info trap = {0};
-	unsigned long out_val = 0;
+	struct sbi_ecall_return out = {0};
 	bool is_0_1_spec = 0;
 
 	ext = sbi_ecall_find_extension(extension_id);
 	if (ext && ext->handle) {
-		ret = ext->handle(extension_id, func_id,
-				  regs, &out_val, &trap);
+		ret = ext->handle(extension_id, func_id, regs, &out);
 		if (extension_id >= SBI_EXT_0_1_SET_TIMER &&
 		    extension_id <= SBI_EXT_0_1_SHUTDOWN)
 			is_0_1_spec = 1;
@@ -116,10 +115,7 @@ int sbi_ecall_handler(struct sbi_trap_regs *regs)
 		ret = SBI_ENOTSUPP;
 	}
 
-	if (ret == SBI_ETRAP) {
-		trap.epc = regs->mepc;
-		sbi_trap_redirect(regs, &trap);
-	} else {
+	if (!out.skip_regs_update) {
 		if (ret < SBI_LAST_ERR ||
 		    (extension_id != SBI_EXT_0_1_CONSOLE_GETCHAR &&
 		     SBI_SUCCESS < ret)) {
@@ -140,7 +136,7 @@ int sbi_ecall_handler(struct sbi_trap_regs *regs)
 		regs->mepc += 4;
 		regs->a0 = ret;
 		if (!is_0_1_spec)
-			regs->a1 = out_val;
+			regs->a1 = out.value;
 	}
 
 	return 0;

lib/sbi/sbi_ecall_base.c

@@ -33,37 +33,36 @@ static int sbi_ecall_base_probe(unsigned long extid, unsigned long *out_val)
 }
 
 static int sbi_ecall_base_handler(unsigned long extid, unsigned long funcid,
-				  const struct sbi_trap_regs *regs,
-				  unsigned long *out_val,
-				  struct sbi_trap_info *out_trap)
+				  struct sbi_trap_regs *regs,
+				  struct sbi_ecall_return *out)
 {
 	int ret = 0;
 
 	switch (funcid) {
 	case SBI_EXT_BASE_GET_SPEC_VERSION:
-		*out_val = (SBI_ECALL_VERSION_MAJOR <<
-			   SBI_SPEC_VERSION_MAJOR_OFFSET) &
-			   (SBI_SPEC_VERSION_MAJOR_MASK <<
-			    SBI_SPEC_VERSION_MAJOR_OFFSET);
-		*out_val = *out_val | SBI_ECALL_VERSION_MINOR;
+		out->value = (SBI_ECALL_VERSION_MAJOR <<
+			      SBI_SPEC_VERSION_MAJOR_OFFSET) &
+			     (SBI_SPEC_VERSION_MAJOR_MASK <<
+			      SBI_SPEC_VERSION_MAJOR_OFFSET);
+		out->value = out->value | SBI_ECALL_VERSION_MINOR;
 		break;
 	case SBI_EXT_BASE_GET_IMP_ID:
-		*out_val = sbi_ecall_get_impid();
+		out->value = sbi_ecall_get_impid();
 		break;
 	case SBI_EXT_BASE_GET_IMP_VERSION:
-		*out_val = OPENSBI_VERSION;
+		out->value = OPENSBI_VERSION;
 		break;
 	case SBI_EXT_BASE_GET_MVENDORID:
-		*out_val = csr_read(CSR_MVENDORID);
+		out->value = csr_read(CSR_MVENDORID);
 		break;
 	case SBI_EXT_BASE_GET_MARCHID:
-		*out_val = csr_read(CSR_MARCHID);
+		out->value = csr_read(CSR_MARCHID);
 		break;
 	case SBI_EXT_BASE_GET_MIMPID:
-		*out_val = csr_read(CSR_MIMPID);
+		out->value = csr_read(CSR_MIMPID);
 		break;
 	case SBI_EXT_BASE_PROBE_EXT:
-		ret = sbi_ecall_base_probe(regs->a0, out_val);
+		ret = sbi_ecall_base_probe(regs->a0, &out->value);
 		break;
 	default:
 		ret = SBI_ENOTSUPP;

lib/sbi/sbi_ecall_cppc.c

@@ -12,9 +12,8 @@
 #include <sbi/sbi_cppc.h>
 
 static int sbi_ecall_cppc_handler(unsigned long extid, unsigned long funcid,
-				  const struct sbi_trap_regs *regs,
-				  unsigned long *out_val,
-				  struct sbi_trap_info *out_trap)
+				  struct sbi_trap_regs *regs,
+				  struct sbi_ecall_return *out)
 {
 	int ret = 0;
 	uint64_t temp;
@@ -22,14 +21,14 @@ static int sbi_ecall_cppc_handler(unsigned long extid, unsigned long funcid,
 	switch (funcid) {
 	case SBI_EXT_CPPC_READ:
 		ret = sbi_cppc_read(regs->a0, &temp);
-		*out_val = temp;
+		out->value = temp;
 		break;
 	case SBI_EXT_CPPC_READ_HI:
 #if __riscv_xlen == 32
 		ret = sbi_cppc_read(regs->a0, &temp);
-		*out_val = temp >> 32;
+		out->value = temp >> 32;
 #else
-		*out_val = 0;
+		out->value = 0;
 #endif
 		break;
 	case SBI_EXT_CPPC_WRITE:
@@ -38,7 +37,7 @@ static int sbi_ecall_cppc_handler(unsigned long extid, unsigned long funcid,
 	case SBI_EXT_CPPC_PROBE:
 		ret = sbi_cppc_probe(regs->a0);
 		if (ret >= 0) {
-			*out_val = ret;
+			out->value = ret;
 			ret = 0;
 		}
 		break;

lib/sbi/sbi_ecall_dbcn.c

@@ -14,11 +14,11 @@
 #include <sbi/sbi_ecall_interface.h>
 #include <sbi/sbi_trap.h>
 #include <sbi/riscv_asm.h>
+#include <sbi/sbi_hart.h>
 
 static int sbi_ecall_dbcn_handler(unsigned long extid, unsigned long funcid,
-				  const struct sbi_trap_regs *regs,
-				  unsigned long *out_val,
-				  struct sbi_trap_info *out_trap)
+				  struct sbi_trap_regs *regs,
+				  struct sbi_ecall_return *out)
 {
 	ulong smode = (csr_read(CSR_MSTATUS) & MSTATUS_MPP) >>
 			MSTATUS_MPP_SHIFT;
@@ -34,10 +34,10 @@ static int sbi_ecall_dbcn_handler(unsigned long extid, unsigned long funcid,
 		 * Based on above, we simply fail if the upper 32bits of
 		 * the physical address (i.e. a2 register) is non-zero on
 		 * RV32.
-                 *
-                 * Analogously, we fail if the upper 64bit of the
-                 * physical address (i.e. a2 register) is non-zero on
-                 * RV64.
+		 *
+		 * Analogously, we fail if the upper 64bit of the
+		 * physical address (i.e. a2 register) is non-zero on
+		 * RV64.
 		 */
 		if (regs->a2)
 			return SBI_ERR_FAILED;
@@ -46,10 +46,12 @@ static int sbi_ecall_dbcn_handler(unsigned long extid, unsigned long funcid,
 					regs->a1, regs->a0, smode,
 					SBI_DOMAIN_READ|SBI_DOMAIN_WRITE))
 			return SBI_ERR_INVALID_PARAM;
+		sbi_hart_map_saddr(regs->a1, regs->a0);
 		if (funcid == SBI_EXT_DBCN_CONSOLE_WRITE)
-			*out_val = sbi_nputs((const char *)regs->a1, regs->a0);
+			out->value = sbi_nputs((const char *)regs->a1, regs->a0);
 		else
-			*out_val = sbi_ngets((char *)regs->a1, regs->a0);
+			out->value = sbi_ngets((char *)regs->a1, regs->a0);
+		sbi_hart_unmap_saddr();
 		return 0;
 	case SBI_EXT_DBCN_CONSOLE_WRITE_BYTE:
 		sbi_putc(regs->a0);

lib/sbi/sbi_ecall_dbtr.c

@@ -0,0 +1,73 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023 Ventana Micro Systems Inc.
+ *
+ * Author(s):
+ *   Himanshu Chauhan <hchauhan@ventanamicro.com>
+ */
+
+#include <sbi/sbi_ecall.h>
+#include <sbi/sbi_ecall_interface.h>
+#include <sbi/sbi_error.h>
+#include <sbi/sbi_trap.h>
+#include <sbi/sbi_domain.h>
+#include <sbi/sbi_dbtr.h>
+
+static int sbi_ecall_dbtr_handler(unsigned long extid, unsigned long funcid,
+				  struct sbi_trap_regs *regs,
+				  struct sbi_ecall_return *out)
+{
+	unsigned long smode = (csr_read(CSR_MSTATUS) & MSTATUS_MPP) >>
+			MSTATUS_MPP_SHIFT;
+	int ret = 0;
+
+	switch (funcid) {
+	case SBI_EXT_DBTR_NUM_TRIGGERS:
+		ret = sbi_dbtr_num_trig(regs->a0, &out->value);
+		break;
+	case SBI_EXT_DBTR_SETUP_SHMEM:
+		ret = sbi_dbtr_setup_shmem(sbi_domain_thishart_ptr(), smode,
+					   regs->a0, regs->a1);
+		break;
+	case SBI_EXT_DBTR_TRIGGER_READ:
+		ret = sbi_dbtr_read_trig(smode, regs->a0, regs->a1);
+		break;
+	case SBI_EXT_DBTR_TRIGGER_INSTALL:
+		ret = sbi_dbtr_install_trig(smode, regs->a0, &out->value);
+		break;
+	case SBI_EXT_DBTR_TRIGGER_UNINSTALL:
+		ret = sbi_dbtr_uninstall_trig(regs->a0, regs->a1);
+		break;
+	case SBI_EXT_DBTR_TRIGGER_ENABLE:
+		ret = sbi_dbtr_enable_trig(regs->a0, regs->a1);
+		break;
+	case SBI_EXT_DBTR_TRIGGER_UPDATE:
+		ret = sbi_dbtr_update_trig(smode, regs->a0, regs->a1);
+		break;
+	case SBI_EXT_DBTR_TRIGGER_DISABLE:
+		ret = sbi_dbtr_disable_trig(regs->a0, regs->a1);
+		break;
+	default:
+		ret = SBI_ENOTSUPP;
+	};
+
+	return ret;
+}
+
+struct sbi_ecall_extension ecall_dbtr;
+
+static int sbi_ecall_dbtr_register_extensions(void)
+{
+	if (sbi_dbtr_get_total_triggers() == 0)
+		return 0;
+
+	return sbi_ecall_register_extension(&ecall_dbtr);
+}
+
+struct sbi_ecall_extension ecall_dbtr = {
+	.extid_start = SBI_EXT_DBTR,
+	.extid_end = SBI_EXT_DBTR,
+	.handle = sbi_ecall_dbtr_handler,
+	.register_extensions = sbi_ecall_dbtr_register_extensions,
+};

lib/sbi/sbi_ecall_fwft.c

@@ -0,0 +1,49 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Rivos Inc.
+ *
+ * Authors:
+ *   Clément Léger <cleger@rivosinc.com>
+ */
+
+#include <sbi/sbi_ecall.h>
+#include <sbi/sbi_ecall_interface.h>
+#include <sbi/sbi_error.h>
+#include <sbi/sbi_fwft.h>
+#include <sbi/sbi_trap.h>
+
+static int sbi_ecall_fwft_handler(unsigned long extid, unsigned long funcid,
+				 struct sbi_trap_regs *regs,
+				 struct sbi_ecall_return *out)
+{
+	int ret = 0;
+
+	switch (funcid) {
+	case SBI_EXT_FWFT_SET:
+		ret = sbi_fwft_set(regs->a0, regs->a1, regs->a2);
+		break;
+	case SBI_EXT_FWFT_GET:
+		ret = sbi_fwft_get(regs->a0, &out->value);
+		break;
+	default:
+		ret = SBI_ENOTSUPP;
+		break;
+	}
+
+	return ret;
+}
+
+struct sbi_ecall_extension ecall_fwft;
+
+static int sbi_ecall_fwft_register_extensions(void)
+{
+	return sbi_ecall_register_extension(&ecall_fwft);
+}
+
+struct sbi_ecall_extension ecall_fwft = {
+	.extid_start = SBI_EXT_FWFT,
+	.extid_end = SBI_EXT_FWFT,
+	.register_extensions = sbi_ecall_fwft_register_extensions,
+	.handle = sbi_ecall_fwft_handler,
+};

lib/sbi/sbi_ecall_hsm.c

@@ -17,9 +17,8 @@
 #include <sbi/riscv_asm.h>
 
 static int sbi_ecall_hsm_handler(unsigned long extid, unsigned long funcid,
-				 const struct sbi_trap_regs *regs,
-				 unsigned long *out_val,
-				 struct sbi_trap_info *out_trap)
+				 struct sbi_trap_regs *regs,
+				 struct sbi_ecall_return *out)
 {
 	int ret = 0;
 	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
@@ -47,7 +46,7 @@ static int sbi_ecall_hsm_handler(unsigned long extid, unsigned long funcid,
 	}
 
 	if (ret >= 0) {
-		*out_val = ret;
+		out->value = ret;
 		ret = 0;
 	}
 

lib/sbi/sbi_ecall_ipi.c

@@ -15,9 +15,8 @@
 #include <sbi/sbi_ipi.h>
 
 static int sbi_ecall_ipi_handler(unsigned long extid, unsigned long funcid,
-				 const struct sbi_trap_regs *regs,
-				 unsigned long *out_val,
-				 struct sbi_trap_info *out_trap)
+				 struct sbi_trap_regs *regs,
+				 struct sbi_ecall_return *out)
 {
 	int ret = 0;
 

lib/sbi/sbi_ecall_legacy.c

@@ -24,32 +24,32 @@
 #include <sbi/sbi_unpriv.h>
 #include <sbi/sbi_hart.h>
 
-static int sbi_load_hart_mask_unpriv(ulong *pmask, ulong *hmask,
-				     struct sbi_trap_info *uptrap)
+static bool sbi_load_hart_mask_unpriv(ulong *pmask, ulong *hmask,
+				      struct sbi_trap_info *uptrap)
 {
 	ulong mask = 0;
 
 	if (pmask) {
 		mask = sbi_load_ulong(pmask, uptrap);
 		if (uptrap->cause)
-			return SBI_ETRAP;
+			return false;
 	} else {
 		sbi_hsm_hart_interruptible_mask(sbi_domain_thishart_ptr(),
 						0, &mask);
 	}
 	*hmask = mask;
 
-	return 0;
+	return true;
 }
 
 static int sbi_ecall_legacy_handler(unsigned long extid, unsigned long funcid,
-				    const struct sbi_trap_regs *regs,
-				    unsigned long *out_val,
-				    struct sbi_trap_info *out_trap)
+				    struct sbi_trap_regs *regs,
+				    struct sbi_ecall_return *out)
 {
 	int ret = 0;
 	struct sbi_tlb_info tlb_info;
 	u32 source_hart = current_hartid();
+	struct sbi_trap_info trap = {0};
 	ulong hmask = 0;
 
 	switch (extid) {
@@ -70,40 +70,47 @@ static int sbi_ecall_legacy_handler(unsigned long extid, unsigned long funcid,
 		sbi_ipi_clear_smode();
 		break;
 	case SBI_EXT_0_1_SEND_IPI:
-		ret = sbi_load_hart_mask_unpriv((ulong *)regs->a0,
-						&hmask, out_trap);
-		if (ret != SBI_ETRAP)
+		if (sbi_load_hart_mask_unpriv((ulong *)regs->a0,
+						&hmask, &trap)) {
 			ret = sbi_ipi_send_smode(hmask, 0);
+		} else {
+			sbi_trap_redirect(regs, &trap);
+			out->skip_regs_update = true;
+		}
 		break;
 	case SBI_EXT_0_1_REMOTE_FENCE_I:
-		ret = sbi_load_hart_mask_unpriv((ulong *)regs->a0,
-						&hmask, out_trap);
-		if (ret != SBI_ETRAP) {
+		if (sbi_load_hart_mask_unpriv((ulong *)regs->a0,
+						&hmask, &trap)) {
 			SBI_TLB_INFO_INIT(&tlb_info, 0, 0, 0, 0,
-					  sbi_tlb_local_fence_i,
-					  source_hart);
+					  SBI_TLB_FENCE_I, source_hart);
 			ret = sbi_tlb_request(hmask, 0, &tlb_info);
+		} else {
+			sbi_trap_redirect(regs, &trap);
+			out->skip_regs_update = true;
 		}
 		break;
 	case SBI_EXT_0_1_REMOTE_SFENCE_VMA:
-		ret = sbi_load_hart_mask_unpriv((ulong *)regs->a0,
-						&hmask, out_trap);
-		if (ret != SBI_ETRAP) {
+		if (sbi_load_hart_mask_unpriv((ulong *)regs->a0,
+						&hmask, &trap)) {
 			SBI_TLB_INFO_INIT(&tlb_info, regs->a1, regs->a2, 0, 0,
-					  sbi_tlb_local_sfence_vma,
-					  source_hart);
+					  SBI_TLB_SFENCE_VMA, source_hart);
 			ret = sbi_tlb_request(hmask, 0, &tlb_info);
+		} else {
+			sbi_trap_redirect(regs, &trap);
+			out->skip_regs_update = true;
 		}
 		break;
 	case SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID:
-		ret = sbi_load_hart_mask_unpriv((ulong *)regs->a0,
-						&hmask, out_trap);
-		if (ret != SBI_ETRAP) {
+		if (sbi_load_hart_mask_unpriv((ulong *)regs->a0,
+						&hmask, &trap)) {
 			SBI_TLB_INFO_INIT(&tlb_info, regs->a1,
 					  regs->a2, regs->a3, 0,
-					  sbi_tlb_local_sfence_vma_asid,
+					  SBI_TLB_SFENCE_VMA_ASID,
 					  source_hart);
 			ret = sbi_tlb_request(hmask, 0, &tlb_info);
+		} else {
+			sbi_trap_redirect(regs, &trap);
+			out->skip_regs_update = true;
 		}
 		break;
 	case SBI_EXT_0_1_SHUTDOWN:

lib/sbi/sbi_ecall_pmu.c

@@ -18,9 +18,8 @@
 #include <sbi/riscv_asm.h>
 
 static int sbi_ecall_pmu_handler(unsigned long extid, unsigned long funcid,
-				 const struct sbi_trap_regs *regs,
-				 unsigned long *out_val,
-				 struct sbi_trap_info *out_trap)
+				 struct sbi_trap_regs *regs,
+				 struct sbi_ecall_return *out)
 {
 	int ret = 0;
 	uint64_t temp;
@@ -29,12 +28,12 @@ static int sbi_ecall_pmu_handler(unsigned long extid, unsigned long funcid,
 	case SBI_EXT_PMU_NUM_COUNTERS:
 		ret = sbi_pmu_num_ctr();
 		if (ret >= 0) {
-			*out_val = ret;
+			out->value = ret;
 			ret = 0;
 		}
 		break;
 	case SBI_EXT_PMU_COUNTER_GET_INFO:
-		ret = sbi_pmu_ctr_get_info(regs->a0, out_val);
+		ret = sbi_pmu_ctr_get_info(regs->a0, &out->value);
 		break;
 	case SBI_EXT_PMU_COUNTER_CFG_MATCH:
 #if __riscv_xlen == 32
@@ -45,21 +44,21 @@ static int sbi_ecall_pmu_handler(unsigned long extid, unsigned long funcid,
 		ret = sbi_pmu_ctr_cfg_match(regs->a0, regs->a1, regs->a2,
 					    regs->a3, temp);
 		if (ret >= 0) {
-			*out_val = ret;
+			out->value = ret;
 			ret = 0;
 		}
 
 		break;
 	case SBI_EXT_PMU_COUNTER_FW_READ:
 		ret = sbi_pmu_ctr_fw_read(regs->a0, &temp);
-		*out_val = temp;
+		out->value = temp;
 		break;
 	case SBI_EXT_PMU_COUNTER_FW_READ_HI:
 #if __riscv_xlen == 32
 		ret = sbi_pmu_ctr_fw_read(regs->a0, &temp);
-		*out_val = temp >> 32;
+		out->value = temp >> 32;
 #else
-		*out_val = 0;
+		out->value = 0;
 #endif
 		break;
 	case SBI_EXT_PMU_COUNTER_START:
@@ -74,6 +73,8 @@ static int sbi_ecall_pmu_handler(unsigned long extid, unsigned long funcid,
 	case SBI_EXT_PMU_COUNTER_STOP:
 		ret = sbi_pmu_ctr_stop(regs->a0, regs->a1, regs->a2);
 		break;
+	case SBI_EXT_PMU_SNAPSHOT_SET_SHMEM:
+		/* fallthrough as OpenSBI doesn't support snapshot yet */
 	default:
 		ret = SBI_ENOTSUPP;
 	}

lib/sbi/sbi_ecall_rfence.c

@@ -16,9 +16,8 @@
 #include <sbi/sbi_tlb.h>
 
 static int sbi_ecall_rfence_handler(unsigned long extid, unsigned long funcid,
-				    const struct sbi_trap_regs *regs,
-				    unsigned long *out_val,
-				    struct sbi_trap_info *out_trap)
+				    struct sbi_trap_regs *regs,
+				    struct sbi_ecall_return *out)
 {
 	int ret = 0;
 	unsigned long vmid;
@@ -33,43 +32,41 @@ static int sbi_ecall_rfence_handler(unsigned long extid, unsigned long funcid,
 	switch (funcid) {
 	case SBI_EXT_RFENCE_REMOTE_FENCE_I:
 		SBI_TLB_INFO_INIT(&tlb_info, 0, 0, 0, 0,
-				  sbi_tlb_local_fence_i, source_hart);
+				  SBI_TLB_FENCE_I, source_hart);
 		ret = sbi_tlb_request(regs->a0, regs->a1, &tlb_info);
 		break;
 	case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA:
 		SBI_TLB_INFO_INIT(&tlb_info, regs->a2, regs->a3, 0, 0,
-				  sbi_tlb_local_hfence_gvma, source_hart);
+				  SBI_TLB_HFENCE_GVMA, source_hart);
 		ret = sbi_tlb_request(regs->a0, regs->a1, &tlb_info);
 		break;
 	case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID:
 		SBI_TLB_INFO_INIT(&tlb_info, regs->a2, regs->a3, 0, regs->a4,
-				  sbi_tlb_local_hfence_gvma_vmid,
-				  source_hart);
+				  SBI_TLB_HFENCE_GVMA_VMID, source_hart);
 		ret = sbi_tlb_request(regs->a0, regs->a1, &tlb_info);
 		break;
 	case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA:
 		vmid = (csr_read(CSR_HGATP) & HGATP_VMID_MASK);
 		vmid = vmid >> HGATP_VMID_SHIFT;
 		SBI_TLB_INFO_INIT(&tlb_info, regs->a2, regs->a3, 0, vmid,
-				  sbi_tlb_local_hfence_vvma, source_hart);
+				  SBI_TLB_HFENCE_VVMA, source_hart);
 		ret = sbi_tlb_request(regs->a0, regs->a1, &tlb_info);
 		break;
 	case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID:
 		vmid = (csr_read(CSR_HGATP) & HGATP_VMID_MASK);
 		vmid = vmid >> HGATP_VMID_SHIFT;
 		SBI_TLB_INFO_INIT(&tlb_info, regs->a2, regs->a3, regs->a4,
-				  vmid, sbi_tlb_local_hfence_vvma_asid,
-				  source_hart);
+				  vmid, SBI_TLB_HFENCE_VVMA_ASID, source_hart);
 		ret = sbi_tlb_request(regs->a0, regs->a1, &tlb_info);
 		break;
 	case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA:
 		SBI_TLB_INFO_INIT(&tlb_info, regs->a2, regs->a3, 0, 0,
-				  sbi_tlb_local_sfence_vma, source_hart);
+				  SBI_TLB_SFENCE_VMA, source_hart);
 		ret = sbi_tlb_request(regs->a0, regs->a1, &tlb_info);
 		break;
 	case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID:
 		SBI_TLB_INFO_INIT(&tlb_info, regs->a2, regs->a3, regs->a4, 0,
-				  sbi_tlb_local_sfence_vma_asid, source_hart);
+				  SBI_TLB_SFENCE_VMA_ASID, source_hart);
 		ret = sbi_tlb_request(regs->a0, regs->a1, &tlb_info);
 		break;
 	default:

lib/sbi/sbi_ecall_srst.c

@@ -15,9 +15,8 @@
 #include <sbi/sbi_system.h>
 
 static int sbi_ecall_srst_handler(unsigned long extid, unsigned long funcid,
-				  const struct sbi_trap_regs *regs,
-				  unsigned long *out_val,
-				  struct sbi_trap_info *out_trap)
+				  struct sbi_trap_regs *regs,
+				  struct sbi_ecall_return *out)
 {
 	if (funcid == SBI_EXT_SRST_RESET) {
 		if ((((u32)-1U) <= ((u64)regs->a0)) ||

lib/sbi/sbi_ecall_sse.c

@@ -0,0 +1,57 @@
+#include <sbi/sbi_error.h>
+#include <sbi/sbi_ecall.h>
+#include <sbi/sbi_trap.h>
+#include <sbi/sbi_sse.h>
+
+static int sbi_ecall_sse_handler(unsigned long extid, unsigned long funcid,
+				  struct sbi_trap_regs *regs,
+				  struct sbi_ecall_return *out)
+{
+	int ret;
+
+	switch (funcid) {
+	case SBI_EXT_SSE_READ_ATTR:
+		ret = sbi_sse_read_attrs(regs->a0, regs->a1, regs->a2,
+					 regs->a3, regs->a4);
+		break;
+	case SBI_EXT_SSE_WRITE_ATTR:
+		ret = sbi_sse_write_attrs(regs->a0, regs->a1, regs->a2,
+					 regs->a3, regs->a4);
+		break;
+	case SBI_EXT_SSE_REGISTER:
+		ret = sbi_sse_register(regs->a0, regs->a1, regs->a2);
+		break;
+	case SBI_EXT_SSE_UNREGISTER:
+		ret = sbi_sse_unregister(regs->a0);
+		break;
+	case SBI_EXT_SSE_ENABLE:
+		ret = sbi_sse_enable(regs->a0);
+		break;
+	case SBI_EXT_SSE_DISABLE:
+		ret = sbi_sse_disable(regs->a0);
+		break;
+	case SBI_EXT_SSE_COMPLETE:
+		ret = sbi_sse_complete(regs, out);
+		break;
+	case SBI_EXT_SSE_INJECT:
+		ret = sbi_sse_inject_from_ecall(regs->a0, regs->a1, out);
+		break;
+	default:
+		ret = SBI_ENOTSUPP;
+	}
+	return ret;
+}
+
+struct sbi_ecall_extension ecall_sse;
+
+static int sbi_ecall_sse_register_extensions(void)
+{
+	return sbi_ecall_register_extension(&ecall_sse);
+}
+
+struct sbi_ecall_extension ecall_sse = {
+	.extid_start		= SBI_EXT_SSE,
+	.extid_end		= SBI_EXT_SSE,
+	.register_extensions	= sbi_ecall_sse_register_extensions,
+	.handle			= sbi_ecall_sse_handler,
+};

lib/sbi/sbi_ecall_susp.c

@@ -6,9 +6,8 @@
 #include <sbi/sbi_system.h>
 
 static int sbi_ecall_susp_handler(unsigned long extid, unsigned long funcid,
-				  const struct sbi_trap_regs *regs,
-				  unsigned long *out_val,
-				  struct sbi_trap_info *out_trap)
+				  struct sbi_trap_regs *regs,
+				  struct sbi_ecall_return *out)
 {
 	int ret = SBI_ENOTSUPP;
 
@@ -16,7 +15,7 @@ static int sbi_ecall_susp_handler(unsigned long extid, unsigned long funcid,
 		ret = sbi_system_suspend(regs->a0, regs->a1, regs->a2);
 
 	if (ret >= 0) {
-		*out_val = ret;
+		out->value = ret;
 		ret = 0;
 	}
 

lib/sbi/sbi_ecall_time.c

@@ -15,9 +15,8 @@
 #include <sbi/sbi_timer.h>
 
 static int sbi_ecall_time_handler(unsigned long extid, unsigned long funcid,
-				  const struct sbi_trap_regs *regs,
-				  unsigned long *out_val,
-				  struct sbi_trap_info *out_trap)
+				  struct sbi_trap_regs *regs,
+				  struct sbi_ecall_return *out)
 {
 	int ret = 0;
 

lib/sbi/sbi_ecall_vendor.c

@@ -23,13 +23,11 @@ static inline unsigned long sbi_ecall_vendor_id(void)
 }
 
 static int sbi_ecall_vendor_handler(unsigned long extid, unsigned long funcid,
-				    const struct sbi_trap_regs *regs,
-				    unsigned long *out_val,
-				    struct sbi_trap_info *out_trap)
+				    struct sbi_trap_regs *regs,
+				    struct sbi_ecall_return *out)
 {
 	return sbi_platform_vendor_ext_provider(sbi_platform_thishart_ptr(),
-						funcid, regs,
-						out_val, out_trap);
+						funcid, regs, out);
 }
 
 struct sbi_ecall_extension ecall_vendor;

lib/sbi/sbi_emulate_csr.c

@@ -109,7 +109,7 @@ int sbi_emulate_csr_read(int csr_num, struct sbi_trap_regs *regs,
 
 #define switchcase_hpm(__uref, __mref, __csr)				\
 	case __csr:							\
-		if ((sbi_hart_mhpm_count(scratch) + 3) <= (__csr - __uref))\
+		if (sbi_hart_mhpm_mask(scratch) & (1 << (__csr - __uref)))\
 			return SBI_ENOTSUPP;				\
 		if (!hpm_allowed(__csr - __uref, prev_mode, virt))	\
 			return SBI_ENOTSUPP;				\

lib/sbi/sbi_expected_trap.S

@@ -23,8 +23,6 @@
 	.global __sbi_expected_trap
 __sbi_expected_trap:
 	/* Without H-extension so, MTVAL2 and MTINST CSRs and GVA not available */
-	csrr	a4, CSR_MEPC
-	REG_S	a4, SBI_TRAP_INFO_OFFSET(epc)(a3)
 	csrr	a4, CSR_MCAUSE
 	REG_S	a4, SBI_TRAP_INFO_OFFSET(cause)(a3)
 	csrr	a4, CSR_MTVAL
@@ -41,8 +39,6 @@ __sbi_expected_trap:
 	.global __sbi_expected_trap_hext
 __sbi_expected_trap_hext:
 	/* With H-extension so, MTVAL2 and MTINST CSRs and GVA available */
-	csrr	a4, CSR_MEPC
-	REG_S	a4, SBI_TRAP_INFO_OFFSET(epc)(a3)
 	csrr	a4, CSR_MCAUSE
 	REG_S	a4, SBI_TRAP_INFO_OFFSET(cause)(a3)
 	csrr	a4, CSR_MTVAL

lib/sbi/sbi_fwft.c

@@ -0,0 +1,266 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Rivos Inc.
+ *
+ * Authors:
+ *   Clément Léger <cleger@rivosinc.com>
+ */
+
+#include <sbi/sbi_console.h>
+#include <sbi/sbi_bitmap.h>
+#include <sbi/sbi_ecall_interface.h>
+#include <sbi/sbi_error.h>
+#include <sbi/sbi_hart.h>
+#include <sbi/sbi_heap.h>
+#include <sbi/sbi_scratch.h>
+#include <sbi/sbi_string.h>
+#include <sbi/sbi_types.h>
+
+#include <sbi/riscv_asm.h>
+#include <sbi/riscv_encoding.h>
+
+/** Offset of pointer to FWFT HART state in scratch space */
+static unsigned long fwft_ptr_offset;
+
+#define fwft_get_hart_state_ptr(__scratch)				\
+	sbi_scratch_read_type((__scratch), void *, fwft_ptr_offset)
+
+#define fwft_thishart_state_ptr()					\
+	fwft_get_hart_state_ptr(sbi_scratch_thishart_ptr())
+
+#define fwft_set_hart_state_ptr(__scratch, __phs)			\
+	sbi_scratch_write_type((__scratch), void *, fwft_ptr_offset, (__phs))
+
+#define MIS_DELEG (1UL << CAUSE_MISALIGNED_LOAD | 1UL << CAUSE_MISALIGNED_STORE)
+
+struct fwft_config;
+
+struct fwft_feature {
+	enum sbi_fwft_feature_t id;
+	int (*supported)(struct fwft_config *conf);
+	int (*set)(struct fwft_config *conf, unsigned long value);
+	int (*get)(struct fwft_config *conf, unsigned long *value);
+};
+
+struct fwft_config {
+	const struct fwft_feature *feature;
+	unsigned long flags;
+};
+
+struct fwft_hart_state {
+	unsigned int config_count;
+	struct fwft_config configs[];
+};
+
+static const unsigned long fwft_defined_features[] = {
+	SBI_FWFT_MISALIGNED_EXC_DELEG,
+	SBI_FWFT_LANDING_PAD,
+	SBI_FWFT_SHADOW_STACK,
+	SBI_FWFT_DOUBLE_TRAP,
+	SBI_FWFT_PTE_AD_HW_UPDATING,
+};
+
+static bool fwft_is_defined_feature(enum sbi_fwft_feature_t feature)
+{
+	int i;
+
+	for (i = 0; i < array_size(fwft_defined_features); i++) {
+		if (fwft_defined_features[i] == feature)
+			return true;
+	}
+
+	return false;
+}
+
+static int fwft_misaligned_delegation_supported(struct fwft_config *conf)
+{
+	if (!misa_extension('S'))
+		return SBI_ENOTSUPP;
+
+	return SBI_OK;
+}
+
+static int fwft_set_misaligned_delegation(struct fwft_config *conf,
+					 unsigned long value)
+{
+	if (value == 1)
+		csr_set(CSR_MEDELEG, MIS_DELEG);
+	else if (value == 0)
+		csr_clear(CSR_MEDELEG, MIS_DELEG);
+	else
+		return SBI_EINVAL;
+
+	return SBI_OK;
+}
+
+static int fwft_get_misaligned_delegation(struct fwft_config *conf,
+					 unsigned long *value)
+{
+	*value = (csr_read(CSR_MEDELEG) & MIS_DELEG) != 0;
+
+	return SBI_OK;
+}
+
+static int fwft_adue_supported(struct fwft_config *conf)
+{
+	if (!sbi_hart_has_extension(sbi_scratch_thishart_ptr(),
+				    SBI_HART_EXT_SVADU))
+		return SBI_ENOTSUPP;
+
+	return SBI_OK;
+}
+
+static int fwft_set_adue(struct fwft_config *conf, unsigned long value)
+{
+	if (value == 1)
+#if __riscv_xlen == 32
+		csr_set(CSR_MENVCFGH, ENVCFG_ADUE >> 32);
+#else
+		csr_set(CSR_MENVCFG, ENVCFG_ADUE);
+#endif
+	else if (value == 0)
+#if __riscv_xlen == 32
+		csr_clear(CSR_MENVCFGH, ENVCFG_ADUE >> 32);
+#else
+		csr_clear(CSR_MENVCFG, ENVCFG_ADUE);
+#endif
+	else
+		return SBI_EINVAL;
+
+	return SBI_OK;
+}
+
+static int fwft_get_adue(struct fwft_config *conf, unsigned long *value)
+{
+	unsigned long cfg;
+
+#if __riscv_xlen == 32
+	cfg = csr_read(CSR_MENVCFGH) & (ENVCFG_ADUE >> 32);
+#else
+	cfg = csr_read(CSR_MENVCFG) & ENVCFG_ADUE;
+#endif
+	*value = cfg != 0;
+
+	return SBI_OK;
+}
+
+static struct fwft_config* get_feature_config(enum sbi_fwft_feature_t feature)
+{
+	int i;
+	struct fwft_hart_state *fhs = fwft_thishart_state_ptr();
+
+	if (feature & SBI_FWFT_GLOBAL_FEATURE_BIT)
+		return NULL;
+
+	for (i = 0; i < fhs->config_count; i++){
+		if (feature == fhs->configs[i].feature->id)
+			return &fhs->configs[i];
+	}
+
+	return NULL;
+}
+
+static int fwft_get_feature(enum sbi_fwft_feature_t feature,
+			    struct fwft_config **conf)
+{
+	int ret;
+	struct fwft_config *tconf;
+
+	tconf = get_feature_config(feature);
+	if (!tconf) {
+		if (fwft_is_defined_feature(feature))
+			return SBI_ENOTSUPP;
+
+		return SBI_EDENIED;
+	}
+
+	if (tconf->feature->supported) {
+		ret = tconf->feature->supported(tconf);
+		if (ret)
+			return ret;
+	}
+	*conf = tconf;
+
+	return SBI_SUCCESS;
+}
+
+int sbi_fwft_set(enum sbi_fwft_feature_t feature, unsigned long value,
+		 unsigned long flags)
+{
+	int ret;
+	struct fwft_config *conf;
+
+	ret = fwft_get_feature(feature, &conf);
+	if (ret)
+		return ret;
+
+	if ((flags & ~SBI_FWFT_SET_FLAG_LOCK) != 0)
+		return SBI_EINVAL;
+
+	if (conf->flags & SBI_FWFT_SET_FLAG_LOCK)
+		return SBI_EDENIED;
+
+	ret = conf->feature->set(conf, value);
+	if (ret)
+		return ret;
+
+	conf->flags = flags;
+
+	return SBI_OK;
+}
+
+int sbi_fwft_get(enum sbi_fwft_feature_t feature, unsigned long *out_val)
+{
+	int ret;
+	struct fwft_config *conf;
+
+	ret = fwft_get_feature(feature, &conf);
+	if (ret)
+		return ret;
+
+	return conf->feature->get(conf, out_val);
+}
+
+static const struct fwft_feature features[] =
+{
+	{
+		.id = SBI_FWFT_MISALIGNED_EXC_DELEG,
+		.supported = fwft_misaligned_delegation_supported,
+		.set = fwft_set_misaligned_delegation,
+		.get = fwft_get_misaligned_delegation,
+	},
+	{
+		.id = SBI_FWFT_PTE_AD_HW_UPDATING,
+		.supported = fwft_adue_supported,
+		.set = fwft_set_adue,
+		.get = fwft_get_adue,
+	},
+};
+
+int sbi_fwft_init(struct sbi_scratch *scratch, bool cold_boot)
+{
+	int i;
+	struct fwft_hart_state *fhs;
+
+	if (cold_boot) {
+		fwft_ptr_offset = sbi_scratch_alloc_type_offset(void *);
+		if (!fwft_ptr_offset)
+			return SBI_ENOMEM;
+	}
+
+	fhs = fwft_get_hart_state_ptr(scratch);
+	if (!fhs) {
+		fhs = sbi_zalloc(sizeof(fhs) + array_size(features) * sizeof(struct fwft_config));
+		if (!fhs)
+			return SBI_ENOMEM;
+
+		fhs->config_count = array_size(features);
+		for (i = 0; i < array_size(features); i++)
+			fhs->configs[i].feature = &features[i];
+
+		fwft_set_hart_state_ptr(scratch, fhs);
+	}
+
+	return 0;
+}

lib/sbi/sbi_hart.c

@@ -33,11 +33,11 @@ static unsigned long hart_features_offset;
 
 static void mstatus_init(struct sbi_scratch *scratch)
 {
-	unsigned long menvcfg_val, mstatus_val = 0;
 	int cidx;
-	unsigned int num_mhpm = sbi_hart_mhpm_count(scratch);
+	unsigned long mstatus_val = 0;
+	unsigned int mhpm_mask = sbi_hart_mhpm_mask(scratch);
 	uint64_t mhpmevent_init_val = 0;
-	uint64_t mstateen_val;
+	uint64_t menvcfg_val, mstateen_val;
 
 	/* Enable FPU */
 	if (misa_extension('D') || misa_extension('F'))
@@ -69,13 +69,14 @@ static void mstatus_init(struct sbi_scratch *scratch)
 	/**
 	 * The mhpmeventn[h] CSR should be initialized with interrupt disabled
 	 * and inhibited running in M-mode during init.
-	 * To keep it simple, only contiguous mhpmcounters are supported as a
-	 * platform with discontiguous mhpmcounters may not make much sense.
 	 */
 	mhpmevent_init_val |= (MHPMEVENT_OF | MHPMEVENT_MINH);
-	for (cidx = 0; cidx < num_mhpm; cidx++) {
+	for (cidx = 0; cidx <= 28; cidx++) {
+		if (!(mhpm_mask & 1 << (cidx + 3)))
+			continue;
 #if __riscv_xlen == 32
-		csr_write_num(CSR_MHPMEVENT3 + cidx, mhpmevent_init_val & 0xFFFFFFFF);
+		csr_write_num(CSR_MHPMEVENT3 + cidx,
+			       mhpmevent_init_val & 0xFFFFFFFF);
 		if (sbi_hart_has_extension(scratch, SBI_HART_EXT_SSCOFPMF))
 			csr_write_num(CSR_MHPMEVENT3H + cidx,
 				      mhpmevent_init_val >> BITS_PER_LONG);
@@ -94,11 +95,16 @@ static void mstatus_init(struct sbi_scratch *scratch)
 		mstateen_val |= SMSTATEEN0_HSENVCFG;
 
 		if (sbi_hart_has_extension(scratch, SBI_HART_EXT_SMAIA))
-			mstateen_val |= (SMSTATEEN0_AIA | SMSTATEEN0_SVSLCT |
-					SMSTATEEN0_IMSIC);
+			mstateen_val |= (SMSTATEEN0_AIA | SMSTATEEN0_IMSIC);
 		else
-			mstateen_val &= ~(SMSTATEEN0_AIA | SMSTATEEN0_SVSLCT |
-					SMSTATEEN0_IMSIC);
+			mstateen_val &= ~(SMSTATEEN0_AIA | SMSTATEEN0_IMSIC);
+
+		if (sbi_hart_has_extension(scratch, SBI_HART_EXT_SMAIA) ||
+		    sbi_hart_has_extension(scratch, SBI_HART_EXT_SMCSRIND))
+			mstateen_val |= (SMSTATEEN0_SVSLCT);
+		else
+			mstateen_val &= ~(SMSTATEEN0_SVSLCT);
+
 		csr_write(CSR_MSTATEEN0, mstateen_val);
 #if __riscv_xlen == 32
 		csr_write(CSR_MSTATEEN0H, mstateen_val >> 32);
@@ -107,58 +113,51 @@ static void mstatus_init(struct sbi_scratch *scratch)
 
 	if (sbi_hart_priv_version(scratch) >= SBI_HART_PRIV_VER_1_12) {
 		menvcfg_val = csr_read(CSR_MENVCFG);
-
-		/*
-		 * Set menvcfg.CBZE == 1
-		 *
-		 * If Zicboz extension is not available then writes to
-		 * menvcfg.CBZE will be ignored because it is a WARL field.
-		 */
-		menvcfg_val |= ENVCFG_CBZE;
-
-		/*
-		 * Set menvcfg.CBCFE == 1
-		 *
-		 * If Zicbom extension is not available then writes to
-		 * menvcfg.CBCFE will be ignored because it is a WARL field.
-		 */
-		menvcfg_val |= ENVCFG_CBCFE;
-
-		/*
-		 * Set menvcfg.CBIE == 3
-		 *
-		 * If Zicbom extension is not available then writes to
-		 * menvcfg.CBIE will be ignored because it is a WARL field.
-		 */
-		menvcfg_val |= ENVCFG_CBIE_INV << ENVCFG_CBIE_SHIFT;
-
-		/*
-		 * Set menvcfg.PBMTE == 1 for RV64 or RV128
-		 *
-		 * If Svpbmt extension is not available then menvcfg.PBMTE
-		 * will be read-only zero.
-		 */
-#if __riscv_xlen > 32
-		menvcfg_val |= ENVCFG_PBMTE;
-#endif
-
-		/*
-		 * The spec doesn't explicitly describe the reset value of menvcfg.
-		 * Enable access to stimecmp if sstc extension is present in the
-		 * hardware.
-		 */
-		if (sbi_hart_has_extension(scratch, SBI_HART_EXT_SSTC)) {
 #if __riscv_xlen == 32
-			unsigned long menvcfgh_val;
-			menvcfgh_val = csr_read(CSR_MENVCFGH);
-			menvcfgh_val |= ENVCFGH_STCE;
-			csr_write(CSR_MENVCFGH, menvcfgh_val);
-#else
-			menvcfg_val |= ENVCFG_STCE;
+		menvcfg_val |= ((uint64_t)csr_read(CSR_MENVCFGH)) << 32;
 #endif
-		}
+
+#define __set_menvcfg_ext(__ext, __bits)				\
+		if (sbi_hart_has_extension(scratch, __ext))		\
+			menvcfg_val |= __bits;
+
+		/*
+		 * Enable access to extensions if they are present in the
+		 * hardware or in the device tree.
+		 */
+
+		__set_menvcfg_ext(SBI_HART_EXT_ZICBOZ, ENVCFG_CBZE)
+		__set_menvcfg_ext(SBI_HART_EXT_ZICBOM, ENVCFG_CBCFE)
+		__set_menvcfg_ext(SBI_HART_EXT_ZICBOM,
+				  ENVCFG_CBIE_INV << ENVCFG_CBIE_SHIFT)
+#if __riscv_xlen > 32
+		__set_menvcfg_ext(SBI_HART_EXT_SVPBMT, ENVCFG_PBMTE)
+#endif
+		__set_menvcfg_ext(SBI_HART_EXT_SSTC, ENVCFG_STCE)
+		__set_menvcfg_ext(SBI_HART_EXT_SMCDELEG, ENVCFG_CDE);
+		__set_menvcfg_ext(SBI_HART_EXT_SVADU, ENVCFG_ADUE);
+
+#undef __set_menvcfg_ext
+
+		/*
+		 * When both Svade and Svadu are present in DT, the default scheme for managing
+		 * the PTE A/D bits should use Svade. Check Svadu before Svade extension to ensure
+		 * that the ADUE bit is cleared when the Svade support are specified.
+		 */
+
+		if (sbi_hart_has_extension(scratch, SBI_HART_EXT_SVADE))
+			menvcfg_val &= ~ENVCFG_ADUE;
 
 		csr_write(CSR_MENVCFG, menvcfg_val);
+#if __riscv_xlen == 32
+		csr_write(CSR_MENVCFGH, menvcfg_val >> 32);
+#endif
+
+		/* Enable S-mode access to seed CSR */
+		if (sbi_hart_has_extension(scratch, SBI_HART_EXT_ZKR)) {
+			csr_set(CSR_MSECCFG, MSECCFG_SSEED);
+			csr_clear(CSR_MSECCFG, MSECCFG_USEED);
+		}
 	}
 
 	/* Disable all interrupts */
@@ -244,12 +243,12 @@ void sbi_hart_delegation_dump(struct sbi_scratch *scratch,
 		   prefix, suffix, csr_read(CSR_MEDELEG));
 }
 
-unsigned int sbi_hart_mhpm_count(struct sbi_scratch *scratch)
+unsigned int sbi_hart_mhpm_mask(struct sbi_scratch *scratch)
 {
 	struct sbi_hart_features *hfeatures =
 			sbi_scratch_offset_ptr(scratch, hart_features_offset);
 
-	return hfeatures->mhpm_count;
+	return hfeatures->mhpm_mask;
 }
 
 unsigned int sbi_hart_pmp_count(struct sbi_scratch *scratch)
@@ -260,12 +259,12 @@ unsigned int sbi_hart_pmp_count(struct sbi_scratch *scratch)
 	return hfeatures->pmp_count;
 }
 
-unsigned long sbi_hart_pmp_granularity(struct sbi_scratch *scratch)
+unsigned int sbi_hart_pmp_log2gran(struct sbi_scratch *scratch)
 {
 	struct sbi_hart_features *hfeatures =
 			sbi_scratch_offset_ptr(scratch, hart_features_offset);
 
-	return hfeatures->pmp_gran;
+	return hfeatures->pmp_log2gran;
 }
 
 unsigned int sbi_hart_pmp_addrbits(struct sbi_scratch *scratch)
@@ -284,20 +283,174 @@ unsigned int sbi_hart_mhpm_bits(struct sbi_scratch *scratch)
 	return hfeatures->mhpm_bits;
 }
 
-int sbi_hart_pmp_configure(struct sbi_scratch *scratch)
+/*
+ * Returns Smepmp flags for a given domain and region based on permissions.
+ */
+static unsigned int sbi_hart_get_smepmp_flags(struct sbi_scratch *scratch,
+					      struct sbi_domain *dom,
+					      struct sbi_domain_memregion *reg)
+{
+	unsigned int pmp_flags = 0;
+
+	if (SBI_DOMAIN_MEMREGION_IS_SHARED(reg->flags)) {
+		/* Read only for both M and SU modes */
+		if (SBI_DOMAIN_MEMREGION_IS_SUR_MR(reg->flags))
+			pmp_flags = (PMP_L | PMP_R | PMP_W | PMP_X);
+
+		/* Execute for SU but Read/Execute for M mode */
+		else if (SBI_DOMAIN_MEMREGION_IS_SUX_MRX(reg->flags))
+			/* locked region */
+			pmp_flags = (PMP_L | PMP_W | PMP_X);
+
+		/* Execute only for both M and SU modes */
+		else if (SBI_DOMAIN_MEMREGION_IS_SUX_MX(reg->flags))
+			pmp_flags = (PMP_L | PMP_W);
+
+		/* Read/Write for both M and SU modes */
+		else if (SBI_DOMAIN_MEMREGION_IS_SURW_MRW(reg->flags))
+			pmp_flags = (PMP_W | PMP_X);
+
+		/* Read only for SU mode but Read/Write for M mode */
+		else if (SBI_DOMAIN_MEMREGION_IS_SUR_MRW(reg->flags))
+			pmp_flags = (PMP_W);
+	} else if (SBI_DOMAIN_MEMREGION_M_ONLY_ACCESS(reg->flags)) {
+		/*
+		 * When smepmp is supported and used, M region cannot have RWX
+		 * permissions on any region.
+		 */
+		if ((reg->flags & SBI_DOMAIN_MEMREGION_M_ACCESS_MASK)
+		    == SBI_DOMAIN_MEMREGION_M_RWX) {
+			sbi_printf("%s: M-mode only regions cannot have"
+				   "RWX permissions\n", __func__);
+			return 0;
+		}
+
+		/* M-mode only access regions are always locked */
+		pmp_flags |= PMP_L;
+
+		if (reg->flags & SBI_DOMAIN_MEMREGION_M_READABLE)
+			pmp_flags |= PMP_R;
+		if (reg->flags & SBI_DOMAIN_MEMREGION_M_WRITABLE)
+			pmp_flags |= PMP_W;
+		if (reg->flags & SBI_DOMAIN_MEMREGION_M_EXECUTABLE)
+			pmp_flags |= PMP_X;
+	} else if (SBI_DOMAIN_MEMREGION_SU_ONLY_ACCESS(reg->flags)) {
+		if (reg->flags & SBI_DOMAIN_MEMREGION_SU_READABLE)
+			pmp_flags |= PMP_R;
+		if (reg->flags & SBI_DOMAIN_MEMREGION_SU_WRITABLE)
+			pmp_flags |= PMP_W;
+		if (reg->flags & SBI_DOMAIN_MEMREGION_SU_EXECUTABLE)
+			pmp_flags |= PMP_X;
+	}
+
+	return pmp_flags;
+}
+
+static void sbi_hart_smepmp_set(struct sbi_scratch *scratch,
+				struct sbi_domain *dom,
+				struct sbi_domain_memregion *reg,
+				unsigned int pmp_idx,
+				unsigned int pmp_flags,
+				unsigned int pmp_log2gran,
+				unsigned long pmp_addr_max)
+{
+	unsigned long pmp_addr = reg->base >> PMP_SHIFT;
+
+	if (pmp_log2gran <= reg->order && pmp_addr < pmp_addr_max) {
+		pmp_set(pmp_idx, pmp_flags, reg->base, reg->order);
+	} else {
+		sbi_printf("Can not configure pmp for domain %s because"
+			   " memory region address 0x%lx or size 0x%lx "
+			   "is not in range.\n", dom->name, reg->base,
+			   reg->order);
+	}
+}
+
+static int sbi_hart_smepmp_configure(struct sbi_scratch *scratch,
+				     unsigned int pmp_count,
+				     unsigned int pmp_log2gran,
+				     unsigned long pmp_addr_max)
 {
 	struct sbi_domain_memregion *reg;
 	struct sbi_domain *dom = sbi_domain_thishart_ptr();
-	unsigned int pmp_idx = 0, pmp_flags, pmp_bits, pmp_gran_log2;
-	unsigned int pmp_count = sbi_hart_pmp_count(scratch);
-	unsigned long pmp_addr = 0, pmp_addr_max = 0;
+	unsigned int pmp_idx, pmp_flags;
 
-	if (!pmp_count)
-		return 0;
+	/*
+	 * Set the RLB so that, we can write to PMP entries without
+	 * enforcement even if some entries are locked.
+	 */
+	csr_set(CSR_MSECCFG, MSECCFG_RLB);
 
-	pmp_gran_log2 = log2roundup(sbi_hart_pmp_granularity(scratch));
-	pmp_bits = sbi_hart_pmp_addrbits(scratch) - 1;
-	pmp_addr_max = (1UL << pmp_bits) | ((1UL << pmp_bits) - 1);
+	/* Disable the reserved entry */
+	pmp_disable(SBI_SMEPMP_RESV_ENTRY);
+
+	/* Program M-only regions when MML is not set. */
+	pmp_idx = 0;
+	sbi_domain_for_each_memregion(dom, reg) {
+		/* Skip reserved entry */
+		if (pmp_idx == SBI_SMEPMP_RESV_ENTRY)
+			pmp_idx++;
+		if (pmp_count <= pmp_idx)
+			break;
+
+		/* Skip shared and SU-only regions */
+		if (!SBI_DOMAIN_MEMREGION_M_ONLY_ACCESS(reg->flags)) {
+			pmp_idx++;
+			continue;
+		}
+
+		pmp_flags = sbi_hart_get_smepmp_flags(scratch, dom, reg);
+		if (!pmp_flags)
+			return 0;
+
+		sbi_hart_smepmp_set(scratch, dom, reg, pmp_idx++, pmp_flags,
+				    pmp_log2gran, pmp_addr_max);
+	}
+
+	/* Set the MML to enforce new encoding */
+	csr_set(CSR_MSECCFG, MSECCFG_MML);
+
+	/* Program shared and SU-only regions */
+	pmp_idx = 0;
+	sbi_domain_for_each_memregion(dom, reg) {
+		/* Skip reserved entry */
+		if (pmp_idx == SBI_SMEPMP_RESV_ENTRY)
+			pmp_idx++;
+		if (pmp_count <= pmp_idx)
+			break;
+
+		/* Skip M-only regions */
+		if (SBI_DOMAIN_MEMREGION_M_ONLY_ACCESS(reg->flags)) {
+			pmp_idx++;
+			continue;
+		}
+
+		pmp_flags = sbi_hart_get_smepmp_flags(scratch, dom, reg);
+		if (!pmp_flags)
+			return 0;
+
+		sbi_hart_smepmp_set(scratch, dom, reg, pmp_idx++, pmp_flags,
+				    pmp_log2gran, pmp_addr_max);
+	}
+
+	/*
+	 * All entries are programmed.
+	 * Keep the RLB bit so that dynamic mappings can be done.
+	 */
+
+	return 0;
+}
+
+static int sbi_hart_oldpmp_configure(struct sbi_scratch *scratch,
+				     unsigned int pmp_count,
+				     unsigned int pmp_log2gran,
+				     unsigned long pmp_addr_max)
+{
+	struct sbi_domain_memregion *reg;
+	struct sbi_domain *dom = sbi_domain_thishart_ptr();
+	unsigned int pmp_idx = 0;
+	unsigned int pmp_flags;
+	unsigned long pmp_addr;
 
 	sbi_domain_for_each_memregion(dom, reg) {
 		if (pmp_count <= pmp_idx)
@@ -306,8 +459,8 @@ int sbi_hart_pmp_configure(struct sbi_scratch *scratch)
 		pmp_flags = 0;
 
 		/*
-		 * If permissions are to be enforced for all modes on this
-		 * region, the lock bit should be set.
+		 * If permissions are to be enforced for all modes on
+		 * this region, the lock bit should be set.
 		 */
 		if (reg->flags & SBI_DOMAIN_MEMREGION_ENF_PERMISSIONS)
 			pmp_flags |= PMP_L;
@@ -319,16 +472,84 @@ int sbi_hart_pmp_configure(struct sbi_scratch *scratch)
 		if (reg->flags & SBI_DOMAIN_MEMREGION_SU_EXECUTABLE)
 			pmp_flags |= PMP_X;
 
-		pmp_addr =  reg->base >> PMP_SHIFT;
-		if (pmp_gran_log2 <= reg->order && pmp_addr < pmp_addr_max)
+		pmp_addr = reg->base >> PMP_SHIFT;
+		if (pmp_log2gran <= reg->order && pmp_addr < pmp_addr_max) {
 			pmp_set(pmp_idx++, pmp_flags, reg->base, reg->order);
-		else {
-			sbi_printf("Can not configure pmp for domain %s", dom->name);
-			sbi_printf(" because memory region address %lx or size %lx is not in range\n",
-				    reg->base, reg->order);
+		} else {
+			sbi_printf("Can not configure pmp for domain %s because"
+				   " memory region address 0x%lx or size 0x%lx "
+				   "is not in range.\n", dom->name, reg->base,
+				   reg->order);
 		}
 	}
 
+	return 0;
+}
+
+int sbi_hart_map_saddr(unsigned long addr, unsigned long size)
+{
+	/* shared R/W access for M and S/U mode */
+	unsigned int pmp_flags = (PMP_W | PMP_X);
+	unsigned long order, base = 0;
+	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
+
+	/* If Smepmp is not supported no special mapping is required */
+	if (!sbi_hart_has_extension(scratch, SBI_HART_EXT_SMEPMP))
+		return SBI_OK;
+
+	if (is_pmp_entry_mapped(SBI_SMEPMP_RESV_ENTRY))
+		return SBI_ENOSPC;
+
+	for (order = MAX(sbi_hart_pmp_log2gran(scratch), log2roundup(size));
+	     order <= __riscv_xlen; order++) {
+		if (order < __riscv_xlen) {
+			base = addr & ~((1UL << order) - 1UL);
+			if ((base <= addr) &&
+			    (addr < (base + (1UL << order))) &&
+			    (base <= (addr + size - 1UL)) &&
+			    ((addr + size - 1UL) < (base + (1UL << order))))
+				break;
+		} else {
+			return SBI_EFAIL;
+		}
+	}
+
+	pmp_set(SBI_SMEPMP_RESV_ENTRY, pmp_flags, base, order);
+
+	return SBI_OK;
+}
+
+int sbi_hart_unmap_saddr(void)
+{
+	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
+
+	if (!sbi_hart_has_extension(scratch, SBI_HART_EXT_SMEPMP))
+		return SBI_OK;
+
+	return pmp_disable(SBI_SMEPMP_RESV_ENTRY);
+}
+
+int sbi_hart_pmp_configure(struct sbi_scratch *scratch)
+{
+	int rc;
+	unsigned int pmp_bits, pmp_log2gran;
+	unsigned int pmp_count = sbi_hart_pmp_count(scratch);
+	unsigned long pmp_addr_max;
+
+	if (!pmp_count)
+		return 0;
+
+	pmp_log2gran = sbi_hart_pmp_log2gran(scratch);
+	pmp_bits = sbi_hart_pmp_addrbits(scratch) - 1;
+	pmp_addr_max = (1UL << pmp_bits) | ((1UL << pmp_bits) - 1);
+
+	if (sbi_hart_has_extension(scratch, SBI_HART_EXT_SMEPMP))
+		rc = sbi_hart_smepmp_configure(scratch, pmp_count,
+						pmp_log2gran, pmp_addr_max);
+	else
+		rc = sbi_hart_oldpmp_configure(scratch, pmp_count,
+						pmp_log2gran, pmp_addr_max);
+
 	/*
 	 * As per section 3.7.2 of privileged specification v1.12,
 	 * virtual address translations can be speculatively performed
@@ -350,7 +571,7 @@ int sbi_hart_pmp_configure(struct sbi_scratch *scratch)
 			__sbi_hfence_gvma_all();
 	}
 
-	return 0;
+	return rc;
 }
 
 int sbi_hart_priv_version(struct sbi_scratch *scratch)
@@ -392,9 +613,9 @@ static inline void __sbi_hart_update_extension(
 					bool enable)
 {
 	if (enable)
-		hfeatures->extensions |= BIT(ext);
+		__set_bit(ext, hfeatures->extensions);
 	else
-		hfeatures->extensions &= ~BIT(ext);
+		__clear_bit(ext, hfeatures->extensions);
 }
 
 /**
@@ -427,39 +648,43 @@ bool sbi_hart_has_extension(struct sbi_scratch *scratch,
 	struct sbi_hart_features *hfeatures =
 			sbi_scratch_offset_ptr(scratch, hart_features_offset);
 
-	if (hfeatures->extensions & BIT(ext))
+	if (__test_bit(ext, hfeatures->extensions))
 		return true;
 	else
 		return false;
 }
 
-static inline char *sbi_hart_extension_id2string(int ext)
-{
-	char *estr = NULL;
-
-	switch (ext) {
-	case SBI_HART_EXT_SSCOFPMF:
-		estr = "sscofpmf";
-		break;
-	case SBI_HART_EXT_TIME:
-		estr = "time";
-		break;
-	case SBI_HART_EXT_SMAIA:
-		estr = "smaia";
-		break;
-	case SBI_HART_EXT_SSTC:
-		estr = "sstc";
-		break;
-	case SBI_HART_EXT_SMSTATEEN:
-		estr = "smstateen";
-		break;
-	default:
-		break;
-	}
-
-	return estr;
+#define __SBI_HART_EXT_DATA(_name, _id) {	\
+	.name = #_name,				\
+	.id = _id,				\
 }
 
+const struct sbi_hart_ext_data sbi_hart_ext[] = {
+	__SBI_HART_EXT_DATA(smaia, SBI_HART_EXT_SMAIA),
+	__SBI_HART_EXT_DATA(smepmp, SBI_HART_EXT_SMEPMP),
+	__SBI_HART_EXT_DATA(smstateen, SBI_HART_EXT_SMSTATEEN),
+	__SBI_HART_EXT_DATA(sscofpmf, SBI_HART_EXT_SSCOFPMF),
+	__SBI_HART_EXT_DATA(sstc, SBI_HART_EXT_SSTC),
+	__SBI_HART_EXT_DATA(zicntr, SBI_HART_EXT_ZICNTR),
+	__SBI_HART_EXT_DATA(zihpm, SBI_HART_EXT_ZIHPM),
+	__SBI_HART_EXT_DATA(zkr, SBI_HART_EXT_ZKR),
+	__SBI_HART_EXT_DATA(smcntrpmf, SBI_HART_EXT_SMCNTRPMF),
+	__SBI_HART_EXT_DATA(xandespmu, SBI_HART_EXT_XANDESPMU),
+	__SBI_HART_EXT_DATA(zicboz, SBI_HART_EXT_ZICBOZ),
+	__SBI_HART_EXT_DATA(zicbom, SBI_HART_EXT_ZICBOM),
+	__SBI_HART_EXT_DATA(svpbmt, SBI_HART_EXT_SVPBMT),
+	__SBI_HART_EXT_DATA(sdtrig, SBI_HART_EXT_SDTRIG),
+	__SBI_HART_EXT_DATA(smcsrind, SBI_HART_EXT_SMCSRIND),
+	__SBI_HART_EXT_DATA(smcdeleg, SBI_HART_EXT_SMCDELEG),
+	__SBI_HART_EXT_DATA(sscsrind, SBI_HART_EXT_SSCSRIND),
+	__SBI_HART_EXT_DATA(ssccfg, SBI_HART_EXT_SSCCFG),
+	__SBI_HART_EXT_DATA(svade, SBI_HART_EXT_SVADE),
+	__SBI_HART_EXT_DATA(svadu, SBI_HART_EXT_SVADU),
+};
+
+_Static_assert(SBI_HART_EXT_MAX == array_size(sbi_hart_ext),
+	       "sbi_hart_ext[]: wrong number of entries");
+
 /**
  * Get the hart extensions in string format
  *
@@ -475,30 +700,18 @@ void sbi_hart_get_extensions_str(struct sbi_scratch *scratch,
 	struct sbi_hart_features *hfeatures =
 			sbi_scratch_offset_ptr(scratch, hart_features_offset);
 	int offset = 0, ext = 0;
-	char *temp;
 
 	if (!extensions_str || nestr <= 0)
 		return;
 	sbi_memset(extensions_str, 0, nestr);
 
-	if (!hfeatures->extensions)
-		goto done;
+	for_each_set_bit(ext, hfeatures->extensions, SBI_HART_EXT_MAX) {
+		sbi_snprintf(extensions_str + offset,
+				 nestr - offset,
+				 "%s,", sbi_hart_ext[ext].name);
+		offset = offset + sbi_strlen(sbi_hart_ext[ext].name) + 1;
+	}
 
-	do {
-		if (hfeatures->extensions & BIT(ext)) {
-			temp = sbi_hart_extension_id2string(ext);
-			if (temp) {
-				sbi_snprintf(extensions_str + offset,
-					     nestr - offset,
-					     "%s,", temp);
-				offset = offset + sbi_strlen(temp) + 1;
-			}
-		}
-
-		ext++;
-	} while (ext < SBI_HART_EXT_MAX);
-
-done:
 	if (offset)
 		extensions_str[offset - 1] = '\0';
 	else
@@ -524,7 +737,7 @@ static unsigned long hart_pmp_get_allowed_addr(void)
 	return val;
 }
 
-static int hart_pmu_get_allowed_bits(void)
+static int hart_mhpm_get_allowed_bits(void)
 {
 	unsigned long val = ~(0UL);
 	struct sbi_trap_info trap = {0};
@@ -561,6 +774,7 @@ static int hart_detect_features(struct sbi_scratch *scratch)
 	struct sbi_hart_features *hfeatures =
 		sbi_scratch_offset_ptr(scratch, hart_features_offset);
 	unsigned long val, oldval;
+	bool has_zicntr = false;
 	int rc;
 
 	/* If hart features already detected then do nothing */
@@ -568,9 +782,32 @@ static int hart_detect_features(struct sbi_scratch *scratch)
 		return 0;
 
 	/* Clear hart features */
-	hfeatures->extensions = 0;
+	sbi_memset(hfeatures->extensions, 0, sizeof(hfeatures->extensions));
 	hfeatures->pmp_count = 0;
-	hfeatures->mhpm_count = 0;
+	hfeatures->mhpm_mask = 0;
+	hfeatures->priv_version = SBI_HART_PRIV_VER_UNKNOWN;
+
+#define __check_hpm_csr(__csr, __mask) 					  \
+	oldval = csr_read_allowed(__csr, (ulong)&trap);			  \
+	if (!trap.cause) {						  \
+		csr_write_allowed(__csr, (ulong)&trap, 1UL);		  \
+		if (!trap.cause && csr_swap(__csr, oldval) == 1UL) {	  \
+			(hfeatures->__mask) |= 1 << (__csr - CSR_MCYCLE); \
+		}							  \
+	}
+
+#define __check_hpm_csr_2(__csr, __mask)	 		  \
+	__check_hpm_csr(__csr + 0, __mask)	 		  \
+	__check_hpm_csr(__csr + 1, __mask)
+#define __check_hpm_csr_4(__csr, __mask)	 		  \
+	__check_hpm_csr_2(__csr + 0, __mask) 			  \
+	__check_hpm_csr_2(__csr + 2, __mask)
+#define __check_hpm_csr_8(__csr, __mask)	 		  \
+	__check_hpm_csr_4(__csr + 0, __mask) 			  \
+	__check_hpm_csr_4(__csr + 4, __mask)
+#define __check_hpm_csr_16(__csr, __mask)	 		  \
+	__check_hpm_csr_8(__csr + 0, __mask) 			  \
+	__check_hpm_csr_8(__csr + 8, __mask)
 
 #define __check_csr(__csr, __rdonly, __wrval, __field, __skip)	\
 	oldval = csr_read_allowed(__csr, (ulong)&trap);			\
@@ -616,28 +853,23 @@ static int hart_detect_features(struct sbi_scratch *scratch)
 	 */
 	val = hart_pmp_get_allowed_addr();
 	if (val) {
-		hfeatures->pmp_gran =  1 << (sbi_ffs(val) + 2);
+		hfeatures->pmp_log2gran = sbi_ffs(val) + 2;
 		hfeatures->pmp_addr_bits = sbi_fls(val) + 1;
 		/* Detect number of PMP regions. At least PMPADDR0 should be implemented*/
 		__check_csr_64(CSR_PMPADDR0, 0, val, pmp_count, __pmp_skip);
 	}
 __pmp_skip:
-
 	/* Detect number of MHPM counters */
-	__check_csr(CSR_MHPMCOUNTER3, 0, 1UL, mhpm_count, __mhpm_skip);
-	hfeatures->mhpm_bits = hart_pmu_get_allowed_bits();
-
-	__check_csr_4(CSR_MHPMCOUNTER4, 0, 1UL, mhpm_count, __mhpm_skip);
-	__check_csr_8(CSR_MHPMCOUNTER8, 0, 1UL, mhpm_count, __mhpm_skip);
-	__check_csr_16(CSR_MHPMCOUNTER16, 0, 1UL, mhpm_count, __mhpm_skip);
+	__check_hpm_csr(CSR_MHPMCOUNTER3, mhpm_mask);
+	hfeatures->mhpm_bits = hart_mhpm_get_allowed_bits();
+	__check_hpm_csr_4(CSR_MHPMCOUNTER4, mhpm_mask);
+	__check_hpm_csr_8(CSR_MHPMCOUNTER8, mhpm_mask);
+	__check_hpm_csr_16(CSR_MHPMCOUNTER16, mhpm_mask);
 
 	/**
 	 * No need to check for MHPMCOUNTERH for RV32 as they are expected to be
 	 * implemented if MHPMCOUNTER is implemented.
 	 */
-
-__mhpm_skip:
-
 #undef __check_csr_64
 #undef __check_csr_32
 #undef __check_csr_16
@@ -646,59 +878,60 @@ __mhpm_skip:
 #undef __check_csr_2
 #undef __check_csr
 
+
+#define __check_priv(__csr, __base_priv, __priv)			\
+	val = csr_read_allowed(__csr, (ulong)&trap);			\
+	if (!trap.cause && (hfeatures->priv_version >= __base_priv)) {	\
+		hfeatures->priv_version = __priv;			\
+	}
+
 	/* Detect if hart supports Priv v1.10 */
-	val = csr_read_allowed(CSR_MCOUNTEREN, (unsigned long)&trap);
-	if (!trap.cause)
-		hfeatures->priv_version = SBI_HART_PRIV_VER_1_10;
-
+	__check_priv(CSR_MCOUNTEREN,
+		     SBI_HART_PRIV_VER_UNKNOWN, SBI_HART_PRIV_VER_1_10);
 	/* Detect if hart supports Priv v1.11 */
-	val = csr_read_allowed(CSR_MCOUNTINHIBIT, (unsigned long)&trap);
-	if (!trap.cause &&
-	    (hfeatures->priv_version >= SBI_HART_PRIV_VER_1_10))
-		hfeatures->priv_version = SBI_HART_PRIV_VER_1_11;
-
+	__check_priv(CSR_MCOUNTINHIBIT,
+		     SBI_HART_PRIV_VER_1_10, SBI_HART_PRIV_VER_1_11);
 	/* Detect if hart supports Priv v1.12 */
-	csr_read_allowed(CSR_MENVCFG, (unsigned long)&trap);
-	if (!trap.cause &&
-	    (hfeatures->priv_version >= SBI_HART_PRIV_VER_1_11))
-		hfeatures->priv_version = SBI_HART_PRIV_VER_1_12;
+	__check_priv(CSR_MENVCFG,
+		     SBI_HART_PRIV_VER_1_11, SBI_HART_PRIV_VER_1_12);
+
+#undef __check_priv_csr
+
+#define __check_ext_csr(__base_priv, __csr, __ext)			\
+	if (hfeatures->priv_version >= __base_priv) {			\
+		csr_read_allowed(__csr, (ulong)&trap);			\
+		if (!trap.cause)					\
+			__sbi_hart_update_extension(hfeatures,		\
+						    __ext, true);	\
+	}
 
 	/* Counter overflow/filtering is not useful without mcounter/inhibit */
-	if (hfeatures->priv_version >= SBI_HART_PRIV_VER_1_12) {
-		/* Detect if hart supports sscofpmf */
-		csr_read_allowed(CSR_SCOUNTOVF, (unsigned long)&trap);
-		if (!trap.cause)
-			__sbi_hart_update_extension(hfeatures,
-					SBI_HART_EXT_SSCOFPMF, true);
-	}
-
+	/* Detect if hart supports sscofpmf */
+	__check_ext_csr(SBI_HART_PRIV_VER_1_11,
+			CSR_SCOUNTOVF, SBI_HART_EXT_SSCOFPMF);
 	/* Detect if hart supports time CSR */
-	csr_read_allowed(CSR_TIME, (unsigned long)&trap);
-	if (!trap.cause)
-		__sbi_hart_update_extension(hfeatures,
-					SBI_HART_EXT_TIME, true);
-
+	__check_ext_csr(SBI_HART_PRIV_VER_UNKNOWN,
+			CSR_TIME, SBI_HART_EXT_ZICNTR);
 	/* Detect if hart has AIA local interrupt CSRs */
-	csr_read_allowed(CSR_MTOPI, (unsigned long)&trap);
-	if (!trap.cause)
-		__sbi_hart_update_extension(hfeatures,
-					SBI_HART_EXT_SMAIA, true);
-
+	__check_ext_csr(SBI_HART_PRIV_VER_UNKNOWN,
+			CSR_MTOPI, SBI_HART_EXT_SMAIA);
 	/* Detect if hart supports stimecmp CSR(Sstc extension) */
-	if (hfeatures->priv_version >= SBI_HART_PRIV_VER_1_12) {
-		csr_read_allowed(CSR_STIMECMP, (unsigned long)&trap);
-		if (!trap.cause)
-			__sbi_hart_update_extension(hfeatures,
-					SBI_HART_EXT_SSTC, true);
-	}
-
+	__check_ext_csr(SBI_HART_PRIV_VER_1_12,
+			CSR_STIMECMP, SBI_HART_EXT_SSTC);
 	/* Detect if hart supports mstateen CSRs */
-	if (hfeatures->priv_version >= SBI_HART_PRIV_VER_1_12) {
-		val = csr_read_allowed(CSR_MSTATEEN0, (unsigned long)&trap);
-		if (!trap.cause)
-			__sbi_hart_update_extension(hfeatures,
-					SBI_HART_EXT_SMSTATEEN, true);
-	}
+	__check_ext_csr(SBI_HART_PRIV_VER_1_12,
+			CSR_MSTATEEN0, SBI_HART_EXT_SMSTATEEN);
+	/* Detect if hart supports smcntrpmf */
+	__check_ext_csr(SBI_HART_PRIV_VER_1_12,
+			CSR_MCYCLECFG, SBI_HART_EXT_SMCNTRPMF);
+	/* Detect if hart support sdtrig (debug triggers) */
+	__check_ext_csr(SBI_HART_PRIV_VER_UNKNOWN,
+			CSR_TSELECT, SBI_HART_EXT_SDTRIG);
+
+#undef __check_ext_csr
+
+	/* Save trap based detection of Zicntr */
+	has_zicntr = sbi_hart_has_extension(scratch, SBI_HART_EXT_ZICNTR);
 
 	/* Let platform populate extensions */
 	rc = sbi_platform_extensions_init(sbi_platform_thishart_ptr(),
@@ -706,9 +939,28 @@ __mhpm_skip:
 	if (rc)
 		return rc;
 
+	/* Zicntr should only be detected using traps */
+	__sbi_hart_update_extension(hfeatures, SBI_HART_EXT_ZICNTR,
+				    has_zicntr);
+
+	/* Extensions implied by other extensions and features */
+	if (hfeatures->mhpm_mask)
+		__sbi_hart_update_extension(hfeatures,
+					SBI_HART_EXT_ZIHPM, true);
+
 	/* Mark hart feature detection done */
 	hfeatures->detected = true;
 
+	/*
+	 * On platforms with Smepmp, the previous booting stage must
+	 * enter OpenSBI with mseccfg.MML == 0. This allows OpenSBI
+	 * to configure it's own M-mode only regions without depending
+	 * on the previous booting stage.
+	 */
+	if (sbi_hart_has_extension(scratch, SBI_HART_EXT_SMEPMP) &&
+	    (csr_read(CSR_MSECCFG) & MSECCFG_MML))
+		return SBI_EILL;
+
 	return 0;
 }
 
@@ -806,10 +1058,17 @@ sbi_hart_switch_mode(unsigned long arg0, unsigned long arg1,
 	csr_write(CSR_MEPC, next_addr);
 
 	if (next_mode == PRV_S) {
-		csr_write(CSR_STVEC, next_addr);
-		csr_write(CSR_SSCRATCH, 0);
-		csr_write(CSR_SIE, 0);
-		csr_write(CSR_SATP, 0);
+		if (next_virt) {
+			csr_write(CSR_VSTVEC, next_addr);
+			csr_write(CSR_VSSCRATCH, 0);
+			csr_write(CSR_VSIE, 0);
+			csr_write(CSR_VSATP, 0);
+		} else {
+			csr_write(CSR_STVEC, next_addr);
+			csr_write(CSR_SSCRATCH, 0);
+			csr_write(CSR_SIE, 0);
+			csr_write(CSR_SATP, 0);
+		}
 	} else if (next_mode == PRV_U) {
 		if (misa_extension('N')) {
 			csr_write(CSR_UTVEC, next_addr);

lib/sbi/sbi_heap.c

@@ -14,8 +14,6 @@
 #include <sbi/sbi_scratch.h>
 #include <sbi/sbi_string.h>
 
-/* Alignment of heap base address and size */
-#define HEAP_BASE_ALIGN			1024
 /* Minimum size and alignment of heap allocations */
 #define HEAP_ALLOC_ALIGN		64
 #define HEAP_HOUSEKEEPING_FACTOR	16

lib/sbi/sbi_hsm.c

@@ -44,6 +44,11 @@ struct sbi_hsm_data {
 	unsigned long suspend_type;
 	unsigned long saved_mie;
 	unsigned long saved_mip;
+	unsigned long saved_medeleg;
+	unsigned long saved_menvcfg;
+#if __riscv_xlen == 32
+	unsigned long saved_menvcfgh;
+#endif
 	atomic_t start_ticket;
 };
 
@@ -115,23 +120,22 @@ int sbi_hsm_hart_interruptible_mask(const struct sbi_domain *dom,
 {
 	int hstate;
 	ulong i, hmask, dmask;
-	ulong hend = sbi_scratch_last_hartid() + 1;
 
 	*out_hmask = 0;
-	if (hend <= hbase)
+	if (!sbi_hartid_valid(hbase))
 		return SBI_EINVAL;
-	if (BITS_PER_LONG < (hend - hbase))
-		hend = hbase + BITS_PER_LONG;
 
 	dmask = sbi_domain_get_assigned_hartmask(dom, hbase);
-	for (i = hbase; i < hend; i++) {
-		hmask = 1UL << (i - hbase);
-		if (dmask & hmask) {
-			hstate = __sbi_hsm_hart_get_state(i);
-			if (hstate == SBI_HSM_STATE_STARTED ||
-			    hstate == SBI_HSM_STATE_SUSPENDED)
-				*out_hmask |= hmask;
-		}
+	for (i = 0; i < BITS_PER_LONG; i++) {
+		hmask = 1UL << i;
+		if (!(dmask & hmask))
+			continue;
+
+		hstate = __sbi_hsm_hart_get_state(hbase + i);
+		if (hstate == SBI_HSM_STATE_STARTED ||
+		    hstate == SBI_HSM_STATE_SUSPENDED ||
+		    hstate == SBI_HSM_STATE_RESUME_PENDING)
+			*out_hmask |= hmask;
 	}
 
 	return 0;
@@ -249,15 +253,15 @@ int sbi_hsm_init(struct sbi_scratch *scratch, u32 hartid, bool cold_boot)
 			return SBI_ENOMEM;
 
 		/* Initialize hart state data for every hart */
-		for (i = 0; i <= sbi_scratch_last_hartid(); i++) {
-			rscratch = sbi_hartid_to_scratch(i);
+		for (i = 0; i <= sbi_scratch_last_hartindex(); i++) {
+			rscratch = sbi_hartindex_to_scratch(i);
 			if (!rscratch)
 				continue;
 
 			hdata = sbi_scratch_offset_ptr(rscratch,
 						       hart_data_offset);
 			ATOMIC_INIT(&hdata->state,
-				    (i == hartid) ?
+				    (sbi_hartindex_to_hartid(i) == hartid) ?
 				    SBI_HSM_STATE_START_PENDING :
 				    SBI_HSM_STATE_STOPPED);
 			ATOMIC_INIT(&hdata->start_ticket, 0);
@@ -356,11 +360,15 @@ int sbi_hsm_hart_start(struct sbi_scratch *scratch,
 	   (hsm_device_has_hart_secondary_boot() && !init_count)) {
 		rc = hsm_device_hart_start(hartid, scratch->warmboot_addr);
 	} else {
-		rc = sbi_ipi_raw_send(hartid);
+		rc = sbi_ipi_raw_send(sbi_hartid_to_hartindex(hartid));
 	}
 
 	if (!rc)
 		return 0;
+
+	/* If it fails to start, change hart state back to stop */
+	__sbi_hsm_hart_change_state(hdata, SBI_HSM_STATE_START_PENDING,
+				    SBI_HSM_STATE_STOPPED);
 err:
 	hsm_start_ticket_release(hdata);
 	return rc;
@@ -414,6 +422,11 @@ void __sbi_hsm_suspend_non_ret_save(struct sbi_scratch *scratch)
 
 	hdata->saved_mie = csr_read(CSR_MIE);
 	hdata->saved_mip = csr_read(CSR_MIP) & (MIP_SSIP | MIP_STIP);
+	hdata->saved_medeleg = csr_read(CSR_MEDELEG);
+#if __riscv_xlen == 32
+	hdata->saved_menvcfgh = csr_read(CSR_MENVCFGH);
+#endif
+	hdata->saved_menvcfg = csr_read(CSR_MENVCFG);
 }
 
 static void __sbi_hsm_suspend_non_ret_restore(struct sbi_scratch *scratch)
@@ -421,6 +434,11 @@ static void __sbi_hsm_suspend_non_ret_restore(struct sbi_scratch *scratch)
 	struct sbi_hsm_data *hdata = sbi_scratch_offset_ptr(scratch,
 							    hart_data_offset);
 
+	csr_write(CSR_MENVCFG, hdata->saved_menvcfg);
+#if __riscv_xlen == 32
+	csr_write(CSR_MENVCFGH, hdata->saved_menvcfgh);
+#endif
+	csr_write(CSR_MEDELEG, hdata->saved_medeleg);
 	csr_write(CSR_MIE, hdata->saved_mie);
 	csr_set(CSR_MIP, (hdata->saved_mip & (MIP_SSIP | MIP_STIP)));
 }

lib/sbi/sbi_illegal_insn.c

@@ -25,7 +25,6 @@ static int truly_illegal_insn(ulong insn, struct sbi_trap_regs *regs)
 {
 	struct sbi_trap_info trap;
 
-	trap.epc = regs->mepc;
 	trap.cause = CAUSE_ILLEGAL_INSTRUCTION;
 	trap.tval = insn;
 	trap.tval2 = 0;
@@ -137,8 +136,10 @@ static const illegal_insn_func illegal_insn_table[32] = {
 	truly_illegal_insn  /* 31 */
 };
 
-int sbi_illegal_insn_handler(ulong insn, struct sbi_trap_regs *regs)
+int sbi_illegal_insn_handler(struct sbi_trap_context *tcntx)
 {
+	struct sbi_trap_regs *regs = &tcntx->regs;
+	ulong insn = tcntx->trap.tval;
 	struct sbi_trap_info uptrap;
 
 	/*
@@ -155,10 +156,8 @@ int sbi_illegal_insn_handler(ulong insn, struct sbi_trap_regs *regs)
 	sbi_pmu_ctr_incr_fw(SBI_PMU_FW_ILLEGAL_INSN);
 	if (unlikely((insn & 3) != 3)) {
 		insn = sbi_get_insn(regs->mepc, &uptrap);
-		if (uptrap.cause) {
-			uptrap.epc = regs->mepc;
+		if (uptrap.cause)
 			return sbi_trap_redirect(regs, &uptrap);
-		}
 		if ((insn & 3) != 3)
 			return truly_illegal_insn(insn, regs);
 	}

lib/sbi/sbi_init.c

@@ -10,11 +10,11 @@
 #include <sbi/riscv_asm.h>
 #include <sbi/riscv_atomic.h>
 #include <sbi/riscv_barrier.h>
-#include <sbi/riscv_locks.h>
 #include <sbi/sbi_console.h>
 #include <sbi/sbi_cppc.h>
 #include <sbi/sbi_domain.h>
 #include <sbi/sbi_ecall.h>
+#include <sbi/sbi_fwft.h>
 #include <sbi/sbi_hart.h>
 #include <sbi/sbi_hartmask.h>
 #include <sbi/sbi_heap.h>
@@ -23,11 +23,14 @@
 #include <sbi/sbi_irqchip.h>
 #include <sbi/sbi_platform.h>
 #include <sbi/sbi_pmu.h>
+#include <sbi/sbi_dbtr.h>
+#include <sbi/sbi_sse.h>
 #include <sbi/sbi_system.h>
 #include <sbi/sbi_string.h>
 #include <sbi/sbi_timer.h>
 #include <sbi/sbi_tlb.h>
 #include <sbi/sbi_version.h>
+#include <sbi/sbi_unit_test.h>
 
 #define BANNER                                              \
 	"   ____                    _____ ____ _____\n"     \
@@ -176,86 +179,31 @@ static void sbi_boot_print_hart(struct sbi_scratch *scratch, u32 hartid)
 	sbi_printf("Boot HART ISA Extensions  : %s\n", str);
 	sbi_printf("Boot HART PMP Count       : %d\n",
 		   sbi_hart_pmp_count(scratch));
-	sbi_printf("Boot HART PMP Granularity : %lu\n",
-		   sbi_hart_pmp_granularity(scratch));
+	sbi_printf("Boot HART PMP Granularity : %u bits\n",
+		   sbi_hart_pmp_log2gran(scratch));
 	sbi_printf("Boot HART PMP Address Bits: %d\n",
 		   sbi_hart_pmp_addrbits(scratch));
-	sbi_printf("Boot HART MHPM Count      : %d\n",
-		   sbi_hart_mhpm_count(scratch));
+	sbi_printf("Boot HART MHPM Info       : %lu (0x%08x)\n",
+		   sbi_popcount(sbi_hart_mhpm_mask(scratch)),
+		   sbi_hart_mhpm_mask(scratch));
+	sbi_printf("Boot HART Debug Triggers  : %d triggers\n",
+		   sbi_dbtr_get_total_triggers());
 	sbi_hart_delegation_dump(scratch, "Boot HART ", "         ");
 }
 
-static spinlock_t coldboot_lock = SPIN_LOCK_INITIALIZER;
-static struct sbi_hartmask coldboot_wait_hmask = { 0 };
-
 static unsigned long coldboot_done;
 
 static void wait_for_coldboot(struct sbi_scratch *scratch, u32 hartid)
 {
-	unsigned long saved_mie, cmip;
-
-	/* Save MIE CSR */
-	saved_mie = csr_read(CSR_MIE);
-
-	/* Set MSIE and MEIE bits to receive IPI */
-	csr_set(CSR_MIE, MIP_MSIP | MIP_MEIP);
-
-	/* Acquire coldboot lock */
-	spin_lock(&coldboot_lock);
-
-	/* Mark current HART as waiting */
-	sbi_hartmask_set_hart(hartid, &coldboot_wait_hmask);
-
-	/* Release coldboot lock */
-	spin_unlock(&coldboot_lock);
-
-	/* Wait for coldboot to finish using WFI */
-	while (!__smp_load_acquire(&coldboot_done)) {
-		do {
-			wfi();
-			cmip = csr_read(CSR_MIP);
-		 } while (!(cmip & (MIP_MSIP | MIP_MEIP)));
-	}
-
-	/* Acquire coldboot lock */
-	spin_lock(&coldboot_lock);
-
-	/* Unmark current HART as waiting */
-	sbi_hartmask_clear_hart(hartid, &coldboot_wait_hmask);
-
-	/* Release coldboot lock */
-	spin_unlock(&coldboot_lock);
-
-	/* Restore MIE CSR */
-	csr_write(CSR_MIE, saved_mie);
-
-	/*
-	 * The wait for coldboot is common for both warm startup and
-	 * warm resume path so clearing IPI here would result in losing
-	 * an IPI in warm resume path.
-	 *
-	 * Also, the sbi_platform_ipi_init() called from sbi_ipi_init()
-	 * will automatically clear IPI for current HART.
-	 */
+	/* Wait for coldboot to finish */
+	while (!__smp_load_acquire(&coldboot_done))
+		cpu_relax();
 }
 
 static void wake_coldboot_harts(struct sbi_scratch *scratch, u32 hartid)
 {
 	/* Mark coldboot done */
 	__smp_store_release(&coldboot_done, 1);
-
-	/* Acquire coldboot lock */
-	spin_lock(&coldboot_lock);
-
-	/* Send an IPI to all HARTs waiting for coldboot */
-	for (u32 i = 0; i <= sbi_scratch_last_hartid(); i++) {
-		if ((i != hartid) &&
-		    sbi_hartmask_test_hart(i, &coldboot_wait_hmask))
-			sbi_ipi_raw_send(i);
-	}
-
-	/* Release coldboot lock */
-	spin_unlock(&coldboot_lock);
 }
 
 static unsigned long entry_count_offset;
@@ -297,6 +245,14 @@ static void __noreturn init_coldboot(struct sbi_scratch *scratch, u32 hartid)
 	if (rc)
 		sbi_hart_hang();
 
+	/*
+	 * All non-coldboot HARTs do HSM initialization (i.e. enter HSM state
+	 * machine) at the start of the warmboot path so it is wasteful to
+	 * have these HARTs busy spin in wait_for_coldboot() until coldboot
+	 * path is completed.
+	 */
+	wake_coldboot_harts(scratch, hartid);
+
 	rc = sbi_platform_early_init(plat, true);
 	if (rc)
 		sbi_hart_hang();
@@ -309,6 +265,12 @@ static void __noreturn init_coldboot(struct sbi_scratch *scratch, u32 hartid)
 	if (rc)
 		sbi_hart_hang();
 
+	rc = sbi_sse_init(scratch, true);
+	if (rc) {
+		sbi_printf("%s: sse init failed (error %d)\n", __func__, rc);
+		sbi_hart_hang();
+	}
+
 	rc = sbi_pmu_init(scratch, true);
 	if (rc) {
 		sbi_printf("%s: pmu init failed (error %d)\n",
@@ -316,6 +278,10 @@ static void __noreturn init_coldboot(struct sbi_scratch *scratch, u32 hartid)
 		sbi_hart_hang();
 	}
 
+	rc = sbi_dbtr_init(scratch, true);
+	if (rc)
+		sbi_hart_hang();
+
 	sbi_boot_print_banner(scratch);
 
 	rc = sbi_irqchip_init(scratch, true);
@@ -343,6 +309,12 @@ static void __noreturn init_coldboot(struct sbi_scratch *scratch, u32 hartid)
 		sbi_hart_hang();
 	}
 
+	rc = sbi_fwft_init(scratch, true);
+	if (rc) {
+		sbi_printf("%s: fwft init failed (error %d)\n", __func__, rc);
+		sbi_hart_hang();
+	}
+
 	/*
 	 * Note: Finalize domains after HSM initialization so that we
 	 * can startup non-root domains.
@@ -356,13 +328,6 @@ static void __noreturn init_coldboot(struct sbi_scratch *scratch, u32 hartid)
 		sbi_hart_hang();
 	}
 
-	rc = sbi_hart_pmp_configure(scratch);
-	if (rc) {
-		sbi_printf("%s: PMP configure failed (error %d)\n",
-			   __func__, rc);
-		sbi_hart_hang();
-	}
-
 	/*
 	 * Note: Platform final initialization should be after finalizing
 	 * domains so that it sees correct domain assignment and PMP
@@ -392,7 +357,18 @@ static void __noreturn init_coldboot(struct sbi_scratch *scratch, u32 hartid)
 
 	sbi_boot_print_hart(scratch, hartid);
 
-	wake_coldboot_harts(scratch, hartid);
+	run_all_tests();
+
+	/*
+	 * Configure PMP at last because if SMEPMP is detected,
+	 * M-mode access to the S/U space will be rescinded.
+	 */
+	rc = sbi_hart_pmp_configure(scratch);
+	if (rc) {
+		sbi_printf("%s: PMP configure failed (error %d)\n",
+			   __func__, rc);
+		sbi_hart_hang();
+	}
 
 	count = sbi_scratch_offset_ptr(scratch, init_count_offset);
 	(*count)++;
@@ -413,6 +389,7 @@ static void __noreturn init_warm_startup(struct sbi_scratch *scratch,
 	count = sbi_scratch_offset_ptr(scratch, entry_count_offset);
 	(*count)++;
 
+	/* Note: This has to be first thing in warmboot init sequence */
 	rc = sbi_hsm_init(scratch, hartid, false);
 	if (rc)
 		sbi_hart_hang();
@@ -425,10 +402,18 @@ static void __noreturn init_warm_startup(struct sbi_scratch *scratch,
 	if (rc)
 		sbi_hart_hang();
 
+	rc = sbi_sse_init(scratch, false);
+	if (rc)
+		sbi_hart_hang();
+
 	rc = sbi_pmu_init(scratch, false);
 	if (rc)
 		sbi_hart_hang();
 
+	rc = sbi_dbtr_init(scratch, false);
+	if (rc)
+		sbi_hart_hang();
+
 	rc = sbi_irqchip_init(scratch, false);
 	if (rc)
 		sbi_hart_hang();
@@ -445,7 +430,7 @@ static void __noreturn init_warm_startup(struct sbi_scratch *scratch,
 	if (rc)
 		sbi_hart_hang();
 
-	rc = sbi_hart_pmp_configure(scratch);
+	rc = sbi_fwft_init(scratch, false);
 	if (rc)
 		sbi_hart_hang();
 
@@ -453,6 +438,14 @@ static void __noreturn init_warm_startup(struct sbi_scratch *scratch,
 	if (rc)
 		sbi_hart_hang();
 
+	/*
+	 * Configure PMP at last because if SMEPMP is detected,
+	 * M-mode access to the S/U space will be rescinded.
+	 */
+	rc = sbi_hart_pmp_configure(scratch);
+	if (rc)
+		sbi_hart_hang();
+
 	count = sbi_scratch_offset_ptr(scratch, init_count_offset);
 	(*count)++;
 
@@ -490,7 +483,7 @@ static void __noreturn init_warmboot(struct sbi_scratch *scratch, u32 hartid)
 	if (hstate == SBI_HSM_STATE_SUSPENDED) {
 		init_warm_resume(scratch, hartid);
 	} else {
-		sbi_ipi_raw_clear(hartid);
+		sbi_ipi_raw_clear(sbi_hartid_to_hartindex(hartid));
 		init_warm_startup(scratch, hartid);
 	}
 }
@@ -511,13 +504,19 @@ static atomic_t coldboot_lottery = ATOMIC_INITIALIZER(0);
  */
 void __noreturn sbi_init(struct sbi_scratch *scratch)
 {
+	u32 i, h;
+	bool hartid_valid		= false;
 	bool next_mode_supported	= false;
 	bool coldboot			= false;
 	u32 hartid			= current_hartid();
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 
-	if ((SBI_HARTMASK_MAX_BITS <= hartid) ||
-	    sbi_platform_hart_invalid(plat, hartid))
+	for (i = 0; i < plat->hart_count; i++) {
+		h = (plat->hart_index2id) ? plat->hart_index2id[i] : i;
+		if (h == hartid)
+			hartid_valid = true;
+	}
+	if (!hartid_valid)
 		sbi_hart_hang();
 
 	switch (scratch->next_mode) {
@@ -614,11 +613,13 @@ void __noreturn sbi_exit(struct sbi_scratch *scratch)
 	u32 hartid			= current_hartid();
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 
-	if (sbi_platform_hart_invalid(plat, hartid))
+	if (!sbi_hartid_valid(hartid))
 		sbi_hart_hang();
 
 	sbi_platform_early_exit(plat);
 
+	sbi_sse_exit(scratch);
+
 	sbi_pmu_exit(scratch);
 
 	sbi_timer_exit(scratch);

lib/sbi/sbi_ipi.c

@@ -27,14 +27,19 @@ struct sbi_ipi_data {
 	unsigned long ipi_type;
 };
 
+_Static_assert(
+	8 * sizeof(((struct sbi_ipi_data*)0)->ipi_type) == SBI_IPI_EVENT_MAX,
+	"type of sbi_ipi_data.ipi_type has changed, please redefine SBI_IPI_EVENT_MAX"
+	);
+
 static unsigned long ipi_data_off;
 static const struct sbi_ipi_device *ipi_dev = NULL;
 static const struct sbi_ipi_event_ops *ipi_ops_array[SBI_IPI_EVENT_MAX];
 
-static int sbi_ipi_send(struct sbi_scratch *scratch, u32 remote_hartid,
+static int sbi_ipi_send(struct sbi_scratch *scratch, u32 remote_hartindex,
 			u32 event, void *data)
 {
-	int ret;
+	int ret = 0;
 	struct sbi_scratch *remote_scratch = NULL;
 	struct sbi_ipi_data *ipi_data;
 	const struct sbi_ipi_event_ops *ipi_ops;
@@ -44,7 +49,7 @@ static int sbi_ipi_send(struct sbi_scratch *scratch, u32 remote_hartid,
 		return SBI_EINVAL;
 	ipi_ops = ipi_ops_array[event];
 
-	remote_scratch = sbi_hartid_to_scratch(remote_hartid);
+	remote_scratch = sbi_hartindex_to_scratch(remote_hartindex);
 	if (!remote_scratch)
 		return SBI_EINVAL;
 
@@ -52,24 +57,34 @@ static int sbi_ipi_send(struct sbi_scratch *scratch, u32 remote_hartid,
 
 	if (ipi_ops->update) {
 		ret = ipi_ops->update(scratch, remote_scratch,
-				      remote_hartid, data);
+				      remote_hartindex, data);
 		if (ret != SBI_IPI_UPDATE_SUCCESS)
 			return ret;
+	} else if (scratch == remote_scratch) {
+		/*
+		 * IPI events with an update() callback are expected to return
+		 * SBI_IPI_UPDATE_BREAK for self-IPIs. For other events, check
+		 * for self-IPI and execute the callback directly here.
+		 */
+		ipi_ops->process(scratch);
+		return 0;
 	}
 
 	/*
 	 * Set IPI type on remote hart's scratch area and
-	 * trigger the interrupt
+	 * trigger the interrupt.
+	 *
+	 * Multiple harts may be trying to send IPI to the
+	 * remote hart so call sbi_ipi_raw_send() only when
+	 * the ipi_type was previously zero.
 	 */
-	atomic_raw_set_bit(event, &ipi_data->ipi_type);
-	smp_wmb();
-
-	if (ipi_dev && ipi_dev->ipi_send)
-		ipi_dev->ipi_send(remote_hartid);
+	if (!__atomic_fetch_or(&ipi_data->ipi_type,
+				BIT(event), __ATOMIC_RELAXED))
+		ret = sbi_ipi_raw_send(remote_hartindex);
 
 	sbi_pmu_ctr_incr_fw(SBI_PMU_FW_IPI_SENT);
 
-	return 0;
+	return ret;
 }
 
 static int sbi_ipi_sync(struct sbi_scratch *scratch, u32 event)
@@ -94,7 +109,7 @@ static int sbi_ipi_sync(struct sbi_scratch *scratch, u32 event)
  */
 int sbi_ipi_send_many(ulong hmask, ulong hbase, u32 event, void *data)
 {
-	int rc;
+	int rc = 0;
 	bool retry_needed;
 	ulong i, m;
 	struct sbi_hartmask target_mask = {0};
@@ -110,14 +125,14 @@ int sbi_ipi_send_many(ulong hmask, ulong hbase, u32 event, void *data)
 
 		for (i = hbase; m; i++, m >>= 1) {
 			if (m & 1UL)
-				sbi_hartmask_set_hart(i, &target_mask);
+				sbi_hartmask_set_hartid(i, &target_mask);
 		}
 	} else {
 		hbase = 0;
 		while (!sbi_hsm_hart_interruptible_mask(dom, hbase, &m)) {
 			for (i = hbase; m; i++, m >>= 1) {
 				if (m & 1UL)
-					sbi_hartmask_set_hart(i, &target_mask);
+					sbi_hartmask_set_hartid(i, &target_mask);
 			}
 			hbase += BITS_PER_LONG;
 		}
@@ -126,19 +141,23 @@ int sbi_ipi_send_many(ulong hmask, ulong hbase, u32 event, void *data)
 	/* Send IPIs */
 	do {
 		retry_needed = false;
-		sbi_hartmask_for_each_hart(i, &target_mask) {
+		sbi_hartmask_for_each_hartindex(i, &target_mask) {
 			rc = sbi_ipi_send(scratch, i, event, data);
+			if (rc < 0)
+				goto done;
 			if (rc == SBI_IPI_UPDATE_RETRY)
 				retry_needed = true;
 			else
-				sbi_hartmask_clear_hart(i, &target_mask);
+				sbi_hartmask_clear_hartindex(i, &target_mask);
+			rc = 0;
 		}
 	} while (retry_needed);
 
+done:
 	/* Sync IPIs */
 	sbi_ipi_sync(scratch, event);
 
-	return 0;
+	return rc;
 }
 
 int sbi_ipi_event_create(const struct sbi_ipi_event_ops *ops)
@@ -214,18 +233,17 @@ void sbi_ipi_process(void)
 	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
 	struct sbi_ipi_data *ipi_data =
 			sbi_scratch_offset_ptr(scratch, ipi_data_off);
-	u32 hartid = current_hartid();
+	u32 hartindex = sbi_hartid_to_hartindex(current_hartid());
 
 	sbi_pmu_ctr_incr_fw(SBI_PMU_FW_IPI_RECVD);
-	if (ipi_dev && ipi_dev->ipi_clear)
-		ipi_dev->ipi_clear(hartid);
+	sbi_ipi_raw_clear(hartindex);
 
 	ipi_type = atomic_raw_xchg_ulong(&ipi_data->ipi_type, 0);
 	ipi_event = 0;
 	while (ipi_type) {
 		if (ipi_type & 1UL) {
 			ipi_ops = ipi_ops_array[ipi_event];
-			if (ipi_ops && ipi_ops->process)
+			if (ipi_ops)
 				ipi_ops->process(scratch);
 		}
 		ipi_type = ipi_type >> 1;
@@ -233,19 +251,41 @@ void sbi_ipi_process(void)
 	}
 }
 
-int sbi_ipi_raw_send(u32 target_hart)
+int sbi_ipi_raw_send(u32 hartindex)
 {
 	if (!ipi_dev || !ipi_dev->ipi_send)
 		return SBI_EINVAL;
 
-	ipi_dev->ipi_send(target_hart);
+	/*
+	 * Ensure that memory or MMIO writes done before
+	 * this function are not observed after the memory
+	 * or MMIO writes done by the ipi_send() device
+	 * callback. This also allows the ipi_send() device
+	 * callback to use relaxed MMIO writes.
+	 *
+	 * This pairs with the wmb() in sbi_ipi_raw_clear().
+	 */
+	wmb();
+
+	ipi_dev->ipi_send(hartindex);
 	return 0;
 }
 
-void sbi_ipi_raw_clear(u32 target_hart)
+void sbi_ipi_raw_clear(u32 hartindex)
 {
 	if (ipi_dev && ipi_dev->ipi_clear)
-		ipi_dev->ipi_clear(target_hart);
+		ipi_dev->ipi_clear(hartindex);
+
+	/*
+	 * Ensure that memory or MMIO writes after this
+	 * function returns are not observed before the
+	 * memory or MMIO writes done by the ipi_clear()
+	 * device callback. This also allows ipi_clear()
+	 * device callback to use relaxed MMIO writes.
+	 *
+	 * This pairs with the wmb() in sbi_ipi_raw_send().
+	 */
+	wmb();
 }
 
 const struct sbi_ipi_device *sbi_ipi_get_device(void)

lib/sbi/sbi_irqchip.c

@@ -10,22 +10,22 @@
 #include <sbi/sbi_irqchip.h>
 #include <sbi/sbi_platform.h>
 
-static int default_irqfn(struct sbi_trap_regs *regs)
+static int default_irqfn(void)
 {
 	return SBI_ENODEV;
 }
 
-static int (*ext_irqfn)(struct sbi_trap_regs *regs) = default_irqfn;
+static int (*ext_irqfn)(void) = default_irqfn;
 
-void sbi_irqchip_set_irqfn(int (*fn)(struct sbi_trap_regs *regs))
+void sbi_irqchip_set_irqfn(int (*fn)(void))
 {
 	if (fn)
 		ext_irqfn = fn;
 }
 
-int sbi_irqchip_process(struct sbi_trap_regs *regs)
+int sbi_irqchip_process(void)
 {
-	return ext_irqfn(regs);
+	return ext_irqfn();
 }
 
 int sbi_irqchip_init(struct sbi_scratch *scratch, bool cold_boot)

lib/sbi/sbi_platform.c

@@ -71,20 +71,3 @@ done:
 	else
 		sbi_strncpy(features_str, "none", nfstr);
 }
-
-u32 sbi_platform_hart_index(const struct sbi_platform *plat, u32 hartid)
-{
-	u32 i;
-
-	if (!plat)
-		return -1U;
-	if (plat->hart_index2id) {
-		for (i = 0; i < plat->hart_count; i++) {
-			if (plat->hart_index2id[i] == hartid)
-				return i;
-		}
-		return -1U;
-	}
-
-	return hartid;
-}

lib/sbi/sbi_pmu.c

@@ -17,6 +17,7 @@
 #include <sbi/sbi_pmu.h>
 #include <sbi/sbi_scratch.h>
 #include <sbi/sbi_string.h>
+#include <sbi/sbi_sse.h>
 
 /** Information about hardware counters */
 struct sbi_pmu_hw_event {
@@ -62,6 +63,8 @@ struct sbi_pmu_hart_state {
 	uint32_t active_events[SBI_PMU_HW_CTR_MAX + SBI_PMU_FW_CTR_MAX];
 	/* Bitmap of firmware counters started */
 	unsigned long fw_counters_started;
+	/* if true, SSE is enabled */
+	bool sse_enabled;
 	/*
 	 * Counter values for SBI firmware events and event codes
 	 * for platform firmware events. Both are mutually exclusive
@@ -74,7 +77,7 @@ struct sbi_pmu_hart_state {
 static unsigned long phs_ptr_offset;
 
 #define pmu_get_hart_state_ptr(__scratch)				\
-	sbi_scratch_read_type((__scratch), void *, phs_ptr_offset)
+	phs_ptr_offset ? sbi_scratch_read_type((__scratch), void *, phs_ptr_offset) : NULL
 
 #define pmu_thishart_state_ptr()					\
 	pmu_get_hart_state_ptr(sbi_scratch_thishart_ptr())
@@ -207,6 +210,9 @@ int sbi_pmu_ctr_fw_read(uint32_t cidx, uint64_t *cval)
 	uint32_t event_code;
 	struct sbi_pmu_hart_state *phs = pmu_thishart_state_ptr();
 
+	if (unlikely(!phs))
+		return SBI_EINVAL;
+
 	event_idx_type = pmu_ctr_validate(phs, cidx, &event_code);
 	if (event_idx_type != SBI_PMU_EVENT_TYPE_FW)
 		return SBI_EINVAL;
@@ -236,8 +242,7 @@ static int pmu_add_hw_event_map(u32 eidx_start, u32 eidx_end, u32 cmap,
 	bool is_overlap;
 	struct sbi_pmu_hw_event *event = &hw_event_map[num_hw_events];
 	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
-	int hw_ctr_avail = sbi_hart_mhpm_count(scratch);
-	uint32_t ctr_avail_mask = ((uint32_t)(~0) >> (32 - (hw_ctr_avail + 3)));
+	uint32_t ctr_avail_mask = sbi_hart_mhpm_mask(scratch) | 0x7;
 
 	/* The first two counters are reserved by priv spec */
 	if (eidx_start > SBI_PMU_HW_INSTRUCTIONS && (cmap & SBI_PMU_FIXED_CTR_MASK))
@@ -298,6 +303,16 @@ int sbi_pmu_add_raw_event_counter_map(uint64_t select, uint64_t select_mask, u32
 				    SBI_PMU_EVENT_RAW_IDX, cmap, select, select_mask);
 }
 
+void sbi_pmu_ovf_irq()
+{
+	/*
+	 * We need to disable LCOFIP before returning to S-mode or we will loop
+	 * on LCOFIP being triggered
+	 */
+	csr_clear(CSR_MIE, MIP_LCOFIP);
+	sbi_sse_inject_event(SBI_SSE_EVENT_LOCAL_PMU);
+}
+
 static int pmu_ctr_enable_irq_hw(int ctr_idx)
 {
 	unsigned long mhpmevent_csr;
@@ -354,8 +369,11 @@ static int pmu_ctr_start_hw(uint32_t cidx, uint64_t ival, bool ival_update)
 	if (cidx >= num_hw_ctrs || cidx == 1)
 		return SBI_EINVAL;
 
-	if (sbi_hart_priv_version(scratch) < SBI_HART_PRIV_VER_1_11)
-		goto skip_inhibit_update;
+	if (sbi_hart_priv_version(scratch) < SBI_HART_PRIV_VER_1_11) {
+		if (ival_update)
+			pmu_ctr_write_hw(cidx, ival);
+		return 0;
+	}
 
 	/*
 	 * Some of the hardware may not support mcountinhibit but perf stat
@@ -369,13 +387,12 @@ static int pmu_ctr_start_hw(uint32_t cidx, uint64_t ival, bool ival_update)
 
 	if (sbi_hart_has_extension(scratch, SBI_HART_EXT_SSCOFPMF))
 		pmu_ctr_enable_irq_hw(cidx);
-	if (pmu_dev && pmu_dev->hw_counter_enable_irq)
-		pmu_dev->hw_counter_enable_irq(cidx);
-	csr_write(CSR_MCOUNTINHIBIT, mctr_inhbt);
-
-skip_inhibit_update:
 	if (ival_update)
 		pmu_ctr_write_hw(cidx, ival);
+	if (pmu_dev && pmu_dev->hw_counter_enable_irq)
+		pmu_dev->hw_counter_enable_irq(cidx);
+
+	csr_write(CSR_MCOUNTINHIBIT, mctr_inhbt);
 
 	return 0;
 }
@@ -431,6 +448,10 @@ int sbi_pmu_ctr_start(unsigned long cbase, unsigned long cmask,
 		      unsigned long flags, uint64_t ival)
 {
 	struct sbi_pmu_hart_state *phs = pmu_thishart_state_ptr();
+
+	if (unlikely(!phs))
+		return SBI_EINVAL;
+
 	int event_idx_type;
 	uint32_t event_code;
 	int ret = SBI_EINVAL;
@@ -441,10 +462,13 @@ int sbi_pmu_ctr_start(unsigned long cbase, unsigned long cmask,
 	if ((cbase + sbi_fls(cmask)) >= total_ctrs)
 		return ret;
 
+	if (flags & SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT)
+		return SBI_ENO_SHMEM;
+
 	if (flags & SBI_PMU_START_FLAG_SET_INIT_VALUE)
 		bUpdate = true;
 
-	for_each_set_bit(i, &cmask, total_ctrs) {
+	for_each_set_bit(i, &cmask, BITS_PER_LONG) {
 		cidx = i + cbase;
 		event_idx_type = pmu_ctr_validate(phs, cidx, &event_code);
 		if (event_idx_type < 0)
@@ -481,6 +505,9 @@ static int pmu_ctr_stop_hw(uint32_t cidx)
 	if (!__test_bit(cidx, &mctr_inhbt)) {
 		__set_bit(cidx, &mctr_inhbt);
 		csr_write(CSR_MCOUNTINHIBIT, mctr_inhbt);
+		if (pmu_dev && pmu_dev->hw_counter_disable_irq) {
+			pmu_dev->hw_counter_disable_irq(cidx);
+		}
 		return 0;
 	} else
 		return SBI_EALREADY_STOPPED;
@@ -528,6 +555,10 @@ int sbi_pmu_ctr_stop(unsigned long cbase, unsigned long cmask,
 		     unsigned long flag)
 {
 	struct sbi_pmu_hart_state *phs = pmu_thishart_state_ptr();
+
+	if (unlikely(!phs))
+		return SBI_EINVAL;
+
 	int ret = SBI_EINVAL;
 	int event_idx_type;
 	uint32_t event_code;
@@ -536,7 +567,10 @@ int sbi_pmu_ctr_stop(unsigned long cbase, unsigned long cmask,
 	if ((cbase + sbi_fls(cmask)) >= total_ctrs)
 		return SBI_EINVAL;
 
-	for_each_set_bit(i, &cmask, total_ctrs) {
+	if (flag & SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT)
+		return SBI_ENO_SHMEM;
+
+	for_each_set_bit(i, &cmask, BITS_PER_LONG) {
 		cidx = i + cbase;
 		event_idx_type = pmu_ctr_validate(phs, cidx, &event_code);
 		if (event_idx_type < 0)
@@ -554,6 +588,10 @@ int sbi_pmu_ctr_stop(unsigned long cbase, unsigned long cmask,
 		}
 	}
 
+	/* Clear MIP_LCOFIP to avoid spurious interrupts */
+	if (phs->sse_enabled)
+		csr_clear(CSR_MIP, MIP_LCOFIP);
+
 	return ret;
 }
 
@@ -595,7 +633,10 @@ static int pmu_update_hw_mhpmevent(struct sbi_pmu_hw_event *hw_evt, int ctr_idx,
 		pmu_dev->hw_counter_disable_irq(ctr_idx);
 
 	/* Update the inhibit flags based on inhibit flags received from supervisor */
-	pmu_update_inhibit_flags(flags, &mhpmevent_val);
+	if (sbi_hart_has_extension(scratch, SBI_HART_EXT_SSCOFPMF))
+		pmu_update_inhibit_flags(flags, &mhpmevent_val);
+	if (pmu_dev && pmu_dev->hw_counter_filter_mode)
+		pmu_dev->hw_counter_filter_mode(flags, ctr_idx);
 
 #if __riscv_xlen == 32
 	csr_write_num(CSR_MHPMEVENT3 + ctr_idx - 3, mhpmevent_val & 0xFFFFFFFF);
@@ -609,7 +650,50 @@ static int pmu_update_hw_mhpmevent(struct sbi_pmu_hw_event *hw_evt, int ctr_idx,
 	return 0;
 }
 
-static int pmu_ctr_find_fixed_fw(unsigned long evt_idx_code)
+static int pmu_fixed_ctr_update_inhibit_bits(int fixed_ctr, unsigned long flags)
+{
+	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
+	uint64_t cfg_val = 0, cfg_csr_no;
+#if __riscv_xlen == 32
+	uint64_t cfgh_csr_no;
+#endif
+	if (!sbi_hart_has_extension(scratch, SBI_HART_EXT_SMCNTRPMF) &&
+		!(pmu_dev && pmu_dev->hw_counter_filter_mode))
+		return fixed_ctr;
+
+	switch (fixed_ctr) {
+	case 0:
+		cfg_csr_no = CSR_MCYCLECFG;
+#if __riscv_xlen == 32
+		cfgh_csr_no = CSR_MCYCLECFGH;
+#endif
+		break;
+	case 2:
+		cfg_csr_no = CSR_MINSTRETCFG;
+#if __riscv_xlen == 32
+		cfgh_csr_no = CSR_MINSTRETCFGH;
+#endif
+		break;
+	default:
+		return SBI_EFAIL;
+	}
+
+	cfg_val |= MHPMEVENT_MINH;
+	if (sbi_hart_has_extension(scratch, SBI_HART_EXT_SMCNTRPMF)) {
+		pmu_update_inhibit_flags(flags, &cfg_val);
+#if __riscv_xlen == 32
+		csr_write_num(cfg_csr_no, cfg_val & 0xFFFFFFFF);
+		csr_write_num(cfgh_csr_no, cfg_val >> BITS_PER_LONG);
+#else
+		csr_write_num(cfg_csr_no, cfg_val);
+#endif
+	}
+	if (pmu_dev && pmu_dev->hw_counter_filter_mode)
+		pmu_dev->hw_counter_filter_mode(flags, fixed_ctr);
+	return fixed_ctr;
+}
+
+static int pmu_ctr_find_fixed_hw(unsigned long evt_idx_code)
 {
 	/* Non-programmables counters are enabled always. No need to do lookup */
 	if (evt_idx_code == SBI_PMU_HW_CPU_CYCLES)
@@ -638,10 +722,10 @@ static int pmu_ctr_find_hw(struct sbi_pmu_hart_state *phs,
 	 * If Sscof is present try to find the programmable counter for
 	 * cycle/instret as well.
 	 */
-	fixed_ctr = pmu_ctr_find_fixed_fw(event_idx);
+	fixed_ctr = pmu_ctr_find_fixed_hw(event_idx);
 	if (fixed_ctr >= 0 &&
 	    !sbi_hart_has_extension(scratch, SBI_HART_EXT_SSCOFPMF))
-		return fixed_ctr;
+		return pmu_fixed_ctr_update_inhibit_bits(fixed_ctr, flags);
 
 	if (sbi_hart_priv_version(scratch) >= SBI_HART_PRIV_VER_1_11)
 		mctr_inhbt = csr_read(CSR_MCOUNTINHIBIT);
@@ -684,7 +768,7 @@ static int pmu_ctr_find_hw(struct sbi_pmu_hart_state *phs,
 		 * Return the fixed counter as they are mandatory anyways.
 		 */
 		if (fixed_ctr >= 0)
-			return fixed_ctr;
+			return pmu_fixed_ctr_update_inhibit_bits(fixed_ctr, flags);
 		else
 			return SBI_EFAIL;
 	}
@@ -738,6 +822,10 @@ int sbi_pmu_ctr_cfg_match(unsigned long cidx_base, unsigned long cidx_mask,
 			  uint64_t event_data)
 {
 	struct sbi_pmu_hart_state *phs = pmu_thishart_state_ptr();
+
+	if (unlikely(!phs))
+		return SBI_EINVAL;
+
 	int ret, event_type, ctr_idx = SBI_ENOTSUPP;
 	u32 event_code;
 
@@ -813,6 +901,9 @@ int sbi_pmu_ctr_incr_fw(enum sbi_pmu_fw_event_code_id fw_id)
 	uint64_t *fcounter = NULL;
 	struct sbi_pmu_hart_state *phs = pmu_thishart_state_ptr();
 
+	if (unlikely(!phs))
+		return 0;
+
 	if (likely(!phs->fw_counters_started))
 		return 0;
 
@@ -843,13 +934,17 @@ int sbi_pmu_ctr_get_info(uint32_t cidx, unsigned long *ctr_info)
 	int width;
 	union sbi_pmu_ctr_info cinfo = {0};
 	struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
+	unsigned long counter_mask = (unsigned long)sbi_hart_mhpm_mask(scratch) |
+				     SBI_PMU_CY_IR_MASK;
 
-	/* Sanity check. Counter1 is not mapped at all */
-	if (cidx >= total_ctrs || cidx == 1)
+	/* Sanity check */
+	if (cidx >= total_ctrs)
 		return SBI_EINVAL;
 
 	/* We have 31 HW counters with 31 being the last index(MHPMCOUNTER31) */
 	if (cidx < num_hw_ctrs) {
+		if (!(__test_bit(cidx, &counter_mask)))
+			return SBI_EINVAL;
 		cinfo.type = SBI_PMU_CTR_TYPE_HW;
 		cinfo.csr = CSR_CYCLE + cidx;
 		/* mcycle & minstret are always 64 bit */
@@ -884,6 +979,7 @@ static void pmu_reset_event_map(struct sbi_pmu_hart_state *phs)
 	for (j = 0; j < SBI_PMU_FW_CTR_MAX; j++)
 		phs->fw_counters_data[j] = 0;
 	phs->fw_counters_started = 0;
+	phs->sse_enabled = 0;
 }
 
 const struct sbi_pmu_device *sbi_pmu_get_device(void)
@@ -901,19 +997,57 @@ void sbi_pmu_set_device(const struct sbi_pmu_device *dev)
 
 void sbi_pmu_exit(struct sbi_scratch *scratch)
 {
+	struct sbi_pmu_hart_state *phs = pmu_get_hart_state_ptr(scratch);
+
 	if (sbi_hart_priv_version(scratch) >= SBI_HART_PRIV_VER_1_11)
 		csr_write(CSR_MCOUNTINHIBIT, 0xFFFFFFF8);
 
 	if (sbi_hart_priv_version(scratch) >= SBI_HART_PRIV_VER_1_10)
 		csr_write(CSR_MCOUNTEREN, -1);
 
-	pmu_reset_event_map(pmu_get_hart_state_ptr(scratch));
+	if (unlikely(!phs))
+		return;
+
+	pmu_reset_event_map(phs);
 }
 
+static void pmu_sse_enable(uint32_t event_id)
+{
+	struct sbi_pmu_hart_state *phs = pmu_thishart_state_ptr();
+
+	phs->sse_enabled = true;
+	csr_clear(CSR_MIDELEG, sbi_pmu_irq_bit());
+	csr_clear(CSR_MIP, MIP_LCOFIP);
+	csr_set(CSR_MIE, MIP_LCOFIP);
+}
+
+static void pmu_sse_disable(uint32_t event_id)
+{
+	struct sbi_pmu_hart_state *phs = pmu_thishart_state_ptr();
+
+	csr_clear(CSR_MIE, MIP_LCOFIP);
+	csr_clear(CSR_MIP, MIP_LCOFIP);
+	csr_set(CSR_MIDELEG, sbi_pmu_irq_bit());
+	phs->sse_enabled = false;
+}
+
+static void pmu_sse_complete(uint32_t event_id)
+{
+	csr_set(CSR_MIE, MIP_LCOFIP);
+}
+
+static const struct sbi_sse_cb_ops pmu_sse_cb_ops = {
+	.enable_cb = pmu_sse_enable,
+	.disable_cb = pmu_sse_disable,
+	.complete_cb = pmu_sse_complete,
+};
+
 int sbi_pmu_init(struct sbi_scratch *scratch, bool cold_boot)
 {
+	int hpm_count = sbi_fls(sbi_hart_mhpm_mask(scratch));
 	struct sbi_pmu_hart_state *phs;
 	const struct sbi_platform *plat;
+	int rc;
 
 	if (cold_boot) {
 		hw_event_map = sbi_calloc(sizeof(*hw_event_map),
@@ -929,13 +1063,26 @@ int sbi_pmu_init(struct sbi_scratch *scratch, bool cold_boot)
 
 		plat = sbi_platform_ptr(scratch);
 		/* Initialize hw pmu events */
-		sbi_platform_pmu_init(plat);
+		rc = sbi_platform_pmu_init(plat);
+		if (rc)
+			sbi_dprintf("%s: platform pmu init failed "
+				    "(error %d)\n", __func__, rc);
 
 		/* mcycle & minstret is available always */
-		num_hw_ctrs = sbi_hart_mhpm_count(scratch) + 3;
+		if (!hpm_count)
+			/* Only CY, TM & IR are implemented in the hw */
+			num_hw_ctrs = 3;
+		else
+			num_hw_ctrs = hpm_count + 1;
+
+		if (num_hw_ctrs > SBI_PMU_HW_CTR_MAX)
+			return SBI_EINVAL;
+
 		total_ctrs = num_hw_ctrs + SBI_PMU_FW_CTR_MAX;
 	}
 
+	sbi_sse_set_cb_ops(SBI_SSE_EVENT_LOCAL_PMU, &pmu_sse_cb_ops);
+
 	phs = pmu_get_hart_state_ptr(scratch);
 	if (!phs) {
 		phs = sbi_zalloc(sizeof(*phs));

lib/sbi/sbi_scratch.c

@@ -14,29 +14,40 @@
 #include <sbi/sbi_scratch.h>
 #include <sbi/sbi_string.h>
 
-u32 last_hartid_having_scratch = SBI_HARTMASK_MAX_BITS - 1;
-struct sbi_scratch *hartid_to_scratch_table[SBI_HARTMASK_MAX_BITS] = { 0 };
+u32 last_hartindex_having_scratch = 0;
+u32 hartindex_to_hartid_table[SBI_HARTMASK_MAX_BITS + 1] = { -1U };
+struct sbi_scratch *hartindex_to_scratch_table[SBI_HARTMASK_MAX_BITS + 1] = { 0 };
 
 static spinlock_t extra_lock = SPIN_LOCK_INITIALIZER;
 static unsigned long extra_offset = SBI_SCRATCH_EXTRA_SPACE_OFFSET;
 
+u32 sbi_hartid_to_hartindex(u32 hartid)
+{
+	u32 i;
+
+	for (i = 0; i <= last_hartindex_having_scratch; i++)
+		if (hartindex_to_hartid_table[i] == hartid)
+			return i;
+
+	return -1U;
+}
+
 typedef struct sbi_scratch *(*hartid2scratch)(ulong hartid, ulong hartindex);
 
 int sbi_scratch_init(struct sbi_scratch *scratch)
 {
-	u32 i;
+	u32 i, h;
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 
-	for (i = 0; i < SBI_HARTMASK_MAX_BITS; i++) {
-		if (sbi_platform_hart_invalid(plat, i))
-			continue;
-		hartid_to_scratch_table[i] =
-			((hartid2scratch)scratch->hartid_to_scratch)(i,
-					sbi_platform_hart_index(plat, i));
-		if (hartid_to_scratch_table[i])
-			last_hartid_having_scratch = i;
+	for (i = 0; i < plat->hart_count; i++) {
+		h = (plat->hart_index2id) ? plat->hart_index2id[i] : i;
+		hartindex_to_hartid_table[i] = h;
+		hartindex_to_scratch_table[i] =
+			((hartid2scratch)scratch->hartid_to_scratch)(h, i);
 	}
 
+	last_hartindex_having_scratch = plat->hart_count - 1;
+
 	return 0;
 }
 
@@ -74,8 +85,8 @@ done:
 	spin_unlock(&extra_lock);
 
 	if (ret) {
-		for (i = 0; i <= sbi_scratch_last_hartid(); i++) {
-			rscratch = sbi_hartid_to_scratch(i);
+		for (i = 0; i <= sbi_scratch_last_hartindex(); i++) {
+			rscratch = sbi_hartindex_to_scratch(i);
 			if (!rscratch)
 				continue;
 			ptr = sbi_scratch_offset_ptr(rscratch, ret);